#!/usr/bin/env bash # Point MM installation at target DB (CNPG pooler or Selectel Managed) and wait until healthy. # Applies canary learnings: long probes, more memory, stale session cleanup (CNPG), orphan RS. # Usage: cutover-one.sh [cnpg|selectel] [--mem-req 256Mi] set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "${DIR}/lib.sh" [[ $# -ge 2 ]] || die "usage: $0 [cnpg|selectel|loop-cnpg-*] [--mem-req 256Mi]" CTX="$1" INSTALLATION_RAW="$2" shift 2 ARGS=() while [[ $# -gt 0 ]]; do case "$1" in --mem-req|--memory-request) CUTOVER_MEMORY_REQUEST="$(normalize_mem_req "${2:-}")" export CUTOVER_MEMORY_REQUEST shift 2 ;; --mem-req=*|--memory-request=*) CUTOVER_MEMORY_REQUEST="$(normalize_mem_req "${1#*=}")" export CUTOVER_MEMORY_REQUEST shift ;; *) ARGS+=("$1") shift ;; esac done set -- "${ARGS[@]+"${ARGS[@]}"}" parse_target_dsn_args "$@" if [[ -n "${PARSE_TARGET}" ]]; then TARGET="${PARSE_TARGET}" fi if [[ -n "${PARSE_CNPG_PROFILE:-}" ]]; then CNPG_CLUSTER_PROFILE="${PARSE_CNPG_PROFILE}" fi INSTALLATION="$(resolve_installation "$CTX" "$INSTALLATION_RAW")" if [[ -z "${PARSE_TARGET:-}" ]]; then TARGET="$(load_installation_target "$INSTALLATION")" fi LOG_FILE="${DIR}/${LOG_DIR}/${INSTALLATION}.cutover.log" exec > >(tee -a "$LOG_FILE") 2>&1 resolve_cnpg_cluster_profile "$INSTALLATION" resolve_target "${TARGET:-cnpg}" log "=== cutover-one ${CTX} ${INSTALLATION} target=${TARGET_KIND} cnpg=${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER:-?}} endpoint=${TARGET_HOST}:${TARGET_PORT} mem-req=${CUTOVER_MEMORY_REQUEST} ===" DSN="$(get_secret_dsn "$CTX" "$INSTALLATION")" parse_dsn "$DSN" NEW_DSN="$(build_target_dsn "$SRC_USER" "$SRC_PASS" "$SRC_DB")" NEW_DSN_RO="$(build_target_dsn_ro "$SRC_USER" "$SRC_PASS" "$SRC_DB")" EXPECTED_HP="${TARGET_HOST}:${TARGET_PORT}" EXPECTED_HP_RO="${TARGET_HOST_RO:-${TARGET_HOST}}:${TARGET_PORT_RO:-${TARGET_PORT}}" if [[ "$SRC_HOST" == "$TARGET_HOST" && "${SRC_PORT}" == "${TARGET_PORT}" ]]; then log "Secret already points at ${TARGET_KIND} ${EXPECTED_HP} — will still retune probes/resources and bounce pods" fi # Drop leftover session-pooler backends before MM reconnects (CNPG only) if [[ "$TARGET_KIND" == "cnpg" ]]; then terminate_stale_cnpg_sessions "$SRC_DB" "$SRC_USER" "${CUTOVER_STALE_IDLE_MINUTES}" fi log "Updating secret ${INSTALLATION}-db-credentials → ${EXPECTED_HP}/${SRC_DB} (ro ${EXPECTED_HP_RO})" kubectl --context "$CTX" -n "$SOURCE_NS" create secret generic "${INSTALLATION}-db-credentials" \ --from-literal=DB_CONNECTION_STRING="$NEW_DSN" \ --dry-run=client -o yaml | kubectl --context "$CTX" -n "$SOURCE_NS" apply -f - # Patch MM CR: DSN env + probes + memory (single replace) TMP_MM="$(mktemp)" kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o json >"$TMP_MM" python3 - "$TMP_MM" "$NEW_DSN" "$NEW_DSN_RO" \ "${CUTOVER_LIVENESS_INITIAL_DELAY}" "${CUTOVER_LIVENESS_FAILURE_THRESHOLD}" \ "${CUTOVER_READINESS_INITIAL_DELAY}" "${CUTOVER_READINESS_FAILURE_THRESHOLD}" \ "${CUTOVER_MEMORY_LIMIT}" "${CUTOVER_MEMORY_REQUEST}" "${CUTOVER_CPU_REQUEST}" <<'PY' import json, os, sys path, new_dsn, new_dsn_ro = sys.argv[1], sys.argv[2], sys.argv[3] live_delay, live_fail = int(sys.argv[4]), int(sys.argv[5]) ready_delay, ready_fail = int(sys.argv[6]), int(sys.argv[7]) mem_limit, mem_req, cpu_req = sys.argv[8], sys.argv[9], sys.argv[10] with open(path) as f: mm = json.load(f) spec = mm.setdefault("spec", {}) # Probes/ресурсы только поднимаем до минимума: свои (больше) у крупных инстансов не трогаем, визор о них не знает def at_least(d, key, val): if int(d.get(key) or 0) < val: d[key] = val probes = spec.setdefault("probes", {}) lp = probes.setdefault("livenessProbe", {}) rp = probes.setdefault("readinessProbe", {}) at_least(lp, "initialDelaySeconds", live_delay); at_least(lp, "failureThreshold", live_fail) at_least(rp, "initialDelaySeconds", ready_delay); at_least(rp, "failureThreshold", ready_fail) sched = spec.setdefault("scheduling", {}) res = sched.setdefault("resources", {}) res.setdefault("limits", {}).setdefault("memory", mem_limit) requests = res.setdefault("requests", {}) requests.setdefault("memory", mem_req) requests.setdefault("cpu", cpu_req) # Force inline DSN via mattermostEnv only (spec.env is not a valid MM CR field) def upsert_env(envs, name, value): for e in envs: if e.get("name") == name: e["value"] = value e.pop("valueFrom", None) return envs envs.append({"name": name, "value": value}) return envs envs = list(spec.get("mattermostEnv") or []) envs = upsert_env(envs, "MM_SQLSETTINGS_DATASOURCE", new_dsn) envs = upsert_env(envs, "MM_SQLSETTINGS_DATASOURCEREPLICAS", new_dsn_ro) spec["mattermostEnv"] = envs # Drop invalid field if present from older script runs / edits spec.pop("env", None) # CUTOVER_REPLICAS (loopctl db move --scale-0) — вернуть число реплик до scale-0. # Иначе replicas не трогаем: приостановленный (0) инстанс остаётся на паузе, ему меняем только DSN. want = os.environ.get("CUTOVER_REPLICAS") if want: spec["replicas"] = int(want) print("replicas=%s" % spec.get("replicas")) with open(path, "w") as f: json.dump(mm, f) print("patched probes+resources+datasource") PY log "Replacing MM CR (DSN + probes + resources)" kubectl --context "$CTX" -n "$SOURCE_NS" replace -f "$TMP_MM" rm -f "$TMP_MM" # Also setenv on Deployment immediately (operator reconcile can lag) if kubectl --context "$CTX" -n "$SOURCE_NS" get deploy "$INSTALLATION" >/dev/null 2>&1; then log "Forcing deploy/${INSTALLATION} MM_SQLSETTINGS_* → ${TARGET_KIND} rw=${EXPECTED_HP} ro=${EXPECTED_HP_RO}" ENV_FILE="$(mktemp)" printf 'MM_SQLSETTINGS_DATASOURCE=%s\nMM_SQLSETTINGS_DATASOURCEREPLICAS=%s\n' "$NEW_DSN" "$NEW_DSN_RO" >"$ENV_FILE" kubectl --context "$CTX" -n "$SOURCE_NS" set env "deploy/${INSTALLATION}" --env - <"$ENV_FILE" >/dev/null rm -f "$ENV_FILE" # ensure deploy is not left at 0 while MM CR says 1 dep_rep="$(kubectl --context "$CTX" -n "$SOURCE_NS" get deploy "$INSTALLATION" \ -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0)" if [[ "${dep_rep:-0}" -lt 1 && "${CUTOVER_REPLICAS:-0}" -ge 1 ]]; then log "Scaling deploy/${INSTALLATION} replicas 0→1" kubectl --context "$CTX" -n "$SOURCE_NS" scale deploy "$INSTALLATION" --replicas=1 >/dev/null fi fi scale_orphan_replicasets "$CTX" "$INSTALLATION" log "Deleting MM pods to pick up new DSN" kubectl --context "$CTX" -n "$SOURCE_NS" delete pod \ -l "installation.mattermost.com/installation=${INSTALLATION}" --wait=false || true # Wait until stable + app ready + DSN on CNPG (+ HTTPS if endpoint known) stale_swept=0 for i in $(seq 1 "${CUTOVER_WAIT_ATTEMPTS}"); do state="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \ -o jsonpath='{.status.state}' 2>/dev/null || echo unknown)" endpoint="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \ -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" ready="false" if mm_app_ready "$CTX" "$INSTALLATION"; then ready="true" fi dsn_hp="$(mm_pod_dsn_hostport "$CTX" "$INSTALLATION" 2>/dev/null || echo no-pod)" ping_ok="no" if mm_https_ping_ok "$endpoint"; then ping_ok="yes" fi restarts="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \ -o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null || echo 0)" mm_rep="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \ -o jsonpath='{.spec.replicas}' 2>/dev/null || echo '?')" log "wait ${i}/${CUTOVER_WAIT_ATTEMPTS}: state=${state} ready=${ready} dsn=${dsn_hp} expected=${EXPECTED_HP} replicas=${mm_rep} https=${ping_ok} restarts=${restarts}" # Приостановленный инстанс не будим: DSN уже в секрете и CR, подов ждать нечего if [[ "${mm_rep:-0}" == "0" ]]; then log "SUCCESS cutover ${INSTALLATION}: на паузе (replicas=0), DSN переключён, поды не поднимаю" mark_cutover_done "$INSTALLATION" exit 0 fi # Periodically collapse duplicate ReplicaSets if (( i % 3 == 0 )); then scale_orphan_replicasets "$CTX" "$INSTALLATION" fi # If pods keep crashing, clear stale DB sessions once if [[ "$TARGET_KIND" == "cnpg" && "${restarts:-0}" -ge 2 && "$stale_swept" -eq 0 ]]; then log "Restarts=${restarts}: sweeping stale CNPG sessions again" terminate_stale_cnpg_sessions "$SRC_DB" "$SRC_USER" "${CUTOVER_STALE_IDLE_MINUTES}" stale_swept=1 fi # Wrong DSN → force again if [[ "$dsn_hp" != "no-pod" && "$dsn_hp" != "unknown" && "$dsn_hp" != "${EXPECTED_HP}" && "$dsn_hp" != ":" ]]; then log "Pod DSN is ${dsn_hp}, expected ${EXPECTED_HP} — re-applying deploy env" ENV_FILE="$(mktemp)" printf 'MM_SQLSETTINGS_DATASOURCE=%s\nMM_SQLSETTINGS_DATASOURCEREPLICAS=%s\n' "$NEW_DSN" "$NEW_DSN_RO" >"$ENV_FILE" kubectl --context "$CTX" -n "$SOURCE_NS" set env "deploy/${INSTALLATION}" --env - <"$ENV_FILE" >/dev/null 2>&1 || true rm -f "$ENV_FILE" kubectl --context "$CTX" -n "$SOURCE_NS" delete pod \ -l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" --wait=false >/dev/null 2>&1 || true fi if [[ "$ready" == "true" && "$dsn_hp" == "${EXPECTED_HP}" ]]; then if [[ "$state" == "stable" ]] || [[ "$ping_ok" == "yes" ]]; then log "SUCCESS cutover ${INSTALLATION} (state=${state} endpoint=${endpoint} dsn=${dsn_hp})" mark_cutover_done "$INSTALLATION" exit 0 fi # ready+correct DSN but state still reconciling — accept after HTTPS OK or enough ready cycles if [[ "$i" -ge 12 && "$ready" == "true" ]]; then # in-pod ping as fallback when ingress endpoint empty pod="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "$pod" ]] && kubectl --context "$CTX" -n "$SOURCE_NS" exec "$pod" -- \ curl -sS -m 5 http://127.0.0.1:8065/api/v4/system/ping 2>/dev/null | grep -q OK; then log "SUCCESS cutover ${INSTALLATION} (app ready+ping; state=${state})" mark_cutover_done "$INSTALLATION" exit 0 fi fi fi sleep "${CUTOVER_WAIT_SLEEP_SEC}" done log "Diagnostics:" kubectl --context "$CTX" -n "$SOURCE_NS" get pods -l "installation.mattermost.com/installation=${INSTALLATION}" || true kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o jsonpath='state={.status.state} endpoint={.status.endpoint}{"\n"}' || true pod="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "$pod" ]]; then kubectl --context "$CTX" -n "$SOURCE_NS" logs "$pod" --tail=40 || true fi die "cutover did not become stable in time for ${INSTALLATION}"