#!/usr/bin/env bash # Shared helpers for MDB → CNPG migration scripts. set -euo pipefail MIGRATE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "${MIGRATE_DIR}/config.env" mkdir -p "${MIGRATE_DIR}/${LOG_DIR}" DONE_PATH="${MIGRATE_DIR}/${DONE_FILE}" FAILED_PATH="${MIGRATE_DIR}/${FAILED_FILE}" LIST_PATH="${MIGRATE_DIR}/${TRANSFER_LIST}" REGISTRY_PATH="${MIGRATE_DIR}/${REGISTRY_FILE}" touch "${DONE_PATH}" "${FAILED_PATH}" if [[ ! -f "${REGISTRY_PATH}" ]]; then printf 'loop_id\tdomain\tkube_context\tyc_cluster\ttarget_cluster\tsize_bytes\tsize_pretty\tbatch_id\tstatus\tnotes\n' \ >"${REGISTRY_PATH}" fi log() { printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } die() { printf '[%s] ERROR: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; exit 1; } # Normalize memory quantity for k8s (250M|256Mi|1G|512 → 250Mi|256Mi|1Gi|512Mi). normalize_mem_req() { local v="${1:-}" [[ -n "$v" ]] || { printf '%s\n' ""; return 0; } if [[ "$v" =~ ^[0-9]+$ ]]; then printf '%sMi\n' "$v" return 0 fi if [[ "$v" =~ ^([0-9]+)([KkMmGg])$ ]]; then printf '%s%si\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" return 0 fi if [[ "$v" =~ ^[0-9]+([Ee][0-9]+)?$ || "$v" =~ ^[0-9]+([KkMmGg]i)$ ]]; then printf '%s\n' "$v" return 0 fi die "invalid memory quantity '${v}' (examples: 256Mi, 250M, 1Gi)" } registry_set() { # registry_set key=value [key=value ...] # done.txt/failed.txt mirrors are updated inside registry_lib.py (flock-safe, shrink-guarded). local installation="$1" shift [[ -n "$installation" ]] || return 0 local attempt=1 err="${MIGRATE_DIR}/${LOG_DIR}/.registry-set.err" while ! python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" set "$installation" "$@" >/dev/null 2>>"$err"; do if [[ "$attempt" -ge 8 ]]; then log "WARN registry_set failed for ${installation} after ${attempt} attempts: $(tail -3 "$err" 2>/dev/null | tr '\n' ' ')" return 1 fi sleep "0.$((RANDOM % 5 + 1))" attempt=$((attempt + 1)) done return 0 } registry_get() { # registry_get [field] python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" get "$@" } sql_quote() { # Escape single quotes for SQL string literals printf "%s" "${1//\'/\'\'}" } url_encode() { python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1" } parse_dsn() { # $1 → exports SRC_USER SRC_PASS SRC_HOST SRC_PORT SRC_DB SRC_QUERY local dsn="${1:-}" [[ -n "$dsn" ]] || die "parse_dsn: empty DSN" local tmp tmp="$(mktemp)" python3 - "$dsn" "$tmp" <<'PY' import sys, urllib.parse, shlex u = urllib.parse.urlparse(sys.argv[1]) db = (u.path or "/").lstrip("/") with open(sys.argv[2], "w") as f: f.write("export SRC_USER=%s\n" % shlex.quote(u.username or "")) f.write("export SRC_PASS=%s\n" % shlex.quote(u.password or "")) f.write("export SRC_HOST=%s\n" % shlex.quote(u.hostname or "")) f.write("export SRC_PORT=%s\n" % shlex.quote(str(u.port or 5432))) f.write("export SRC_DB=%s\n" % shlex.quote(db)) f.write("export SRC_QUERY=%s\n" % shlex.quote(u.query or "")) PY # shellcheck disable=SC1090 source "$tmp" rm -f "$tmp" } get_secret_dsn() { local ctx="$1" installation="$2" kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${installation}-db-credentials" \ -o jsonpath='{.data.DB_CONNECTION_STRING}' | base64 --decode } # Resolve cloud UUID / partial id → Mattermost installation name (e.g. loop036d0e599ad3). # Accepts: loop… id, raw UUID, or unique substring of the installation name. resolve_installation() { local ctx="$1" raw="$2" local id="$raw" # Already a MM installation with credentials secret if kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${id}-db-credentials" >/dev/null 2>&1; then echo "$id" return 0 fi # UUID → loop + last 12 hex chars (Loop naming convention) local hex last12 candidate hex="$(printf '%s' "$raw" | tr -d '-' | tr '[:upper:]' '[:lower:]')" if [[ "$hex" =~ ^[0-9a-f]{32}$ ]]; then last12="${hex: -12}" candidate="loop${last12}" if kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${candidate}-db-credentials" >/dev/null 2>&1; then log "Resolved ${raw} → ${candidate}" >&2 echo "$candidate" return 0 fi fi # Unique substring match on MM CR names local matches matches="$(kubectl --context "$ctx" -n "$SOURCE_NS" get mm -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ | grep -Fi "$raw" || true)" local count count="$(printf '%s\n' "$matches" | grep -c . || true)" if [[ "$count" == "1" ]]; then candidate="$(printf '%s\n' "$matches" | head -1)" log "Resolved ${raw} → ${candidate}" >&2 echo "$candidate" return 0 fi if [[ "$count" -gt 1 ]]; then die "ambiguous installation '${raw}', matches: $(printf '%s' "$matches" | tr '\n' ' ')" fi die "installation not found for '${raw}' (need MM id like loop036d0e599ad3, or cloud UUID). Secret ${id}-db-credentials missing in ${ctx}/${SOURCE_NS}" } is_done() { # Source of truth: registry status only (migrated|verified|cutover|done). # done.txt is a write-only mirror and must NOT gate re-runs. local installation="$1" python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" is-done "$installation" 2>/dev/null } mark_done() { local installation="$1" local cur # do not downgrade cutover/done/verified cur="$(registry_get "$installation" status 2>/dev/null || true)" if [[ "$cur" == "done" || "$cur" == "cutover" || "$cur" == "verified" ]]; then registry_set "$installation" "status=${cur}" || true return 0 fi registry_set "$installation" "status=migrated" || true } mark_failed() { local installation="$1" registry_set "$installation" "status=failed" || true } mark_cutover_done() { local installation="$1" registry_set "$installation" "status=done" || true } resolve_cnpg_host() { # CNPG_PREFER_CONFIG_HOST=1 → always use CNPG_HOST (public VIP / DNAT; InternalIP may be unreachable) if [[ "${CNPG_PREFER_CONFIG_HOST:-0}" == "1" ]]; then echo "${CNPG_HOST}" return 0 fi local pooler="${CNPG_POOLER_RW:-pooler-cnpg-wilix-rw}" local ip ip="$(kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" get pods \ -l "cnpg.io/poolerName=${pooler}" \ -o jsonpath='{.items[0].status.hostIP}' 2>/dev/null || true)" if [[ -n "$ip" ]]; then echo "$ip" else echo "${CNPG_HOST}" fi } cnpg_primary_pod() { kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" \ get clusters.postgresql.cnpg.io "${CNPG_CLUSTER}" \ -o jsonpath='{.status.currentPrimary}' } cnpg_admin_psql() { # Run SQL as postgres superuser inside primary pod. Args: SQL string local sql="$1" local pod pod="$(cnpg_primary_pod)" [[ -n "$pod" ]] || die "cannot resolve CNPG primary" kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \ psql -U postgres -d postgres -v ON_ERROR_STOP=1 -c "$sql" } cnpg_psql_db() { local db="$1" sql="$2" local pod pod="$(cnpg_primary_pod)" kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \ psql -U postgres -d "$db" -v ON_ERROR_STOP=1 -c "$sql" } ensure_role_and_db() { local user="$1" pass="$2" db="$3" local quser qpass qdb quser="$(sql_quote "$user")" qpass="$(sql_quote "$pass")" qdb="$(sql_quote "$db")" log "Ensuring role/database ${user}/${db} on CNPG" cnpg_admin_psql " DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${quser}') THEN CREATE ROLE \"${quser}\" LOGIN PASSWORD '${qpass}' CREATEDB; ELSE ALTER ROLE \"${quser}\" WITH LOGIN PASSWORD '${qpass}' CREATEDB; END IF; END \$\$; " # Create DB if missing local exists exists="$(kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$(cnpg_primary_pod)" -c postgres -- \ psql -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='${qdb}'" | tr -d '[:space:]')" if [[ "$exists" != "1" ]]; then cnpg_admin_psql "CREATE DATABASE \"${qdb}\" OWNER \"${quser}\" ENCODING 'UTF8' LC_COLLATE 'ru_RU.UTF-8' LC_CTYPE 'ru_RU.UTF-8' TEMPLATE template0;" else cnpg_admin_psql "ALTER DATABASE \"${qdb}\" OWNER TO \"${quser}\";" fi cnpg_psql_db "$db" "CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"; DO \$\$ BEGIN CREATE EXTENSION IF NOT EXISTS vector; EXCEPTION WHEN OTHERS THEN NULL; END \$\$;" install_pooler_auth "$db" cnpg_admin_psql "GRANT ALL PRIVILEGES ON DATABASE \"${qdb}\" TO \"${quser}\";" } install_pooler_auth() { # Must be owned by a superuser (vizorcnpgadmin); tenant-owned SECURITY DEFINER cannot read pg_shadow. local db="$1" cnpg_psql_db "$db" " CREATE OR REPLACE FUNCTION public.user_search(uname text) RETURNS TABLE(usename name, passwd text) LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog AS \$\$SELECT usename, passwd FROM pg_catalog.pg_shadow WHERE usename=\$1;\$\$; ALTER FUNCTION public.user_search(text) OWNER TO vizorcnpgadmin; REVOKE ALL ON FUNCTION public.user_search(text) FROM PUBLIC; GRANT USAGE ON SCHEMA public TO cnpg_pooler_pgbouncer; GRANT EXECUTE ON FUNCTION public.user_search(text) TO cnpg_pooler_pgbouncer; " } set_role_superuser() { local user="$1" onoff="$2" # onoff: ON|OFF local quser quser="$(sql_quote "$user")" if [[ "$onoff" == "ON" ]]; then cnpg_admin_psql "ALTER ROLE \"${quser}\" WITH SUPERUSER;" else cnpg_admin_psql "ALTER ROLE \"${quser}\" WITH NOSUPERUSER;" fi } # Apply CNPG endpoint/admin profile by logical cluster name (registry target_cluster). # Does not touch TRANSFER_* knobs. Override anytime with CNPG_CLUSTER_PROFILE=... apply_cnpg_cluster_profile() { local name="${1:-}" [[ -n "$name" ]] || return 0 # normalize aliases case "$name" in wilix|cnpg-wilix|loop-cnpg-wilix) name="loop-cnpg-wilix-1" ;; lm|cnpg-lm) name="loop-cnpg-lm" ;; ff|cnpg-ff) name="loop-cnpg-ff" ;; lp3|lp-3|cnpg-lp3|NEW|new|cnpg-new|loop-cnpg-NEW) name="loop-cnpg-lp3" ;; esac case "$name" in loop-cnpg-wilix-1) CNPG_HOST="${CNPG_WILIX_HOST:-95.143.178.247}" CNPG_PORT="${CNPG_WILIX_PORT:-31574}" CNPG_HOST_RO="${CNPG_WILIX_HOST_RO:-${CNPG_HOST}}" CNPG_PORT_RO="${CNPG_WILIX_PORT_RO:-31575}" CNPG_SSLMODE="${CNPG_WILIX_SSLMODE:-disable}" CNPG_PREFER_CONFIG_HOST="${CNPG_WILIX_PREFER_CONFIG_HOST:-0}" CNPG_KUBE_CONTEXT="${CNPG_WILIX_KUBE_CONTEXT:-local}" CNPG_NAMESPACE="${CNPG_WILIX_NAMESPACE:-cnpg-loop-wilix}" CNPG_CLUSTER="loop-cnpg-wilix-1" CNPG_ADMIN_SECRET="${CNPG_WILIX_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}" CNPG_POOLER_RW="pooler-cnpg-wilix-rw" CNPG_POOLER_RO="pooler-cnpg-wilix-ro" ;; loop-cnpg-lm) CNPG_HOST="${CNPG_LM_HOST:-135.106.201.40}" CNPG_PORT="${CNPG_LM_PORT:-30434}" CNPG_HOST_RO="${CNPG_LM_HOST_RO:-135.106.201.40}" CNPG_PORT_RO="${CNPG_LM_PORT_RO:-30435}" CNPG_SSLMODE="${CNPG_LM_SSLMODE:-disable}" CNPG_PREFER_CONFIG_HOST="${CNPG_LM_PREFER_CONFIG_HOST:-1}" CNPG_KUBE_CONTEXT="${CNPG_LM_KUBE_CONTEXT:-loop-cnpg}" CNPG_NAMESPACE="${CNPG_LM_NAMESPACE:-cnpg-loop-lm}" CNPG_CLUSTER="loop-cnpg-lm" CNPG_ADMIN_SECRET="${CNPG_LM_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}" CNPG_POOLER_RW="pooler-cnpg-lm-rw" CNPG_POOLER_RO="pooler-cnpg-lm-ro" ;; loop-cnpg-ff) CNPG_HOST="${CNPG_FF_HOST:-135.106.201.38}" CNPG_PORT="${CNPG_FF_PORT:-30432}" CNPG_HOST_RO="${CNPG_FF_HOST_RO:-135.106.201.38}" CNPG_PORT_RO="${CNPG_FF_PORT_RO:-30433}" CNPG_SSLMODE="${CNPG_FF_SSLMODE:-disable}" CNPG_PREFER_CONFIG_HOST="${CNPG_FF_PREFER_CONFIG_HOST:-1}" CNPG_KUBE_CONTEXT="${CNPG_FF_KUBE_CONTEXT:-loop-cnpg}" CNPG_NAMESPACE="${CNPG_FF_NAMESPACE:-cnpg-loop-ff}" CNPG_CLUSTER="loop-cnpg-ff" CNPG_ADMIN_SECRET="${CNPG_FF_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}" CNPG_POOLER_RW="pooler-cnpg-ff-rw" CNPG_POOLER_RO="pooler-cnpg-ff-ro" ;; loop-cnpg-lp3) CNPG_HOST="${CNPG_LP3_HOST:-135.106.201.42}" CNPG_PORT="${CNPG_LP3_PORT:-30436}" CNPG_HOST_RO="${CNPG_LP3_HOST_RO:-135.106.201.42}" CNPG_PORT_RO="${CNPG_LP3_PORT_RO:-30437}" CNPG_SSLMODE="${CNPG_LP3_SSLMODE:-disable}" CNPG_PREFER_CONFIG_HOST="${CNPG_LP3_PREFER_CONFIG_HOST:-1}" CNPG_KUBE_CONTEXT="${CNPG_LP3_KUBE_CONTEXT:-loop-cnpg}" CNPG_NAMESPACE="${CNPG_LP3_NAMESPACE:-cnpg-loop-lp3}" CNPG_CLUSTER="loop-cnpg-lp3" CNPG_ADMIN_SECRET="${CNPG_LP3_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}" CNPG_POOLER_RW="pooler-cnpg-lp3-rw" CNPG_POOLER_RO="pooler-cnpg-lp3-ro" ;; *) die "unknown CNPG cluster profile '${name}' (loop-cnpg-wilix-1|loop-cnpg-lm|loop-cnpg-ff|loop-cnpg-lp3)" ;; esac CNPG_CLUSTER_PROFILE="$name" export CNPG_HOST CNPG_PORT CNPG_HOST_RO CNPG_PORT_RO CNPG_SSLMODE CNPG_PREFER_CONFIG_HOST \ CNPG_KUBE_CONTEXT CNPG_NAMESPACE CNPG_CLUSTER CNPG_ADMIN_SECRET CNPG_POOLER_RW CNPG_POOLER_RO \ CNPG_CLUSTER_PROFILE log "CNPG profile=${CNPG_CLUSTER_PROFILE} rw=${CNPG_HOST}:${CNPG_PORT} ro=${CNPG_HOST_RO}:${CNPG_PORT_RO} admin=${CNPG_KUBE_CONTEXT}/${CNPG_NAMESPACE}/${CNPG_CLUSTER}" } # Pick CNPG profile: explicit CNPG_CLUSTER_PROFILE > registry.target_cluster > logs/.cnpg-cluster # Skip when TARGET=selectel or CNPG_SKIP_REGISTRY=1. resolve_cnpg_cluster_profile() { local installation="${1:-}" local profile="${CNPG_CLUSTER_PROFILE:-}" local from="env" if [[ "${TARGET:-cnpg}" == "selectel" || "${CNPG_SKIP_REGISTRY:-0}" == "1" ]]; then return 0 fi if [[ -z "$profile" && -n "$installation" ]]; then profile="$(registry_get "$installation" target_cluster 2>/dev/null || true)" [[ -n "$profile" ]] && from="registry" fi if [[ -z "$profile" && -n "$installation" ]]; then local f="${MIGRATE_DIR}/${LOG_DIR}/${installation}.cnpg-cluster" if [[ -f "$f" ]]; then profile="$(tr -d '[:space:]' <"$f")" from="log" fi fi # fall back to current CNPG_CLUSTER name if it looks like a known profile if [[ -z "$profile" && -n "${CNPG_CLUSTER:-}" ]]; then case "${CNPG_CLUSTER}" in loop-cnpg-wilix-1|loop-cnpg-lm|loop-cnpg-ff|loop-cnpg-lp3) profile="${CNPG_CLUSTER}"; from="config" ;; esac fi [[ -n "$profile" ]] || return 0 log "Resolving CNPG profile from ${from}: ${profile}" apply_cnpg_cluster_profile "$profile" } # Resolve TARGET (cnpg|selectel) → TARGET_KIND TARGET_HOST TARGET_PORT TARGET_SSLMODE resolve_target() { local kind="${1:-${TARGET:-cnpg}}" kind="$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" case "$kind" in cnpg|cn) TARGET_KIND="cnpg" TARGET_HOST="$(resolve_cnpg_host)" TARGET_PORT="${CNPG_PORT}" TARGET_HOST_RO="${CNPG_HOST_RO:-${TARGET_HOST}}" TARGET_PORT_RO="${CNPG_PORT_RO:-${TARGET_PORT}}" TARGET_SSLMODE="${CNPG_SSLMODE}" ;; selectel|sel|managed) TARGET_KIND="selectel" [[ -n "${SELECTEL_HOST:-}" ]] || die "TARGET=selectel requires SELECTEL_HOST in config.env or env" TARGET_HOST="${SELECTEL_HOST}" TARGET_PORT="${SELECTEL_PORT:-5432}" TARGET_HOST_RO="${SELECTEL_HOST_RO:-${TARGET_HOST}}" TARGET_PORT_RO="${SELECTEL_PORT_RO:-${TARGET_PORT}}" TARGET_SSLMODE="${SELECTEL_SSLMODE:-require}" ;; *) die "unknown TARGET='${kind}' (use cnpg or selectel)" ;; esac export TARGET_KIND TARGET_HOST TARGET_PORT TARGET_HOST_RO TARGET_PORT_RO TARGET_SSLMODE } save_installation_target() { local installation="$1" kind="$2" printf '%s\n' "$kind" >"${MIGRATE_DIR}/${LOG_DIR}/${installation}.target" if [[ "$kind" == "cnpg" && -n "${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER:-}}" ]]; then printf '%s\n' "${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER}}" \ >"${MIGRATE_DIR}/${LOG_DIR}/${installation}.cnpg-cluster" fi } load_installation_target() { local installation="$1" local f="${MIGRATE_DIR}/${LOG_DIR}/${installation}.target" if [[ -f "$f" ]]; then tr -d '[:space:]' <"$f" return 0 fi printf '%s\n' "${TARGET:-cnpg}" } # Parse migrate-list / CLI extras: [target] [dsn] in any order after installation id. # Sets PARSE_TARGET PARSE_DSN (empty if absent). parse_target_dsn_args() { PARSE_TARGET="" PARSE_DSN="" PARSE_CNPG_PROFILE="" local a lower for a in "$@"; do [[ -z "$a" ]] && continue lower="$(printf '%s' "$a" | tr '[:upper:]' '[:lower:]')" case "$lower" in cnpg|cn) PARSE_TARGET="cnpg" ;; selectel|sel|managed) PARSE_TARGET="selectel" ;; loop-cnpg-wilix-1|wilix|cnpg-wilix|loop-cnpg-wilix) PARSE_TARGET="cnpg" PARSE_CNPG_PROFILE="loop-cnpg-wilix-1" ;; loop-cnpg-lm|lm|cnpg-lm) PARSE_TARGET="cnpg" PARSE_CNPG_PROFILE="loop-cnpg-lm" ;; loop-cnpg-ff|ff|cnpg-ff) PARSE_TARGET="cnpg" PARSE_CNPG_PROFILE="loop-cnpg-ff" ;; loop-cnpg-lp3|lp3|lp-3|cnpg-lp3|loop-cnpg-new|new|cnpg-new) PARSE_TARGET="cnpg" PARSE_CNPG_PROFILE="loop-cnpg-lp3" ;; postgres://*|postgresql://*) PARSE_DSN="$a" ;; *) die "unexpected argument '${a}' (expected cnpg|selectel|loop-cnpg-lm|loop-cnpg-wilix-1|loop-cnpg-ff|loop-cnpg-lp3 and/or postgres:// DSN)" ;; esac done return 0 } build_target_dsn() { local user="$1" pass="$2" db="$3" local host="${4:-${TARGET_HOST}}" local port="${5:-${TARGET_PORT}}" local ssl="${6:-${TARGET_SSLMODE}}" local enc_pass enc_pass="$(url_encode "$pass")" printf 'postgres://%s:%s@%s:%s/%s?connect_timeout=10&sslmode=%s' \ "$user" "$enc_pass" "$host" "$port" "$db" "$ssl" } # RO pooler DSN for MM_SQLSETTINGS_DATASOURCEREPLICAS (falls back to RW if RO unset). build_target_dsn_ro() { local user="$1" pass="$2" db="$3" build_target_dsn "$user" "$pass" "$db" \ "${TARGET_HOST_RO:-${TARGET_HOST}}" \ "${TARGET_PORT_RO:-${TARGET_PORT}}" \ "${TARGET_SSLMODE}" } # Target-side row counts (users/channels/posts). Uses CNPG superuser or tenant→Selectel Job. target_counts() { local ctx="$1" installation="$2" db="$3" user="$4" pass="$5" local kind="${TARGET_KIND:-cnpg}" if [[ "$kind" == "cnpg" ]]; then local pod pod="$(cnpg_primary_pod)" kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \ psql -U postgres -d "$db" -Atc \ "SELECT (SELECT count(*) FROM users)||E'\\t'||(SELECT count(*) FROM channels)||E'\\t'||(SELECT count(*) FROM posts);" return 0 fi # selectel: count via short Job with tenant credentials local job secret jfile job="pgtcount-${installation:0:16}-$(date +%s | tail -c 4)" secret="${job}-env" jfile="$(mktemp)" kubectl --context "$ctx" -n "$JOB_NS" create secret generic "$secret" \ --from-literal=DST_HOST="$TARGET_HOST" \ --from-literal=DST_PORT="$TARGET_PORT" \ --from-literal=DST_SSLMODE="$TARGET_SSLMODE" \ --from-literal=SRC_USER="$user" \ --from-literal=SRC_PASS="$pass" \ --from-literal=SRC_DB="$db" \ --dry-run=client -o yaml | kubectl --context "$ctx" -n "$JOB_NS" apply -f - >/dev/null JOB_NAME="$job" JOB_NS="$JOB_NS" PG_IMAGE="$PG_IMAGE" SECRET_NAME="$secret" python3 - "$jfile" <<'PY' import json, os, sys script = r"""set -euo pipefail export PGSSLMODE="${DST_SSLMODE}" PGPASSWORD="${SRC_PASS}" psql -h "${DST_HOST}" -p "${DST_PORT}" -U "${SRC_USER}" -d "${SRC_DB}" -Atc \ "SELECT (SELECT count(*) FROM users)||E'\t'||(SELECT count(*) FROM channels)||E'\t'||(SELECT count(*) FROM posts);" """ json.dump({ "apiVersion": "batch/v1", "kind": "Job", "metadata": {"name": os.environ["JOB_NAME"], "namespace": os.environ["JOB_NS"]}, "spec": { "ttlSecondsAfterFinished": 600, "backoffLimit": 1, "template": {"spec": { "restartPolicy": "Never", "containers": [{ "name": "count", "image": os.environ["PG_IMAGE"], "envFrom": [{"secretRef": {"name": os.environ["SECRET_NAME"]}}], "command": ["bash", "-c"], "args": [script], }], }}, }, }, open(sys.argv[1], "w")) PY kubectl --context "$ctx" -n "$JOB_NS" apply -f "$jfile" >/dev/null if wait_job "$ctx" "$JOB_NS" "$job" 600; then kubectl --context "$ctx" -n "$JOB_NS" logs "job/${job}" | tail -1 else kubectl --context "$ctx" -n "$JOB_NS" delete job "$job" --ignore-not-found --wait=false >/dev/null 2>&1 || true kubectl --context "$ctx" -n "$JOB_NS" delete secret "$secret" --ignore-not-found --wait=false >/dev/null 2>&1 || true rm -f "$jfile" die "target count job failed for ${installation}" fi kubectl --context "$ctx" -n "$JOB_NS" delete job "$job" --ignore-not-found --wait=false >/dev/null 2>&1 || true kubectl --context "$ctx" -n "$JOB_NS" delete secret "$secret" --ignore-not-found --wait=false >/dev/null 2>&1 || true rm -f "$jfile" } # Wait for a Job. Returns: 0=succeeded, 1=failed, 2=timeout while still active/unknown. # timeout=0 (or empty/negative) → wait forever. wait_job() { local ctx="$1" ns="$2" job="$3" timeout="${4:-0}" local start now elapsed succeeded failed active phase start="$(date +%s)" local last_log=0 while true; do succeeded="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.succeeded}' 2>/dev/null || echo 0)" failed="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.failed}' 2>/dev/null || echo 0)" active="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.active}' 2>/dev/null || echo 0)" if [[ "${succeeded:-0}" -ge 1 ]]; then return 0 fi if [[ "${failed:-0}" -ge 1 && "${active:-0}" -eq 0 ]]; then return 1 fi now="$(date +%s)" elapsed=$((now - start)) if (( now - last_log >= 60 )); then phase="$(kubectl --context "$ctx" -n "$ns" get pods -l "job-name=${job}" \ -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo unknown)" if [[ "${timeout:-0}" -gt 0 ]]; then log "wait_job ${job}: elapsed=${elapsed}s/timeout=${timeout}s active=${active:-0} pod=${phase}" else log "wait_job ${job}: elapsed=${elapsed}s (no timeout) active=${active:-0} pod=${phase}" fi last_log=$now fi if [[ "${timeout:-0}" -gt 0 && "$elapsed" -ge "$timeout" ]]; then if [[ "${active:-0}" -ge 1 ]]; then log "wait_job timeout after ${elapsed}s for ${job} (still active — not marking failed)" return 2 fi log "wait_job timeout after ${elapsed}s for ${job}" return 1 fi sleep 5 done } # Terminate idle backends for a tenant DB (session-pooler leftovers after MM crash). terminate_stale_cnpg_sessions() { local db="$1" user="$2" minutes="${3:-${CUTOVER_STALE_IDLE_MINUTES:-5}}" local qdb quser qdb="$(sql_quote "$db")" quser="$(sql_quote "$user")" log "Terminating idle CNPG sessions for ${user}/${db} older than ${minutes}m" cnpg_psql_db "$db" " SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${qdb}' AND usename = '${quser}' AND state = 'idle' AND pid <> pg_backend_pid() AND state_change < now() - interval '${minutes} minutes'; DELETE FROM db_lock WHERE true; " 2>/dev/null || cnpg_psql_db "$db" " SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${qdb}' AND usename = '${quser}' AND state = 'idle' AND pid <> pg_backend_pid() AND state_change < now() - interval '${minutes} minutes'; " || true } # Current MM CR spec.replicas (default 1 if unset). mm_get_replicas() { local ctx="$1" installation="$2" local r r="$(kubectl --context "$ctx" -n "$SOURCE_NS" get mm "$installation" \ -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" if [[ -z "$r" ]]; then echo 1 else echo "$r" fi } # Patch MM CR replicas and scale Deployment immediately (operator may lag). mm_set_replicas() { local ctx="$1" installation="$2" replicas="$3" kubectl --context "$ctx" -n "$SOURCE_NS" patch mm "$installation" --type merge \ -p "{\"spec\":{\"replicas\":${replicas}}}" >/dev/null if kubectl --context "$ctx" -n "$SOURCE_NS" get deploy "$installation" >/dev/null 2>&1; then kubectl --context "$ctx" -n "$SOURCE_NS" scale deploy "$installation" \ --replicas="$replicas" >/dev/null 2>&1 || true fi } # Scale app to 0 before pg_dump so MM releases DB sessions. # Saves previous replica count to logs/.scale0-prev and SCALE_0_PREV. # Idempotent if already 0. Sets SCALE_0_ACTIVE=1 when we scaled (or were already 0 under --scale-0). scale_mm_app_to_zero() { local ctx="$1" installation="$2" local prev wait_sec="${SCALE_0_WAIT_SEC:-180}" i ready prev="$(mm_get_replicas "$ctx" "$installation")" SCALE_0_PREV="$prev" printf '%s\n' "$prev" >"${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev" SCALE_0_ACTIVE=1 export SCALE_0_PREV SCALE_0_ACTIVE if [[ "$prev" -le 0 ]]; then log "scale-0: ${installation} already replicas=0 — leave hibernated during transfer" return 0 fi log "scale-0: ${installation} replicas ${prev}→0 (MM CR + deploy)" mm_set_replicas "$ctx" "$installation" 0 for i in $(seq 1 "$wait_sec"); do ready="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${installation}" \ --field-selector=status.phase=Running \ -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)" if [[ -z "$ready" ]]; then log "scale-0: ${installation} pods stopped (${i}s)" return 0 fi sleep 1 done log "scale-0: warn — pods still present after ${wait_sec}s, continuing transfer anyway" } # Restore replicas after transfer. No-op unless SCALE_0_ACTIVE=1. # Skips restore while transfer Job is still running (KEEP_JOB / active). restore_mm_app_replicas() { local ctx="${1:-${CTX:-}}" installation="${2:-${INSTALLATION:-}}" [[ "${SCALE_0_ACTIVE:-0}" == "1" ]] || return 0 [[ -n "$ctx" && -n "$installation" ]] || return 0 local prev="${SCALE_0_PREV:-}" if [[ -z "$prev" && -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev" ]]; then prev="$(tr -d '[:space:]' <"${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev")" fi [[ -n "$prev" ]] || prev=1 if [[ "${MIGRATE_KEEP_SCALED:-0}" == "1" ]]; then log "scale-0: ${installation} остаётся 0 до cutover (было ${prev}, сохранено в ${installation}.scale0-prev)" return 0 fi if [[ "${KEEP_JOB:-0}" == "1" ]]; then log "scale-0: keeping ${installation} at 0 while Job still running (was ${prev})" return 0 fi local active=0 if [[ -n "${JOB_NAME:-}" ]]; then active="$(kubectl --context "$ctx" -n "${JOB_NS:-${SOURCE_NS}}" get job "$JOB_NAME" \ -o jsonpath='{.status.active}' 2>/dev/null || echo 0)" fi if [[ "${active:-0}" -ge 1 ]]; then log "scale-0: keeping ${installation} at 0 — Job ${JOB_NAME} still active (was ${prev})" return 0 fi if [[ "$prev" -le 0 ]]; then log "scale-0: ${installation} was hibernated (0) — leave at 0" SCALE_0_ACTIVE=0 rm -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev" return 0 fi log "scale-0: restoring ${installation} replicas 0→${prev}" mm_set_replicas "$ctx" "$installation" "$prev" || true SCALE_0_ACTIVE=0 rm -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev" } # Scale non-current app ReplicaSets to 0 (operator can leave orphans during cutover). # Keeps the ReplicaSet with the highest deployment.kubernetes.io/revision among those with replicas>0. scale_orphan_replicasets() { local ctx="$1" installation="$2" local rs_json rs_json="$(kubectl --context "$ctx" -n "$SOURCE_NS" get rs \ -l "installation.mattermost.com/installation=${installation}" -o json 2>/dev/null || echo '{}')" CTX="$ctx" NS="$SOURCE_NS" python3 -c ' import json, os, subprocess, sys data = json.loads(sys.argv[1]) ctx, ns = os.environ["CTX"], os.environ["NS"] candidates = [] for rs in data.get("items") or []: name = rs["metadata"]["name"] if "jobserver" in name: continue des = int(rs.get("spec", {}).get("replicas") or 0) if des <= 0: continue rev = int((rs.get("metadata", {}).get("annotations") or {}).get("deployment.kubernetes.io/revision") or 0) ready = int((rs.get("status") or {}).get("readyReplicas") or 0) candidates.append((rev, ready, name, des)) if len(candidates) <= 1: sys.exit(0) candidates.sort(reverse=True) keep = candidates[0][2] for rev, ready, name, des in candidates[1:]: print("Scaling orphan ReplicaSet %s → 0 (was %s, keep=%s)" % (name, des, keep), flush=True) subprocess.run( ["kubectl", "--context", ctx, "-n", ns, "scale", "rs", name, "--replicas=0"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) ' "$rs_json" || true } # Return host:port from MM app pod DATASOURCE (first running mattermost pod). # Falls back to Deployment env when no Running pod (hibernated / starting). mm_pod_dsn_hostport() { local ctx="$1" installation="$2" local pod dsn pod="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${installation}" \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" if [[ -n "$pod" ]]; then dsn="$(kubectl --context "$ctx" -n "$SOURCE_NS" exec "$pod" -- printenv MM_SQLSETTINGS_DATASOURCE 2>/dev/null || true)" else dsn="$(kubectl --context "$ctx" -n "$SOURCE_NS" get deploy "$installation" \ -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="MM_SQLSETTINGS_DATASOURCE")]}{.value}{end}' 2>/dev/null || true)" [[ -n "$dsn" ]] || return 1 fi [[ -n "$dsn" ]] || return 1 printf '%s' "$dsn" | python3 -c 'import sys,urllib.parse; u=urllib.parse.urlparse(sys.stdin.read().strip()); print("%s:%s" % (u.hostname or "", u.port or ""))' 2>/dev/null } mm_app_ready() { local ctx="$1" installation="$2" local ready ready="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${installation}" \ -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null || echo false)" [[ "$ready" == "true" ]] } mm_https_ping_ok() { local endpoint="$1" [[ -n "$endpoint" ]] || return 1 local body body="$(curl -sS -m 8 "https://${endpoint}/api/v4/system/ping" 2>/dev/null || true)" [[ "$body" == *'"status":"OK"'* ]] || [[ "$body" == *'"status": "OK"'* ]] }