loop/loop-ops
3
0
Fork 0
Code Pull Requests Releases Activity
Files
1769946c99b22c2517a4f9859109aa9a2a1276ae
loop-ops/db/lib.sh
T
kirill.moos 1769946c99 init
2026-10-27 14:29:43 +03:00

801 lines
30 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Shared helpers for MDB → CNPG migration scripts.
set -euo pipefail
MIGRATE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${MIGRATE_DIR}/config.env"
mkdir -p "${MIGRATE_DIR}/${LOG_DIR}"
DONE_PATH="${MIGRATE_DIR}/${DONE_FILE}"
FAILED_PATH="${MIGRATE_DIR}/${FAILED_FILE}"
LIST_PATH="${MIGRATE_DIR}/${TRANSFER_LIST}"
REGISTRY_PATH="${MIGRATE_DIR}/${REGISTRY_FILE}"
touch "${DONE_PATH}" "${FAILED_PATH}"
if [[ ! -f "${REGISTRY_PATH}" ]]; then
printf 'loop_id\tdomain\tkube_context\tyc_cluster\ttarget_cluster\tsize_bytes\tsize_pretty\tbatch_id\tstatus\tnotes\n' \
>"${REGISTRY_PATH}"
fi
log() { printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
die() { printf '[%s] ERROR: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; exit 1; }
# Normalize memory quantity for k8s (250M|256Mi|1G|512 → 250Mi|256Mi|1Gi|512Mi).
normalize_mem_req() {
local v="${1:-}"
[[ -n "$v" ]] || { printf '%s\n' ""; return 0; }
if [[ "$v" =~ ^[0-9]+$ ]]; then
printf '%sMi\n' "$v"
return 0
fi
if [[ "$v" =~ ^([0-9]+)([KkMmGg])$ ]]; then
printf '%s%si\n' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
return 0
fi
if [[ "$v" =~ ^[0-9]+([Ee][0-9]+)?$ || "$v" =~ ^[0-9]+([KkMmGg]i)$ ]]; then
printf '%s\n' "$v"
return 0
fi
die "invalid memory quantity '${v}' (examples: 256Mi, 250M, 1Gi)"
}
registry_set() {
# registry_set <loop_id> key=value [key=value ...]
# done.txt/failed.txt mirrors are updated inside registry_lib.py (flock-safe, shrink-guarded).
local installation="$1"
shift
[[ -n "$installation" ]] || return 0
local attempt=1 err="${MIGRATE_DIR}/${LOG_DIR}/.registry-set.err"
while ! python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" set "$installation" "$@" >/dev/null 2>>"$err"; do
if [[ "$attempt" -ge 8 ]]; then
log "WARN registry_set failed for ${installation} after ${attempt} attempts: $(tail -3 "$err" 2>/dev/null | tr '\n' ' ')"
return 1
fi
sleep "0.$((RANDOM % 5 + 1))"
attempt=$((attempt + 1))
done
return 0
}
registry_get() {
# registry_get <loop_id> [field]
python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" get "$@"
}
sql_quote() {
# Escape single quotes for SQL string literals
printf "%s" "${1//\'/\'\'}"
}
url_encode() {
python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
}
parse_dsn() {
# $1 → exports SRC_USER SRC_PASS SRC_HOST SRC_PORT SRC_DB SRC_QUERY
local dsn="${1:-}"
[[ -n "$dsn" ]] || die "parse_dsn: empty DSN"
local tmp
tmp="$(mktemp)"
python3 - "$dsn" "$tmp" <<'PY'
import sys, urllib.parse, shlex
u = urllib.parse.urlparse(sys.argv[1])
db = (u.path or "/").lstrip("/")
with open(sys.argv[2], "w") as f:
f.write("export SRC_USER=%s\n" % shlex.quote(u.username or ""))
f.write("export SRC_PASS=%s\n" % shlex.quote(u.password or ""))
f.write("export SRC_HOST=%s\n" % shlex.quote(u.hostname or ""))
f.write("export SRC_PORT=%s\n" % shlex.quote(str(u.port or 5432)))
f.write("export SRC_DB=%s\n" % shlex.quote(db))
f.write("export SRC_QUERY=%s\n" % shlex.quote(u.query or ""))
PY
# shellcheck disable=SC1090
source "$tmp"
rm -f "$tmp"
}
get_secret_dsn() {
local ctx="$1" installation="$2"
kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${installation}-db-credentials" \
-o jsonpath='{.data.DB_CONNECTION_STRING}' | base64 --decode
}
# Resolve cloud UUID / partial id → Mattermost installation name (e.g. loop036d0e599ad3).
# Accepts: loop… id, raw UUID, or unique substring of the installation name.
resolve_installation() {
local ctx="$1" raw="$2"
local id="$raw"
# Already a MM installation with credentials secret
if kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${id}-db-credentials" >/dev/null 2>&1; then
echo "$id"
return 0
fi
# UUID → loop + last 12 hex chars (Loop naming convention)
local hex last12 candidate
hex="$(printf '%s' "$raw" | tr -d '-' | tr '[:upper:]' '[:lower:]')"
if [[ "$hex" =~ ^[0-9a-f]{32}$ ]]; then
last12="${hex: -12}"
candidate="loop${last12}"
if kubectl --context "$ctx" -n "$SOURCE_NS" get secret "${candidate}-db-credentials" >/dev/null 2>&1; then
log "Resolved ${raw} → ${candidate}" >&2
echo "$candidate"
return 0
fi
fi
# Unique substring match on MM CR names
local matches
matches="$(kubectl --context "$ctx" -n "$SOURCE_NS" get mm -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \
| grep -Fi "$raw" || true)"
local count
count="$(printf '%s\n' "$matches" | grep -c . || true)"
if [[ "$count" == "1" ]]; then
candidate="$(printf '%s\n' "$matches" | head -1)"
log "Resolved ${raw} → ${candidate}" >&2
echo "$candidate"
return 0
fi
if [[ "$count" -gt 1 ]]; then
die "ambiguous installation '${raw}', matches: $(printf '%s' "$matches" | tr '\n' ' ')"
fi
die "installation not found for '${raw}' (need MM id like loop036d0e599ad3, or cloud UUID). Secret ${id}-db-credentials missing in ${ctx}/${SOURCE_NS}"
}
is_done() {
# Source of truth: registry status only (migrated|verified|cutover|done).
# done.txt is a write-only mirror and must NOT gate re-runs.
local installation="$1"
python3 "${MIGRATE_DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" is-done "$installation" 2>/dev/null
}
mark_done() {
local installation="$1"
local cur
# do not downgrade cutover/done/verified
cur="$(registry_get "$installation" status 2>/dev/null || true)"
if [[ "$cur" == "done" || "$cur" == "cutover" || "$cur" == "verified" ]]; then
registry_set "$installation" "status=${cur}" || true
return 0
fi
registry_set "$installation" "status=migrated" || true
}
mark_failed() {
local installation="$1"
registry_set "$installation" "status=failed" || true
}
mark_cutover_done() {
local installation="$1"
registry_set "$installation" "status=done" || true
}
resolve_cnpg_host() {
# CNPG_PREFER_CONFIG_HOST=1 → always use CNPG_HOST (public VIP / DNAT; InternalIP may be unreachable)
if [[ "${CNPG_PREFER_CONFIG_HOST:-0}" == "1" ]]; then
echo "${CNPG_HOST}"
return 0
fi
local pooler="${CNPG_POOLER_RW:-pooler-cnpg-wilix-rw}"
local ip
ip="$(kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" get pods \
-l "cnpg.io/poolerName=${pooler}" \
-o jsonpath='{.items[0].status.hostIP}' 2>/dev/null || true)"
if [[ -n "$ip" ]]; then
echo "$ip"
else
echo "${CNPG_HOST}"
fi
}
cnpg_primary_pod() {
kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" \
get clusters.postgresql.cnpg.io "${CNPG_CLUSTER}" \
-o jsonpath='{.status.currentPrimary}'
}
cnpg_admin_psql() {
# Run SQL as postgres superuser inside primary pod. Args: SQL string
local sql="$1"
local pod
pod="$(cnpg_primary_pod)"
[[ -n "$pod" ]] || die "cannot resolve CNPG primary"
kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \
psql -U postgres -d postgres -v ON_ERROR_STOP=1 -c "$sql"
}
cnpg_psql_db() {
local db="$1" sql="$2"
local pod
pod="$(cnpg_primary_pod)"
kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \
psql -U postgres -d "$db" -v ON_ERROR_STOP=1 -c "$sql"
}
ensure_role_and_db() {
local user="$1" pass="$2" db="$3"
local quser qpass qdb
quser="$(sql_quote "$user")"
qpass="$(sql_quote "$pass")"
qdb="$(sql_quote "$db")"
log "Ensuring role/database ${user}/${db} on CNPG"
cnpg_admin_psql "
DO \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '${quser}') THEN
CREATE ROLE \"${quser}\" LOGIN PASSWORD '${qpass}' CREATEDB;
ELSE
ALTER ROLE \"${quser}\" WITH LOGIN PASSWORD '${qpass}' CREATEDB;
END IF;
END
\$\$;
"
# Create DB if missing
local exists
exists="$(kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$(cnpg_primary_pod)" -c postgres -- \
psql -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='${qdb}'" | tr -d '[:space:]')"
if [[ "$exists" != "1" ]]; then
cnpg_admin_psql "CREATE DATABASE \"${qdb}\" OWNER \"${quser}\" ENCODING 'UTF8' LC_COLLATE 'ru_RU.UTF-8' LC_CTYPE 'ru_RU.UTF-8' TEMPLATE template0;"
else
cnpg_admin_psql "ALTER DATABASE \"${qdb}\" OWNER TO \"${quser}\";"
fi
cnpg_psql_db "$db" "CREATE EXTENSION IF NOT EXISTS pg_trgm; CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\"; DO \$\$ BEGIN CREATE EXTENSION IF NOT EXISTS vector; EXCEPTION WHEN OTHERS THEN NULL; END \$\$;"
install_pooler_auth "$db"
cnpg_admin_psql "GRANT ALL PRIVILEGES ON DATABASE \"${qdb}\" TO \"${quser}\";"
}
install_pooler_auth() {
# Must be owned by a superuser (vizorcnpgadmin); tenant-owned SECURITY DEFINER cannot read pg_shadow.
local db="$1"
cnpg_psql_db "$db" "
CREATE OR REPLACE FUNCTION public.user_search(uname text)
RETURNS TABLE(usename name, passwd text)
LANGUAGE sql
SECURITY DEFINER
SET search_path = pg_catalog
AS \$\$SELECT usename, passwd FROM pg_catalog.pg_shadow WHERE usename=\$1;\$\$;
ALTER FUNCTION public.user_search(text) OWNER TO vizorcnpgadmin;
REVOKE ALL ON FUNCTION public.user_search(text) FROM PUBLIC;
GRANT USAGE ON SCHEMA public TO cnpg_pooler_pgbouncer;
GRANT EXECUTE ON FUNCTION public.user_search(text) TO cnpg_pooler_pgbouncer;
"
}
set_role_superuser() {
local user="$1" onoff="$2" # onoff: ON|OFF
local quser
quser="$(sql_quote "$user")"
if [[ "$onoff" == "ON" ]]; then
cnpg_admin_psql "ALTER ROLE \"${quser}\" WITH SUPERUSER;"
else
cnpg_admin_psql "ALTER ROLE \"${quser}\" WITH NOSUPERUSER;"
fi
}
# Apply CNPG endpoint/admin profile by logical cluster name (registry target_cluster).
# Does not touch TRANSFER_* knobs. Override anytime with CNPG_CLUSTER_PROFILE=...
apply_cnpg_cluster_profile() {
local name="${1:-}"
[[ -n "$name" ]] || return 0
# normalize aliases
case "$name" in
wilix|cnpg-wilix|loop-cnpg-wilix) name="loop-cnpg-wilix-1" ;;
lm|cnpg-lm) name="loop-cnpg-lm" ;;
ff|cnpg-ff) name="loop-cnpg-ff" ;;
lp3|lp-3|cnpg-lp3|NEW|new|cnpg-new|loop-cnpg-NEW) name="loop-cnpg-lp3" ;;
esac
case "$name" in
loop-cnpg-wilix-1)
CNPG_HOST="${CNPG_WILIX_HOST:-95.143.178.247}"
CNPG_PORT="${CNPG_WILIX_PORT:-31574}"
CNPG_HOST_RO="${CNPG_WILIX_HOST_RO:-${CNPG_HOST}}"
CNPG_PORT_RO="${CNPG_WILIX_PORT_RO:-31575}"
CNPG_SSLMODE="${CNPG_WILIX_SSLMODE:-disable}"
CNPG_PREFER_CONFIG_HOST="${CNPG_WILIX_PREFER_CONFIG_HOST:-0}"
CNPG_KUBE_CONTEXT="${CNPG_WILIX_KUBE_CONTEXT:-local}"
CNPG_NAMESPACE="${CNPG_WILIX_NAMESPACE:-cnpg-loop-wilix}"
CNPG_CLUSTER="loop-cnpg-wilix-1"
CNPG_ADMIN_SECRET="${CNPG_WILIX_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}"
CNPG_POOLER_RW="pooler-cnpg-wilix-rw"
CNPG_POOLER_RO="pooler-cnpg-wilix-ro"
;;
loop-cnpg-lm)
CNPG_HOST="${CNPG_LM_HOST:-135.106.201.40}"
CNPG_PORT="${CNPG_LM_PORT:-30434}"
CNPG_HOST_RO="${CNPG_LM_HOST_RO:-135.106.201.40}"
CNPG_PORT_RO="${CNPG_LM_PORT_RO:-30435}"
CNPG_SSLMODE="${CNPG_LM_SSLMODE:-disable}"
CNPG_PREFER_CONFIG_HOST="${CNPG_LM_PREFER_CONFIG_HOST:-1}"
CNPG_KUBE_CONTEXT="${CNPG_LM_KUBE_CONTEXT:-loop-cnpg}"
CNPG_NAMESPACE="${CNPG_LM_NAMESPACE:-cnpg-loop-lm}"
CNPG_CLUSTER="loop-cnpg-lm"
CNPG_ADMIN_SECRET="${CNPG_LM_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}"
CNPG_POOLER_RW="pooler-cnpg-lm-rw"
CNPG_POOLER_RO="pooler-cnpg-lm-ro"
;;
loop-cnpg-ff)
CNPG_HOST="${CNPG_FF_HOST:-135.106.201.38}"
CNPG_PORT="${CNPG_FF_PORT:-30432}"
CNPG_HOST_RO="${CNPG_FF_HOST_RO:-135.106.201.38}"
CNPG_PORT_RO="${CNPG_FF_PORT_RO:-30433}"
CNPG_SSLMODE="${CNPG_FF_SSLMODE:-disable}"
CNPG_PREFER_CONFIG_HOST="${CNPG_FF_PREFER_CONFIG_HOST:-1}"
CNPG_KUBE_CONTEXT="${CNPG_FF_KUBE_CONTEXT:-loop-cnpg}"
CNPG_NAMESPACE="${CNPG_FF_NAMESPACE:-cnpg-loop-ff}"
CNPG_CLUSTER="loop-cnpg-ff"
CNPG_ADMIN_SECRET="${CNPG_FF_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}"
CNPG_POOLER_RW="pooler-cnpg-ff-rw"
CNPG_POOLER_RO="pooler-cnpg-ff-ro"
;;
loop-cnpg-lp3)
CNPG_HOST="${CNPG_LP3_HOST:-135.106.201.42}"
CNPG_PORT="${CNPG_LP3_PORT:-30436}"
CNPG_HOST_RO="${CNPG_LP3_HOST_RO:-135.106.201.42}"
CNPG_PORT_RO="${CNPG_LP3_PORT_RO:-30437}"
CNPG_SSLMODE="${CNPG_LP3_SSLMODE:-disable}"
CNPG_PREFER_CONFIG_HOST="${CNPG_LP3_PREFER_CONFIG_HOST:-1}"
CNPG_KUBE_CONTEXT="${CNPG_LP3_KUBE_CONTEXT:-loop-cnpg}"
CNPG_NAMESPACE="${CNPG_LP3_NAMESPACE:-cnpg-loop-lp3}"
CNPG_CLUSTER="loop-cnpg-lp3"
CNPG_ADMIN_SECRET="${CNPG_LP3_ADMIN_SECRET:-cnpg-vizor-admin-role-secret}"
CNPG_POOLER_RW="pooler-cnpg-lp3-rw"
CNPG_POOLER_RO="pooler-cnpg-lp3-ro"
;;
*)
die "unknown CNPG cluster profile '${name}' (loop-cnpg-wilix-1|loop-cnpg-lm|loop-cnpg-ff|loop-cnpg-lp3)"
;;
esac
CNPG_CLUSTER_PROFILE="$name"
export CNPG_HOST CNPG_PORT CNPG_HOST_RO CNPG_PORT_RO CNPG_SSLMODE CNPG_PREFER_CONFIG_HOST \
CNPG_KUBE_CONTEXT CNPG_NAMESPACE CNPG_CLUSTER CNPG_ADMIN_SECRET CNPG_POOLER_RW CNPG_POOLER_RO \
CNPG_CLUSTER_PROFILE
log "CNPG profile=${CNPG_CLUSTER_PROFILE} rw=${CNPG_HOST}:${CNPG_PORT} ro=${CNPG_HOST_RO}:${CNPG_PORT_RO} admin=${CNPG_KUBE_CONTEXT}/${CNPG_NAMESPACE}/${CNPG_CLUSTER}"
}
# Pick CNPG profile: explicit CNPG_CLUSTER_PROFILE > registry.target_cluster > logs/<id>.cnpg-cluster
# Skip when TARGET=selectel or CNPG_SKIP_REGISTRY=1.
resolve_cnpg_cluster_profile() {
local installation="${1:-}"
local profile="${CNPG_CLUSTER_PROFILE:-}"
local from="env"
if [[ "${TARGET:-cnpg}" == "selectel" || "${CNPG_SKIP_REGISTRY:-0}" == "1" ]]; then
return 0
fi
if [[ -z "$profile" && -n "$installation" ]]; then
profile="$(registry_get "$installation" target_cluster 2>/dev/null || true)"
[[ -n "$profile" ]] && from="registry"
fi
if [[ -z "$profile" && -n "$installation" ]]; then
local f="${MIGRATE_DIR}/${LOG_DIR}/${installation}.cnpg-cluster"
if [[ -f "$f" ]]; then
profile="$(tr -d '[:space:]' <"$f")"
from="log"
fi
fi
# fall back to current CNPG_CLUSTER name if it looks like a known profile
if [[ -z "$profile" && -n "${CNPG_CLUSTER:-}" ]]; then
case "${CNPG_CLUSTER}" in
loop-cnpg-wilix-1|loop-cnpg-lm|loop-cnpg-ff|loop-cnpg-lp3) profile="${CNPG_CLUSTER}"; from="config" ;;
esac
fi
[[ -n "$profile" ]] || return 0
log "Resolving CNPG profile from ${from}: ${profile}"
apply_cnpg_cluster_profile "$profile"
}
# Resolve TARGET (cnpg|selectel) → TARGET_KIND TARGET_HOST TARGET_PORT TARGET_SSLMODE
resolve_target() {
local kind="${1:-${TARGET:-cnpg}}"
kind="$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')"
case "$kind" in
cnpg|cn)
TARGET_KIND="cnpg"
TARGET_HOST="$(resolve_cnpg_host)"
TARGET_PORT="${CNPG_PORT}"
TARGET_HOST_RO="${CNPG_HOST_RO:-${TARGET_HOST}}"
TARGET_PORT_RO="${CNPG_PORT_RO:-${TARGET_PORT}}"
TARGET_SSLMODE="${CNPG_SSLMODE}"
;;
selectel|sel|managed)
TARGET_KIND="selectel"
[[ -n "${SELECTEL_HOST:-}" ]] || die "TARGET=selectel requires SELECTEL_HOST in config.env or env"
TARGET_HOST="${SELECTEL_HOST}"
TARGET_PORT="${SELECTEL_PORT:-5432}"
TARGET_HOST_RO="${SELECTEL_HOST_RO:-${TARGET_HOST}}"
TARGET_PORT_RO="${SELECTEL_PORT_RO:-${TARGET_PORT}}"
TARGET_SSLMODE="${SELECTEL_SSLMODE:-require}"
;;
*)
die "unknown TARGET='${kind}' (use cnpg or selectel)"
;;
esac
export TARGET_KIND TARGET_HOST TARGET_PORT TARGET_HOST_RO TARGET_PORT_RO TARGET_SSLMODE
}
save_installation_target() {
local installation="$1" kind="$2"
printf '%s\n' "$kind" >"${MIGRATE_DIR}/${LOG_DIR}/${installation}.target"
if [[ "$kind" == "cnpg" && -n "${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER:-}}" ]]; then
printf '%s\n' "${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER}}" \
>"${MIGRATE_DIR}/${LOG_DIR}/${installation}.cnpg-cluster"
fi
}
load_installation_target() {
local installation="$1"
local f="${MIGRATE_DIR}/${LOG_DIR}/${installation}.target"
if [[ -f "$f" ]]; then
tr -d '[:space:]' <"$f"
return 0
fi
printf '%s\n' "${TARGET:-cnpg}"
}
# Parse migrate-list / CLI extras: [target] [dsn] in any order after installation id.
# Sets PARSE_TARGET PARSE_DSN (empty if absent).
parse_target_dsn_args() {
PARSE_TARGET=""
PARSE_DSN=""
PARSE_CNPG_PROFILE=""
local a lower
for a in "$@"; do
[[ -z "$a" ]] && continue
lower="$(printf '%s' "$a" | tr '[:upper:]' '[:lower:]')"
case "$lower" in
cnpg|cn)
PARSE_TARGET="cnpg"
;;
selectel|sel|managed)
PARSE_TARGET="selectel"
;;
loop-cnpg-wilix-1|wilix|cnpg-wilix|loop-cnpg-wilix)
PARSE_TARGET="cnpg"
PARSE_CNPG_PROFILE="loop-cnpg-wilix-1"
;;
loop-cnpg-lm|lm|cnpg-lm)
PARSE_TARGET="cnpg"
PARSE_CNPG_PROFILE="loop-cnpg-lm"
;;
loop-cnpg-ff|ff|cnpg-ff)
PARSE_TARGET="cnpg"
PARSE_CNPG_PROFILE="loop-cnpg-ff"
;;
loop-cnpg-lp3|lp3|lp-3|cnpg-lp3|loop-cnpg-new|new|cnpg-new)
PARSE_TARGET="cnpg"
PARSE_CNPG_PROFILE="loop-cnpg-lp3"
;;
postgres://*|postgresql://*)
PARSE_DSN="$a"
;;
*)
die "unexpected argument '${a}' (expected cnpg|selectel|loop-cnpg-lm|loop-cnpg-wilix-1|loop-cnpg-ff|loop-cnpg-lp3 and/or postgres:// DSN)"
;;
esac
done
return 0
}
build_target_dsn() {
local user="$1" pass="$2" db="$3"
local host="${4:-${TARGET_HOST}}"
local port="${5:-${TARGET_PORT}}"
local ssl="${6:-${TARGET_SSLMODE}}"
local enc_pass
enc_pass="$(url_encode "$pass")"
printf 'postgres://%s:%s@%s:%s/%s?connect_timeout=10&sslmode=%s' \
"$user" "$enc_pass" "$host" "$port" "$db" "$ssl"
}
# RO pooler DSN for MM_SQLSETTINGS_DATASOURCEREPLICAS (falls back to RW if RO unset).
build_target_dsn_ro() {
local user="$1" pass="$2" db="$3"
build_target_dsn "$user" "$pass" "$db" \
"${TARGET_HOST_RO:-${TARGET_HOST}}" \
"${TARGET_PORT_RO:-${TARGET_PORT}}" \
"${TARGET_SSLMODE}"
}
# Target-side row counts (users/channels/posts). Uses CNPG superuser or tenant→Selectel Job.
target_counts() {
local ctx="$1" installation="$2" db="$3" user="$4" pass="$5"
local kind="${TARGET_KIND:-cnpg}"
if [[ "$kind" == "cnpg" ]]; then
local pod
pod="$(cnpg_primary_pod)"
kubectl --context "${CNPG_KUBE_CONTEXT}" -n "${CNPG_NAMESPACE}" exec "$pod" -c postgres -- \
psql -U postgres -d "$db" -Atc \
"SELECT (SELECT count(*) FROM users)||E'\\t'||(SELECT count(*) FROM channels)||E'\\t'||(SELECT count(*) FROM posts);"
return 0
fi
# selectel: count via short Job with tenant credentials
local job secret jfile
job="pgtcount-${installation:0:16}-$(date +%s | tail -c 4)"
secret="${job}-env"
jfile="$(mktemp)"
kubectl --context "$ctx" -n "$JOB_NS" create secret generic "$secret" \
--from-literal=DST_HOST="$TARGET_HOST" \
--from-literal=DST_PORT="$TARGET_PORT" \
--from-literal=DST_SSLMODE="$TARGET_SSLMODE" \
--from-literal=SRC_USER="$user" \
--from-literal=SRC_PASS="$pass" \
--from-literal=SRC_DB="$db" \
--dry-run=client -o yaml | kubectl --context "$ctx" -n "$JOB_NS" apply -f - >/dev/null
JOB_NAME="$job" JOB_NS="$JOB_NS" PG_IMAGE="$PG_IMAGE" SECRET_NAME="$secret" python3 - "$jfile" <<'PY'
import json, os, sys
script = r"""set -euo pipefail
export PGSSLMODE="${DST_SSLMODE}"
PGPASSWORD="${SRC_PASS}" psql -h "${DST_HOST}" -p "${DST_PORT}" -U "${SRC_USER}" -d "${SRC_DB}" -Atc \
"SELECT (SELECT count(*) FROM users)||E'\t'||(SELECT count(*) FROM channels)||E'\t'||(SELECT count(*) FROM posts);"
"""
json.dump({
"apiVersion": "batch/v1", "kind": "Job",
"metadata": {"name": os.environ["JOB_NAME"], "namespace": os.environ["JOB_NS"]},
"spec": {
"ttlSecondsAfterFinished": 600, "backoffLimit": 1,
"template": {"spec": {
"restartPolicy": "Never",
"containers": [{
"name": "count", "image": os.environ["PG_IMAGE"],
"envFrom": [{"secretRef": {"name": os.environ["SECRET_NAME"]}}],
"command": ["bash", "-c"], "args": [script],
}],
}},
},
}, open(sys.argv[1], "w"))
PY
kubectl --context "$ctx" -n "$JOB_NS" apply -f "$jfile" >/dev/null
if wait_job "$ctx" "$JOB_NS" "$job" 600; then
kubectl --context "$ctx" -n "$JOB_NS" logs "job/${job}" | tail -1
else
kubectl --context "$ctx" -n "$JOB_NS" delete job "$job" --ignore-not-found --wait=false >/dev/null 2>&1 || true
kubectl --context "$ctx" -n "$JOB_NS" delete secret "$secret" --ignore-not-found --wait=false >/dev/null 2>&1 || true
rm -f "$jfile"
die "target count job failed for ${installation}"
fi
kubectl --context "$ctx" -n "$JOB_NS" delete job "$job" --ignore-not-found --wait=false >/dev/null 2>&1 || true
kubectl --context "$ctx" -n "$JOB_NS" delete secret "$secret" --ignore-not-found --wait=false >/dev/null 2>&1 || true
rm -f "$jfile"
}
# Wait for a Job. Returns: 0=succeeded, 1=failed, 2=timeout while still active/unknown.
# timeout=0 (or empty/negative) → wait forever.
wait_job() {
local ctx="$1" ns="$2" job="$3" timeout="${4:-0}"
local start now elapsed succeeded failed active phase
start="$(date +%s)"
local last_log=0
while true; do
succeeded="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.succeeded}' 2>/dev/null || echo 0)"
failed="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.failed}' 2>/dev/null || echo 0)"
active="$(kubectl --context "$ctx" -n "$ns" get job "$job" -o jsonpath='{.status.active}' 2>/dev/null || echo 0)"
if [[ "${succeeded:-0}" -ge 1 ]]; then
return 0
fi
if [[ "${failed:-0}" -ge 1 && "${active:-0}" -eq 0 ]]; then
return 1
fi
now="$(date +%s)"
elapsed=$((now - start))
if (( now - last_log >= 60 )); then
phase="$(kubectl --context "$ctx" -n "$ns" get pods -l "job-name=${job}" \
-o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo unknown)"
if [[ "${timeout:-0}" -gt 0 ]]; then
log "wait_job ${job}: elapsed=${elapsed}s/timeout=${timeout}s active=${active:-0} pod=${phase}"
else
log "wait_job ${job}: elapsed=${elapsed}s (no timeout) active=${active:-0} pod=${phase}"
fi
last_log=$now
fi
if [[ "${timeout:-0}" -gt 0 && "$elapsed" -ge "$timeout" ]]; then
if [[ "${active:-0}" -ge 1 ]]; then
log "wait_job timeout after ${elapsed}s for ${job} (still active — not marking failed)"
return 2
fi
log "wait_job timeout after ${elapsed}s for ${job}"
return 1
fi
sleep 5
done
}
# Terminate idle backends for a tenant DB (session-pooler leftovers after MM crash).
terminate_stale_cnpg_sessions() {
local db="$1" user="$2" minutes="${3:-${CUTOVER_STALE_IDLE_MINUTES:-5}}"
local qdb quser
qdb="$(sql_quote "$db")"
quser="$(sql_quote "$user")"
log "Terminating idle CNPG sessions for ${user}/${db} older than ${minutes}m"
cnpg_psql_db "$db" "
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = '${qdb}'
AND usename = '${quser}'
AND state = 'idle'
AND pid <> pg_backend_pid()
AND state_change < now() - interval '${minutes} minutes';
DELETE FROM db_lock WHERE true;
" 2>/dev/null || cnpg_psql_db "$db" "
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = '${qdb}'
AND usename = '${quser}'
AND state = 'idle'
AND pid <> pg_backend_pid()
AND state_change < now() - interval '${minutes} minutes';
" || true
}
# Current MM CR spec.replicas (default 1 if unset).
mm_get_replicas() {
local ctx="$1" installation="$2"
local r
r="$(kubectl --context "$ctx" -n "$SOURCE_NS" get mm "$installation" \
-o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
if [[ -z "$r" ]]; then
echo 1
else
echo "$r"
fi
}
# Patch MM CR replicas and scale Deployment immediately (operator may lag).
mm_set_replicas() {
local ctx="$1" installation="$2" replicas="$3"
kubectl --context "$ctx" -n "$SOURCE_NS" patch mm "$installation" --type merge \
-p "{\"spec\":{\"replicas\":${replicas}}}" >/dev/null
if kubectl --context "$ctx" -n "$SOURCE_NS" get deploy "$installation" >/dev/null 2>&1; then
kubectl --context "$ctx" -n "$SOURCE_NS" scale deploy "$installation" \
--replicas="$replicas" >/dev/null 2>&1 || true
fi
}
# Scale app to 0 before pg_dump so MM releases DB sessions.
# Saves previous replica count to logs/<id>.scale0-prev and SCALE_0_PREV.
# Idempotent if already 0. Sets SCALE_0_ACTIVE=1 when we scaled (or were already 0 under --scale-0).
scale_mm_app_to_zero() {
local ctx="$1" installation="$2"
local prev wait_sec="${SCALE_0_WAIT_SEC:-180}" i ready
prev="$(mm_get_replicas "$ctx" "$installation")"
SCALE_0_PREV="$prev"
printf '%s\n' "$prev" >"${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev"
SCALE_0_ACTIVE=1
export SCALE_0_PREV SCALE_0_ACTIVE
if [[ "$prev" -le 0 ]]; then
log "scale-0: ${installation} already replicas=0 — leave hibernated during transfer"
return 0
fi
log "scale-0: ${installation} replicas ${prev}→0 (MM CR + deploy)"
mm_set_replicas "$ctx" "$installation" 0
for i in $(seq 1 "$wait_sec"); do
ready="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \
-l "app=mattermost,installation.mattermost.com/installation=${installation}" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)"
if [[ -z "$ready" ]]; then
log "scale-0: ${installation} pods stopped (${i}s)"
return 0
fi
sleep 1
done
log "scale-0: warn — pods still present after ${wait_sec}s, continuing transfer anyway"
}
# Restore replicas after transfer. No-op unless SCALE_0_ACTIVE=1.
# Skips restore while transfer Job is still running (KEEP_JOB / active).
restore_mm_app_replicas() {
local ctx="${1:-${CTX:-}}" installation="${2:-${INSTALLATION:-}}"
[[ "${SCALE_0_ACTIVE:-0}" == "1" ]] || return 0
[[ -n "$ctx" && -n "$installation" ]] || return 0
local prev="${SCALE_0_PREV:-}"
if [[ -z "$prev" && -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev" ]]; then
prev="$(tr -d '[:space:]' <"${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev")"
fi
[[ -n "$prev" ]] || prev=1
if [[ "${MIGRATE_KEEP_SCALED:-0}" == "1" ]]; then
log "scale-0: ${installation} остаётся 0 до cutover (было ${prev}, сохранено в ${installation}.scale0-prev)"
return 0
fi
if [[ "${KEEP_JOB:-0}" == "1" ]]; then
log "scale-0: keeping ${installation} at 0 while Job still running (was ${prev})"
return 0
fi
local active=0
if [[ -n "${JOB_NAME:-}" ]]; then
active="$(kubectl --context "$ctx" -n "${JOB_NS:-${SOURCE_NS}}" get job "$JOB_NAME" \
-o jsonpath='{.status.active}' 2>/dev/null || echo 0)"
fi
if [[ "${active:-0}" -ge 1 ]]; then
log "scale-0: keeping ${installation} at 0 — Job ${JOB_NAME} still active (was ${prev})"
return 0
fi
if [[ "$prev" -le 0 ]]; then
log "scale-0: ${installation} was hibernated (0) — leave at 0"
SCALE_0_ACTIVE=0
rm -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev"
return 0
fi
log "scale-0: restoring ${installation} replicas 0→${prev}"
mm_set_replicas "$ctx" "$installation" "$prev" || true
SCALE_0_ACTIVE=0
rm -f "${MIGRATE_DIR}/${LOG_DIR}/${installation}.scale0-prev"
}
# Scale non-current app ReplicaSets to 0 (operator can leave orphans during cutover).
# Keeps the ReplicaSet with the highest deployment.kubernetes.io/revision among those with replicas>0.
scale_orphan_replicasets() {
local ctx="$1" installation="$2"
local rs_json
rs_json="$(kubectl --context "$ctx" -n "$SOURCE_NS" get rs \
-l "installation.mattermost.com/installation=${installation}" -o json 2>/dev/null || echo '{}')"
CTX="$ctx" NS="$SOURCE_NS" python3 -c '
import json, os, subprocess, sys
data = json.loads(sys.argv[1])
ctx, ns = os.environ["CTX"], os.environ["NS"]
candidates = []
for rs in data.get("items") or []:
name = rs["metadata"]["name"]
if "jobserver" in name:
continue
des = int(rs.get("spec", {}).get("replicas") or 0)
if des <= 0:
continue
rev = int((rs.get("metadata", {}).get("annotations") or {}).get("deployment.kubernetes.io/revision") or 0)
ready = int((rs.get("status") or {}).get("readyReplicas") or 0)
candidates.append((rev, ready, name, des))
if len(candidates) <= 1:
sys.exit(0)
candidates.sort(reverse=True)
keep = candidates[0][2]
for rev, ready, name, des in candidates[1:]:
print("Scaling orphan ReplicaSet %s → 0 (was %s, keep=%s)" % (name, des, keep), flush=True)
subprocess.run(
["kubectl", "--context", ctx, "-n", ns, "scale", "rs", name, "--replicas=0"],
check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
' "$rs_json" || true
}
# Return host:port from MM app pod DATASOURCE (first running mattermost pod).
# Falls back to Deployment env when no Running pod (hibernated / starting).
mm_pod_dsn_hostport() {
local ctx="$1" installation="$2"
local pod dsn
pod="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \
-l "app=mattermost,installation.mattermost.com/installation=${installation}" \
--field-selector=status.phase=Running \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
if [[ -n "$pod" ]]; then
dsn="$(kubectl --context "$ctx" -n "$SOURCE_NS" exec "$pod" -- printenv MM_SQLSETTINGS_DATASOURCE 2>/dev/null || true)"
else
dsn="$(kubectl --context "$ctx" -n "$SOURCE_NS" get deploy "$installation" \
-o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="MM_SQLSETTINGS_DATASOURCE")]}{.value}{end}' 2>/dev/null || true)"
[[ -n "$dsn" ]] || return 1
fi
[[ -n "$dsn" ]] || return 1
printf '%s' "$dsn" | python3 -c 'import sys,urllib.parse; u=urllib.parse.urlparse(sys.stdin.read().strip());
print("%s:%s" % (u.hostname or "", u.port or ""))' 2>/dev/null
}
mm_app_ready() {
local ctx="$1" installation="$2"
local ready
ready="$(kubectl --context "$ctx" -n "$SOURCE_NS" get pods \
-l "app=mattermost,installation.mattermost.com/installation=${installation}" \
-o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null || echo false)"
[[ "$ready" == "true" ]]
}
mm_https_ping_ok() {
local endpoint="$1"
[[ -n "$endpoint" ]] || return 1
local body
body="$(curl -sS -m 8 "https://${endpoint}/api/v4/system/ping" 2>/dev/null || true)"
[[ "$body" == *'"status":"OK"'* ]] || [[ "$body" == *'"status": "OK"'* ]]
}
View Git Blame Copy Permalink