262 lines
11 KiB
Bash
Executable File
262 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Point MM installation at target DB (CNPG pooler or Selectel Managed) and wait until healthy.
|
|
# Applies canary learnings: long probes, more memory, stale session cleanup (CNPG), orphan RS.
|
|
# Usage: cutover-one.sh <kube_context> <installation_id|uuid> [cnpg|selectel] [--mem-req 256Mi]
|
|
set -euo pipefail
|
|
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "${DIR}/lib.sh"
|
|
|
|
[[ $# -ge 2 ]] || die "usage: $0 <kube_context> <installation_id|uuid> [cnpg|selectel|loop-cnpg-*] [--mem-req 256Mi]"
|
|
CTX="$1"
|
|
INSTALLATION_RAW="$2"
|
|
shift 2
|
|
|
|
ARGS=()
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--mem-req|--memory-request)
|
|
CUTOVER_MEMORY_REQUEST="$(normalize_mem_req "${2:-}")"
|
|
export CUTOVER_MEMORY_REQUEST
|
|
shift 2
|
|
;;
|
|
--mem-req=*|--memory-request=*)
|
|
CUTOVER_MEMORY_REQUEST="$(normalize_mem_req "${1#*=}")"
|
|
export CUTOVER_MEMORY_REQUEST
|
|
shift
|
|
;;
|
|
*)
|
|
ARGS+=("$1")
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
set -- "${ARGS[@]+"${ARGS[@]}"}"
|
|
|
|
parse_target_dsn_args "$@"
|
|
if [[ -n "${PARSE_TARGET}" ]]; then
|
|
TARGET="${PARSE_TARGET}"
|
|
fi
|
|
if [[ -n "${PARSE_CNPG_PROFILE:-}" ]]; then
|
|
CNPG_CLUSTER_PROFILE="${PARSE_CNPG_PROFILE}"
|
|
fi
|
|
|
|
INSTALLATION="$(resolve_installation "$CTX" "$INSTALLATION_RAW")"
|
|
if [[ -z "${PARSE_TARGET:-}" ]]; then
|
|
TARGET="$(load_installation_target "$INSTALLATION")"
|
|
fi
|
|
|
|
LOG_FILE="${DIR}/${LOG_DIR}/${INSTALLATION}.cutover.log"
|
|
exec > >(tee -a "$LOG_FILE") 2>&1
|
|
|
|
resolve_cnpg_cluster_profile "$INSTALLATION"
|
|
resolve_target "${TARGET:-cnpg}"
|
|
|
|
log "=== cutover-one ${CTX} ${INSTALLATION} target=${TARGET_KIND} cnpg=${CNPG_CLUSTER_PROFILE:-${CNPG_CLUSTER:-?}} endpoint=${TARGET_HOST}:${TARGET_PORT} mem-req=${CUTOVER_MEMORY_REQUEST} ==="
|
|
|
|
DSN="$(get_secret_dsn "$CTX" "$INSTALLATION")"
|
|
parse_dsn "$DSN"
|
|
|
|
NEW_DSN="$(build_target_dsn "$SRC_USER" "$SRC_PASS" "$SRC_DB")"
|
|
NEW_DSN_RO="$(build_target_dsn_ro "$SRC_USER" "$SRC_PASS" "$SRC_DB")"
|
|
EXPECTED_HP="${TARGET_HOST}:${TARGET_PORT}"
|
|
EXPECTED_HP_RO="${TARGET_HOST_RO:-${TARGET_HOST}}:${TARGET_PORT_RO:-${TARGET_PORT}}"
|
|
|
|
if [[ "$SRC_HOST" == "$TARGET_HOST" && "${SRC_PORT}" == "${TARGET_PORT}" ]]; then
|
|
log "Secret already points at ${TARGET_KIND} ${EXPECTED_HP} — will still retune probes/resources and bounce pods"
|
|
fi
|
|
|
|
# Drop leftover session-pooler backends before MM reconnects (CNPG only)
|
|
if [[ "$TARGET_KIND" == "cnpg" ]]; then
|
|
terminate_stale_cnpg_sessions "$SRC_DB" "$SRC_USER" "${CUTOVER_STALE_IDLE_MINUTES}"
|
|
fi
|
|
|
|
log "Updating secret ${INSTALLATION}-db-credentials → ${EXPECTED_HP}/${SRC_DB} (ro ${EXPECTED_HP_RO})"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" create secret generic "${INSTALLATION}-db-credentials" \
|
|
--from-literal=DB_CONNECTION_STRING="$NEW_DSN" \
|
|
--dry-run=client -o yaml | kubectl --context "$CTX" -n "$SOURCE_NS" apply -f -
|
|
|
|
# Patch MM CR: DSN env + probes + memory (single replace)
|
|
TMP_MM="$(mktemp)"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o json >"$TMP_MM"
|
|
|
|
python3 - "$TMP_MM" "$NEW_DSN" "$NEW_DSN_RO" \
|
|
"${CUTOVER_LIVENESS_INITIAL_DELAY}" "${CUTOVER_LIVENESS_FAILURE_THRESHOLD}" \
|
|
"${CUTOVER_READINESS_INITIAL_DELAY}" "${CUTOVER_READINESS_FAILURE_THRESHOLD}" \
|
|
"${CUTOVER_MEMORY_LIMIT}" "${CUTOVER_MEMORY_REQUEST}" "${CUTOVER_CPU_REQUEST}" <<'PY'
|
|
import json, os, sys
|
|
|
|
path, new_dsn, new_dsn_ro = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
live_delay, live_fail = int(sys.argv[4]), int(sys.argv[5])
|
|
ready_delay, ready_fail = int(sys.argv[6]), int(sys.argv[7])
|
|
mem_limit, mem_req, cpu_req = sys.argv[8], sys.argv[9], sys.argv[10]
|
|
|
|
with open(path) as f:
|
|
mm = json.load(f)
|
|
spec = mm.setdefault("spec", {})
|
|
|
|
# Probes/ресурсы только поднимаем до минимума: свои (больше) у крупных инстансов не трогаем, визор о них не знает
|
|
def at_least(d, key, val):
|
|
if int(d.get(key) or 0) < val:
|
|
d[key] = val
|
|
|
|
probes = spec.setdefault("probes", {})
|
|
lp = probes.setdefault("livenessProbe", {})
|
|
rp = probes.setdefault("readinessProbe", {})
|
|
at_least(lp, "initialDelaySeconds", live_delay); at_least(lp, "failureThreshold", live_fail)
|
|
at_least(rp, "initialDelaySeconds", ready_delay); at_least(rp, "failureThreshold", ready_fail)
|
|
|
|
sched = spec.setdefault("scheduling", {})
|
|
res = sched.setdefault("resources", {})
|
|
res.setdefault("limits", {}).setdefault("memory", mem_limit)
|
|
requests = res.setdefault("requests", {})
|
|
requests.setdefault("memory", mem_req)
|
|
requests.setdefault("cpu", cpu_req)
|
|
|
|
# Force inline DSN via mattermostEnv only (spec.env is not a valid MM CR field)
|
|
def upsert_env(envs, name, value):
|
|
for e in envs:
|
|
if e.get("name") == name:
|
|
e["value"] = value
|
|
e.pop("valueFrom", None)
|
|
return envs
|
|
envs.append({"name": name, "value": value})
|
|
return envs
|
|
|
|
envs = list(spec.get("mattermostEnv") or [])
|
|
envs = upsert_env(envs, "MM_SQLSETTINGS_DATASOURCE", new_dsn)
|
|
envs = upsert_env(envs, "MM_SQLSETTINGS_DATASOURCEREPLICAS", new_dsn_ro)
|
|
spec["mattermostEnv"] = envs
|
|
# Drop invalid field if present from older script runs / edits
|
|
spec.pop("env", None)
|
|
|
|
# CUTOVER_REPLICAS (loopctl db move --scale-0) — вернуть число реплик до scale-0.
|
|
# Иначе replicas не трогаем: приостановленный (0) инстанс остаётся на паузе, ему меняем только DSN.
|
|
want = os.environ.get("CUTOVER_REPLICAS")
|
|
if want:
|
|
spec["replicas"] = int(want)
|
|
print("replicas=%s" % spec.get("replicas"))
|
|
|
|
with open(path, "w") as f:
|
|
json.dump(mm, f)
|
|
print("patched probes+resources+datasource")
|
|
PY
|
|
|
|
log "Replacing MM CR (DSN + probes + resources)"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" replace -f "$TMP_MM"
|
|
rm -f "$TMP_MM"
|
|
|
|
# Also setenv on Deployment immediately (operator reconcile can lag)
|
|
if kubectl --context "$CTX" -n "$SOURCE_NS" get deploy "$INSTALLATION" >/dev/null 2>&1; then
|
|
log "Forcing deploy/${INSTALLATION} MM_SQLSETTINGS_* → ${TARGET_KIND} rw=${EXPECTED_HP} ro=${EXPECTED_HP_RO}"
|
|
ENV_FILE="$(mktemp)"
|
|
printf 'MM_SQLSETTINGS_DATASOURCE=%s\nMM_SQLSETTINGS_DATASOURCEREPLICAS=%s\n' "$NEW_DSN" "$NEW_DSN_RO" >"$ENV_FILE"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" set env "deploy/${INSTALLATION}" --env - <"$ENV_FILE" >/dev/null
|
|
rm -f "$ENV_FILE"
|
|
# ensure deploy is not left at 0 while MM CR says 1
|
|
dep_rep="$(kubectl --context "$CTX" -n "$SOURCE_NS" get deploy "$INSTALLATION" \
|
|
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0)"
|
|
if [[ "${dep_rep:-0}" -lt 1 && "${CUTOVER_REPLICAS:-0}" -ge 1 ]]; then
|
|
log "Scaling deploy/${INSTALLATION} replicas 0→1"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" scale deploy "$INSTALLATION" --replicas=1 >/dev/null
|
|
fi
|
|
fi
|
|
|
|
scale_orphan_replicasets "$CTX" "$INSTALLATION"
|
|
|
|
log "Deleting MM pods to pick up new DSN"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" delete pod \
|
|
-l "installation.mattermost.com/installation=${INSTALLATION}" --wait=false || true
|
|
|
|
# Wait until stable + app ready + DSN on CNPG (+ HTTPS if endpoint known)
|
|
stale_swept=0
|
|
for i in $(seq 1 "${CUTOVER_WAIT_ATTEMPTS}"); do
|
|
state="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \
|
|
-o jsonpath='{.status.state}' 2>/dev/null || echo unknown)"
|
|
endpoint="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \
|
|
-o jsonpath='{.status.endpoint}' 2>/dev/null || true)"
|
|
ready="false"
|
|
if mm_app_ready "$CTX" "$INSTALLATION"; then
|
|
ready="true"
|
|
fi
|
|
dsn_hp="$(mm_pod_dsn_hostport "$CTX" "$INSTALLATION" 2>/dev/null || echo no-pod)"
|
|
ping_ok="no"
|
|
if mm_https_ping_ok "$endpoint"; then
|
|
ping_ok="yes"
|
|
fi
|
|
|
|
restarts="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \
|
|
-l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \
|
|
-o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null || echo 0)"
|
|
mm_rep="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" \
|
|
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo '?')"
|
|
|
|
log "wait ${i}/${CUTOVER_WAIT_ATTEMPTS}: state=${state} ready=${ready} dsn=${dsn_hp} expected=${EXPECTED_HP} replicas=${mm_rep} https=${ping_ok} restarts=${restarts}"
|
|
|
|
# Приостановленный инстанс не будим: DSN уже в секрете и CR, подов ждать нечего
|
|
if [[ "${mm_rep:-0}" == "0" ]]; then
|
|
log "SUCCESS cutover ${INSTALLATION}: на паузе (replicas=0), DSN переключён, поды не поднимаю"
|
|
mark_cutover_done "$INSTALLATION"
|
|
exit 0
|
|
fi
|
|
|
|
# Periodically collapse duplicate ReplicaSets
|
|
if (( i % 3 == 0 )); then
|
|
scale_orphan_replicasets "$CTX" "$INSTALLATION"
|
|
fi
|
|
|
|
# If pods keep crashing, clear stale DB sessions once
|
|
if [[ "$TARGET_KIND" == "cnpg" && "${restarts:-0}" -ge 2 && "$stale_swept" -eq 0 ]]; then
|
|
log "Restarts=${restarts}: sweeping stale CNPG sessions again"
|
|
terminate_stale_cnpg_sessions "$SRC_DB" "$SRC_USER" "${CUTOVER_STALE_IDLE_MINUTES}"
|
|
stale_swept=1
|
|
fi
|
|
|
|
# Wrong DSN → force again
|
|
if [[ "$dsn_hp" != "no-pod" && "$dsn_hp" != "unknown" && "$dsn_hp" != "${EXPECTED_HP}" && "$dsn_hp" != ":" ]]; then
|
|
log "Pod DSN is ${dsn_hp}, expected ${EXPECTED_HP} — re-applying deploy env"
|
|
ENV_FILE="$(mktemp)"
|
|
printf 'MM_SQLSETTINGS_DATASOURCE=%s\nMM_SQLSETTINGS_DATASOURCEREPLICAS=%s\n' "$NEW_DSN" "$NEW_DSN_RO" >"$ENV_FILE"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" set env "deploy/${INSTALLATION}" --env - <"$ENV_FILE" >/dev/null 2>&1 || true
|
|
rm -f "$ENV_FILE"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" delete pod \
|
|
-l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" --wait=false >/dev/null 2>&1 || true
|
|
fi
|
|
|
|
if [[ "$ready" == "true" && "$dsn_hp" == "${EXPECTED_HP}" ]]; then
|
|
if [[ "$state" == "stable" ]] || [[ "$ping_ok" == "yes" ]]; then
|
|
log "SUCCESS cutover ${INSTALLATION} (state=${state} endpoint=${endpoint} dsn=${dsn_hp})"
|
|
mark_cutover_done "$INSTALLATION"
|
|
exit 0
|
|
fi
|
|
# ready+correct DSN but state still reconciling — accept after HTTPS OK or enough ready cycles
|
|
if [[ "$i" -ge 12 && "$ready" == "true" ]]; then
|
|
# in-pod ping as fallback when ingress endpoint empty
|
|
pod="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \
|
|
-l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
if [[ -n "$pod" ]] && kubectl --context "$CTX" -n "$SOURCE_NS" exec "$pod" -- \
|
|
curl -sS -m 5 http://127.0.0.1:8065/api/v4/system/ping 2>/dev/null | grep -q OK; then
|
|
log "SUCCESS cutover ${INSTALLATION} (app ready+ping; state=${state})"
|
|
mark_cutover_done "$INSTALLATION"
|
|
exit 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
sleep "${CUTOVER_WAIT_SLEEP_SEC}"
|
|
done
|
|
|
|
log "Diagnostics:"
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" get pods -l "installation.mattermost.com/installation=${INSTALLATION}" || true
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o jsonpath='state={.status.state} endpoint={.status.endpoint}{"\n"}' || true
|
|
pod="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \
|
|
-l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
if [[ -n "$pod" ]]; then
|
|
kubectl --context "$CTX" -n "$SOURCE_NS" logs "$pod" --tail=40 || true
|
|
fi
|
|
|
|
die "cutover did not become stable in time for ${INSTALLATION}"
|