#!/usr/bin/env bash # Post-cutover health check for one Mattermost installation. # Verifies kube readiness + HTTP API (/api/v4/system/ping, config/client). # Writes registry notes: live | err: # # Usage: # ./healthcheck-mm-one.sh # ./healthcheck-mm-one.sh --require-dsn-host # ./healthcheck-mm-one.sh --skip-https # ./healthcheck-mm-one.sh --no-registry # do not update notes # # Exit 0 if Ready + API answers OK (HTTPS preferred, in-pod fallback). Exit 1 otherwise. set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=lib.sh source "${DIR}/lib.sh" CTX="" INSTALLATION="" REQUIRE_DSN_HOST="" SKIP_HTTPS=0 NO_REGISTRY=0 while [[ $# -gt 0 ]]; do case "$1" in --require-dsn-host) REQUIRE_DSN_HOST="${2:-}"; shift 2 ;; --skip-https) SKIP_HTTPS=1; shift ;; --no-registry) NO_REGISTRY=1; shift ;; -h|--help) sed -n '2,14p' "$0" exit 0 ;; *) if [[ -z "$CTX" ]]; then CTX="$1" elif [[ -z "$INSTALLATION" ]]; then INSTALLATION="$1" else die "unknown arg: $1" fi shift ;; esac done [[ -n "$CTX" && -n "$INSTALLATION" ]] || die "usage: $0 [--require-dsn-host HOST]" SOURCE_NS="${SOURCE_NS:-mm-cloud}" fail=0 api_ok=0 ERR_REASONS=() API_VIA="" MM_VERSION="" note_sanitize() { # single-line TSV-safe note printf '%s' "$1" | tr '\t\n\r' ' ' | sed 's/ */ /g' | cut -c1-200 } hc_finish() { local note="$1" rc="$2" note="$(note_sanitize "$note")" if [[ "$NO_REGISTRY" != "1" ]]; then if registry_set "$INSTALLATION" "notes=${note}"; then log "REGISTRY ${INSTALLATION} notes=${note}" else log "WARN ${INSTALLATION}: failed to write notes=${note}" fi fi exit "$rc" } push_err() { ERR_REASONS+=("$1"); fail=1; } err_note() { local joined if [[ ${#ERR_REASONS[@]} -eq 0 ]]; then printf 'err:unknown' return fi joined="$(IFS=','; echo "${ERR_REASONS[*]}")" printf 'err:%s' "$joined" } mm_json="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o json 2>/dev/null || true)" if [[ -z "$mm_json" ]]; then log "FAIL ${INSTALLATION}: Mattermost CR not found in ${CTX}/${SOURCE_NS}" hc_finish "err:no_cr" 1 fi MM_STATE="$(python3 -c 'import json,sys; print((json.loads(sys.argv[1]).get("status") or {}).get("state") or "")' "$mm_json")" MM_ENDPOINT="$(python3 -c 'import json,sys; print((json.loads(sys.argv[1]).get("status") or {}).get("endpoint") or "")' "$mm_json")" MM_REPLICAS="$(python3 -c 'import json,sys; r=(json.loads(sys.argv[1]).get("spec") or {}).get("replicas"); print("" if r is None else r)' "$mm_json")" MM_MEM_REQ="$(python3 -c 'import json,sys mm=json.loads(sys.argv[1]); spec=mm.get("spec") or {} res=(spec.get("scheduling") or {}).get("resources") or spec.get("resources") or {} print((res.get("requests") or {}).get("memory") or "")' "$mm_json")" log "=== healthcheck ${CTX} ${INSTALLATION} state=${MM_STATE:-?} endpoint=${MM_ENDPOINT:-?} replicas=${MM_REPLICAS:-?} mem=${MM_MEM_REQ:-?} ===" pod_line="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \ -l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\t"}{.status.podIP}{"\n"}{end}' 2>/dev/null || true)" pod_name=""; pod_phase=""; pod_ready="false"; pod_ip="" if [[ -n "$pod_line" ]]; then while IFS=$'\t' read -r n ph rd ip; do [[ -z "${n:-}" ]] && continue pod_name="$n"; pod_phase="$ph"; pod_ready="${rd:-false}"; pod_ip="${ip:-}" [[ "$pod_ready" == "true" ]] && break done <<<"$pod_line" fi if [[ -z "$pod_name" ]]; then log "FAIL ${INSTALLATION}: no mattermost pod" push_err "no_pod" else log "POD ${pod_name} phase=${pod_phase} ready=${pod_ready} ip=${pod_ip:-?}" if [[ "$pod_ready" != "true" ]]; then log "WARN ${INSTALLATION}: container not Ready" push_err "not_ready" fi fi dsn_hp="$(mm_pod_dsn_hostport "$CTX" "$INSTALLATION" 2>/dev/null || true)" if [[ -n "$dsn_hp" ]]; then log "DSN hostport=${dsn_hp}" if [[ -n "$REQUIRE_DSN_HOST" ]]; then dsn_host="${dsn_hp%%:*}" if [[ "$dsn_host" != "$REQUIRE_DSN_HOST" ]]; then log "FAIL ${INSTALLATION}: DSN host ${dsn_host} != required ${REQUIRE_DSN_HOST}" push_err "dsn_host=${dsn_host}" else log "OK DSN host matches ${REQUIRE_DSN_HOST}" fi fi else log "WARN ${INSTALLATION}: could not read DATASOURCE hostport" fi mm_https_api_check() { local endpoint="$1" local url body code version [[ -n "$endpoint" ]] || return 1 url="https://${endpoint}/api/v4/system/ping" body="$(curl -sS -m 10 -w '\n%{http_code}' "$url" 2>/dev/null || true)" code="$(printf '%s' "$body" | tail -n1)" body="$(printf '%s' "$body" | sed '$d')" log "HTTPS GET ${url} → HTTP ${code:-?} body=${body:-}" if [[ "$code" != "200" ]]; then return 1 fi if [[ "$body" != *'"status":"OK"'* && "$body" != *'"status": "OK"'* && "$body" != *OK* ]]; then return 1 fi url="https://${endpoint}/api/v4/config/client?format=old" body="$(curl -sS -m 10 -w '\n%{http_code}' "$url" 2>/dev/null || true)" code="$(printf '%s' "$body" | tail -n1)" body="$(printf '%s' "$body" | sed '$d')" version="$(printf '%s' "$body" | python3 -c 'import sys,json try: d=json.load(sys.stdin) print(d.get("Version") or d.get("version") or "") except Exception: print("")' 2>/dev/null || true)" MM_VERSION="$version" log "HTTPS GET config/client → HTTP ${code:-?} Version=${version:-?}" [[ "$code" == "200" && -n "$version" ]] || log "WARN ${INSTALLATION}: config/client incomplete (ping OK is enough)" return 0 } mm_inpod_ping() { local pod="$1" [[ -n "$pod" ]] || return 1 local body body="$(kubectl --context "$CTX" -n "$SOURCE_NS" exec "$pod" -- \ curl -sS -m 5 http://127.0.0.1:8065/api/v4/system/ping 2>/dev/null || true)" log "IN-POD ping → ${body:-}" [[ "$body" == *OK* ]] } if [[ "$SKIP_HTTPS" != "1" ]]; then if mm_https_api_check "${MM_ENDPOINT:-}"; then api_ok=1 API_VIA="https" log "OK ${INSTALLATION}: HTTPS API healthy" else log "WARN ${INSTALLATION}: HTTPS API failed or no endpoint; trying in-pod" if [[ -n "$pod_name" ]] && mm_inpod_ping "$pod_name"; then api_ok=1 API_VIA="inpod" log "OK ${INSTALLATION}: in-pod API healthy (HTTPS unavailable)" else log "FAIL ${INSTALLATION}: API not responding" push_err "api" fi fi else if [[ -n "$pod_name" ]] && mm_inpod_ping "$pod_name"; then api_ok=1 API_VIA="inpod" log "OK ${INSTALLATION}: in-pod API healthy (--skip-https)" else log "FAIL ${INSTALLATION}: in-pod API failed" push_err "api" fi fi case "${MM_STATE:-}" in stable|Stable|"") ;; *) log "WARN ${INSTALLATION}: MM state=${MM_STATE} (expected stable)" push_err "state=${MM_STATE}" ;; esac if [[ "$api_ok" -eq 1 && "$fail" -eq 0 ]]; then log "SUCCESS healthcheck ${INSTALLATION}" hc_finish "live" 0 fi if [[ "$api_ok" -eq 1 ]]; then log "PARTIAL ${INSTALLATION}: API OK but other checks failed" hc_finish "$(err_note),api_ok=${API_VIA}" 1 fi log "FAILED healthcheck ${INSTALLATION}" hc_finish "$(err_note)" 1