224 lines
7.2 KiB
Bash
Executable File
224 lines
7.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Post-cutover health check for one Mattermost installation.
|
|
# Verifies kube readiness + HTTP API (/api/v4/system/ping, config/client).
|
|
# Writes registry notes: live | err:<reason>
|
|
#
|
|
# Usage:
|
|
# ./healthcheck-mm-one.sh <kube_context> <installation_id>
|
|
# ./healthcheck-mm-one.sh <ctx> <id> --require-dsn-host <host>
|
|
# ./healthcheck-mm-one.sh <ctx> <id> --skip-https
|
|
# ./healthcheck-mm-one.sh <ctx> <id> --no-registry # do not update notes
|
|
#
|
|
# Exit 0 if Ready + API answers OK (HTTPS preferred, in-pod fallback). Exit 1 otherwise.
|
|
set -euo pipefail
|
|
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=lib.sh
|
|
source "${DIR}/lib.sh"
|
|
|
|
CTX=""
|
|
INSTALLATION=""
|
|
REQUIRE_DSN_HOST=""
|
|
SKIP_HTTPS=0
|
|
NO_REGISTRY=0
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--require-dsn-host) REQUIRE_DSN_HOST="${2:-}"; shift 2 ;;
|
|
--skip-https) SKIP_HTTPS=1; shift ;;
|
|
--no-registry) NO_REGISTRY=1; shift ;;
|
|
-h|--help)
|
|
sed -n '2,14p' "$0"
|
|
exit 0
|
|
;;
|
|
*)
|
|
if [[ -z "$CTX" ]]; then CTX="$1"
|
|
elif [[ -z "$INSTALLATION" ]]; then INSTALLATION="$1"
|
|
else die "unknown arg: $1"
|
|
fi
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[[ -n "$CTX" && -n "$INSTALLATION" ]] || die "usage: $0 <kube_context> <installation_id> [--require-dsn-host HOST]"
|
|
|
|
SOURCE_NS="${SOURCE_NS:-mm-cloud}"
|
|
fail=0
|
|
api_ok=0
|
|
ERR_REASONS=()
|
|
API_VIA=""
|
|
MM_VERSION=""
|
|
|
|
note_sanitize() {
|
|
# single-line TSV-safe note
|
|
printf '%s' "$1" | tr '\t\n\r' ' ' | sed 's/ */ /g' | cut -c1-200
|
|
}
|
|
|
|
hc_finish() {
|
|
local note="$1" rc="$2"
|
|
note="$(note_sanitize "$note")"
|
|
if [[ "$NO_REGISTRY" != "1" ]]; then
|
|
if registry_set "$INSTALLATION" "notes=${note}"; then
|
|
log "REGISTRY ${INSTALLATION} notes=${note}"
|
|
else
|
|
log "WARN ${INSTALLATION}: failed to write notes=${note}"
|
|
fi
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
push_err() { ERR_REASONS+=("$1"); fail=1; }
|
|
|
|
err_note() {
|
|
local joined
|
|
if [[ ${#ERR_REASONS[@]} -eq 0 ]]; then
|
|
printf 'err:unknown'
|
|
return
|
|
fi
|
|
joined="$(IFS=','; echo "${ERR_REASONS[*]}")"
|
|
printf 'err:%s' "$joined"
|
|
}
|
|
|
|
mm_json="$(kubectl --context "$CTX" -n "$SOURCE_NS" get mm "$INSTALLATION" -o json 2>/dev/null || true)"
|
|
if [[ -z "$mm_json" ]]; then
|
|
log "FAIL ${INSTALLATION}: Mattermost CR not found in ${CTX}/${SOURCE_NS}"
|
|
hc_finish "err:no_cr" 1
|
|
fi
|
|
|
|
MM_STATE="$(python3 -c 'import json,sys; print((json.loads(sys.argv[1]).get("status") or {}).get("state") or "")' "$mm_json")"
|
|
MM_ENDPOINT="$(python3 -c 'import json,sys; print((json.loads(sys.argv[1]).get("status") or {}).get("endpoint") or "")' "$mm_json")"
|
|
MM_REPLICAS="$(python3 -c 'import json,sys; r=(json.loads(sys.argv[1]).get("spec") or {}).get("replicas"); print("" if r is None else r)' "$mm_json")"
|
|
MM_MEM_REQ="$(python3 -c 'import json,sys
|
|
mm=json.loads(sys.argv[1]); spec=mm.get("spec") or {}
|
|
res=(spec.get("scheduling") or {}).get("resources") or spec.get("resources") or {}
|
|
print((res.get("requests") or {}).get("memory") or "")' "$mm_json")"
|
|
|
|
log "=== healthcheck ${CTX} ${INSTALLATION} state=${MM_STATE:-?} endpoint=${MM_ENDPOINT:-?} replicas=${MM_REPLICAS:-?} mem=${MM_MEM_REQ:-?} ==="
|
|
|
|
pod_line="$(kubectl --context "$CTX" -n "$SOURCE_NS" get pods \
|
|
-l "app=mattermost,installation.mattermost.com/installation=${INSTALLATION}" \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{.status.containerStatuses[0].ready}{"\t"}{.status.podIP}{"\n"}{end}' 2>/dev/null || true)"
|
|
pod_name=""; pod_phase=""; pod_ready="false"; pod_ip=""
|
|
if [[ -n "$pod_line" ]]; then
|
|
while IFS=$'\t' read -r n ph rd ip; do
|
|
[[ -z "${n:-}" ]] && continue
|
|
pod_name="$n"; pod_phase="$ph"; pod_ready="${rd:-false}"; pod_ip="${ip:-}"
|
|
[[ "$pod_ready" == "true" ]] && break
|
|
done <<<"$pod_line"
|
|
fi
|
|
|
|
if [[ -z "$pod_name" ]]; then
|
|
log "FAIL ${INSTALLATION}: no mattermost pod"
|
|
push_err "no_pod"
|
|
else
|
|
log "POD ${pod_name} phase=${pod_phase} ready=${pod_ready} ip=${pod_ip:-?}"
|
|
if [[ "$pod_ready" != "true" ]]; then
|
|
log "WARN ${INSTALLATION}: container not Ready"
|
|
push_err "not_ready"
|
|
fi
|
|
fi
|
|
|
|
dsn_hp="$(mm_pod_dsn_hostport "$CTX" "$INSTALLATION" 2>/dev/null || true)"
|
|
if [[ -n "$dsn_hp" ]]; then
|
|
log "DSN hostport=${dsn_hp}"
|
|
if [[ -n "$REQUIRE_DSN_HOST" ]]; then
|
|
dsn_host="${dsn_hp%%:*}"
|
|
if [[ "$dsn_host" != "$REQUIRE_DSN_HOST" ]]; then
|
|
log "FAIL ${INSTALLATION}: DSN host ${dsn_host} != required ${REQUIRE_DSN_HOST}"
|
|
push_err "dsn_host=${dsn_host}"
|
|
else
|
|
log "OK DSN host matches ${REQUIRE_DSN_HOST}"
|
|
fi
|
|
fi
|
|
else
|
|
log "WARN ${INSTALLATION}: could not read DATASOURCE hostport"
|
|
fi
|
|
|
|
mm_https_api_check() {
|
|
local endpoint="$1"
|
|
local url body code version
|
|
[[ -n "$endpoint" ]] || return 1
|
|
url="https://${endpoint}/api/v4/system/ping"
|
|
body="$(curl -sS -m 10 -w '\n%{http_code}' "$url" 2>/dev/null || true)"
|
|
code="$(printf '%s' "$body" | tail -n1)"
|
|
body="$(printf '%s' "$body" | sed '$d')"
|
|
log "HTTPS GET ${url} → HTTP ${code:-?} body=${body:-<empty>}"
|
|
if [[ "$code" != "200" ]]; then
|
|
return 1
|
|
fi
|
|
if [[ "$body" != *'"status":"OK"'* && "$body" != *'"status": "OK"'* && "$body" != *OK* ]]; then
|
|
return 1
|
|
fi
|
|
url="https://${endpoint}/api/v4/config/client?format=old"
|
|
body="$(curl -sS -m 10 -w '\n%{http_code}' "$url" 2>/dev/null || true)"
|
|
code="$(printf '%s' "$body" | tail -n1)"
|
|
body="$(printf '%s' "$body" | sed '$d')"
|
|
version="$(printf '%s' "$body" | python3 -c 'import sys,json
|
|
try:
|
|
d=json.load(sys.stdin)
|
|
print(d.get("Version") or d.get("version") or "")
|
|
except Exception:
|
|
print("")' 2>/dev/null || true)"
|
|
MM_VERSION="$version"
|
|
log "HTTPS GET config/client → HTTP ${code:-?} Version=${version:-?}"
|
|
[[ "$code" == "200" && -n "$version" ]] || log "WARN ${INSTALLATION}: config/client incomplete (ping OK is enough)"
|
|
return 0
|
|
}
|
|
|
|
mm_inpod_ping() {
|
|
local pod="$1"
|
|
[[ -n "$pod" ]] || return 1
|
|
local body
|
|
body="$(kubectl --context "$CTX" -n "$SOURCE_NS" exec "$pod" -- \
|
|
curl -sS -m 5 http://127.0.0.1:8065/api/v4/system/ping 2>/dev/null || true)"
|
|
log "IN-POD ping → ${body:-<empty>}"
|
|
[[ "$body" == *OK* ]]
|
|
}
|
|
|
|
if [[ "$SKIP_HTTPS" != "1" ]]; then
|
|
if mm_https_api_check "${MM_ENDPOINT:-}"; then
|
|
api_ok=1
|
|
API_VIA="https"
|
|
log "OK ${INSTALLATION}: HTTPS API healthy"
|
|
else
|
|
log "WARN ${INSTALLATION}: HTTPS API failed or no endpoint; trying in-pod"
|
|
if [[ -n "$pod_name" ]] && mm_inpod_ping "$pod_name"; then
|
|
api_ok=1
|
|
API_VIA="inpod"
|
|
log "OK ${INSTALLATION}: in-pod API healthy (HTTPS unavailable)"
|
|
else
|
|
log "FAIL ${INSTALLATION}: API not responding"
|
|
push_err "api"
|
|
fi
|
|
fi
|
|
else
|
|
if [[ -n "$pod_name" ]] && mm_inpod_ping "$pod_name"; then
|
|
api_ok=1
|
|
API_VIA="inpod"
|
|
log "OK ${INSTALLATION}: in-pod API healthy (--skip-https)"
|
|
else
|
|
log "FAIL ${INSTALLATION}: in-pod API failed"
|
|
push_err "api"
|
|
fi
|
|
fi
|
|
|
|
case "${MM_STATE:-}" in
|
|
stable|Stable|"") ;;
|
|
*)
|
|
log "WARN ${INSTALLATION}: MM state=${MM_STATE} (expected stable)"
|
|
push_err "state=${MM_STATE}"
|
|
;;
|
|
esac
|
|
|
|
if [[ "$api_ok" -eq 1 && "$fail" -eq 0 ]]; then
|
|
log "SUCCESS healthcheck ${INSTALLATION}"
|
|
hc_finish "live" 0
|
|
fi
|
|
if [[ "$api_ok" -eq 1 ]]; then
|
|
log "PARTIAL ${INSTALLATION}: API OK but other checks failed"
|
|
hc_finish "$(err_note),api_ok=${API_VIA}" 1
|
|
fi
|
|
log "FAILED healthcheck ${INSTALLATION}"
|
|
hc_finish "$(err_note)" 1
|