loop-ops/db/verify-list.sh

#!/usr/bin/env bash
# Run verify-one for each line in a list file OR a registry batch.
#
# Usage:
#   ./verify-list.sh --batch wave4-lp-3
#   ./verify-list.sh batches/foo.txt
#   PARALLELISM=4 ./verify-list.sh --batch wave4-lp-3
#   ./verify-list.sh --batch wave4-lp-3 --force   # also re-verify status=verified
#
# Without --force: ONLY registry status=migrated (batch export + per-row check).
# With --force: also queue status=verified (re-verify).
set -euo pipefail

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1091
source "${DIR}/lib.sh"

LIST=""
BATCH=""
FORCE=0
STATUS="migrated"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --batch) BATCH="${2:-}"; shift 2 ;;
    --status)
      STATUS="${2:-}"
      shift 2
      ;;
    --force) FORCE=1; shift ;;
    -h|--help)
      sed -n '2,14p' "$0"
      exit 0
      ;;
    *)
      LIST="$1"
      shift
      ;;
  esac
done

if [[ "$FORCE" != "1" ]]; then
  STATUS="migrated"
else
  if [[ "$STATUS" == "migrated" ]]; then
    STATUS="migrated,verified"
  fi
fi

registry_status() {
  local installation="$1" st
  st="$(registry_get "$installation" status 2>/dev/null || true)"
  printf '%s' "$st" | tr -d '[:space:]'
}

verify_allowed() {
  local installation="$1" st
  st="$(registry_status "$installation")"
  if [[ "$FORCE" == "1" ]]; then
    case "$st" in
      migrated|verified) return 0 ;;
      *)
        IFS=',' read -r -a allowed <<<"$STATUS"
        for a in "${allowed[@]}"; do
          [[ "$st" == "$a" ]] && return 0
        done
        return 1
        ;;
    esac
  fi
  [[ "$st" == "migrated" ]]
}

WORK="$(mktemp)"
NORM="$(mktemp)"
trap 'rm -f "$WORK" "$NORM"' EXIT

if [[ -n "$BATCH" ]]; then
  # --include-done: migrated/verified are in DONE_LIKE and would be skipped otherwise
  "${DIR}/registry-export-batch.sh" "$BATCH" --status "$STATUS" --include-done -o "$WORK"
  log "=== verify-list batch=${BATCH} status=${STATUS} parallelism=${PARALLELISM} force=${FORCE} ==="
elif [[ -n "$LIST" ]]; then
  [[ -f "$LIST" ]] || die "list file not found: $LIST"
  while IFS= read -r line || [[ -n "$line" ]]; do
    [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
    echo "$line" >>"${WORK}.raw"
  done <"$LIST"
  mv "${WORK}.raw" "$WORK"
  log "=== verify-list ${LIST} parallelism=${PARALLELISM} force=${FORCE} (filter status=${STATUS}) ==="
else
  die "usage: $0 --batch <id> | <list_file> [--force]"
fi

skipped=0
while IFS= read -r line || [[ -n "$line" ]]; do
  [[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
  # shellcheck disable=SC2086
  set -- $line
  ctx="${1:-}"; installation="${2:-}"
  [[ -n "$ctx" && -n "$installation" ]] || continue
  shift 2 2>/dev/null || true
  if ! verify_allowed "$installation"; then
    st="$(registry_status "$installation")"
    if [[ "$FORCE" == "1" ]]; then
      log "SKIP ${installation} (status=${st:-empty}; --force allows migrated|verified)"
    else
      log "SKIP ${installation} (status=${st:-empty}; need migrated, or pass --force)"
    fi
    skipped=$((skipped + 1))
    continue
  fi
  printf '%s\t%s' "$ctx" "$installation" >>"$NORM"
  for a in "$@"; do
    printf '\t%s' "$a" >>"$NORM"
  done
  printf '\n' >>"$NORM"
done <"$WORK"

total="$(wc -l <"$NORM" | tr -d ' ')"
log "Queued ${total} installations for verify (skipped=${skipped})"
[[ "$total" -gt 0 ]] || { log "nothing to verify"; exit 0; }

run_one() {
  local ctx="$1" installation="$2" st
  shift 2
  if ! verify_allowed "$installation"; then
    st="$(registry_status "$installation")"
    log "SKIP ${installation} (status=${st:-empty} at run time)"
    return 0
  fi
  "${DIR}/verify-one.sh" "$ctx" "$installation" "$@" || log "FAILED verify $installation (continuing)"
}

if [[ "${PARALLELISM}" -le 1 ]]; then
  while IFS=$'\t' read -r ctx installation a3 a4 || [[ -n "${ctx:-}" ]]; do
    [[ -z "${ctx:-}" ]] && continue
    args=()
    [[ -n "${a3:-}" ]] && args+=("$a3")
    [[ -n "${a4:-}" ]] && args+=("$a4")
    run_one "$ctx" "$installation" "${args[@]+"${args[@]}"}"
  done <"$NORM"
else
  while IFS=$'\t' read -r ctx installation a3 a4 || [[ -n "${ctx:-}" ]]; do
    [[ -z "${ctx:-}" ]] && continue
    while [[ "$(jobs -pr | wc -l | tr -d ' ')" -ge "${PARALLELISM}" ]]; do
      sleep 5
    done
    args=()
    [[ -n "${a3:-}" ]] && args+=("$a3")
    [[ -n "${a4:-}" ]] && args+=("$a4")
    run_one "$ctx" "$installation" "${args[@]+"${args[@]}"}" &
  done <"$NORM"
  wait || true
fi

log "=== verify-list finished ==="
python3 "${DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" summary || true