124 lines
3.8 KiB
Bash
Executable File
124 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run migrate-one for each line in a list file OR a registry batch.
|
|
#
|
|
# Usage:
|
|
# ./migrate-list.sh batches/foo.txt
|
|
# ./migrate-list.sh --batch wave1-lm
|
|
# PARALLELISM=4 ./migrate-list.sh --batch wave1-lm --status pending,sized
|
|
# ./migrate-list.sh --batch wave1-lm --scale-0 # hibernate MM during dump, restore after
|
|
set -euo pipefail
|
|
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck disable=SC1091
|
|
source "${DIR}/lib.sh"
|
|
|
|
LIST=""
|
|
BATCH=""
|
|
# include migrating (stuck mid-run) and size_error so list is registry-driven
|
|
STATUS="pending,sized,failed,size_error,migrating"
|
|
SCALE_0=0
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--batch) BATCH="${2:-}"; shift 2 ;;
|
|
--status) STATUS="${2:-}"; shift 2 ;;
|
|
--scale-0) SCALE_0=1; shift ;;
|
|
-h|--help)
|
|
sed -n '2,12p' "$0"
|
|
exit 0
|
|
;;
|
|
*)
|
|
LIST="$1"
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$SCALE_0" == "1" ]]; then
|
|
export MIGRATE_SCALE_0=1
|
|
fi
|
|
|
|
WORK="$(mktemp)"
|
|
trap 'rm -f "$WORK"' EXIT
|
|
|
|
if [[ -n "$BATCH" ]]; then
|
|
"${DIR}/registry-export-batch.sh" "$BATCH" --status "$STATUS" -o "$WORK"
|
|
LIST="$WORK"
|
|
log "=== migrate-list batch=${BATCH} status=${STATUS} parallelism=${PARALLELISM} scale-0=${SCALE_0} ==="
|
|
elif [[ -n "$LIST" ]]; then
|
|
[[ -f "$LIST" ]] || die "list file not found: $LIST"
|
|
# copy filtering comments
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
[[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
|
|
echo "$line" >>"${WORK}.raw"
|
|
done <"$LIST"
|
|
mv "${WORK}.raw" "$WORK"
|
|
log "=== migrate-list ${LIST} parallelism=${PARALLELISM} default_target=${TARGET:-cnpg} scale-0=${SCALE_0} ==="
|
|
else
|
|
LIST="${LIST_PATH}"
|
|
[[ -f "$LIST" ]] || die "list file not found: $LIST (or pass --batch)"
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
[[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
|
|
echo "$line" >>"${WORK}.raw"
|
|
done <"$LIST"
|
|
mv "${WORK}.raw" "$WORK"
|
|
log "=== migrate-list ${LIST} parallelism=${PARALLELISM} default_target=${TARGET:-cnpg} scale-0=${SCALE_0} ==="
|
|
fi
|
|
|
|
# normalize to TSV ctx/installation/extras
|
|
NORM="$(mktemp)"
|
|
trap 'rm -f "$WORK" "$NORM"' EXIT
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
[[ -z "$line" || "$line" =~ ^[[:space:]]*# ]] && continue
|
|
# shellcheck disable=SC2086
|
|
set -- $line
|
|
ctx="${1:-}"; installation="${2:-}"
|
|
[[ -n "$ctx" && -n "$installation" ]] || continue
|
|
shift 2 2>/dev/null || true
|
|
printf '%s\t%s' "$ctx" "$installation" >>"$NORM"
|
|
for a in "$@"; do
|
|
printf '\t%s' "$a" >>"$NORM"
|
|
done
|
|
printf '\n' >>"$NORM"
|
|
done <"$WORK"
|
|
|
|
total="$(wc -l <"$NORM" | tr -d ' ')"
|
|
log "Queued ${total} installations"
|
|
[[ "$total" -gt 0 ]] || { log "nothing to migrate"; exit 0; }
|
|
|
|
run_one() {
|
|
local ctx="$1" installation="$2"
|
|
shift 2
|
|
local extra=()
|
|
if [[ "$SCALE_0" == "1" ]]; then
|
|
extra+=(--scale-0)
|
|
fi
|
|
# do not pre-set migrating — migrate-one owns status (and may SKIP)
|
|
"${DIR}/migrate-one.sh" "$ctx" "$installation" "${extra[@]+"${extra[@]}"}" "$@" || log "FAILED $installation (continuing)"
|
|
}
|
|
|
|
if [[ "${PARALLELISM}" -le 1 ]]; then
|
|
while IFS=$'\t' read -r ctx installation a3 a4 || [[ -n "${ctx:-}" ]]; do
|
|
[[ -z "${ctx:-}" ]] && continue
|
|
args=()
|
|
if [[ -n "${a3:-}" ]]; then args+=("$a3"); fi
|
|
if [[ -n "${a4:-}" ]]; then args+=("$a4"); fi
|
|
run_one "$ctx" "$installation" "${args[@]+"${args[@]}"}"
|
|
done <"$NORM"
|
|
else
|
|
while IFS=$'\t' read -r ctx installation a3 a4 || [[ -n "${ctx:-}" ]]; do
|
|
[[ -z "${ctx:-}" ]] && continue
|
|
while [[ "$(jobs -pr | wc -l | tr -d ' ')" -ge "${PARALLELISM}" ]]; do
|
|
sleep 5
|
|
done
|
|
args=()
|
|
if [[ -n "${a3:-}" ]]; then args+=("$a3"); fi
|
|
if [[ -n "${a4:-}" ]]; then args+=("$a4"); fi
|
|
run_one "$ctx" "$installation" "${args[@]+"${args[@]}"}" &
|
|
done <"$NORM"
|
|
wait || true
|
|
fi
|
|
|
|
log "=== migrate-list finished ==="
|
|
python3 "${DIR}/registry_lib.py" --registry "${REGISTRY_PATH}" summary || true
|