修复异机备份凭证与中断恢复（07-27-offhost-backup-final-fixes）

scripts/receive-offhost-backup.sh

@@ -15,6 +15,8 @@ RRSYNC_BIN="${RRSYNC_BIN:-/usr/bin/rrsync}"
 PYTHON_BIN="${PYTHON_BIN:-python3}"
 DAILY_KEEP="${DAILY_KEEP:-7}"
 WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
+declare -A RECONCILED_WEEK_SOURCE=()
+declare -a DAILY_ORPHANS_PENDING_DELETE=()
 
 fail() {
   echo "[offhost-receiver][ERROR] $1" >&2
@@ -96,7 +98,8 @@ validate_batch() {
   [[ "$actual_manifest_sha" == "$expected_manifest_sha" ]] \
     || fail "manifest checksum mismatch"
 
-  "$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON'
+  "$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON' \
+    || fail "batch manifest or artifact validation failed"
 import hashlib
 import json
 import os
@@ -135,6 +138,128 @@ for key, expected_name in expected.items():
 PYTHON
 }
 
+validated_batch_identity() {
+  local batch_dir="$1"
+  [[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "recovery candidate is not a regular directory"
+
+  local run_id manifest_sha identity
+  run_id="$($PYTHON_BIN - "$batch_dir/manifest.json" <<'PYTHON'
+import json
+import re
+import sys
+
+with open(sys.argv[1], "r", encoding="utf-8") as handle:
+    manifest = json.load(handle)
+run_id = manifest.get("runId")
+if not isinstance(run_id, str) or not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", run_id):
+    raise SystemExit("manifest run id is invalid")
+print(run_id)
+PYTHON
+  )" || fail "recovery candidate manifest identity is invalid"
+  manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
+  validate_batch "$batch_dir" "$run_id" "$manifest_sha"
+  identity="$($PYTHON_BIN - "$run_id" <<'PYTHON'
+from datetime import datetime
+import sys
+
+run_id = sys.argv[1]
+day = run_id.removeprefix("auto-").split("T", 1)[0]
+iso = datetime.strptime(day, "%Y%m%d").date().isocalendar()
+print(f"{run_id}\t{day}\tweek-{iso.year:04d}-W{iso.week:02d}")
+PYTHON
+  )" || fail "recovery candidate date identity is invalid"
+  printf '%s\t%s\n' "$identity" "$manifest_sha"
+}
+
+validate_temp_directory() {
+  local target="$1"
+  local root="$2"
+  local pattern="$3"
+  [[ "$(dirname -- "$target")" == "$root" ]] || fail "orphan target escaped its managed root"
+  [[ "$(basename -- "$target")" =~ $pattern ]] || fail "orphan target name is invalid"
+  [[ -d "$target" && ! -L "$target" ]] || fail "orphan target is not a regular directory"
+}
+
+remove_temp_directory() {
+  local target="$1"
+  local root="$2"
+  local pattern="$3"
+  validate_temp_directory "$target" "$root" "$pattern"
+  rm -rf -- "$target"
+}
+
+canonical_daily_for_day() {
+  local run_day="$1"
+  local -a entries=()
+  mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
+    -name "hyb-oa-backup-auto-${run_day}T*Z" -print | LC_ALL=C sort)
+  [[ "${#entries[@]}" -le 1 ]] || fail "daily archive invariant is ambiguous during recovery"
+  if [[ "${#entries[@]}" -eq 1 ]]; then
+    printf '%s\n' "${entries[0]}"
+  fi
+}
+
+reconcile_daily_orphans() {
+  local -a orphans=()
+  mapfile -t orphans < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
+    -name '.daily-replacement-*' -print | LC_ALL=C sort)
+  local orphan base run_day identity orphan_run_id orphan_day orphan_week orphan_sha
+  local canonical canonical_identity canonical_run_id canonical_day canonical_week canonical_sha
+  local -A seen_days=() identities=() canonicals=()
+  for orphan in "${orphans[@]}"; do
+    base="$(basename -- "$orphan")"
+    [[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
+      || fail "daily replacement orphan name is invalid"
+    run_day="${BASH_REMATCH[1]}"
+    [[ -z "${seen_days[$run_day]:-}" ]] || fail "multiple daily replacement orphans exist for one day"
+    seen_days[$run_day]=1
+    validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
+      '^\.daily-replacement-[0-9]{8}\.[0-9]+$'
+    identity="$(validated_batch_identity "$orphan")" \
+      || fail "daily replacement orphan validation failed"
+    IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
+    [[ "$orphan_day" == "$run_day" ]] || fail "daily replacement orphan day does not match its manifest"
+    identities[$orphan]="$identity"
+
+    canonical="$(canonical_daily_for_day "$run_day")" \
+      || fail "daily archive lookup failed during recovery"
+    canonicals[$run_day]="$canonical"
+    if [[ -n "$canonical" ]]; then
+      [[ -d "$canonical" && ! -L "$canonical" ]] || fail "daily archive recovery target is invalid"
+      canonical_identity="$(validated_batch_identity "$canonical")" \
+        || fail "daily archive validation failed during recovery"
+      IFS=$'\t' read -r canonical_run_id canonical_day canonical_week canonical_sha <<<"$canonical_identity"
+      [[ "$canonical_day" == "$run_day" \
+        && "$(basename -- "$canonical")" == "hyb-oa-backup-$canonical_run_id" ]] \
+        || fail "daily archive identity is inconsistent during recovery"
+      [[ "$canonical_week" == "$orphan_week" ]] || fail "daily replacement week identity is inconsistent"
+    else
+      canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
+      [[ ! -e "$canonical" ]] || fail "daily recovery target already exists"
+    fi
+  done
+
+  for orphan in "${orphans[@]}"; do
+    base="$(basename -- "$orphan")"
+    [[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
+      || fail "daily replacement orphan name changed during recovery"
+    run_day="${BASH_REMATCH[1]}"
+    identity="${identities[$orphan]}"
+    IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
+    canonical="${canonicals[$run_day]}"
+    if [[ -z "$canonical" ]]; then
+      canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
+      validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
+        '^\.daily-replacement-[0-9]{8}\.[0-9]+$'
+      [[ ! -e "$canonical" ]] || fail "daily recovery target appeared during recovery"
+      mv -- "$orphan" "$canonical"
+    else
+      DAILY_ORPHANS_PENDING_DELETE+=("$orphan")
+    fi
+    RECONCILED_WEEK_SOURCE[$orphan_week]="$canonical"
+  done
+}
+
 prune_named_directories() {
   local root="$1"
   local prefix="$2"
@@ -186,6 +311,138 @@ publish_weekly_checkpoint() {
   fi
 }
 
+unique_daily_source_for_week() {
+  local expected_week="$1"
+  local -a candidates=() entries=()
+  mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
+    -name 'hyb-oa-backup-auto-*' -print | LC_ALL=C sort)
+  local entry identity run_id run_day iso_week manifest_sha
+  for entry in "${entries[@]}"; do
+    [[ -d "$entry" && ! -L "$entry" ]] || fail "daily archive candidate is invalid during weekly recovery"
+    identity="$(validated_batch_identity "$entry")" \
+      || fail "daily archive validation failed during weekly recovery"
+    IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
+    [[ "$(basename -- "$entry")" == "hyb-oa-backup-$run_id" ]] \
+      || fail "daily archive name does not match its manifest"
+    if [[ "$iso_week" == "$expected_week" ]]; then
+      candidates+=("$entry")
+    fi
+  done
+  [[ "${#candidates[@]}" -eq 1 ]] || fail "weekly recovery source is not unique"
+  printf '%s\n' "${candidates[0]}"
+}
+
+reconcile_weekly_orphans() {
+  local -a entries=()
+  mapfile -t entries < <(find "$BACKUP_WEEKLY_DIR" -mindepth 1 -maxdepth 1 \
+    \( -name 'week-*-W*.new.*' -o -name 'week-*-W*.rollback.*' \) -print | LC_ALL=C sort)
+  local -A new_by_week=() rollback_by_week=() weeks=()
+  local entry base week kind
+  for entry in "${entries[@]}"; do
+    base="$(basename -- "$entry")"
+    if [[ "$base" =~ ^(week-[0-9]{4}-W[0-9]{2})\.(new|rollback)\.([0-9]+)$ ]]; then
+      week="${BASH_REMATCH[1]}"
+      kind="${BASH_REMATCH[2]}"
+    else
+      fail "weekly orphan name is invalid"
+    fi
+    validate_temp_directory "$entry" "$BACKUP_WEEKLY_DIR" \
+      '^week-[0-9]{4}-W[0-9]{2}\.(new|rollback)\.[0-9]+$'
+    weeks[$week]=1
+    if [[ "$kind" == "new" ]]; then
+      [[ -z "${new_by_week[$week]:-}" ]] || fail "multiple weekly staging orphans exist"
+      new_by_week[$week]="$entry"
+    else
+      [[ -z "${rollback_by_week[$week]:-}" ]] || fail "multiple weekly rollback orphans exist"
+      rollback_by_week[$week]="$entry"
+    fi
+  done
+  for week in "${!RECONCILED_WEEK_SOURCE[@]}"; do
+    weeks[$week]=1
+  done
+
+  local -A sources=()
+  local canonical new_dir rollback_dir identity run_id run_day iso_week manifest_sha source
+  for week in "${!weeks[@]}"; do
+    canonical="$BACKUP_WEEKLY_DIR/$week"
+    new_dir="${new_by_week[$week]:-}"
+    rollback_dir="${rollback_by_week[$week]:-}"
+
+    if [[ -e "$canonical" ]]; then
+      [[ -d "$canonical" && ! -L "$canonical" ]] || fail "weekly checkpoint is not a regular directory"
+      identity="$(validated_batch_identity "$canonical")" \
+        || fail "weekly checkpoint validation failed during recovery"
+      IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
+      [[ "$iso_week" == "$week" ]] || fail "weekly checkpoint manifest belongs to another week"
+    fi
+    if [[ -n "$rollback_dir" ]]; then
+      identity="$(validated_batch_identity "$rollback_dir")" \
+        || fail "weekly rollback validation failed"
+      IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
+      [[ "$iso_week" == "$week" ]] || fail "weekly rollback manifest belongs to another week"
+    fi
+
+    source="${RECONCILED_WEEK_SOURCE[$week]:-}"
+    if [[ -z "$source" && ! -e "$canonical" && -z "$rollback_dir" && -n "$new_dir" ]]; then
+      source="$(unique_daily_source_for_week "$week")" \
+        || fail "weekly staging recovery source could not be resolved"
+    fi
+    if [[ -n "$source" ]]; then
+      [[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
+      identity="$(validated_batch_identity "$source")" \
+        || fail "weekly rebuild source validation failed"
+      IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
+      [[ "$iso_week" == "$week" ]] || fail "weekly rebuild source belongs to another week"
+      sources[$week]="$source"
+    fi
+    [[ -e "$canonical" || -n "$rollback_dir" || -n "$source" ]] \
+      || fail "weekly checkpoint has no authoritative recovery source"
+  done
+
+  for week in "${!weeks[@]}"; do
+    canonical="$BACKUP_WEEKLY_DIR/$week"
+    new_dir="${new_by_week[$week]:-}"
+    rollback_dir="${rollback_by_week[$week]:-}"
+
+    if [[ ! -e "$canonical" && -n "$rollback_dir" ]]; then
+      validate_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
+        '^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
+      mv -- "$rollback_dir" "$canonical"
+      rollback_dir=""
+    fi
+
+    source="${sources[$week]:-}"
+    if [[ -n "$source" ]]; then
+      [[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
+      if ! publish_weekly_checkpoint "$source" "$canonical" "$([[ -e "$canonical" ]] && echo 1 || echo 0)"; then
+        fail "weekly checkpoint reconciliation failed"
+      fi
+    fi
+
+    [[ -e "$canonical" ]] || fail "weekly checkpoint is missing after reconciliation"
+    if [[ -n "$rollback_dir" ]]; then
+      remove_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
+        '^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
+    fi
+    if [[ -n "$new_dir" ]]; then
+      remove_temp_directory "$new_dir" "$BACKUP_WEEKLY_DIR" \
+        '^week-[0-9]{4}-W[0-9]{2}\.new\.[0-9]+$'
+    fi
+  done
+  local pending_daily_orphan
+  for pending_daily_orphan in "${DAILY_ORPHANS_PENDING_DELETE[@]}"; do
+    remove_temp_directory "$pending_daily_orphan" "$BACKUP_ARCHIVE_DIR" \
+      '^\.daily-replacement-[0-9]{8}\.[0-9]+$'
+  done
+  DAILY_ORPHANS_PENDING_DELETE=()
+  RECONCILED_WEEK_SOURCE=()
+}
+
+reconcile_interrupted_finalize() {
+  reconcile_daily_orphans
+  reconcile_weekly_orphans
+}
+
 finalize_batch() {
   local run_id="$1"
   local expected_manifest_sha="$2"
@@ -194,6 +451,7 @@ finalize_batch() {
   exec 9>"$BACKUP_ARCHIVE_DIR/.finalize.lock"
   chmod 0600 "$BACKUP_ARCHIVE_DIR/.finalize.lock"
   flock -n 9 || fail "another finalize operation is already running"
+  reconcile_interrupted_finalize
 
   local batch_name="hyb-oa-backup-$run_id"
   local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"