#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TEST_DIR="$(mktemp -d)"
PASS_COUNT=0

cleanup() {
  if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" && "$TEST_DIR" == /tmp/* ]]; then
    rm -rf -- "$TEST_DIR"
  fi
}
trap cleanup EXIT

pass() {
  PASS_COUNT=$((PASS_COUNT + 1))
  printf '[PASS] %s\n' "$1"
}

expect_failure() {
  local label="$1"
  shift
  if "$@" >"$TEST_DIR/last-command.out" 2>&1; then
    printf '[FAIL] expected rejection: %s\n' "$label" >&2
    exit 1
  fi
  pass "$label"
}

make_layout() {
  local root="$1"
  mkdir -p "$root/inbox" "$root/archive" "$root/weekly"
}

make_batch() {
  local target="$1"
  local run_id="$2"
  mkdir -p "$target"
  printf 'synthetic database payload\n' >"$target/database.dump"
  printf 'synthetic storage payload\n' >"$target/storage.tar.gz"
  python3 - "$target" "$run_id" <<'PYTHON'
import hashlib
import json
import os
import sys

target, run_id = sys.argv[1:]
def artifact(file_name):
    file_path = os.path.join(target, file_name)
    with open(file_path, "rb") as handle:
        content = handle.read()
    return {"file": file_name, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}

manifest = {
    "schemaVersion": 1,
    "runId": run_id,
    "createdAt": "2026-01-01T00:00:00Z",
    "buildSha": "a" * 40,
    "sourceImageDigest": None,
    "status": "complete",
    "artifacts": {
        "database": artifact("database.dump"),
        "storage": artifact("storage.tar.gz"),
    },
}
with open(os.path.join(target, "manifest.json"), "x", encoding="utf-8") as handle:
    json.dump(manifest, handle, indent=2)
    handle.write("\n")
PYTHON
}

receiver() {
  local layout="$1"
  shift
  env RECEIVER_TEST_MODE=1 RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
    BACKUP_INBOX_DIR="$layout/inbox" BACKUP_ARCHIVE_DIR="$layout/archive" \
    BACKUP_WEEKLY_DIR="$layout/weekly" DAILY_KEEP=7 WEEKLY_KEEP=4 \
    "$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
}

CURRENT_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"

VALID_LAYOUT="$TEST_DIR/receiver-valid"
make_layout "$VALID_LAYOUT"
make_batch "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
VALID_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
[[ -d "$VALID_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" ]]
pass "receiver finalizes one complete synthetic batch"

receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
pass "receiver treats an identical repeated finalize as idempotent"

SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA" >/dev/null
[[ ! -e "$VALID_LAYOUT/archive/$CURRENT_BATCH" && -d "$VALID_LAYOUT/archive/$SECOND_BATCH" ]]
WEEKLY_BATCH="$(find "$VALID_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
[[ -n "$WEEKLY_BATCH" ]]
[[ "$(stat -c %i "$WEEKLY_BATCH/database.dump")" == "$(stat -c %i "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump")" ]]
pass "receiver atomically replaces a different batch for the same UTC day and rebuilds its weekly checkpoint"

THIRD_RUN_ID="auto-$(date -u +%Y%m%d)T000002Z"
THIRD_BATCH="hyb-oa-backup-$THIRD_RUN_ID"
make_batch "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
printf 'corrupt candidate\n' >>"$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/database.dump"
THIRD_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver validates a same-day replacement before touching the current archive" \
  receiver "$VALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$THIRD_MANIFEST_SHA"
[[ -d "$VALID_LAYOUT/archive/$SECOND_BATCH" && ! -e "$VALID_LAYOUT/archive/$THIRD_BATCH" ]]
pass "receiver preserves the current daily archive when a replacement candidate is corrupt"

ROLLBACK_LAYOUT="$TEST_DIR/receiver-replacement-rollback"
make_layout "$ROLLBACK_LAYOUT"
make_batch "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
ROLLBACK_FIRST_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$ROLLBACK_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$ROLLBACK_FIRST_SHA" >/dev/null
ROLLBACK_WEEKLY="$(find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -exec rm -rf -- {} +
printf 'invalid weekly target\n' >"$ROLLBACK_WEEKLY"
make_batch "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
ROLLBACK_SECOND_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver rolls back the daily archive when weekly checkpoint publication fails" \
  receiver "$ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$ROLLBACK_SECOND_SHA"
[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
pass "receiver restores the prior daily point after a later replacement step fails"

negative_receiver_case() {
  local label="$1"
  local mutation="$2"
  local layout="$TEST_DIR/receiver-negative-${PASS_COUNT}"
  make_layout "$layout"
  make_batch "$layout/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
  case "$mutation" in
    corrupt-artifact) printf 'changed\n' >>"$layout/inbox/${CURRENT_BATCH}.partial/database.dump" ;;
    corrupt-manifest) printf ' ' >>"$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" ;;
    extra-file) printf 'extra\n' >"$layout/inbox/${CURRENT_BATCH}.partial/extra.txt" ;;
    symlink) rm "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"; ln -s database.dump "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" ;;
  esac
  local sha
  sha="$(sha256sum "$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
  if [[ "$mutation" == "corrupt-manifest" ]]; then
    sha="$VALID_MANIFEST_SHA"
  fi
  expect_failure "$label" receiver "$layout" finalize-test "$CURRENT_RUN_ID" "$sha"
  [[ ! -e "$layout/archive/$CURRENT_BATCH" ]]
}

negative_receiver_case "receiver rejects an artifact checksum mismatch" corrupt-artifact
negative_receiver_case "receiver rejects a manifest checksum mismatch" corrupt-manifest
negative_receiver_case "receiver rejects an extra file" extra-file
negative_receiver_case "receiver rejects links in a batch" symlink

RESUME_LAYOUT="$TEST_DIR/receiver-resume"
make_layout "$RESUME_LAYOUT"
make_batch "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
RESUME_MANIFEST_SHA="$(sha256sum "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
mv "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" "$TEST_DIR/resume-storage.tar.gz"
expect_failure "receiver keeps an incomplete partial batch out of archive" \
  receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA"
mv "$TEST_DIR/resume-storage.tar.gz" "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" >/dev/null
[[ -d "$RESUME_LAYOUT/archive/$CURRENT_BATCH" ]]
pass "receiver finalizes the same partial directory after transfer resumes"

FORCED_LAYOUT="$TEST_DIR/receiver-forced"
make_layout "$FORCED_LAYOUT"
expect_failure "receiver rejects rsync sender/read mode" env \
  RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
  BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
  BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
  SSH_ORIGINAL_COMMAND="rsync --server --sender -logDtpre.iLsfxCIvu . ." \
  "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects a sender flag in a later rsync position" env \
  RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
  BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
  BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
  SSH_ORIGINAL_COMMAND="rsync --server -logDtpre.iLsfxCIvu --sender . ." \
  "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects destructive rsync flags" env \
  RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
  BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
  BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
  SSH_ORIGINAL_COMMAND="rsync --server --delete -logDtpre.iLsfxCIvu . ." \
  "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects an ordinary shell command" env \
  RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
  BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
  BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" SSH_ORIGINAL_COMMAND="sh -c id" \
  "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced

NESTED_LAYOUT="$TEST_DIR/receiver-nested"
mkdir -p "$NESTED_LAYOUT/inbox/archive" "$NESTED_LAYOUT/weekly"
expect_failure "receiver rejects archive nested inside the rsync inbox" env \
  RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
  BACKUP_INBOX_DIR="$NESTED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$NESTED_LAYOUT/inbox/archive" \
  BACKUP_WEEKLY_DIR="$NESTED_LAYOUT/weekly" \
  "$ROOT_DIR/scripts/receive-offhost-backup.sh" validate-config

RETENTION_LAYOUT="$TEST_DIR/receiver-retention"
make_layout "$RETENTION_LAYOUT"
for day in 01 02 03 04 05 06 07; do
  mkdir "$RETENTION_LAYOUT/archive/hyb-oa-backup-auto-202601${day}T023000Z"
done
for week in 01 02 03 04; do
  mkdir "$RETENTION_LAYOUT/weekly/week-2026-W${week}"
done
make_batch "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
RETENTION_SHA="$(sha256sum "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$RETENTION_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RETENTION_SHA" >/dev/null
[[ "$(find "$RETENTION_LAYOUT/archive" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "7" ]]
[[ "$(find "$RETENTION_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "4" ]]
[[ -d "$RETENTION_LAYOUT/archive/$CURRENT_BATCH" ]]
pass "receiver retains seven daily and four weekly recovery points while protecting current success"

EMPTY_SOURCE="$TEST_DIR/empty-rsync-source"
EMPTY_TARGET="$TEST_DIR/empty-rsync-target"
mkdir "$EMPTY_SOURCE" "$EMPTY_TARGET"
rsync -a --dry-run --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= "$EMPTY_SOURCE/" "$EMPTY_TARGET/"
pass "installed rsync accepts the production chmod contract"

WRAPPER_ROOT="$TEST_DIR/wrapper-root"
WRAPPER_PG="$TEST_DIR/wrapper-pg"
WRAPPER_STORAGE="$TEST_DIR/wrapper-storage"
WRAPPER_BACKUPS="$TEST_DIR/wrapper-backups"
WRAPPER_STATE="$TEST_DIR/wrapper-state"
MOCK_BIN="$TEST_DIR/mock-bin"
mkdir -p "$WRAPPER_ROOT/deploy/trialrun" "$WRAPPER_PG" "$WRAPPER_STORAGE" \
  "$WRAPPER_BACKUPS" "$WRAPPER_STATE" "$MOCK_BIN"
printf '16\n' >"$WRAPPER_PG/PG_VERSION"
printf 'synthetic attachment\n' >"$WRAPPER_STORAGE/marker.txt"
printf 'synthetic env\n' >"$WRAPPER_ROOT/deploy/trialrun/.env"
ln -s "$ROOT_DIR/scripts" "$WRAPPER_ROOT/scripts"
printf 'private-key-fixture\n' >"$TEST_DIR/offhost-key"
chmod 0600 "$TEST_DIR/offhost-key"
printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"

BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG

cat >"$MOCK_BIN/docker" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then exit 0; fi
if [[ "${1:-}" == "compose" ]]; then
  args=" $* "
  if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
  if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
  if [[ "$args" == *" exec -T server "* ]]; then exit 99; fi
fi
if [[ "${1:-}" == "inspect" ]]; then
  template="${3:-}"
  container="${4:-}"
  case "$template" in
    *State.Status*) echo running ;;
    *Config.Env*)
      if [[ "$container" == "server-id" ]]; then
        printf 'BUILD_SHA=%s\nSTORAGE_DIR=/app/storage\nDATABASE_URL=postgresql://backup_user:fixture@db:5432/hyb_oa\n' "$MOCK_BUILD_SHA"
      else
        printf 'POSTGRES_DB=hyb_oa\nPOSTGRES_USER=backup_user\nPOSTGRES_PASSWORD=fixture\n'
      fi
      ;;
    *json*.Mounts*)
      if [[ "$container" == "server-id" ]]; then
        printf '[{"Destination":"/app/storage","Source":"%s"}]\n' "$MOCK_STORAGE"
      else
        printf '[{"Destination":"/var/lib/postgresql/data","Source":"%s"}]\n' "$MOCK_PG"
      fi
      ;;
    *Image*) printf 'sha256:%064d\n' 0 ;;
    *) exit 1 ;;
  esac
  exit 0
fi
if [[ "${1:-}" == "exec" ]]; then
  if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
  if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
fi
exit 1
MOCK

cat >"$MOCK_BIN/rsync" <<'MOCK'
#!/usr/bin/env bash
echo 'synthetic transfer diagnostic' >&2
exit "${MOCK_RSYNC_EXIT:-1}"
MOCK
cat >"$MOCK_BIN/ssh" <<'MOCK'
#!/usr/bin/env bash
exit "${MOCK_SSH_EXIT:-0}"
MOCK
chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
WECOM_TEST_HOST='qyapi.weixin.qq.com'
WECOM_TEST_PATH='/cgi-bin/webhook/send'
printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
  '00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook"
chmod 0600 "$TEST_DIR/wecom-webhook"
ALERT_CAPTURE="$TEST_DIR/alert-capture.jsonl"
cat >"$TEST_DIR/failure-summary.json" <<'JSON'
{"issueId":"AUTO_BACKUP","status":"FAIL_AUTOMATIC_BACKUP","conclusion":"FAIL_AUTOMATIC_BACKUP","findingCount":1,"findings":[{"source":"automatic_backup","status":"FAIL_AUTOMATIC_BACKUP","severity":"P0","detail":"synthetic_failure"}]}
JSON
chmod 0600 "$TEST_DIR/failure-summary.json"
chmod 0644 "$TEST_DIR/wecom-webhook"
expect_failure "group alert helper rejects a webhook file that is not mode 0600" env \
  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook" \
  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
chmod 0600 "$TEST_DIR/wecom-webhook"
ln -s "$TEST_DIR/wecom-webhook" "$TEST_DIR/wecom-webhook-link"
expect_failure "group alert helper rejects a symbolic-link webhook file" env \
  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-link" \
  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
printf 'https://%s%s?key=%s&extra=forbidden\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
  '00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook-invalid"
chmod 0600 "$TEST_DIR/wecom-webhook-invalid"
expect_failure "group alert helper rejects a webhook URL with an unexpected query field" env \
  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-invalid" \
  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"

wrapper_env=(
  env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
  REMOTE_ENV_FILE="$WRAPPER_ROOT/deploy/trialrun/.env"
  COMPOSE_PROJECT_NAME="hyb-oa-current" POSTGRES_DATA_DIR="$WRAPPER_PG"
  STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
  AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
  OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
  OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_WEBHOOK_FILE="$TEST_DIR/wecom-webhook"
  ALERT_HELPER="$ROOT_DIR/scripts/send-wecom-group-alert.py"
  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE"
)

exec 8>"$WRAPPER_STATE/automatic-backup.lock"
flock -n 8
expect_failure "wrapper rejects a concurrent automatic-backup run" \
  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" run
flock -u 8

rm -f "$WRAPPER_STATE/last-success.json"
for day in 01 02 03 04 05 06 07; do
  mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
done
expect_failure "unreachable rsync fails without writing off-host success" \
  "${wrapper_env[@]}" MOCK_RSYNC_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
pass "source retention does not run before off-host finalize succeeds"
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
    payload = json.load(handle)
serialized = json.dumps(payload)
assert payload["issueId"] == "AUTO_BACKUP"
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
PYTHON
pass "wrapper failure summary is category-only and redacted"
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
pass "wrapper retains the latest command diagnostic locally with mode 0600"

find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
for day in 01 02 03 04 05 06 07; do
  mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
done
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=0 \
  "$ROOT_DIR/scripts/run-automatic-backup.sh" run >/dev/null
[[ -f "$WRAPPER_STATE/last-success.json" ]]
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
pass "off-host finalize success writes state and then prunes source retention to seven"
[[ ! -e "$WRAPPER_STATE/last-error.log" ]]
pass "wrapper clears the retained diagnostic after a successful off-host run"

rm -f "$WRAPPER_STATE/last-success.json"
expect_failure "stale watchdog rejects a missing success record" \
  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale

python3 - "$WRAPPER_STATE/last-success.json" "$BUILD_SHA" <<'PYTHON'
import json
import sys
from datetime import datetime, timezone
with open(sys.argv[1], "w", encoding="utf-8") as handle:
    json.dump({
        "schemaVersion": 1,
        "status": "complete",
        "completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
        "runId": "auto-20260727T000000Z",
        "manifestSha256": "0" * 64,
        "composeProject": "hyb-oa-current",
        "buildSha": sys.argv[2],
    }, handle)
PYTHON
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
    payload = json.load(handle)
payload["buildSha"] = "0" * 40
with open(sys.argv[1], "w", encoding="utf-8") as handle:
    json.dump(payload, handle)
PYTHON
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
    payload = json.load(handle)
assert payload["status"] == "PASS_WITH_FINDINGS"
assert payload["findings"][0]["detail"] == "recovery_point_build_differs"
PYTHON
pass "stale watchdog downgrades a fresh recovery point from another build to one finding"
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
pass "stale watchdog accepts a fresh project/build-bound success record"

cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
    payload = json.load(handle)
payload["composeProject"] = "different-project"
with open(sys.argv[1], "w", encoding="utf-8") as handle:
    json.dump(payload, handle)
PYTHON
expect_failure "stale watchdog hard-fails a fresh record from another Compose project" \
  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
    payload = json.load(handle)
payload["manifestSha256"] = "invalid"
with open(sys.argv[1], "w", encoding="utf-8") as handle:
    json.dump(payload, handle)
PYTHON
expect_failure "stale watchdog hard-fails a malformed success record" \
  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"

DEADMAN_LAYOUT="$TEST_DIR/deadman"
mkdir -p "$DEADMAN_LAYOUT/archive" "$DEADMAN_LAYOUT/state"
cp -a "$VALID_LAYOUT/archive/$SECOND_BATCH" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
DEADMAN_CAPTURE="$TEST_DIR/deadman-alerts.jsonl"
deadman_env=(env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CAPTURE")
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
  --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
  --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
[[ ! -e "$DEADMAN_CAPTURE" ]]
pass "ARM dead-man accepts a fresh verified archive without sending a healthy baseline"
DEADMAN_CORRUPT_STATE="$TEST_DIR/deadman-corrupt-state"
DEADMAN_CORRUPT_CAPTURE="$TEST_DIR/deadman-corrupt-alerts.jsonl"
mkdir "$DEADMAN_CORRUPT_STATE"
chmod 0600 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
printf 'corruption\n' >>"$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
expect_failure "ARM dead-man rejects a corrupt newest finalized archive" env \
  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CORRUPT_CAPTURE" \
  python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_CORRUPT_STATE" \
    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
cp "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
chmod 0400 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
pass "ARM dead-man detects archive corruption independently of source state"
mv "$DEADMAN_LAYOUT/archive/$SECOND_BATCH" "$TEST_DIR/deadman-withheld-batch"
expect_failure "ARM dead-man detects that the source no longer produces a finalized archive" \
  "${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
expect_failure "ARM dead-man suppresses an identical repeated failure alert" \
  "${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
pass "ARM dead-man keeps repeated identical failure alerts deduplicated"
mv "$TEST_DIR/deadman-withheld-batch" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
  --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
  --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "2" ]]
pass "ARM dead-man sends one recovery notification after finalized archives resume"

DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
  "${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \
  "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale

"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
FIRST_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
SECOND_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
[[ "$FIRST_REMINDER" == "$SECOND_REMINDER" && "$FIRST_REMINDER" == quarterly_shadow_restore_due_* ]]
pass "quarterly reminder finding key is stable within the same quarter"

expect_failure "receiver installer uninstall rejects an unsafe target path" env \
  BACKUP_USER=hybbackup RECEIVER_TARGET=/ CONFIG_FILE=/etc/hyb-oa-test.conf \
  "$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" uninstall

expect_failure "receiver installer rejects a system-managed backup root before mutation" env \
  BACKUP_ROOT=/etc/hyb-oa-backups PUBLIC_KEY_FILE="$TEST_DIR/known-hosts" \
  RECEIVER_SOURCE="$ROOT_DIR/scripts/receive-offhost-backup.sh" \
  "$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" install

printf '[PASS] automatic off-host backup contract completed (%d checks)\n' "$PASS_COUNT"