#!/usr/bin/env bash set -eEuo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" INSPECTOR="$ROOT_DIR/scripts/qa/inspect-backup-transfer-incident.py" TEST_ROOT="$(mktemp -d)" PASS_COUNT=0 cleanup() { if [[ -n "${TEST_ROOT:-}" && -d "$TEST_ROOT" && "$TEST_ROOT" == /tmp/* ]]; then rm -rf -- "$TEST_ROOT" fi } trap cleanup EXIT pass() { PASS_COUNT=$((PASS_COUNT + 1)) printf '[PASS] %s\n' "$1" } assert_line() { local file="$1" local expected="$2" grep -Fxq -- "$expected" "$file" || { printf '[FAIL] missing field: %s\n' "$expected" >&2 exit 1 } } make_batch() { local target="$1" local run_id="$2" local db_bytes="${3:-1024}" local storage_bytes="${4:-2048}" mkdir -p "$target" head -c "$db_bytes" /dev/zero >"$target/database.dump" head -c "$storage_bytes" /dev/zero >"$target/storage.tar.gz" python3 - "$target" "$run_id" <<'PY' import hashlib import json import sys from pathlib import Path target = Path(sys.argv[1]) run_id = sys.argv[2] def item(name): data = (target / name).read_bytes() return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} payload = { "schemaVersion": 1, "runId": run_id, "createdAt": "2026-08-28T00:00:00Z", "buildSha": "a" * 40, "sourceImageDigest": None, "status": "complete", "artifacts": {"database": item("database.dump"), "storage": item("storage.tar.gz")}, } (target / "manifest.json").write_text(json.dumps(payload) + "\n", encoding="utf-8") PY } run_production() { local fixture="$1" local output="$2" python3 -B "$INSPECTOR" --now-epoch 1787875200 production \ --backup-root "$fixture/backups" \ --state-dir "$fixture/state" \ --wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \ --backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$output" } make_production_layout() { local fixture="$1" mkdir -p "$fixture/backups" "$fixture/state" } write_transfer_failure_state() { local fixture="$1" printf '%s\n' '{"findings":[{"detail":"phase_offhost_transfer_failed"}]}' >"$fixture/state/last-run.json" } PRODUCTION_VALID="$TEST_ROOT/production-valid" make_production_layout "$PRODUCTION_VALID" make_batch "$PRODUCTION_VALID/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" write_transfer_failure_state "$PRODUCTION_VALID" printf 'rsync error: synthetic redacted fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_VALID/state/last-error.log" run_production "$PRODUCTION_VALID" "$TEST_ROOT/production-valid.out" assert_line "$TEST_ROOT/production-valid.out" "complete_local_recovery_point=yes" assert_line "$TEST_ROOT/production-valid.out" "backup_risk_classification=production_single_point" assert_line "$TEST_ROOT/production-valid.out" "transition_recovery_point_action=wait_natural" assert_line "$TEST_ROOT/production-valid.out" "transfer_exit_code=23" assert_line "$TEST_ROOT/production-valid.out" "transfer_failure_category=partial_transfer_error" assert_line "$TEST_ROOT/production-valid.out" "transfer_duration_seconds=NOT_AVAILABLE" pass "生产存在完整本地恢复点时识别为单点风险并保留未测时长" PRODUCTION_MULTIPLE="$TEST_ROOT/production-multiple" make_production_layout "$PRODUCTION_MULTIPLE" make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z" make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" printf 'X' | dd of="$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none run_production "$PRODUCTION_MULTIPLE" "$TEST_ROOT/production-multiple.out" assert_line "$TEST_ROOT/production-multiple.out" "complete_local_recovery_point=yes" assert_line "$TEST_ROOT/production-multiple.out" "fully_verified_local_batch_count=1" assert_line "$TEST_ROOT/production-multiple.out" "metadata_complete_local_batch_count=2" assert_line "$TEST_ROOT/production-multiple.out" "older_metadata_batch_hash_status=NOT_RUN" assert_line "$TEST_ROOT/production-multiple.out" "latest_local_candidate_hash_status=PASS" pass "生产多个批次时只对最新元数据合格候选执行全量 SHA-256" PRODUCTION_LATEST_HASH_FAIL="$TEST_ROOT/production-latest-hash-fail" make_production_layout "$PRODUCTION_LATEST_HASH_FAIL" make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z" make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" printf 'X' | dd of="$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none run_production "$PRODUCTION_LATEST_HASH_FAIL" "$TEST_ROOT/production-latest-hash-fail.out" assert_line "$TEST_ROOT/production-latest-hash-fail.out" "complete_local_recovery_point=no" assert_line "$TEST_ROOT/production-latest-hash-fail.out" "fully_verified_local_batch_count=0" assert_line "$TEST_ROOT/production-latest-hash-fail.out" "metadata_complete_local_batch_count=2" assert_line "$TEST_ROOT/production-latest-hash-fail.out" "older_metadata_batch_hash_status=NOT_RUN" assert_line "$TEST_ROOT/production-latest-hash-fail.out" "latest_local_candidate_hash_status=FAIL" pass "最新候选哈希失败时不回退扫描旧批次并保持无已证明恢复点" PRODUCTION_EMPTY="$TEST_ROOT/production-empty" make_production_layout "$PRODUCTION_EMPTY" run_production "$PRODUCTION_EMPTY" "$TEST_ROOT/production-empty.out" assert_line "$TEST_ROOT/production-empty.out" "complete_local_recovery_point=no" assert_line "$TEST_ROOT/production-empty.out" "backup_risk_classification=no_proven_recovery_point" assert_line "$TEST_ROOT/production-empty.out" "transition_recovery_point_action=request_manual_run" pass "生产无完整本地恢复点时升级风险与过渡恢复点建议" PRODUCTION_CORRUPT="$TEST_ROOT/production-corrupt" make_production_layout "$PRODUCTION_CORRUPT" make_batch "$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" printf 'tamper' >>"$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump" run_production "$PRODUCTION_CORRUPT" "$TEST_ROOT/production-corrupt.out" assert_line "$TEST_ROOT/production-corrupt.out" "complete_local_recovery_point=no" assert_line "$TEST_ROOT/production-corrupt.out" "invalid_local_batch_count=1" pass "manifest 大小或 SHA 篡改不得冒充完整本地恢复点" PRODUCTION_MALFORMED="$TEST_ROOT/production-malformed" make_production_layout "$PRODUCTION_MALFORMED" make_batch "$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" printf '%s\n' '[]' >"$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json" run_production "$PRODUCTION_MALFORMED" "$TEST_ROOT/production-malformed.out" assert_line "$TEST_ROOT/production-malformed.out" "complete_local_recovery_point=no" assert_line "$TEST_ROOT/production-malformed.out" "invalid_local_batch_count=1" pass "畸形 manifest 安全归类为无效批次且不输出 traceback" PRODUCTION_UNREADABLE="$TEST_ROOT/production-unreadable" make_production_layout "$PRODUCTION_UNREADABLE" make_batch "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z" chmod 000 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json" if [[ "$(id -u)" == 0 ]] && command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1; then chmod 0755 "$TEST_ROOT" "$PRODUCTION_UNREADABLE" "$PRODUCTION_UNREADABLE/backups" "$PRODUCTION_UNREADABLE/state" \ "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z" if runuser -u nobody -- python3 -B "$INSPECTOR" --now-epoch 1787875200 production \ --backup-root "$PRODUCTION_UNREADABLE/backups" \ --state-dir "$PRODUCTION_UNREADABLE/state" \ --wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \ --backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/production-unreadable.out"; then printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2 exit 1 fi else if run_production "$PRODUCTION_UNREADABLE" "$TEST_ROOT/production-unreadable.out"; then printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2 exit 1 fi fi chmod 0600 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json" assert_line "$TEST_ROOT/production-unreadable.out" "probe_category=access_denied" pass "访问拒绝保持为 unknown/ACCESS_DENIED 而不是伪装成无有效备份" assert_line "$TEST_ROOT/production-valid.out" "retained_after_transfer_failure=yes" assert_line "$TEST_ROOT/production-valid.out" "pruned_after_offhost_success=yes" assert_line "$TEST_ROOT/production-valid.out" "generation_temp_cleanup_on_failure=yes" assert_line "$TEST_ROOT/production-valid.out" "cleanup_contract_source_verified=yes" pass "生产脚本顺序机械证明失败保留、成功后 prune 与生成临时清理" SOURCE_MUTATION="$TEST_ROOT/source-mutation" cp "$ROOT_DIR/scripts/run-automatic-backup.sh" "$SOURCE_MUTATION" python3 - "$SOURCE_MUTATION" <<'PY' import sys from pathlib import Path path = Path(sys.argv[1]) text = path.read_text(encoding="utf-8") transfer = text.index("capture_command offhost_transfer rsync") text = text[:transfer] + 'prune_local_backups "$batch_name"\n ' + text[transfer:] path.write_text(text, encoding="utf-8") PY python3 -B "$INSPECTOR" --now-epoch 1787875200 production \ --backup-root "$PRODUCTION_VALID/backups" \ --state-dir "$PRODUCTION_VALID/state" \ --wrapper-script "$SOURCE_MUTATION" \ --backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/source-mutation.out" assert_line "$TEST_ROOT/source-mutation.out" "retained_after_transfer_failure=no" assert_line "$TEST_ROOT/source-mutation.out" "pruned_after_offhost_success=no" pass "源码顺序漂移会关闭失败保留与成功后 prune 结论" for code_category in \ 23:partial_transfer_error \ 30:transport_timeout \ 12:protocol_stream_error \ 11:file_io_error \ 255:ssh_session_error; do code="${code_category%%:*}" category="${code_category#*:}" fixture="$TEST_ROOT/rsync-$code" make_production_layout "$fixture" write_transfer_failure_state "$fixture" printf 'rsync error: synthetic redacted fixture (code %s) at fixture.c(1)\n' "$code" >"$fixture/state/last-error.log" run_production "$fixture" "$TEST_ROOT/rsync-$code.out" assert_line "$TEST_ROOT/rsync-$code.out" "transfer_exit_code=$code" assert_line "$TEST_ROOT/rsync-$code.out" "transfer_failure_category=$category" done pass "rsync 23/30/12/11/255 均映射为稳定脱敏类别" PRODUCTION_UNKNOWN="$TEST_ROOT/production-unknown" make_production_layout "$PRODUCTION_UNKNOWN" write_transfer_failure_state "$PRODUCTION_UNKNOWN" printf 'unstructured synthetic transport failure\n' >"$PRODUCTION_UNKNOWN/state/last-error.log" run_production "$PRODUCTION_UNKNOWN" "$TEST_ROOT/production-unknown.out" assert_line "$TEST_ROOT/production-unknown.out" "transfer_exit_code=NOT_AVAILABLE" assert_line "$TEST_ROOT/production-unknown.out" "transfer_failure_category=not_available" pass "原始 rsync 码不存在时安全失败为 NOT_AVAILABLE" PRODUCTION_PHASE_MISMATCH="$TEST_ROOT/production-phase-mismatch" make_production_layout "$PRODUCTION_PHASE_MISMATCH" printf '%s\n' '{"findings":[{"detail":"phase_offhost_finalize_failed"}]}' >"$PRODUCTION_PHASE_MISMATCH/state/last-run.json" printf 'rsync error: stale synthetic fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_PHASE_MISMATCH/state/last-error.log" run_production "$PRODUCTION_PHASE_MISMATCH" "$TEST_ROOT/production-phase-mismatch.out" assert_line "$TEST_ROOT/production-phase-mismatch.out" "last_backup_failure_phase=offhost_finalize" assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code=NOT_AVAILABLE" assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code_source=NOT_AVAILABLE" pass "私有诊断只在安全摘要明确为 offhost_transfer 时绑定 rsync 原始码" ARM_SIMILAR="$TEST_ROOT/arm-similar" mkdir -p "$ARM_SIMILAR/inbox" "$ARM_SIMILAR/archive" make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000 make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 1040 1000 rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz" rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz" python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_SIMILAR/inbox" --archive-root "$ARM_SIMILAR/archive" >"$TEST_ROOT/arm-similar.out" assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_count=2" assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_stop_offset_class=similar" assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_incomplete_contract_count=2" pass "两个 partial 接近停止偏移按预先冻结阈值分类为 similar" ARM_DIFFERENT="$TEST_ROOT/arm-different" mkdir -p "$ARM_DIFFERENT/inbox" "$ARM_DIFFERENT/archive" make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000 make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 70000000 1000 rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz" rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz" python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_DIFFERENT/inbox" --archive-root "$ARM_DIFFERENT/archive" >"$TEST_ROOT/arm-different.out" assert_line "$TEST_ROOT/arm-different.out" "arm_partial_stop_offset_class=different" pass "两个 partial 明显不同停止偏移分类为 different" python3 -B - "$INSPECTOR" <<'PY' import importlib.util import sys spec = importlib.util.spec_from_file_location("backup_transfer_incident", sys.argv[1]) module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) def sample(total_bytes): return module.BatchResult(False, total_bytes, None, None, total_bytes) cases = [ ("现场量级的 2.68MB 与 0 字节必须不同", [sample(0), sample(2_682_168)], "different"), ("大文件差异同时落在 5% 与 64MiB 内才接近", [sample(1_000_000_000), sample(1_040_000_000)], "similar"), ("即使落在 64MiB 内但超过 5% 仍不同", [sample(10_000_000), sample(10_600_000)], "different"), ("即使落在 5% 内但超过 64MiB 仍不同", [sample(2_000_000_000), sample(2_080_000_000)], "different"), ] for label, samples, expected in cases: actual = module.stopping_class(samples) if actual != expected: raise SystemExit(f"{label}: expected {expected}, got {actual}") PY pass "停止偏移分类同时满足相对与绝对阈值并覆盖现场 2.68MB/0 字节回归" ARM_COMPLETE="$TEST_ROOT/arm-complete" mkdir -p "$ARM_COMPLETE/inbox" "$ARM_COMPLETE/archive" make_batch "$ARM_COMPLETE/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" make_batch "$ARM_COMPLETE/archive/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z" python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_COMPLETE/inbox" --archive-root "$ARM_COMPLETE/archive" >"$TEST_ROOT/arm-complete.out" assert_line "$TEST_ROOT/arm-complete.out" "arm_partial_complete_contract_count=1" assert_line "$TEST_ROOT/arm-complete.out" "arm_valid_atomic_archive_count=1" pass "完整 partial 与正式原子归档分别按同一三文件身份和哈希合同验证" PERMISSION_FIXTURE="$TEST_ROOT/permission" mkdir -p "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/archive" chmod 000 "$PERMISSION_FIXTURE/inbox" if command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1 && [[ "$(id -u)" == 0 ]]; then if runuser -u nobody -- python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then printf '[FAIL] permission denial unexpectedly passed\n' >&2 exit 1 fi assert_line "$TEST_ROOT/permission.out" "probe_category=access_denied" else chmod 0700 "$PERMISSION_FIXTURE/inbox" ln -s "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/inbox-link" if python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox-link" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then printf '[FAIL] invalid root unexpectedly passed\n' >&2 exit 1 fi assert_line "$TEST_ROOT/permission.out" "probe_category=root_invalid" fi chmod 0700 "$PERMISSION_FIXTURE/inbox" pass "不可读或不安全根目录明确拒绝且不伪装为不存在" if rg -n --fixed-strings \ -e "$TEST_ROOT" \ -e "auto-20260828T000000Z" \ -e "database.dump" \ -e "storage.tar.gz" \ -e "unstructured synthetic transport failure" \ "$TEST_ROOT"/*.out >/dev/null; then printf '[FAIL] redacted output leaked fixture identity, path, artifact name, or diagnostic text\n' >&2 exit 1 fi pass "stdout allowlist 不含路径、run id、产物名或原始诊断" PYTHONPYCACHEPREFIX="$TEST_ROOT/pycache" python3 -m py_compile "$INSPECTOR" bash -n "$ROOT_DIR/scripts/qa/run-backup-transfer-incident-contract.sh" pass "Python 与 Bash 语法通过" cleanup [[ ! -e "$TEST_ROOT" ]] TEST_ROOT="" trap - EXIT pass "合成 fixture 与 Python cache 零残留" printf '[PASS] backup transfer incident contract: %s checks\n' "$PASS_COUNT"