#!/usr/bin/env bash set -Eeuo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" RUNNER_TEMPLATE="$ROOT_DIR/scripts/qa/offhost-shadow-restore-runner.template.sh" ATTACHMENT_VALIDATOR="$ROOT_DIR/scripts/qa/validate-offhost-shadow-restore-attachments.py" RESTORE_DB="$ROOT_DIR/scripts/restore-db.sh" RESTORE_STORAGE="$ROOT_DIR/scripts/restore-storage.sh" TEST_DIR="$(mktemp -d)" RUNNER="$TEST_DIR/rendered-runner.sh" PRIVATE_REPO_TEST_DIR="$ROOT_DIR/.restore-drill-private/contract-$$-$RANDOM" UNIGNORED_CONFIG="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.json" UNIGNORED_OUTPUT="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.sh" CONTAINER_NAME="$(basename "$0" .sh)-$RANDOM-$$" POSTGRES_USER="fixture_user_$RANDOM" POSTGRES_PASSWORD="$(openssl rand -hex 16)" POSTGRES_DB="fixture_restore_drill_$RANDOM" cleanup() { [[ -z "$CONTAINER_NAME" ]] \ || docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true rm -rf -- "$TEST_DIR" rm -rf -- "$PRIVATE_REPO_TEST_DIR" rm -f -- "$UNIGNORED_CONFIG" "$UNIGNORED_OUTPUT" } trap cleanup EXIT HUP INT TERM fail() { echo "[runner-contract][FAIL] $1" >&2 exit 1 } pass() { echo "[runner-contract][PASS] $1" } require_ignored_if_inside_repo() { local candidate_path="$1" local resolved_parent resolved_path relative_path resolved_parent="$(realpath -e "$(dirname "$candidate_path")")" \ || return 1 resolved_path="$resolved_parent/$(basename "$candidate_path")" case "$resolved_path" in "$ROOT_DIR"/*) relative_path="${resolved_path#"$ROOT_DIR"/}" git -C "$ROOT_DIR" check-ignore -q --no-index -- "$relative_path" \ || return 1 ;; esac } render_runner() { local config_file="$1" local output_file="$2" require_ignored_if_inside_repo "$config_file" || return 1 require_ignored_if_inside_repo "$output_file" || return 1 python3 - "$RUNNER_TEMPLATE" "$config_file" "$output_file" <<'PY' import json import os import pathlib import re import sys template_path, config_path, output_path = map(pathlib.Path, sys.argv[1:]) template = template_path.read_text(encoding="utf-8") config = json.loads(config_path.read_text(encoding="utf-8")) tokens = set(re.findall(r"@@([A-Z0-9_]+)@@", template)) if set(config) != tokens: raise SystemExit("render_config_keys") path_keys = { "RECEIVER_CONFIG_FILE", "RESTORE_LIB_DIR", "SELF_PATH", "SUDOERS_PATH", "SUDOERS_MAIN_FILE", "WORK_ROOT", } prefix_keys = {"WORK_PREFIX", "CONTAINER_PREFIX", "ARCHIVE_BATCH_PREFIX", "RUN_ID_PREFIX"} sha_keys = { "EXPECTED_DB_RESTORE_SHA", "EXPECTED_STORAGE_RESTORE_SHA", "EXPECTED_ATTACHMENT_VALIDATOR_SHA", "EXPECTED_MIGRATION_SET_SHA", } for key, value in config.items(): if not isinstance(value, str) or not value or "\n" in value or "\r" in value: raise SystemExit("render_config_value") if key in path_keys: parsed = pathlib.PurePosixPath(value) if not parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts[1:]): raise SystemExit("render_path_value") if not re.fullmatch(r"/[A-Za-z0-9_./-]+", value): raise SystemExit("render_path_charset") elif key in prefix_keys and not re.fullmatch(r"[a-z0-9][a-z0-9-]*-", value): raise SystemExit("render_prefix_value") elif key == "SHADOW_DB_USER" and not re.fullmatch(r"[a-z_][a-z0-9_]*", value): raise SystemExit("render_db_user") elif key == "SHADOW_DB_NAME" and ( not re.fullmatch(r"[a-z_][a-z0-9_]*", value) or not re.search(r"(^|_)(shadow|drill|restore_test)(_|$)", value) ): raise SystemExit("render_db_name") elif key == "POSTGRES_IMAGE" and not re.fullmatch(r"[a-z0-9._/-]+:[A-Za-z0-9._-]+", value): raise SystemExit("render_postgres_image") elif key in {"DEADMAN_TIMER_UNIT", "HEALTH_TIMER_UNIT"} and not re.fullmatch( r"[A-Za-z0-9_.@-]+\.timer", value ): raise SystemExit("render_timer_unit") elif key in sha_keys and not re.fullmatch(r"[0-9a-f]{64}", value): raise SystemExit("render_sha") elif key == "EXPECTED_BUILD_PREFIX" and not re.fullmatch(r"[0-9a-f]{12}", value): raise SystemExit("render_build_prefix") template = template.replace(f"@@{key}@@", value) if re.search(r"@@[A-Z0-9_]+@@", template): raise SystemExit("render_placeholder_remaining") descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o700) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: handle.write(template) PY } if [[ "${1:-}" == "render" ]]; then [[ "$#" -eq 3 ]] || fail "usage: $0 render " render_runner "$2" "$3" || fail "private render paths must be outside the repository or ignored" bash -n "$3" candidate_sha="$(sha256sum "$3" | awk '{print $1}')" echo "[runner-contract][PASS] rendered candidate syntax" echo "candidate_sha256=$candidate_sha" exit 0 fi [[ "$#" -eq 0 ]] || fail "usage: $0 [render ]" db_restore_sha="$(sha256sum "$RESTORE_DB" | awk '{print $1}')" storage_restore_sha="$(sha256sum "$RESTORE_STORAGE" | awk '{print $1}')" attachment_validator_sha="$(sha256sum "$ATTACHMENT_VALIDATOR" | awk '{print $1}')" migration_fixture_names=$'20260101000000_fixture_one\n20260102000000_fixture_two' migration_fixture_sha="$(printf '%s\n' "$migration_fixture_names" | sha256sum | awk '{print $1}')" python3 - "$TEST_DIR/render-config.json" "$db_restore_sha" \ "$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" \ "$POSTGRES_USER" "$POSTGRES_DB" <<'PY' import json import pathlib import sys output, db_sha, storage_sha, validator_sha, migration_sha, db_user, db_name = sys.argv[1:] payload = { "RECEIVER_CONFIG_FILE": "/synthetic/receiver.conf", "RESTORE_LIB_DIR": "/synthetic/restore-lib", "SELF_PATH": "/synthetic/restore-entry", "SUDOERS_PATH": "/synthetic/restore-rule", "SUDOERS_MAIN_FILE": "/synthetic/sudoers", "WORK_ROOT": "/tmp", "WORK_PREFIX": "restore-contract-work-", "CONTAINER_PREFIX": "restore-contract-container-", "SHADOW_DB_USER": db_user, "SHADOW_DB_NAME": db_name, "POSTGRES_IMAGE": "postgres:16-alpine", "DEADMAN_TIMER_UNIT": "fixture-deadman.timer", "HEALTH_TIMER_UNIT": "fixture-health.timer", "ARCHIVE_BATCH_PREFIX": "fixture-backup-", "RUN_ID_PREFIX": "auto-", "EXPECTED_DB_RESTORE_SHA": db_sha, "EXPECTED_STORAGE_RESTORE_SHA": storage_sha, "EXPECTED_ATTACHMENT_VALIDATOR_SHA": validator_sha, "EXPECTED_BUILD_PREFIX": "a" * 12, "EXPECTED_MIGRATION_SET_SHA": migration_sha, } pathlib.Path(output).write_text(json.dumps(payload), encoding="utf-8") PY cp "$TEST_DIR/render-config.json" "$UNIGNORED_CONFIG" if render_runner "$UNIGNORED_CONFIG" "$TEST_DIR/unignored-config-output.sh" \ >/dev/null 2>&1; then fail "renderer accepted an unignored in-repository private config" fi [[ ! -e "$TEST_DIR/unignored-config-output.sh" ]] \ || fail "renderer published output for an unignored private config" pass "unignored in-repository private config fails before candidate creation" if render_runner "$TEST_DIR/render-config.json" "$UNIGNORED_OUTPUT" \ >/dev/null 2>&1; then fail "renderer accepted an unignored in-repository candidate path" fi [[ ! -e "$UNIGNORED_OUTPUT" ]] \ || fail "renderer published an unignored in-repository candidate" pass "unignored in-repository candidate path fails before creation" mkdir -m 0700 -p "$PRIVATE_REPO_TEST_DIR" cp "$TEST_DIR/render-config.json" "$PRIVATE_REPO_TEST_DIR/render-config.json" render_runner "$PRIVATE_REPO_TEST_DIR/render-config.json" \ "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" [[ -s "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" ]] \ || fail "ignored private render path did not produce a candidate" pass "ignored in-repository private config and candidate paths are accepted" render_runner "$TEST_DIR/render-config.json" "$RUNNER" render_runner "$TEST_DIR/render-config.json" "$TEST_DIR/rendered-runner-repeat.sh" cmp -s "$RUNNER" "$TEST_DIR/rendered-runner-repeat.sh" \ || fail "identical private config did not render a deterministic candidate" pass "identical private config renders a deterministic candidate SHA-256" python3 - "$TEST_DIR/render-config.json" "$TEST_DIR/unsafe-render-config.json" <<'PY' import json import pathlib import sys source, target = map(pathlib.Path, sys.argv[1:]) payload = json.loads(source.read_text(encoding="utf-8")) payload["SELF_PATH"] = '/synthetic/entry";unsafe-command' target.write_text(json.dumps(payload), encoding="utf-8") PY if render_runner "$TEST_DIR/unsafe-render-config.json" \ "$TEST_DIR/unsafe-rendered-runner.sh" >/dev/null 2>&1; then fail "renderer accepted shell syntax in a runtime path" fi [[ ! -e "$TEST_DIR/unsafe-rendered-runner.sh" ]] \ || fail "renderer published an unsafe partial candidate" pass "renderer rejects shell syntax before publishing a candidate" bash -n "$RUNNER" "$RUNNER_TEMPLATE" "$RESTORE_DB" "$RESTORE_STORAGE" python3 - "$ATTACHMENT_VALIDATOR" <<'PY' import pathlib import sys compile(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"), sys.argv[1], "exec") PY python3 - "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" "$0" <<'PY' import ipaddress import pathlib import re import sys for source in sys.argv[1:]: text = pathlib.Path(source).read_text(encoding="utf-8") for raw in re.findall(r"(?/dev/null; then fail "tracked runner assets contain the local account name" fi pass "tracked template renders one syntax-valid candidate without site-specific values" python3 - "$RUNNER_TEMPLATE" <<'PY' import pathlib import re import sys text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") truncating = [ line.strip() for line in text.splitlines() if re.search(r"(?)>\"\$private_log\"", line) ] if truncating != [': >"$private_log"']: raise SystemExit("private_log_truncation_contract") required_markers = { "[stage=postgres_container_start]", "[stage=dump_list]", "[stage=target_confirmation]", "[stage=restore_apply]", "[stage=restore_storage]", } missing = {marker for marker in required_markers if marker not in text} if missing: raise SystemExit("private_log_stage_markers_missing") PY pass "private raw log is initialized once and appends fixed stage markers" python3 - "$RUNNER" "$TEST_DIR/confirm-postgres-target.sh" <<'PY' import pathlib import re import sys runner_path, output_path = map(pathlib.Path, sys.argv[1:]) text = runner_path.read_text(encoding="utf-8") match = re.search( r"\n(confirm_postgres_target\(\) \{.*?\n\})\n\ncleanup_runtime\(\)", text, re.S, ) if not match: raise SystemExit("confirm_postgres_target_body_missing") output_path.write_text(match.group(1) + "\n", encoding="utf-8") PY if rg -n 'pg_isready' "$RUNNER" >/dev/null; then fail "socket-only pg_isready remains in rendered runner" fi pass "rendered runner exposes the single TCP target-confirmation implementation" python3 - "$RUNNER" "$TEST_DIR/validate-batch.py" <<'PY' import pathlib import re import sys runner_path, output_path = map(pathlib.Path, sys.argv[1:]) text = runner_path.read_text(encoding="utf-8") function_text = text.split("validate_batch() {", 1)[1] match = re.search(r"<<'PY'\n(?P.*?)\nPY\n\}", function_text, re.S) if not match: raise SystemExit("validate_batch_body_missing") output_path.write_text(match.group("body") + "\n", encoding="utf-8") PY python3 - "$TEST_DIR/batches" <<'PY' import hashlib import json import pathlib import shutil import sys root = pathlib.Path(sys.argv[1]) run_id = "auto-20260825T000000Z" source = root / f"fixture-backup-{run_id}" source.mkdir(parents=True) (source / "database.dump").write_bytes(b"database-fixture") (source / "storage.tar.gz").write_bytes(b"storage-fixture") def artifact(name): data = (source / name).read_bytes() return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} manifest = { "schemaVersion": 1, "status": "complete", "runId": run_id, "createdAt": "2026-08-25T00:00:00Z", "buildSha": "a" * 40, "sourceImageDigest": "sha256:" + "c" * 64, "artifacts": { "database": artifact("database.dump"), "storage": artifact("storage.tar.gz"), }, } (source / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") shutil.copytree(source, root / "private-copy") shutil.copytree(source, root / "wrong-source-name") PY source_batch="$TEST_DIR/batches/fixture-backup-auto-20260825T000000Z" copy_batch="$TEST_DIR/batches/private-copy" validator_args=("$(printf 'a%.0s' {1..12})" "fixture-backup-" "auto-") source_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$source_batch" \ "${validator_args[0]}" source '' "${validator_args[1]}" "${validator_args[2]}")" copy_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ "${validator_args[0]}" copy auto-20260825T000000Z \ "${validator_args[1]}" "${validator_args[2]}")" [[ "$source_metadata" == "$copy_metadata" ]] \ || fail "source and renamed copy metadata differ" pass "source identity and renamed private copy both pass with source binding" if python3 "$TEST_DIR/validate-batch.py" \ "$TEST_DIR/batches/wrong-source-name" "${validator_args[0]}" source '' \ "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then fail "source basename/run-id mismatch passed" fi pass "source basename/run-id mismatch fails closed" printf 'tamper' >>"$copy_batch/storage.tar.gz" if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ "${validator_args[0]}" copy auto-20260825T000000Z \ "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then fail "copied content tamper passed" fi pass "copied content tamper fails closed" python3 - "$source_batch/storage.tar.gz" "$copy_batch/storage.tar.gz" \ "$copy_batch/manifest.json" <<'PY' import json import pathlib import sys source_storage, copied_storage, path = map(pathlib.Path, sys.argv[1:]) copied_storage.write_bytes(source_storage.read_bytes()) payload = json.loads(path.read_text(encoding="utf-8")) payload["runId"] = "auto-20260825T010000Z" path.write_text(json.dumps(payload), encoding="utf-8") PY if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ "${validator_args[0]}" copy auto-20260825T000000Z \ "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then fail "copy run-id different from verified source passed" fi pass "copy run-id different from verified source fails closed" rendered_migration_sha="$(sed -nE \ 's/^EXPECTED_MIGRATION_SET_SHA="([0-9a-f]{64})"$/\1/p' "$RUNNER")" [[ "$rendered_migration_sha" == "$migration_fixture_sha" ]] \ || fail "rendered dynamic migration fingerprint mismatch" mutated_migration_sha="$(printf '%s\n%s\n' "$migration_fixture_names" \ '20260103000000_fixture_three' | sha256sum | awk '{print $1}')" [[ "$mutated_migration_sha" != "$rendered_migration_sha" ]] \ || fail "migration set mutation was not detected" if rg -n 'EXPECTED_MIGRATIONS|migration[^\n]*==[[:space:]]*65' "$RUNNER" >/dev/null; then fail "runner contains a hard-coded migration count" fi pass "dynamic migration fingerprint accepts exact set and rejects mutation" mkdir -p "$TEST_DIR/attachment-root/nested" printf 'attachment-fixture' >"$TEST_DIR/attachment-root/nested/file.bin" attachment_path_b64="$(printf 'nested/file.bin' | base64 -w0)" attachment_size="$(stat -c %s "$TEST_DIR/attachment-root/nested/file.bin")" attachment_sha="$(sha256sum "$TEST_DIR/attachment-root/nested/file.bin" | awk '{print $1}')" attachment_ok="$(printf '%s\t%s\t%s\n' "$attachment_path_b64" \ "$attachment_size" "$attachment_sha" \ | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" [[ "$attachment_ok" == "1 0 0" ]] \ || fail "attachment validator rejected a valid local reference" unsafe_path_b64="$(printf '../escape.bin' | base64 -w0)" attachment_unsafe="$(printf '%s\t\t\n' "$unsafe_path_b64" \ | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" [[ "$attachment_unsafe" == "1 1 0" ]] \ || fail "attachment validator accepted path traversal" s3_path_b64="$(printf 's3:external-object' | base64 -w0)" attachment_s3="$(printf '%s\t\t\n' "$s3_path_b64" \ | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" [[ "$attachment_s3" == "1 1 1" ]] \ || fail "attachment validator accepted an external object reference" pass "attachment validator accepts exact local hash and rejects traversal/S3" python3 - "$RUNNER" <<'PY' import pathlib import re import sys text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") finish = re.search(r"\nfinish\(\) \{(?P.*?)\n\}\ntrap finish", text, re.S) if not finish: raise SystemExit("finish function not found") body = finish.group("body") finalize_position = body.find("\n if ! rto_finalize; then") emit_position = body.find("\n emit_restore_diagnostics") cleanup_position = body.find("\n cleanup_runtime ||") if finalize_position == -1: raise SystemExit("RTO finalization missing") if emit_position == -1: raise SystemExit("diagnostic emission missing") if cleanup_position == -1: raise SystemExit("runtime cleanup call missing") if not (finalize_position < emit_position < cleanup_position): raise SystemExit("RTO/stage diagnostics are not finalized before cleanup") required = { 'echo "dump_list=$DUMP_LIST_STATUS"', 'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"', 'echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE"', 'echo "target_name_match=$TARGET_NAME_MATCH"', 'echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT"', 'echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK"', 'echo "restore_execution_path=$RESTORE_EXECUTION_PATH"', 'echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"', 'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"', 'echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"', 'echo "restore_error_category=$RESTORE_ERROR_CATEGORY"', 'echo "rto_batch_copy_seconds=$RTO_BATCH_COPY_SECONDS"', 'echo "rto_postgres_container_start_seconds=$RTO_POSTGRES_CONTAINER_START_SECONDS"', 'echo "rto_dump_list_seconds=$RTO_DUMP_LIST_SECONDS"', 'echo "rto_target_confirmation_seconds=$RTO_TARGET_CONFIRMATION_SECONDS"', 'echo "rto_restore_apply_seconds=$RTO_RESTORE_APPLY_SECONDS"', 'echo "rto_database_verify_seconds=$RTO_DATABASE_VERIFY_SECONDS"', 'echo "rto_storage_attachment_verify_seconds=$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS"', 'echo "rto_final_integrity_seconds=$RTO_FINAL_INTEGRITY_SECONDS"', 'echo "data_restore_rto_seconds=$RTO_SECONDS"', } missing = sorted(value for value in required if value not in text) if missing: raise SystemExit(f"diagnostic fields missing: {missing}") if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text: raise SystemExit("non-zero apply classification missing") transitions = [ "rto_begin batch_copy", "rto_transition batch_copy postgres_container_start", "rto_transition postgres_container_start dump_list", "rto_transition dump_list target_confirmation", "rto_transition target_confirmation restore_apply", "rto_transition restore_apply database_verify", "rto_transition database_verify storage_attachment_verify", "rto_transition storage_attachment_verify final_integrity", ] positions = [text.find(value) for value in transitions] if any(position == -1 for position in positions) or positions != sorted(positions): raise SystemExit("RTO stage transitions missing or out of order") target_call_position = text.find("if ! confirm_postgres_target; then") if not positions[3] < target_call_position < positions[4]: raise SystemExit("TCP target confirmation is outside its RTO stage boundary") if text.count('echo "data_restore_rto_seconds=$RTO_SECONDS"') != 1: raise SystemExit("data restore RTO must be emitted exactly once") PY pass "eight-stage RTO and restore diagnostics precede cleanup with one vocabulary" cat >"$TEST_DIR/mock-target-docker" <<'MOCK_DOCKER' #!/usr/bin/env bash set -euo pipefail has_tcp_loopback=0 previous="" for argument in "$@"; do if [[ "$previous" == "-h" && "$argument" == "127.0.0.1" ]]; then has_tcp_loopback=1 fi previous="$argument" done if [[ "$has_tcp_loopback" -ne 1 ]]; then printf '%s\n' "$MOCK_EXPECTED_DATABASE" exit 0 fi index="$(cat "$MOCK_QUERY_STATE")" index="$((index + 1))" printf '%s\n' "$index" >"$MOCK_QUERY_STATE" line="$(sed -n "${index}p" "$MOCK_QUERY_SEQUENCE")" [[ -n "$line" ]] || exit 99 exit_code="${line%%|*}" result="${line#*|}" [[ "$exit_code" =~ ^[0-9]+$ ]] || exit 98 if [[ "$exit_code" -eq 0 ]]; then printf '%s\n' "$result" fi exit "$exit_code" MOCK_DOCKER chmod 0700 "$TEST_DIR/mock-target-docker" capture_target_confirmation_case() { local label="$1" local sequence="$2" local output_file="$3" local sequence_file="$TEST_DIR/$label.sequence" local state_file="$TEST_DIR/$label.state" printf '%s' "$sequence" >"$sequence_file" printf '0\n' >"$state_file" ( # shellcheck disable=SC1090 source "$TEST_DIR/confirm-postgres-target.sh" docker() { "$TEST_DIR/mock-target-docker" "$@" } sleep() { : } export MOCK_EXPECTED_DATABASE="$POSTGRES_DB" export MOCK_QUERY_SEQUENCE="$sequence_file" export MOCK_QUERY_STATE="$state_file" private_log="$TEST_DIR/$label.private.log" : >"$private_log" postgres_password="$POSTGRES_PASSWORD" CONTAINER_NAME="fixture-container" SHADOW_DB_USER="$POSTGRES_USER" SHADOW_DB_NAME="$POSTGRES_DB" if confirm_postgres_target; then function_exit=0 else function_exit="$?" fi echo "function_exit=$function_exit" echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE" echo "target_name_match=$TARGET_NAME_MATCH" echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT" echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK" echo "target_confirmation_failure_reason=$TARGET_CONFIRMATION_FAILURE_REASON" ) >"$output_file" 2>&1 } capture_target_confirmation_case recovery \ "$(printf '1|\n0|%s\n0|%s\n' "$POSTGRES_DB" "$POSTGRES_DB")" \ "$TEST_DIR/readiness-recovery.out" grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-recovery.out" \ || fail "TCP target confirmation did not recover after an initial query failure" grep -Fxq 'target_confirmation_attempt_count=3' "$TEST_DIR/readiness-recovery.out" \ || fail "TCP target confirmation recovery attempt count mismatch" grep -Fxq 'target_confirmation_success_streak=2' "$TEST_DIR/readiness-recovery.out" \ || fail "TCP target confirmation recovery streak mismatch" pass "rendered TCP target confirmation recovers only after two consecutive successes" capture_target_confirmation_case streak-reset \ "$(printf '0|%s\n1|\n0|%s\n0|%s\n' \ "$POSTGRES_DB" "$POSTGRES_DB" "$POSTGRES_DB")" \ "$TEST_DIR/readiness-streak-reset.out" grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-streak-reset.out" \ || fail "TCP target confirmation did not recover after streak reset" grep -Fxq 'target_confirmation_attempt_count=4' \ "$TEST_DIR/readiness-streak-reset.out" \ || fail "query failure did not reset the consecutive-success streak" pass "rendered TCP target confirmation resets its streak after a query failure" capture_target_confirmation_case name-mismatch \ '0|wrong_restore_drill ' "$TEST_DIR/readiness-name-mismatch.out" grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-name-mismatch.out" \ || fail "target-name mismatch did not fail" grep -Fxq 'target_name_match=no' "$TEST_DIR/readiness-name-mismatch.out" \ || fail "target-name mismatch diagnostic missing" grep -Fxq 'target_confirmation_attempt_count=1' \ "$TEST_DIR/readiness-name-mismatch.out" \ || fail "target-name mismatch was retried" grep -Fxq 'target_confirmation_failure_reason=TARGET_NAME_MISMATCH' \ "$TEST_DIR/readiness-name-mismatch.out" \ || fail "target-name mismatch category missing" pass "rendered TCP target confirmation fails immediately on an exact-name mismatch" query_failures="" for _ in $(seq 1 60); do query_failures+=$'1|\n' done capture_target_confirmation_case tcp-unavailable "$query_failures" \ "$TEST_DIR/readiness-tcp-unavailable.out" grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-tcp-unavailable.out" \ || fail "socket-only mock passed without TCP loopback readiness" grep -Fxq 'target_confirmation_attempt_count=60' \ "$TEST_DIR/readiness-tcp-unavailable.out" \ || fail "permanent TCP failure did not stop at the bounded attempt count" grep -Fxq 'target_confirmation_failure_reason=TARGET_QUERY_NONZERO' \ "$TEST_DIR/readiness-tcp-unavailable.out" \ || fail "permanent TCP failure category missing" pass "socket-only availability cannot bypass the rendered TCP readiness gate" docker run -d --rm --name "$CONTAINER_NAME" -p 127.0.0.1::5432 \ -e "POSTGRES_USER=$POSTGRES_USER" \ -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ -e "POSTGRES_DB=$POSTGRES_DB" \ postgres:16-alpine >/dev/null private_log="$TEST_DIR/fixture-private.log" : >"$private_log" postgres_password="$POSTGRES_PASSWORD" SHADOW_DB_USER="$POSTGRES_USER" SHADOW_DB_NAME="$POSTGRES_DB" # shellcheck disable=SC1090 source "$TEST_DIR/confirm-postgres-target.sh" confirm_postgres_target \ || fail "PostgreSQL 16 fixture did not pass rendered TCP target confirmation" pass "real PostgreSQL 16 fixture readiness uses the rendered runner implementation" HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp \ | awk -F: 'NR == 1 {print $NF}')" [[ "$HOST_PORT" =~ ^[0-9]+$ ]] \ || fail "loopback PostgreSQL 16 port unavailable" HOST_DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${HOST_PORT}/${POSTGRES_DB}" docker exec -i "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' >/dev/null CREATE TABLE restore_contract_marker (value text NOT NULL); INSERT INTO restore_contract_marker(value) VALUES ('direct-real-script'); SQL docker exec "$CONTAINER_NAME" pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ --format=c --no-owner --no-acl >"$TEST_DIR/fixture.dump" docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -c 'DROP TABLE restore_contract_marker' >/dev/null DOCKER_DATABASE_URL="postgresql://shadow.invalid/$POSTGRES_DB" DB_TOOL_MODE=docker \ DB_CONTAINER="$CONTAINER_NAME" \ POSTGRES_DB="$POSTGRES_DB" \ POSTGRES_USER="$POSTGRES_USER" \ POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ DATABASE_URL="$DOCKER_DATABASE_URL" \ RESTORE_SCOPE=shadow \ RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null [[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -Atqc 'SELECT value FROM restore_contract_marker')" \ == "direct-real-script" ]] || fail "Docker stdin restore result mismatch" pass "real restore-db.sh Docker stdin path restores a PostgreSQL 16 custom dump" head -c 32 "$TEST_DIR/fixture.dump" >"$TEST_DIR/corrupt.dump" if DB_TOOL_MODE=docker \ DB_CONTAINER="$CONTAINER_NAME" \ POSTGRES_DB="$POSTGRES_DB" \ POSTGRES_USER="$POSTGRES_USER" \ POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ DATABASE_URL="$DOCKER_DATABASE_URL" \ RESTORE_SCOPE=shadow \ RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$RESTORE_DB" "$TEST_DIR/corrupt.dump" >/dev/null 2>&1; then fail "corrupt custom dump passed list validation" fi pass "real restore-db.sh rejects a dump-list failure" if DB_TOOL_MODE=docker \ DB_CONTAINER="$CONTAINER_NAME" \ POSTGRES_DB="$POSTGRES_DB" \ POSTGRES_USER="$POSTGRES_USER" \ POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ DATABASE_URL="$DOCKER_DATABASE_URL" \ RESTORE_SCOPE=shadow \ RESTORE_CONFIRM_DATABASE=wrong_restore_drill \ "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null 2>&1; then fail "wrong target confirmation was accepted" fi pass "real restore-db.sh rejects target confirmation mismatch" REAL_DOCKER="$(command -v docker)" mkdir -m 0700 "$TEST_DIR/proxy" cat >"$TEST_DIR/proxy/docker" <<'PROXY' #!/usr/bin/env bash set +e is_pg_restore=0 is_list=0 for argument in "$@"; do [[ "$argument" == "pg_restore" ]] && is_pg_restore=1 [[ "$argument" == "--list" ]] && is_list=1 done if [[ "$is_pg_restore" -eq 1 && "$is_list" -eq 0 ]]; then cat >/dev/null exit 23 fi "$REAL_DOCKER" "$@" exit "$?" PROXY chmod 0700 "$TEST_DIR/proxy/docker" set +e proxy_output="$(REAL_DOCKER="$REAL_DOCKER" \ PATH="$TEST_DIR/proxy:$PATH" \ DB_TOOL_MODE=docker \ DB_CONTAINER="$CONTAINER_NAME" \ POSTGRES_DB="$POSTGRES_DB" \ POSTGRES_USER="$POSTGRES_USER" \ POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ DATABASE_URL="$DOCKER_DATABASE_URL" \ RESTORE_SCOPE=shadow \ RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$RESTORE_DB" "$TEST_DIR/fixture.dump" 2>&1)" proxy_exit="$?" set -e [[ "$proxy_exit" -eq 23 ]] || fail "apply failure exit code was not preserved" [[ "$proxy_output" != *"$POSTGRES_PASSWORD"* \ && "$proxy_output" != *"$DOCKER_DATABASE_URL"* ]] \ || fail "apply failure output exposed fixture credentials" pass "real restore-db.sh preserves an injected apply exit and exposes no fixture secret" mkdir -m 0700 "$TEST_DIR/host-tools" cat >"$TEST_DIR/host-tools/psql" <<'HOST_PSQL' #!/usr/bin/env bash exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \ postgres:16-alpine psql "$@" HOST_PSQL cat >"$TEST_DIR/host-tools/pg_restore" <<'HOST_RESTORE' #!/usr/bin/env bash exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \ -v "$HOST_TOOL_MOUNT:$HOST_TOOL_MOUNT:ro" \ postgres:16-alpine pg_restore "$@" HOST_RESTORE chmod 0700 "$TEST_DIR/host-tools/psql" "$TEST_DIR/host-tools/pg_restore" export PGPASSWORD="$POSTGRES_PASSWORD" REAL_DOCKER HOST_TOOL_MOUNT="$TEST_DIR" docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -c 'DROP TABLE restore_contract_marker' >/dev/null PATH="$TEST_DIR/host-tools:$PATH" \ DB_TOOL_MODE=host \ DATABASE_URL="$HOST_DATABASE_URL" \ RESTORE_SCOPE=shadow \ RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null [[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -Atqc 'SELECT value FROM restore_contract_marker')" == "direct-real-script" ]] \ || fail "host-file restore result mismatch" pass "real restore-db.sh host-file path restores through PostgreSQL 16 tools" docker rm -f "$CONTAINER_NAME" >/dev/null if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then fail "fixture container cleanup failed" fi CONTAINER_NAME="" sed -n '1,/^trap finish EXIT HUP INT TERM$/p' "$RUNNER" \ | sed '$d' >"$TEST_DIR/runner-finalization-functions.sh" ( # shellcheck disable=SC1090 source "$TEST_DIR/runner-finalization-functions.sh" RTO_TEST_NOW=100 rto_now() { printf '%s\n' "$RTO_TEST_NOW" } rto_begin batch_copy RTO_TEST_NOW=103 rto_transition batch_copy postgres_container_start RTO_TEST_NOW=108 rto_transition postgres_container_start dump_list RTO_TEST_NOW=110 rto_transition dump_list target_confirmation RTO_TEST_NOW=121 rto_transition target_confirmation restore_apply RTO_TEST_NOW=128 rto_transition restore_apply database_verify RTO_TEST_NOW=141 rto_transition database_verify storage_attachment_verify RTO_TEST_NOW=158 rto_transition storage_attachment_verify final_integrity RTO_TEST_NOW=161 rto_finalize emit_restore_diagnostics ) >"$TEST_DIR/captured-rto-success" 2>&1 for expected in \ 'rto_batch_copy_seconds=3' \ 'rto_postgres_container_start_seconds=5' \ 'rto_dump_list_seconds=2' \ 'rto_target_confirmation_seconds=11' \ 'rto_restore_apply_seconds=7' \ 'rto_database_verify_seconds=13' \ 'rto_storage_attachment_verify_seconds=17' \ 'rto_final_integrity_seconds=3' \ 'data_restore_rto_seconds=61'; do grep -Fxq "$expected" "$TEST_DIR/captured-rto-success" \ || fail "successful eight-stage RTO total or boundary mismatch" done [[ "$(grep -c '^data_restore_rto_seconds=' "$TEST_DIR/captured-rto-success")" -eq 1 ]] \ || fail "successful RTO total was not emitted exactly once" pass "rendered runner measures eight contiguous stages with an exact total" set +e ( # shellcheck disable=SC1090 source "$TEST_DIR/runner-finalization-functions.sh" cleanup_runtime() { echo "cleanup_marker=runtime" return 0 } cleanup_access() { echo "cleanup_marker=access" return 0 } RTO_BATCH_COPY_SECONDS="2" RTO_POSTGRES_CONTAINER_START_SECONDS="4" RTO_DUMP_LIST_SECONDS="1" RTO_ACTIVE_STAGE="target_confirmation" RTO_STAGE_STARTED_EPOCH="200" RTO_TEST_NOW=209 rto_now() { printf '%s\n' "$RTO_TEST_NOW" } FAILURE_CATEGORY="restore_target_confirmation_failed" DUMP_LIST_STATUS="PASS" DUMP_LIST_EXIT_CODE="0" TARGET_CONFIRMATION_STATUS="FAIL" TARGET_CONFIRMATION_EXIT_CODE="1" TARGET_CONFIRMATION_ATTEMPT_COUNT="60" RESTORE_EXECUTION_PATH="DOCKER_STDIN" RESTORE_ERROR_CATEGORY="TARGET_QUERY_NONZERO" trap finish EXIT exit 23 ) >"$TEST_DIR/captured-rto-target-failure" 2>&1 rto_target_failure_exit="$?" set -e [[ "$rto_target_failure_exit" -eq 23 ]] \ || fail "target-confirmation timing failure did not preserve exit code" for expected in \ 'rto_batch_copy_seconds=2' \ 'rto_postgres_container_start_seconds=4' \ 'rto_dump_list_seconds=1' \ 'rto_target_confirmation_seconds=9' \ 'rto_restore_apply_seconds=NOT_RUN' \ 'rto_database_verify_seconds=NOT_RUN' \ 'rto_storage_attachment_verify_seconds=NOT_RUN' \ 'rto_final_integrity_seconds=NOT_RUN' \ 'data_restore_rto_seconds=16'; do grep -Fxq "$expected" "$TEST_DIR/captured-rto-target-failure" \ || fail "target-confirmation failure RTO boundary mismatch" done rto_line="$(grep -n '^rto_target_confirmation_seconds=9$' \ "$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)" rto_cleanup_line="$(grep -n '^cleanup_marker=runtime$' \ "$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)" [[ "$rto_line" =~ ^[0-9]+$ && "$rto_cleanup_line" =~ ^[0-9]+$ \ && "$rto_line" -lt "$rto_cleanup_line" ]] \ || fail "failed-stage RTO was not emitted before teardown" pass "target-confirmation failure preserves elapsed stages and later NOT_RUN before cleanup" capture_finalization_case() { local output_file="$1" local failure_category="$2" local dump_status="$3" local dump_exit="$4" local target_status="$5" local apply_status="$6" local script_exit="$7" local pg_restore_exit="$8" local error_count="$9" local error_category="${10}" set +e ( # shellcheck disable=SC1090 source "$TEST_DIR/runner-finalization-functions.sh" cleanup_runtime() { echo "cleanup_marker=runtime" return 0 } cleanup_access() { echo "cleanup_marker=access" return 0 } FAILURE_CATEGORY="$failure_category" DUMP_LIST_STATUS="$dump_status" DUMP_LIST_EXIT_CODE="$dump_exit" TARGET_CONFIRMATION_STATUS="$target_status" case "$target_status" in PASS) TARGET_CONFIRMATION_EXIT_CODE="0" TARGET_NAME_MATCH="yes" TARGET_CONFIRMATION_ATTEMPT_COUNT="2" TARGET_CONFIRMATION_SUCCESS_STREAK="2" ;; FAIL) TARGET_CONFIRMATION_EXIT_CODE="1" TARGET_NAME_MATCH="NOT_RUN" TARGET_CONFIRMATION_ATTEMPT_COUNT="60" TARGET_CONFIRMATION_SUCCESS_STREAK="0" ;; esac RESTORE_EXECUTION_PATH="DOCKER_STDIN" RESTORE_APPLY_STATUS="$apply_status" RESTORE_SCRIPT_EXIT_CODE="$script_exit" PG_RESTORE_EXIT_CODE="$pg_restore_exit" PG_RESTORE_ERROR_COUNT="$error_count" RESTORE_ERROR_CATEGORY="$error_category" trap finish EXIT exit 23 ) >"$output_file" 2>&1 local case_exit="$?" set -e [[ "$case_exit" -eq 23 ]] \ || fail "finalization did not preserve the injected exit code" } capture_finalization_case \ "$TEST_DIR/captured-dump-list-failure" \ dump_list_failed FAIL 1 NOT_RUN NOT_RUN NOT_RUN NOT_RUN NOT_RUN \ DUMP_LIST_NONZERO grep -Fxq 'dump_list=FAIL' "$TEST_DIR/captured-dump-list-failure" \ || fail "dump-list failure stage missing" grep -Fxq 'target_confirmation=NOT_RUN' \ "$TEST_DIR/captured-dump-list-failure" \ || fail "target stage ran after dump-list failure" grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-dump-list-failure" \ || fail "apply stage ran after dump-list failure" grep -Fxq 'rto_batch_copy_seconds=NOT_RUN' \ "$TEST_DIR/captured-dump-list-failure" \ || fail "pre-RTO failure unexpectedly reported a measured stage" grep -Fxq 'data_restore_rto_seconds=NOT_RUN' \ "$TEST_DIR/captured-dump-list-failure" \ || fail "pre-RTO failure unexpectedly reported a measured total" pass "pre-RTO failure keeps timing, target and apply fields NOT_RUN" capture_finalization_case \ "$TEST_DIR/captured-target-failure" \ restore_target_confirmation_failed PASS 0 FAIL NOT_RUN NOT_RUN NOT_RUN \ NOT_RUN TARGET_QUERY_NONZERO grep -Fxq 'target_confirmation=FAIL' "$TEST_DIR/captured-target-failure" \ || fail "target-confirmation failure stage missing" grep -Fxq 'target_confirmation_exit_code=1' \ "$TEST_DIR/captured-target-failure" \ || fail "target-confirmation failure exit code missing" grep -Fxq 'target_confirmation_attempt_count=60' \ "$TEST_DIR/captured-target-failure" \ || fail "target-confirmation attempt count missing" grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-target-failure" \ || fail "apply stage ran after target-confirmation failure" pass "target-confirmation failure keeps apply stage NOT_RUN" set +e ( # shellcheck disable=SC1090 source "$TEST_DIR/runner-finalization-functions.sh" cleanup_runtime() { echo "cleanup_marker=runtime" return 0 } cleanup_access() { echo "cleanup_marker=access" return 0 } FAILURE_CATEGORY="restore_apply_failed_after_list_pass" DUMP_LIST_STATUS="PASS" DUMP_LIST_EXIT_CODE="0" TARGET_CONFIRMATION_STATUS="PASS" TARGET_CONFIRMATION_EXIT_CODE="0" TARGET_NAME_MATCH="yes" TARGET_CONFIRMATION_ATTEMPT_COUNT="2" TARGET_CONFIRMATION_SUCCESS_STREAK="2" RESTORE_EXECUTION_PATH="DOCKER_STDIN" RESTORE_APPLY_STATUS="FAIL" RESTORE_SCRIPT_EXIT_CODE="23" PG_RESTORE_EXIT_CODE="23" PG_RESTORE_ERROR_COUNT="UNKNOWN" RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO" RTO_BATCH_COPY_SECONDS="2" RTO_POSTGRES_CONTAINER_START_SECONDS="4" RTO_DUMP_LIST_SECONDS="1" RTO_TARGET_CONFIRMATION_SECONDS="9" RTO_ACTIVE_STAGE="restore_apply" RTO_STAGE_STARTED_EPOCH="300" RTO_TEST_NOW=312 rto_now() { printf '%s\n' "$RTO_TEST_NOW" } trap finish EXIT exit 23 ) >"$TEST_DIR/captured-finalization" 2>&1 finalization_exit="$?" set -e [[ "$finalization_exit" -eq 23 ]] \ || fail "finalization did not preserve the injected exit code" diagnostic_line="$(grep -n '^restore_apply=FAIL$' \ "$TEST_DIR/captured-finalization" | cut -d: -f1)" cleanup_line="$(grep -n '^cleanup_marker=runtime$' \ "$TEST_DIR/captured-finalization" | cut -d: -f1)" [[ "$diagnostic_line" =~ ^[0-9]+$ && "$cleanup_line" =~ ^[0-9]+$ \ && "$diagnostic_line" -lt "$cleanup_line" ]] \ || fail "stage diagnostics were not emitted before teardown" grep -Fxq 'runtime_cleanup=PASS' "$TEST_DIR/captured-finalization" \ || fail "runtime teardown result was not retained" grep -Fxq 'temporary_access_cleanup=PASS' "$TEST_DIR/captured-finalization" \ || fail "access teardown result was not retained" grep -Fxq 'pg_restore_exit_code=23' "$TEST_DIR/captured-finalization" \ || fail "pg_restore exit code disappeared after teardown" grep -Fxq 'rto_restore_apply_seconds=12' "$TEST_DIR/captured-finalization" \ || fail "restore-apply failure duration missing" grep -Fxq 'rto_database_verify_seconds=NOT_RUN' "$TEST_DIR/captured-finalization" \ || fail "database verification ran after restore-apply failure" grep -Fxq 'data_restore_rto_seconds=28' "$TEST_DIR/captured-finalization" \ || fail "restore-apply failure RTO total mismatch" pass "injected restore-apply failure preserves diagnostics and RTO before teardown" if grep -Eiq 'postgresql://|SELECT[[:space:]]|/tmp/|local_fixture_password' \ "$TEST_DIR/captured-dump-list-failure" \ "$TEST_DIR/captured-target-failure" \ "$TEST_DIR/captured-finalization"; then fail "allowlisted diagnostics exposed a path, credential or SQL" fi pass "captured stage diagnostics contain no path, credential or SQL" echo "[runner-contract][PASS] all off-host shadow-restore runner contracts completed"