建立每日自动异机备份基线（07-27-automatic-offhost-backup）
这个提交包含在：
+375
@@ -0,0 +1,375 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TEST_DIR="$(mktemp -d)"
|
||||
PASS_COUNT=0
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" && "$TEST_DIR" == /tmp/* ]]; then
|
||||
rm -rf -- "$TEST_DIR"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
pass() {
|
||||
PASS_COUNT=$((PASS_COUNT + 1))
|
||||
printf '[PASS] %s\n' "$1"
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
local label="$1"
|
||||
shift
|
||||
if "$@" >"$TEST_DIR/last-command.out" 2>&1; then
|
||||
printf '[FAIL] expected rejection: %s\n' "$label" >&2
|
||||
exit 1
|
||||
fi
|
||||
pass "$label"
|
||||
}
|
||||
|
||||
make_layout() {
|
||||
local root="$1"
|
||||
mkdir -p "$root/inbox" "$root/archive" "$root/weekly"
|
||||
}
|
||||
|
||||
make_batch() {
|
||||
local target="$1"
|
||||
local run_id="$2"
|
||||
mkdir -p "$target"
|
||||
printf 'synthetic database payload\n' >"$target/database.dump"
|
||||
printf 'synthetic storage payload\n' >"$target/storage.tar.gz"
|
||||
python3 - "$target" "$run_id" <<'PYTHON'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
target, run_id = sys.argv[1:]
|
||||
def artifact(file_name):
|
||||
file_path = os.path.join(target, file_name)
|
||||
with open(file_path, "rb") as handle:
|
||||
content = handle.read()
|
||||
return {"file": file_name, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"runId": run_id,
|
||||
"createdAt": "2026-01-01T00:00:00Z",
|
||||
"buildSha": "a" * 40,
|
||||
"sourceImageDigest": None,
|
||||
"status": "complete",
|
||||
"artifacts": {
|
||||
"database": artifact("database.dump"),
|
||||
"storage": artifact("storage.tar.gz"),
|
||||
},
|
||||
}
|
||||
with open(os.path.join(target, "manifest.json"), "x", encoding="utf-8") as handle:
|
||||
json.dump(manifest, handle, indent=2)
|
||||
handle.write("\n")
|
||||
PYTHON
|
||||
}
|
||||
|
||||
receiver() {
|
||||
local layout="$1"
|
||||
shift
|
||||
env RECEIVER_TEST_MODE=1 RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$layout/inbox" BACKUP_ARCHIVE_DIR="$layout/archive" \
|
||||
BACKUP_WEEKLY_DIR="$layout/weekly" DAILY_KEEP=7 WEEKLY_KEEP=4 \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
|
||||
}
|
||||
|
||||
CURRENT_RUN_ID="auto-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"
|
||||
|
||||
VALID_LAYOUT="$TEST_DIR/receiver-valid"
|
||||
make_layout "$VALID_LAYOUT"
|
||||
make_batch "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
||||
VALID_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
|
||||
[[ -d "$VALID_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" ]]
|
||||
pass "receiver finalizes one complete synthetic batch"
|
||||
|
||||
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
|
||||
pass "receiver treats an identical repeated finalize as idempotent"
|
||||
|
||||
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
|
||||
if [[ "$SECOND_RUN_ID" == "$CURRENT_RUN_ID" ]]; then
|
||||
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
|
||||
fi
|
||||
SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
|
||||
make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
||||
SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver rejects a different automatic batch for the same UTC day" \
|
||||
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA"
|
||||
|
||||
negative_receiver_case() {
|
||||
local label="$1"
|
||||
local mutation="$2"
|
||||
local layout="$TEST_DIR/receiver-negative-${PASS_COUNT}"
|
||||
make_layout "$layout"
|
||||
make_batch "$layout/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
||||
case "$mutation" in
|
||||
corrupt-artifact) printf 'changed\n' >>"$layout/inbox/${CURRENT_BATCH}.partial/database.dump" ;;
|
||||
corrupt-manifest) printf ' ' >>"$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" ;;
|
||||
extra-file) printf 'extra\n' >"$layout/inbox/${CURRENT_BATCH}.partial/extra.txt" ;;
|
||||
symlink) rm "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"; ln -s database.dump "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" ;;
|
||||
esac
|
||||
local sha
|
||||
sha="$(sha256sum "$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
if [[ "$mutation" == "corrupt-manifest" ]]; then
|
||||
sha="$VALID_MANIFEST_SHA"
|
||||
fi
|
||||
expect_failure "$label" receiver "$layout" finalize-test "$CURRENT_RUN_ID" "$sha"
|
||||
[[ ! -e "$layout/archive/$CURRENT_BATCH" ]]
|
||||
}
|
||||
|
||||
negative_receiver_case "receiver rejects an artifact checksum mismatch" corrupt-artifact
|
||||
negative_receiver_case "receiver rejects a manifest checksum mismatch" corrupt-manifest
|
||||
negative_receiver_case "receiver rejects an extra file" extra-file
|
||||
negative_receiver_case "receiver rejects links in a batch" symlink
|
||||
|
||||
RESUME_LAYOUT="$TEST_DIR/receiver-resume"
|
||||
make_layout "$RESUME_LAYOUT"
|
||||
make_batch "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
||||
RESUME_MANIFEST_SHA="$(sha256sum "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
mv "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" "$TEST_DIR/resume-storage.tar.gz"
|
||||
expect_failure "receiver keeps an incomplete partial batch out of archive" \
|
||||
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA"
|
||||
mv "$TEST_DIR/resume-storage.tar.gz" "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"
|
||||
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" >/dev/null
|
||||
[[ -d "$RESUME_LAYOUT/archive/$CURRENT_BATCH" ]]
|
||||
pass "receiver finalizes the same partial directory after transfer resumes"
|
||||
|
||||
FORCED_LAYOUT="$TEST_DIR/receiver-forced"
|
||||
make_layout "$FORCED_LAYOUT"
|
||||
expect_failure "receiver rejects rsync sender/read mode" env \
|
||||
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
||||
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
||||
SSH_ORIGINAL_COMMAND="rsync --server --sender -logDtpre.iLsfxCIvu . ." \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
||||
expect_failure "receiver rejects a sender flag in a later rsync position" env \
|
||||
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
||||
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
||||
SSH_ORIGINAL_COMMAND="rsync --server -logDtpre.iLsfxCIvu --sender . ." \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
||||
expect_failure "receiver rejects destructive rsync flags" env \
|
||||
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
||||
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
||||
SSH_ORIGINAL_COMMAND="rsync --server --delete -logDtpre.iLsfxCIvu . ." \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
||||
expect_failure "receiver rejects an ordinary shell command" env \
|
||||
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
||||
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" SSH_ORIGINAL_COMMAND="sh -c id" \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
||||
|
||||
NESTED_LAYOUT="$TEST_DIR/receiver-nested"
|
||||
mkdir -p "$NESTED_LAYOUT/inbox/archive" "$NESTED_LAYOUT/weekly"
|
||||
expect_failure "receiver rejects archive nested inside the rsync inbox" env \
|
||||
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
||||
BACKUP_INBOX_DIR="$NESTED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$NESTED_LAYOUT/inbox/archive" \
|
||||
BACKUP_WEEKLY_DIR="$NESTED_LAYOUT/weekly" \
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" validate-config
|
||||
|
||||
RETENTION_LAYOUT="$TEST_DIR/receiver-retention"
|
||||
make_layout "$RETENTION_LAYOUT"
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
mkdir "$RETENTION_LAYOUT/archive/hyb-oa-backup-auto-202601${day}T023000Z"
|
||||
done
|
||||
for week in 01 02 03 04; do
|
||||
mkdir "$RETENTION_LAYOUT/weekly/week-2026-W${week}"
|
||||
done
|
||||
make_batch "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
||||
RETENTION_SHA="$(sha256sum "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
receiver "$RETENTION_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RETENTION_SHA" >/dev/null
|
||||
[[ "$(find "$RETENTION_LAYOUT/archive" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "7" ]]
|
||||
[[ "$(find "$RETENTION_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "4" ]]
|
||||
[[ -d "$RETENTION_LAYOUT/archive/$CURRENT_BATCH" ]]
|
||||
pass "receiver retains seven daily and four weekly recovery points while protecting current success"
|
||||
|
||||
EMPTY_SOURCE="$TEST_DIR/empty-rsync-source"
|
||||
EMPTY_TARGET="$TEST_DIR/empty-rsync-target"
|
||||
mkdir "$EMPTY_SOURCE" "$EMPTY_TARGET"
|
||||
rsync -a --dry-run --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= "$EMPTY_SOURCE/" "$EMPTY_TARGET/"
|
||||
pass "installed rsync accepts the production chmod contract"
|
||||
|
||||
WRAPPER_ROOT="$TEST_DIR/wrapper-root"
|
||||
WRAPPER_PG="$TEST_DIR/wrapper-pg"
|
||||
WRAPPER_STORAGE="$TEST_DIR/wrapper-storage"
|
||||
WRAPPER_BACKUPS="$TEST_DIR/wrapper-backups"
|
||||
WRAPPER_STATE="$TEST_DIR/wrapper-state"
|
||||
MOCK_BIN="$TEST_DIR/mock-bin"
|
||||
mkdir -p "$WRAPPER_ROOT/deploy/trialrun" "$WRAPPER_PG" "$WRAPPER_STORAGE" \
|
||||
"$WRAPPER_BACKUPS" "$WRAPPER_STATE" "$MOCK_BIN"
|
||||
printf '16\n' >"$WRAPPER_PG/PG_VERSION"
|
||||
printf 'synthetic attachment\n' >"$WRAPPER_STORAGE/marker.txt"
|
||||
printf 'synthetic env\n' >"$WRAPPER_ROOT/deploy/trialrun/.env"
|
||||
ln -s "$ROOT_DIR/scripts" "$WRAPPER_ROOT/scripts"
|
||||
printf 'private-key-fixture\n' >"$TEST_DIR/offhost-key"
|
||||
chmod 0600 "$TEST_DIR/offhost-key"
|
||||
printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"
|
||||
|
||||
BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
|
||||
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
|
||||
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
|
||||
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG
|
||||
|
||||
cat >"$MOCK_BIN/docker" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then exit 0; fi
|
||||
if [[ "${1:-}" == "compose" ]]; then
|
||||
args=" $* "
|
||||
if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
|
||||
if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
|
||||
if [[ "$args" == *" exec -T server "* ]]; then cat >>"$MOCK_ALERT_LOG"; exit 0; fi
|
||||
fi
|
||||
if [[ "${1:-}" == "inspect" ]]; then
|
||||
template="${3:-}"
|
||||
container="${4:-}"
|
||||
case "$template" in
|
||||
*State.Status*) echo running ;;
|
||||
*Config.Env*)
|
||||
if [[ "$container" == "server-id" ]]; then
|
||||
printf 'BUILD_SHA=%s\nSTORAGE_DIR=/app/storage\nDATABASE_URL=postgresql://backup_user:fixture@db:5432/hyb_oa\n' "$MOCK_BUILD_SHA"
|
||||
else
|
||||
printf 'POSTGRES_DB=hyb_oa\nPOSTGRES_USER=backup_user\nPOSTGRES_PASSWORD=fixture\n'
|
||||
fi
|
||||
;;
|
||||
*Mounts*)
|
||||
if [[ "$container" == "server-id" ]]; then
|
||||
printf '/app/storage\t%s\n' "$MOCK_STORAGE"
|
||||
else
|
||||
printf '/var/lib/postgresql/data\t%s\n' "$MOCK_PG"
|
||||
fi
|
||||
;;
|
||||
*Image*) printf 'sha256:%064d\n' 0 ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${1:-}" == "exec" ]]; then
|
||||
if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
|
||||
if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
|
||||
fi
|
||||
exit 1
|
||||
MOCK
|
||||
|
||||
cat >"$MOCK_BIN/rsync" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
exit "${MOCK_RSYNC_EXIT:-1}"
|
||||
MOCK
|
||||
cat >"$MOCK_BIN/ssh" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
exit "${MOCK_SSH_EXIT:-0}"
|
||||
MOCK
|
||||
chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
|
||||
|
||||
wrapper_env=(
|
||||
env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
|
||||
REMOTE_ENV_FILE="$WRAPPER_ROOT/deploy/trialrun/.env"
|
||||
COMPOSE_PROJECT_NAME="hyb-oa-current" POSTGRES_DATA_DIR="$WRAPPER_PG"
|
||||
STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
|
||||
AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
|
||||
OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
|
||||
OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_RECIPIENT_EMPLOYEE_NO="OPS001"
|
||||
)
|
||||
|
||||
exec 8>"$WRAPPER_STATE/automatic-backup.lock"
|
||||
flock -n 8
|
||||
expect_failure "wrapper rejects a concurrent automatic-backup run" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
||||
flock -u 8
|
||||
|
||||
rm -f "$WRAPPER_STATE/last-success.json"
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
|
||||
done
|
||||
expect_failure "unreachable rsync fails without writing off-host success" \
|
||||
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
||||
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
|
||||
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
|
||||
pass "source retention does not run before off-host finalize succeeds"
|
||||
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
serialized = json.dumps(payload)
|
||||
assert payload["issueId"] == "AUTO_BACKUP"
|
||||
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
|
||||
assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
|
||||
PYTHON
|
||||
pass "wrapper failure summary is category-only and redacted"
|
||||
|
||||
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
|
||||
done
|
||||
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=0 \
|
||||
"$ROOT_DIR/scripts/run-automatic-backup.sh" run >/dev/null
|
||||
[[ -f "$WRAPPER_STATE/last-success.json" ]]
|
||||
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
|
||||
pass "off-host finalize success writes state and then prunes source retention to seven"
|
||||
|
||||
rm -f "$WRAPPER_STATE/last-success.json"
|
||||
expect_failure "stale watchdog rejects a missing success record" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
|
||||
python3 - "$WRAPPER_STATE/last-success.json" "$BUILD_SHA" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump({
|
||||
"schemaVersion": 1,
|
||||
"status": "complete",
|
||||
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"runId": "auto-synthetic",
|
||||
"manifestSha256": "0" * 64,
|
||||
"composeProject": "hyb-oa-current",
|
||||
"buildSha": sys.argv[2],
|
||||
}, handle)
|
||||
PYTHON
|
||||
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
||||
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
payload["buildSha"] = "0" * 40
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
PYTHON
|
||||
expect_failure "stale watchdog rejects a success record from another build" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
|
||||
pass "stale watchdog accepts a fresh project/build-bound success record"
|
||||
|
||||
DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
|
||||
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
|
||||
"${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \
|
||||
"$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
|
||||
FIRST_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
|
||||
SECOND_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
|
||||
[[ "$FIRST_REMINDER" == "$SECOND_REMINDER" && "$FIRST_REMINDER" == quarterly_shadow_restore_due_* ]]
|
||||
pass "quarterly reminder finding key is stable within the same quarter"
|
||||
|
||||
expect_failure "receiver installer uninstall rejects an unsafe target path" env \
|
||||
BACKUP_USER=hybbackup RECEIVER_TARGET=/ CONFIG_FILE=/etc/hyb-oa-test.conf \
|
||||
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" uninstall
|
||||
|
||||
expect_failure "receiver installer rejects a system-managed backup root before mutation" env \
|
||||
BACKUP_ROOT=/etc/hyb-oa-backups PUBLIC_KEY_FILE="$TEST_DIR/known-hosts" \
|
||||
RECEIVER_SOURCE="$ROOT_DIR/scripts/receive-offhost-backup.sh" \
|
||||
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" install
|
||||
|
||||
printf '[PASS] automatic off-host backup contract completed (%d checks)\n' "$PASS_COUNT"
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_FILE="${RECEIVER_CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}"
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
# Root-owned installer output with path-only assignments.
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
fi
|
||||
|
||||
BACKUP_INBOX_DIR="${BACKUP_INBOX_DIR:-}"
|
||||
BACKUP_ARCHIVE_DIR="${BACKUP_ARCHIVE_DIR:-}"
|
||||
BACKUP_WEEKLY_DIR="${BACKUP_WEEKLY_DIR:-}"
|
||||
RRSYNC_BIN="${RRSYNC_BIN:-/usr/bin/rrsync}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
DAILY_KEEP="${DAILY_KEEP:-7}"
|
||||
WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
|
||||
|
||||
fail() {
|
||||
echo "[offhost-receiver][ERROR] $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
validate_absolute_path() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
|
||||
fail "$name must be an explicit safe absolute path"
|
||||
fi
|
||||
if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
|
||||
|| "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
|
||||
fail "$name contains an unsafe path segment"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_layout() {
|
||||
local name value
|
||||
for name in BACKUP_INBOX_DIR BACKUP_ARCHIVE_DIR BACKUP_WEEKLY_DIR; do
|
||||
value="${!name:-}"
|
||||
validate_absolute_path "$name" "$value"
|
||||
[[ -d "$value" && ! -L "$value" ]] || fail "$name is not an existing regular directory"
|
||||
done
|
||||
if [[ "$BACKUP_INBOX_DIR" == "$BACKUP_ARCHIVE_DIR" \
|
||||
|| "$BACKUP_INBOX_DIR" == "$BACKUP_WEEKLY_DIR" \
|
||||
|| "$BACKUP_ARCHIVE_DIR" == "$BACKUP_WEEKLY_DIR" ]]; then
|
||||
fail "backup receiver directories must be distinct"
|
||||
fi
|
||||
local first second
|
||||
for first in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do
|
||||
for second in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do
|
||||
if [[ "$first" != "$second" && ( "$first" == "$second"/* || "$second" == "$first"/* ) ]]; then
|
||||
fail "backup receiver directories must not be nested"
|
||||
fi
|
||||
done
|
||||
done
|
||||
if [[ "$(stat -c %d "$BACKUP_INBOX_DIR")" != "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" \
|
||||
|| "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" != "$(stat -c %d "$BACKUP_WEEKLY_DIR")" ]]; then
|
||||
fail "receiver directories must share a filesystem for atomic finalize and weekly hardlinks"
|
||||
fi
|
||||
[[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "DAILY_KEEP must be positive"
|
||||
[[ "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "WEEKLY_KEEP must be positive"
|
||||
command -v "$PYTHON_BIN" >/dev/null 2>&1 || fail "python3 is required"
|
||||
}
|
||||
|
||||
validate_run_id() {
|
||||
local run_id="$1"
|
||||
[[ "$run_id" =~ ^auto-[0-9]{8}T[0-9]{6}Z$ ]] \
|
||||
|| fail "run id must use auto-YYYYMMDDTHHMMSSZ"
|
||||
local run_day="${run_id#auto-}"
|
||||
run_day="${run_day%%T*}"
|
||||
[[ "$run_day" == "$(date -u +%Y%m%d)" || "$run_day" == "$(date -u -d yesterday +%Y%m%d)" ]] \
|
||||
|| fail "run id UTC day is outside the accepted receiver window"
|
||||
}
|
||||
|
||||
validate_batch() {
|
||||
local batch_dir="$1"
|
||||
local run_id="$2"
|
||||
local expected_manifest_sha="$3"
|
||||
|
||||
[[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "partial batch is missing"
|
||||
[[ "$expected_manifest_sha" =~ ^[0-9a-f]{64}$ ]] || fail "manifest SHA-256 is invalid"
|
||||
|
||||
local entry_count regular_count
|
||||
entry_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -printf '.' | wc -c | tr -d ' ')"
|
||||
regular_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -type f -printf '.' | wc -c | tr -d ' ')"
|
||||
[[ "$entry_count" == "3" && "$regular_count" == "3" ]] \
|
||||
|| fail "batch must contain exactly three regular files"
|
||||
for file_name in database.dump storage.tar.gz manifest.json; do
|
||||
[[ -f "$batch_dir/$file_name" && ! -L "$batch_dir/$file_name" ]] \
|
||||
|| fail "batch required file is missing"
|
||||
done
|
||||
|
||||
local actual_manifest_sha
|
||||
actual_manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
||||
[[ "$actual_manifest_sha" == "$expected_manifest_sha" ]] \
|
||||
|| fail "manifest checksum mismatch"
|
||||
|
||||
"$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
batch_dir, expected_run_id = sys.argv[1:]
|
||||
manifest_path = os.path.join(batch_dir, "manifest.json")
|
||||
with open(manifest_path, "r", encoding="utf-8") as handle:
|
||||
manifest = json.load(handle)
|
||||
|
||||
if manifest.get("schemaVersion") != 1:
|
||||
raise SystemExit("unsupported manifest schema")
|
||||
if manifest.get("status") != "complete" or manifest.get("runId") != expected_run_id:
|
||||
raise SystemExit("manifest identity mismatch")
|
||||
if not manifest.get("buildSha") and not manifest.get("sourceImageDigest"):
|
||||
raise SystemExit("manifest source identity missing")
|
||||
|
||||
artifacts = manifest.get("artifacts") or {}
|
||||
expected = {
|
||||
"database": "database.dump",
|
||||
"storage": "storage.tar.gz",
|
||||
}
|
||||
for key, expected_name in expected.items():
|
||||
artifact = artifacts.get(key) or {}
|
||||
if artifact.get("file") != expected_name:
|
||||
raise SystemExit("manifest artifact name mismatch")
|
||||
file_path = os.path.join(batch_dir, expected_name)
|
||||
if os.path.getsize(file_path) != artifact.get("bytes"):
|
||||
raise SystemExit("manifest artifact size mismatch")
|
||||
digest = hashlib.sha256()
|
||||
with open(file_path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != artifact.get("sha256"):
|
||||
raise SystemExit("manifest artifact checksum mismatch")
|
||||
PYTHON
|
||||
}
|
||||
|
||||
prune_named_directories() {
|
||||
local root="$1"
|
||||
local prefix="$2"
|
||||
local keep="$3"
|
||||
local -a entries=()
|
||||
mapfile -t entries < <(find "$root" -mindepth 1 -maxdepth 1 -type d \
|
||||
-name "${prefix}*" -printf '%f\n' | LC_ALL=C sort)
|
||||
local remove_count=$((${#entries[@]} - keep))
|
||||
if (( remove_count <= 0 )); then
|
||||
return 0
|
||||
fi
|
||||
local index entry target
|
||||
for ((index = 0; index < remove_count; index += 1)); do
|
||||
entry="${entries[$index]}"
|
||||
target="$root/$entry"
|
||||
[[ "$entry" == "$prefix"* && -d "$target" && ! -L "$target" ]] \
|
||||
|| fail "retention target failed safety validation"
|
||||
rm -rf -- "$target"
|
||||
done
|
||||
}
|
||||
|
||||
finalize_batch() {
|
||||
local run_id="$1"
|
||||
local expected_manifest_sha="$2"
|
||||
validate_run_id "$run_id"
|
||||
validate_layout
|
||||
|
||||
local batch_name="hyb-oa-backup-$run_id"
|
||||
local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"
|
||||
local archive_dir="$BACKUP_ARCHIVE_DIR/$batch_name"
|
||||
local already_archived=0
|
||||
if [[ -e "$archive_dir" ]]; then
|
||||
[[ -d "$archive_dir" && ! -L "$archive_dir" ]] || fail "archive target is invalid"
|
||||
validate_batch "$archive_dir" "$run_id" "$expected_manifest_sha"
|
||||
already_archived=1
|
||||
fi
|
||||
|
||||
local run_day="${run_id#auto-}"
|
||||
run_day="${run_day%%T*}"
|
||||
local existing_for_day
|
||||
existing_for_day="$(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
|
||||
-name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print -quit)"
|
||||
[[ -z "$existing_for_day" ]] || fail "another automatic archive already exists for this UTC day"
|
||||
|
||||
if [[ "$already_archived" == "0" ]]; then
|
||||
validate_batch "$partial_dir" "$run_id" "$expected_manifest_sha"
|
||||
chmod 0400 "$partial_dir/database.dump" "$partial_dir/storage.tar.gz" "$partial_dir/manifest.json"
|
||||
mv -- "$partial_dir" "$archive_dir"
|
||||
fi
|
||||
|
||||
local iso_week
|
||||
iso_week="$($PYTHON_BIN - "$run_day" <<'PYTHON'
|
||||
from datetime import datetime
|
||||
import sys
|
||||
day = datetime.strptime(sys.argv[1], "%Y%m%d").date()
|
||||
iso = day.isocalendar()
|
||||
print(f"week-{iso.year:04d}-W{iso.week:02d}")
|
||||
PYTHON
|
||||
)"
|
||||
local weekly_dir="$BACKUP_WEEKLY_DIR/$iso_week"
|
||||
if [[ ! -e "$weekly_dir" ]]; then
|
||||
cp -al -- "$archive_dir" "$weekly_dir"
|
||||
fi
|
||||
|
||||
prune_named_directories "$BACKUP_ARCHIVE_DIR" "hyb-oa-backup-auto-" "$DAILY_KEEP"
|
||||
prune_named_directories "$BACKUP_WEEKLY_DIR" "week-" "$WEEKLY_KEEP"
|
||||
echo "[offhost-receiver][PASS] finalized batch with verified manifest"
|
||||
}
|
||||
|
||||
run_forced_command() {
|
||||
validate_layout
|
||||
local original_command="${SSH_ORIGINAL_COMMAND:-}"
|
||||
if [[ "$original_command" == rsync\ --server\ * ]]; then
|
||||
if [[ "$original_command" =~ (^|[[:space:]])--sender([[:space:]]|$) ]]; then
|
||||
fail "read access is forbidden"
|
||||
fi
|
||||
if [[ "$original_command" =~ (^|[[:space:]])--(delete|delete-before|delete-after|delete-during|delete-delay|delete-excluded|remove-source-files|remove-sent-files|inplace)(=|[[:space:]]|$) ]]; then
|
||||
fail "destructive rsync option is forbidden"
|
||||
fi
|
||||
[[ -x "$RRSYNC_BIN" ]] || fail "rrsync is required"
|
||||
exec "$RRSYNC_BIN" -wo "$BACKUP_INBOX_DIR"
|
||||
fi
|
||||
|
||||
if [[ "$original_command" =~ ^hyb-backup-finalize[[:space:]](auto-[0-9]{8}T[0-9]{6}Z)[[:space:]]([0-9a-f]{64})$ ]]; then
|
||||
finalize_batch "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
||||
return 0
|
||||
fi
|
||||
fail "unsupported forced command"
|
||||
}
|
||||
|
||||
case "${1:-forced}" in
|
||||
forced)
|
||||
run_forced_command
|
||||
;;
|
||||
finalize-test)
|
||||
[[ "${RECEIVER_TEST_MODE:-0}" == "1" ]] || fail "test mode is disabled"
|
||||
finalize_batch "${2:-}" "${3:-}"
|
||||
;;
|
||||
validate-config)
|
||||
validate_layout
|
||||
echo "[offhost-receiver][PASS] configuration valid"
|
||||
;;
|
||||
*)
|
||||
fail "unsupported mode"
|
||||
;;
|
||||
esac
|
||||
+425
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eEuo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REMOTE_DIR="${REMOTE_DIR:-$ROOT_DIR}"
|
||||
REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}"
|
||||
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}"
|
||||
POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}"
|
||||
STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}"
|
||||
BUILD_SHA="${BUILD_SHA:-}"
|
||||
VITE_BUILD_SHA="${VITE_BUILD_SHA:-}"
|
||||
AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}"
|
||||
AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
|
||||
OFFHOST_TARGET="${OFFHOST_TARGET:-}"
|
||||
OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
|
||||
OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
|
||||
AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}"
|
||||
AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}"
|
||||
ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}"
|
||||
MODE="${1:-run}"
|
||||
|
||||
CURRENT_PHASE="preflight"
|
||||
SERVER_CONTAINER=""
|
||||
DB_CONTAINER=""
|
||||
SUMMARY_FILE=""
|
||||
COMPOSE_CMD=()
|
||||
|
||||
fail() {
|
||||
echo "[automatic-backup][ERROR] $1" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
validate_absolute_path() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
|
||||
fail "$name must be an explicit safe absolute path"
|
||||
fi
|
||||
if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
|
||||
|| "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
|
||||
fail "$name contains an unsafe path segment"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_managed_directory_path() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
validate_absolute_path "$name" "$value"
|
||||
[[ "$value" =~ ^/[^/]+/[^/]+ ]] \
|
||||
|| fail "$name must be a dedicated path below a top-level directory"
|
||||
case "$value" in
|
||||
/bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*)
|
||||
fail "$name must not be inside a system-managed directory"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
validate_common() {
|
||||
local name value
|
||||
for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \
|
||||
AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR; do
|
||||
value="${!name:-}"
|
||||
validate_absolute_path "$name" "$value"
|
||||
done
|
||||
validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR"
|
||||
validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR"
|
||||
[[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" ]] || fail "deployment root or env file is missing"
|
||||
[[ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]] || fail "active PGDATA marker is missing"
|
||||
[[ -d "$STORAGE_HOST_DIR" ]] || fail "active storage directory is missing"
|
||||
[[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid"
|
||||
[[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \
|
||||
&& "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \
|
||||
|| fail "Compose file must be a safe relative path"
|
||||
[[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid"
|
||||
BUILD_SHA="${BUILD_SHA,,}"
|
||||
VITE_BUILD_SHA="${VITE_BUILD_SHA,,}"
|
||||
[[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \
|
||||
|| fail "frontend build SHA must match the full build prefix"
|
||||
[[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive"
|
||||
[[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive"
|
||||
[[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid"
|
||||
if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \
|
||||
|| "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \
|
||||
|| "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then
|
||||
fail "backup and state directories must be independent"
|
||||
fi
|
||||
mkdir -p "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
|
||||
[[ -d "$AUTOMATIC_BACKUP_DIR" && ! -L "$AUTOMATIC_BACKUP_DIR" \
|
||||
&& -d "$AUTOMATIC_BACKUP_STATE_DIR" && ! -L "$AUTOMATIC_BACKUP_STATE_DIR" ]] \
|
||||
|| fail "backup and state directories must not be symbolic links"
|
||||
chmod 0700 "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
|
||||
command -v python3 >/dev/null 2>&1 || fail "python3 is required"
|
||||
}
|
||||
|
||||
validate_transfer_config() {
|
||||
validate_absolute_path OFFHOST_SSH_KEY "$OFFHOST_SSH_KEY"
|
||||
validate_absolute_path OFFHOST_KNOWN_HOSTS "$OFFHOST_KNOWN_HOSTS"
|
||||
[[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" ]] || fail "off-host private key is missing"
|
||||
[[ "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] || fail "off-host private key mode must be 0600"
|
||||
[[ -f "$OFFHOST_KNOWN_HOSTS" && -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing"
|
||||
[[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \
|
||||
|| fail "off-host target must use a safe user@alias value"
|
||||
command -v ssh >/dev/null 2>&1 || fail "ssh is required"
|
||||
command -v rsync >/dev/null 2>&1 || fail "rsync is required"
|
||||
}
|
||||
|
||||
select_compose() {
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker compose)
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD=(docker-compose)
|
||||
else
|
||||
fail "Compose runtime is unavailable"
|
||||
fi
|
||||
}
|
||||
|
||||
compose() {
|
||||
POSTGRES_DATA_DIR="$POSTGRES_DATA_DIR" \
|
||||
STORAGE_HOST_DIR="$STORAGE_HOST_DIR" \
|
||||
BUILD_SHA="$BUILD_SHA" \
|
||||
VITE_BUILD_SHA="$VITE_BUILD_SHA" \
|
||||
"${COMPOSE_CMD[@]}" -p "$COMPOSE_PROJECT_NAME" \
|
||||
-f "$REMOTE_DIR/$COMPOSE_FILE" --env-file "$REMOTE_ENV_FILE" "$@"
|
||||
}
|
||||
|
||||
container_env() {
|
||||
local container_id="$1"
|
||||
local key="$2"
|
||||
docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \
|
||||
| sed -n "s/^${key}=//p" | head -n 1
|
||||
}
|
||||
|
||||
mount_source() {
|
||||
local container_id="$1"
|
||||
local destination="$2"
|
||||
docker inspect -f '{{range .Mounts}}{{println .Destination "\t" .Source}}{{end}}' "$container_id" \
|
||||
| awk -F '\t' -v expected="$destination" '$1 == expected { print $2 }'
|
||||
}
|
||||
|
||||
resolve_active_runtime() {
|
||||
select_compose
|
||||
local -a servers=() dbs=()
|
||||
mapfile -t servers < <(compose ps -q server)
|
||||
mapfile -t dbs < <(compose ps -q db)
|
||||
[[ "${#servers[@]}" -eq 1 && -n "${servers[0]}" ]] || fail "expected exactly one active server container"
|
||||
[[ "${#dbs[@]}" -eq 1 && -n "${dbs[0]}" ]] || fail "expected exactly one active database container"
|
||||
SERVER_CONTAINER="${servers[0]}"
|
||||
DB_CONTAINER="${dbs[0]}"
|
||||
[[ "$(docker inspect -f '{{.State.Status}}' "$SERVER_CONTAINER")" == "running" ]] \
|
||||
|| fail "server container is not running"
|
||||
[[ "$(docker inspect -f '{{.State.Status}}' "$DB_CONTAINER")" == "running" ]] \
|
||||
|| fail "database container is not running"
|
||||
|
||||
local actual_build storage_destination actual_storage actual_pgdata
|
||||
actual_build="$(container_env "$SERVER_CONTAINER" BUILD_SHA)"
|
||||
storage_destination="$(container_env "$SERVER_CONTAINER" STORAGE_DIR)"
|
||||
[[ -n "$storage_destination" && "$storage_destination" == /* ]] || fail "server storage destination is invalid"
|
||||
actual_storage="$(mount_source "$SERVER_CONTAINER" "$storage_destination")"
|
||||
actual_pgdata="$(mount_source "$DB_CONTAINER" /var/lib/postgresql/data)"
|
||||
[[ "$actual_build" == "$BUILD_SHA" ]] || fail "active server build does not match the installed backup contract"
|
||||
[[ "$actual_storage" == "$STORAGE_HOST_DIR" ]] || fail "active storage mount does not match the installed backup contract"
|
||||
[[ "$actual_pgdata" == "$POSTGRES_DATA_DIR" ]] || fail "active PGDATA mount does not match the installed backup contract"
|
||||
}
|
||||
|
||||
write_summary() {
|
||||
local status="$1"
|
||||
local finding_count="$2"
|
||||
local detail="$3"
|
||||
local output_file="$AUTOMATIC_BACKUP_STATE_DIR/last-run.json"
|
||||
local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-run.json.tmp.$$"
|
||||
python3 - "$temp_file" "$status" "$finding_count" "$detail" "$VITE_BUILD_SHA" <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
file_path, status, finding_count, detail, build_sha = sys.argv[1:]
|
||||
finding_count = int(finding_count)
|
||||
findings = []
|
||||
if finding_count:
|
||||
findings.append({
|
||||
"source": "automatic_backup",
|
||||
"status": status,
|
||||
"severity": "P0",
|
||||
"detail": detail,
|
||||
})
|
||||
payload = {
|
||||
"issueId": "AUTO_BACKUP",
|
||||
"status": status,
|
||||
"conclusion": status,
|
||||
"findingCount": finding_count,
|
||||
"findings": findings,
|
||||
"generatedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"reportDir": "host-backup-state",
|
||||
"buildSha": build_sha,
|
||||
"privacy": "status/count/category only; no credentials, host addresses, absolute paths, file names, or payloads",
|
||||
}
|
||||
with open(file_path, "x", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
os.chmod(file_path, 0o600)
|
||||
PYTHON
|
||||
mv -f -- "$temp_file" "$output_file"
|
||||
SUMMARY_FILE="$output_file"
|
||||
}
|
||||
|
||||
send_summary_alert() {
|
||||
local state_name="$1"
|
||||
[[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1
|
||||
[[ -n "$SERVER_CONTAINER" ]] || resolve_active_runtime
|
||||
compose exec -T server sh -lc \
|
||||
"cd /app/server && WARCH3_OPS_ALERT_SEND=1 npm run send:wecom-ops-alert -- --summary-file=- --recipient-employee-no=${ALERT_RECIPIENT_EMPLOYEE_NO} --send --state-file=/app/artifacts/qa-flow/${state_name} --format=json" \
|
||||
<"$SUMMARY_FILE" >/dev/null
|
||||
}
|
||||
|
||||
on_error() {
|
||||
local exit_code="$?"
|
||||
trap - ERR
|
||||
set +e
|
||||
write_summary "FAIL_AUTOMATIC_BACKUP" 1 "phase_${CURRENT_PHASE}_failed"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
echo "[automatic-backup][FAIL] phase=${CURRENT_PHASE}" >&2
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
prune_local_backups() {
|
||||
local current_name="$1"
|
||||
local -a entries=()
|
||||
mapfile -t entries < <(find "$AUTOMATIC_BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d \
|
||||
-name 'hyb-oa-backup-auto-*' -printf '%f\n' | LC_ALL=C sort)
|
||||
local remove_count=$((${#entries[@]} - AUTOMATIC_BACKUP_KEEP))
|
||||
if (( remove_count <= 0 )); then
|
||||
return 0
|
||||
fi
|
||||
local index entry target
|
||||
for ((index = 0; index < remove_count; index += 1)); do
|
||||
entry="${entries[$index]}"
|
||||
[[ "$entry" != "$current_name" && "$entry" =~ ^hyb-oa-backup-auto-[0-9]{8}T[0-9]{6}Z$ ]] \
|
||||
|| fail "local retention target failed safety validation"
|
||||
target="$AUTOMATIC_BACKUP_DIR/$entry"
|
||||
[[ -d "$target" && ! -L "$target" ]] || fail "local retention target is invalid"
|
||||
rm -rf -- "$target"
|
||||
done
|
||||
}
|
||||
|
||||
write_success_state() {
|
||||
local run_id="$1"
|
||||
local manifest_sha="$2"
|
||||
local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-success.json.tmp.$$"
|
||||
python3 - "$temp_file" "$run_id" "$manifest_sha" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
file_path, run_id, manifest_sha, project, build_sha = sys.argv[1:]
|
||||
payload = {
|
||||
"schemaVersion": 1,
|
||||
"status": "complete",
|
||||
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"runId": run_id,
|
||||
"manifestSha256": manifest_sha,
|
||||
"composeProject": project,
|
||||
"buildSha": build_sha,
|
||||
}
|
||||
with open(file_path, "x", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
os.chmod(file_path, 0o600)
|
||||
PYTHON
|
||||
mv -f -- "$temp_file" "$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
|
||||
}
|
||||
|
||||
run_backup() {
|
||||
validate_common
|
||||
validate_transfer_config
|
||||
exec 9>"$AUTOMATIC_BACKUP_STATE_DIR/automatic-backup.lock"
|
||||
flock -n 9 || fail "another automatic backup is already running"
|
||||
trap on_error ERR
|
||||
|
||||
CURRENT_PHASE="runtime_identity"
|
||||
resolve_active_runtime
|
||||
|
||||
local database_url postgres_db postgres_user postgres_password parsed_identity url_user url_db
|
||||
database_url="$(container_env "$SERVER_CONTAINER" DATABASE_URL)"
|
||||
postgres_db="$(container_env "$DB_CONTAINER" POSTGRES_DB)"
|
||||
postgres_user="$(container_env "$DB_CONTAINER" POSTGRES_USER)"
|
||||
postgres_password="$(container_env "$DB_CONTAINER" POSTGRES_PASSWORD)"
|
||||
[[ -n "$database_url" && -n "$postgres_db" && -n "$postgres_user" && -n "$postgres_password" ]] \
|
||||
|| fail "active database identity is incomplete"
|
||||
parsed_identity="$(printf '%s' "$database_url" | python3 -c '
|
||||
import sys
|
||||
from urllib.parse import unquote, urlparse
|
||||
value = urlparse(sys.stdin.read())
|
||||
if value.scheme not in {"postgres", "postgresql"} or not value.username or not value.path.strip("/"):
|
||||
raise SystemExit(1)
|
||||
database_name = unquote(value.path.strip("/"))
|
||||
print(f"{unquote(value.username)}\t{database_name}")
|
||||
')"
|
||||
IFS=$'\t' read -r url_user url_db <<<"$parsed_identity"
|
||||
[[ "$url_user" == "$postgres_user" && "$url_db" == "$postgres_db" ]] \
|
||||
|| fail "active server and database container identities do not match"
|
||||
|
||||
local run_id batch_name batch_dir source_image_digest manifest_sha
|
||||
run_id="auto-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
batch_name="hyb-oa-backup-$run_id"
|
||||
batch_dir="$AUTOMATIC_BACKUP_DIR/$batch_name"
|
||||
source_image_digest="$(docker inspect -f '{{.Image}}' "$SERVER_CONTAINER")"
|
||||
[[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid"
|
||||
|
||||
CURRENT_PHASE="paired_backup"
|
||||
if ! DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
|
||||
POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" POSTGRES_PASSWORD="$postgres_password" \
|
||||
STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" \
|
||||
BACKUP_RUN_ID="$run_id" BUILD_SHA="$BUILD_SHA" SOURCE_IMAGE_DIGEST="$source_image_digest" \
|
||||
"$ROOT_DIR/scripts/backup-all.sh" >/dev/null 2>&1; then
|
||||
fail "paired backup command failed"
|
||||
fi
|
||||
[[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published"
|
||||
manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
||||
|
||||
local -a ssh_options=(
|
||||
-i "$OFFHOST_SSH_KEY"
|
||||
-o BatchMode=yes
|
||||
-o IdentitiesOnly=yes
|
||||
-o StrictHostKeyChecking=yes
|
||||
-o "UserKnownHostsFile=$OFFHOST_KNOWN_HOSTS"
|
||||
-o ConnectTimeout=15
|
||||
)
|
||||
local rsync_shell
|
||||
printf -v rsync_shell 'ssh -i %q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%q -o ConnectTimeout=15' \
|
||||
"$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS"
|
||||
|
||||
CURRENT_PHASE="offhost_transfer"
|
||||
if ! rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
|
||||
"$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/" >/dev/null 2>&1; then
|
||||
fail "off-host transfer failed"
|
||||
fi
|
||||
|
||||
CURRENT_PHASE="offhost_finalize"
|
||||
if ! ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
|
||||
"hyb-backup-finalize $run_id $manifest_sha" >/dev/null 2>&1; then
|
||||
fail "off-host finalize failed"
|
||||
fi
|
||||
|
||||
CURRENT_PHASE="success_state"
|
||||
write_success_state "$run_id" "$manifest_sha"
|
||||
prune_local_backups "$batch_name"
|
||||
write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
trap - ERR
|
||||
echo "[automatic-backup][PASS] paired batch finalized off-host"
|
||||
}
|
||||
|
||||
check_stale() {
|
||||
validate_common
|
||||
trap on_error ERR
|
||||
CURRENT_PHASE="stale_check"
|
||||
resolve_active_runtime
|
||||
local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
|
||||
if ! python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
file_path, max_age, expected_project, expected_build = sys.argv[1:]
|
||||
if not os.path.isfile(file_path):
|
||||
raise SystemExit(1)
|
||||
with open(file_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("schemaVersion") != 1 or payload.get("status") != "complete":
|
||||
raise SystemExit(1)
|
||||
if payload.get("composeProject") != expected_project or payload.get("buildSha") != expected_build:
|
||||
raise SystemExit(1)
|
||||
completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00"))
|
||||
age = (datetime.now(timezone.utc) - completed).total_seconds()
|
||||
if age < 0 or age > int(max_age):
|
||||
raise SystemExit(1)
|
||||
PYTHON
|
||||
then
|
||||
fail "latest off-host success is missing, invalid, mismatched, or stale"
|
||||
fi
|
||||
write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
trap - ERR
|
||||
echo "[automatic-backup][PASS] off-host recovery point is fresh"
|
||||
}
|
||||
|
||||
remind_drill() {
|
||||
validate_common
|
||||
trap on_error ERR
|
||||
CURRENT_PHASE="quarterly_reminder"
|
||||
resolve_active_runtime
|
||||
local quarter
|
||||
quarter="$(python3 - <<'PYTHON'
|
||||
from datetime import datetime, timezone
|
||||
now = datetime.now(timezone.utc)
|
||||
print(f"{now.year}-Q{((now.month - 1) // 3) + 1}")
|
||||
PYTHON
|
||||
)"
|
||||
write_summary "PASS_WITH_FINDINGS" 1 "quarterly_shadow_restore_due_${quarter}"
|
||||
send_summary_alert "automatic-backup-drill-alert-state.json"
|
||||
trap - ERR
|
||||
echo "[automatic-backup][PASS] quarterly restore reminder evaluated"
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
run)
|
||||
run_backup
|
||||
;;
|
||||
check-stale)
|
||||
check_stale
|
||||
;;
|
||||
remind-drill)
|
||||
remind_drill
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: run-automatic-backup.sh [run|check-stale|remind-drill]"
|
||||
;;
|
||||
*)
|
||||
fail "unsupported mode"
|
||||
;;
|
||||
esac
|
||||
在新工单中引用
屏蔽一个用户