完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）
这个提交包含在：
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate the newest ARM archive and drive an independent dead-man alert."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BATCH_RE = re.compile(r"^hyb-oa-backup-(auto-(\d{8}T\d{6}Z))$")
|
||||
|
||||
|
||||
def fail(message: str) -> "NoReturn":
|
||||
print(f"[offhost-deadman][ERROR] {message}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def validate_latest(archive_root: Path, max_age: int) -> tuple[str, str]:
|
||||
if not archive_root.is_dir() or archive_root.is_symlink():
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_root_invalid"
|
||||
candidates = sorted(
|
||||
(entry for entry in archive_root.iterdir() if entry.is_dir() and not entry.is_symlink() and BATCH_RE.fullmatch(entry.name)),
|
||||
key=lambda entry: entry.name,
|
||||
)
|
||||
if not candidates:
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_missing"
|
||||
batch = candidates[-1]
|
||||
match = BATCH_RE.fullmatch(batch.name)
|
||||
assert match is not None
|
||||
run_id, stamp = match.groups()
|
||||
expected_names = {"database.dump", "storage.tar.gz", "manifest.json"}
|
||||
entries = list(batch.iterdir())
|
||||
if {entry.name for entry in entries} != expected_names or any(
|
||||
entry.is_symlink() or not stat.S_ISREG(entry.stat().st_mode) for entry in entries
|
||||
):
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_files_invalid"
|
||||
try:
|
||||
manifest = json.loads((batch / "manifest.json").read_text(encoding="utf-8"))
|
||||
if manifest.get("schemaVersion") != 1 or manifest.get("status") != "complete" or manifest.get("runId") != run_id:
|
||||
raise ValueError
|
||||
artifacts = manifest.get("artifacts") or {}
|
||||
for key, name in (("database", "database.dump"), ("storage", "storage.tar.gz")):
|
||||
item = artifacts.get(key) or {}
|
||||
target = batch / name
|
||||
if item.get("file") != name or item.get("bytes") != target.stat().st_size or item.get("sha256") != sha256(target):
|
||||
raise ValueError
|
||||
completed = datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
|
||||
except (OSError, ValueError, TypeError, json.JSONDecodeError):
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_manifest_invalid"
|
||||
age = (datetime.now(timezone.utc) - completed).total_seconds()
|
||||
if age < 0:
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_time_in_future"
|
||||
if age > max_age:
|
||||
return "FAIL_OFFHOST_DEADMAN", "archive_stale"
|
||||
return "PASS_OFFHOST_DEADMAN_FRESH", "fresh_offhost_archive"
|
||||
|
||||
|
||||
def write_summary(path: Path, status: str, category: str) -> None:
|
||||
finding = status.startswith("FAIL_")
|
||||
payload = {
|
||||
"issueId": "AUTO_BACKUP",
|
||||
"status": status,
|
||||
"conclusion": status,
|
||||
"findingCount": 1 if finding else 0,
|
||||
"findings": ([{"source": "offhost_deadman", "status": status, "severity": "P0", "detail": category}] if finding else []),
|
||||
"privacy": "status/count/category only; no credentials, host addresses, absolute paths, file names, or payloads",
|
||||
}
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
fd, temporary = tempfile.mkstemp(prefix=".deadman-summary.", dir=path.parent)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
os.chmod(temporary, 0o600)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--archive-root", required=True, type=Path)
|
||||
parser.add_argument("--state-dir", required=True, type=Path)
|
||||
parser.add_argument("--webhook-file", required=True, type=Path)
|
||||
parser.add_argument("--alert-helper", required=True, type=Path)
|
||||
parser.add_argument("--max-age-seconds", type=int, default=93600)
|
||||
args = parser.parse_args()
|
||||
if args.max_age_seconds <= 0:
|
||||
fail("max age must be positive")
|
||||
try:
|
||||
args.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if args.state_dir.is_symlink() or not args.state_dir.is_dir():
|
||||
fail("state directory is invalid")
|
||||
os.chmod(args.state_dir, 0o700)
|
||||
status, category = validate_latest(args.archive_root, args.max_age_seconds)
|
||||
summary = args.state_dir / "last-deadman.json"
|
||||
write_summary(summary, status, category)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(args.alert_helper), "--webhook-file", str(args.webhook_file), "--summary-file", str(summary), "--state-file", str(args.state_dir / "deadman-alert-state.json")],
|
||||
check=False,
|
||||
)
|
||||
except OSError:
|
||||
fail("dead-man local file operation failed")
|
||||
if result.returncode != 0:
|
||||
fail("alert evaluation failed")
|
||||
if status.startswith("FAIL_"):
|
||||
fail(category)
|
||||
print("[offhost-deadman][PASS] newest archive is fresh and valid")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -78,7 +78,7 @@ receiver() {
|
||||
"$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
|
||||
}
|
||||
|
||||
CURRENT_RUN_ID="auto-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
CURRENT_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
|
||||
CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"
|
||||
|
||||
VALID_LAYOUT="$TEST_DIR/receiver-valid"
|
||||
@@ -92,15 +92,41 @@ pass "receiver finalizes one complete synthetic batch"
|
||||
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
|
||||
pass "receiver treats an identical repeated finalize as idempotent"
|
||||
|
||||
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
|
||||
if [[ "$SECOND_RUN_ID" == "$CURRENT_RUN_ID" ]]; then
|
||||
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
|
||||
fi
|
||||
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
|
||||
SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
|
||||
make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
||||
SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver rejects a different automatic batch for the same UTC day" \
|
||||
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA"
|
||||
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA" >/dev/null
|
||||
[[ ! -e "$VALID_LAYOUT/archive/$CURRENT_BATCH" && -d "$VALID_LAYOUT/archive/$SECOND_BATCH" ]]
|
||||
WEEKLY_BATCH="$(find "$VALID_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
|
||||
[[ -n "$WEEKLY_BATCH" ]]
|
||||
[[ "$(stat -c %i "$WEEKLY_BATCH/database.dump")" == "$(stat -c %i "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump")" ]]
|
||||
pass "receiver atomically replaces a different batch for the same UTC day and rebuilds its weekly checkpoint"
|
||||
|
||||
THIRD_RUN_ID="auto-$(date -u +%Y%m%d)T000002Z"
|
||||
THIRD_BATCH="hyb-oa-backup-$THIRD_RUN_ID"
|
||||
make_batch "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
||||
printf 'corrupt candidate\n' >>"$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/database.dump"
|
||||
THIRD_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver validates a same-day replacement before touching the current archive" \
|
||||
receiver "$VALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$THIRD_MANIFEST_SHA"
|
||||
[[ -d "$VALID_LAYOUT/archive/$SECOND_BATCH" && ! -e "$VALID_LAYOUT/archive/$THIRD_BATCH" ]]
|
||||
pass "receiver preserves the current daily archive when a replacement candidate is corrupt"
|
||||
|
||||
ROLLBACK_LAYOUT="$TEST_DIR/receiver-replacement-rollback"
|
||||
make_layout "$ROLLBACK_LAYOUT"
|
||||
make_batch "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
||||
ROLLBACK_FIRST_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
receiver "$ROLLBACK_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$ROLLBACK_FIRST_SHA" >/dev/null
|
||||
ROLLBACK_WEEKLY="$(find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
|
||||
find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -exec rm -rf -- {} +
|
||||
printf 'invalid weekly target\n' >"$ROLLBACK_WEEKLY"
|
||||
make_batch "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
||||
ROLLBACK_SECOND_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver rolls back the daily archive when weekly checkpoint publication fails" \
|
||||
receiver "$ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$ROLLBACK_SECOND_SHA"
|
||||
[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
|
||||
pass "receiver restores the prior daily point after a later replacement step fails"
|
||||
|
||||
negative_receiver_case() {
|
||||
local label="$1"
|
||||
@@ -225,7 +251,7 @@ if [[ "${1:-}" == "compose" ]]; then
|
||||
args=" $* "
|
||||
if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
|
||||
if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
|
||||
if [[ "$args" == *" exec -T server "* ]]; then cat >>"$MOCK_ALERT_LOG"; exit 0; fi
|
||||
if [[ "$args" == *" exec -T server "* ]]; then exit 99; fi
|
||||
fi
|
||||
if [[ "${1:-}" == "inspect" ]]; then
|
||||
template="${3:-}"
|
||||
@@ -260,6 +286,7 @@ MOCK
|
||||
|
||||
cat >"$MOCK_BIN/rsync" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
echo 'synthetic transfer diagnostic' >&2
|
||||
exit "${MOCK_RSYNC_EXIT:-1}"
|
||||
MOCK
|
||||
cat >"$MOCK_BIN/ssh" <<'MOCK'
|
||||
@@ -267,6 +294,34 @@ cat >"$MOCK_BIN/ssh" <<'MOCK'
|
||||
exit "${MOCK_SSH_EXIT:-0}"
|
||||
MOCK
|
||||
chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
|
||||
WECOM_TEST_HOST='qyapi.weixin.qq.com'
|
||||
WECOM_TEST_PATH='/cgi-bin/webhook/send'
|
||||
printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
|
||||
'00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook"
|
||||
chmod 0600 "$TEST_DIR/wecom-webhook"
|
||||
ALERT_CAPTURE="$TEST_DIR/alert-capture.jsonl"
|
||||
cat >"$TEST_DIR/failure-summary.json" <<'JSON'
|
||||
{"issueId":"AUTO_BACKUP","status":"FAIL_AUTOMATIC_BACKUP","conclusion":"FAIL_AUTOMATIC_BACKUP","findingCount":1,"findings":[{"source":"automatic_backup","status":"FAIL_AUTOMATIC_BACKUP","severity":"P0","detail":"synthetic_failure"}]}
|
||||
JSON
|
||||
chmod 0600 "$TEST_DIR/failure-summary.json"
|
||||
chmod 0644 "$TEST_DIR/wecom-webhook"
|
||||
expect_failure "group alert helper rejects a webhook file that is not mode 0600" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
||||
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook" \
|
||||
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
||||
chmod 0600 "$TEST_DIR/wecom-webhook"
|
||||
ln -s "$TEST_DIR/wecom-webhook" "$TEST_DIR/wecom-webhook-link"
|
||||
expect_failure "group alert helper rejects a symbolic-link webhook file" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
||||
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-link" \
|
||||
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
||||
printf 'https://%s%s?key=%s&extra=forbidden\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
|
||||
'00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook-invalid"
|
||||
chmod 0600 "$TEST_DIR/wecom-webhook-invalid"
|
||||
expect_failure "group alert helper rejects a webhook URL with an unexpected query field" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
||||
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-invalid" \
|
||||
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
||||
|
||||
wrapper_env=(
|
||||
env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
|
||||
@@ -275,7 +330,9 @@ wrapper_env=(
|
||||
STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
|
||||
AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
|
||||
OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
|
||||
OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_RECIPIENT_EMPLOYEE_NO="OPS001"
|
||||
OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_WEBHOOK_FILE="$TEST_DIR/wecom-webhook"
|
||||
ALERT_HELPER="$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE"
|
||||
)
|
||||
|
||||
exec 8>"$WRAPPER_STATE/automatic-backup.lock"
|
||||
@@ -305,6 +362,9 @@ assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
|
||||
assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
|
||||
PYTHON
|
||||
pass "wrapper failure summary is category-only and redacted"
|
||||
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
|
||||
grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
|
||||
pass "wrapper retains the latest command diagnostic locally with mode 0600"
|
||||
|
||||
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
@@ -315,6 +375,8 @@ done
|
||||
[[ -f "$WRAPPER_STATE/last-success.json" ]]
|
||||
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
|
||||
pass "off-host finalize success writes state and then prunes source retention to seven"
|
||||
[[ ! -e "$WRAPPER_STATE/last-error.log" ]]
|
||||
pass "wrapper clears the retained diagnostic after a successful off-host run"
|
||||
|
||||
rm -f "$WRAPPER_STATE/last-success.json"
|
||||
expect_failure "stale watchdog rejects a missing success record" \
|
||||
@@ -329,7 +391,7 @@ with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
"schemaVersion": 1,
|
||||
"status": "complete",
|
||||
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
"runId": "auto-synthetic",
|
||||
"runId": "auto-20260727T000000Z",
|
||||
"manifestSha256": "0" * 64,
|
||||
"composeProject": "hyb-oa-current",
|
||||
"buildSha": sys.argv[2],
|
||||
@@ -345,12 +407,89 @@ payload["buildSha"] = "0" * 40
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
PYTHON
|
||||
expect_failure "stale watchdog rejects a success record from another build" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
|
||||
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
assert payload["status"] == "PASS_WITH_FINDINGS"
|
||||
assert payload["findings"][0]["detail"] == "recovery_point_build_differs"
|
||||
PYTHON
|
||||
pass "stale watchdog downgrades a fresh recovery point from another build to one finding"
|
||||
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
|
||||
pass "stale watchdog accepts a fresh project/build-bound success record"
|
||||
|
||||
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
||||
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
payload["composeProject"] = "different-project"
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
PYTHON
|
||||
expect_failure "stale watchdog hard-fails a fresh record from another Compose project" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
||||
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
||||
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
payload["manifestSha256"] = "invalid"
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
PYTHON
|
||||
expect_failure "stale watchdog hard-fails a malformed success record" \
|
||||
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
||||
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
||||
|
||||
DEADMAN_LAYOUT="$TEST_DIR/deadman"
|
||||
mkdir -p "$DEADMAN_LAYOUT/archive" "$DEADMAN_LAYOUT/state"
|
||||
cp -a "$VALID_LAYOUT/archive/$SECOND_BATCH" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
|
||||
DEADMAN_CAPTURE="$TEST_DIR/deadman-alerts.jsonl"
|
||||
deadman_env=(env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CAPTURE")
|
||||
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
||||
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
|
||||
[[ ! -e "$DEADMAN_CAPTURE" ]]
|
||||
pass "ARM dead-man accepts a fresh verified archive without sending a healthy baseline"
|
||||
DEADMAN_CORRUPT_STATE="$TEST_DIR/deadman-corrupt-state"
|
||||
DEADMAN_CORRUPT_CAPTURE="$TEST_DIR/deadman-corrupt-alerts.jsonl"
|
||||
mkdir "$DEADMAN_CORRUPT_STATE"
|
||||
chmod 0600 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
||||
printf 'corruption\n' >>"$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
||||
expect_failure "ARM dead-man rejects a corrupt newest finalized archive" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CORRUPT_CAPTURE" \
|
||||
python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
||||
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_CORRUPT_STATE" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
||||
cp "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
||||
chmod 0400 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
||||
pass "ARM dead-man detects archive corruption independently of source state"
|
||||
mv "$DEADMAN_LAYOUT/archive/$SECOND_BATCH" "$TEST_DIR/deadman-withheld-batch"
|
||||
expect_failure "ARM dead-man detects that the source no longer produces a finalized archive" \
|
||||
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
||||
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
||||
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
|
||||
expect_failure "ARM dead-man suppresses an identical repeated failure alert" \
|
||||
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
||||
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
||||
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
|
||||
pass "ARM dead-man keeps repeated identical failure alerts deduplicated"
|
||||
mv "$TEST_DIR/deadman-withheld-batch" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
|
||||
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
||||
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
|
||||
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "2" ]]
|
||||
pass "ARM dead-man sends one recovery notification after finalized archives resume"
|
||||
|
||||
DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
|
||||
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
|
||||
"${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \
|
||||
|
||||
@@ -60,6 +60,7 @@ validate_layout() {
|
||||
[[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "DAILY_KEEP must be positive"
|
||||
[[ "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "WEEKLY_KEEP must be positive"
|
||||
command -v "$PYTHON_BIN" >/dev/null 2>&1 || fail "python3 is required"
|
||||
command -v flock >/dev/null 2>&1 || fail "flock is required"
|
||||
}
|
||||
|
||||
validate_run_id() {
|
||||
@@ -155,11 +156,44 @@ prune_named_directories() {
|
||||
done
|
||||
}
|
||||
|
||||
publish_weekly_checkpoint() {
|
||||
local archive_dir="$1"
|
||||
local weekly_dir="$2"
|
||||
local replace_existing="$3"
|
||||
if [[ -e "$weekly_dir" && "$replace_existing" != "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
local weekly_temp="${weekly_dir}.new.$$"
|
||||
local weekly_rollback="${weekly_dir}.rollback.$$"
|
||||
[[ ! -e "$weekly_temp" && ! -e "$weekly_rollback" ]] || return 1
|
||||
if ! cp -al -- "$archive_dir" "$weekly_temp"; then
|
||||
rm -rf -- "$weekly_temp"
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$weekly_dir" ]]; then
|
||||
[[ -d "$weekly_dir" && ! -L "$weekly_dir" ]] || return 1
|
||||
mv -- "$weekly_dir" "$weekly_rollback" || return 1
|
||||
fi
|
||||
if ! mv -- "$weekly_temp" "$weekly_dir"; then
|
||||
rm -rf -- "$weekly_temp"
|
||||
if [[ -e "$weekly_rollback" ]]; then
|
||||
mv -- "$weekly_rollback" "$weekly_dir"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
if [[ -e "$weekly_rollback" ]]; then
|
||||
rm -rf -- "$weekly_rollback"
|
||||
fi
|
||||
}
|
||||
|
||||
finalize_batch() {
|
||||
local run_id="$1"
|
||||
local expected_manifest_sha="$2"
|
||||
validate_run_id "$run_id"
|
||||
validate_layout
|
||||
exec 9>"$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
||||
chmod 0600 "$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
||||
flock -n 9 || fail "another finalize operation is already running"
|
||||
|
||||
local batch_name="hyb-oa-backup-$run_id"
|
||||
local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"
|
||||
@@ -173,15 +207,32 @@ finalize_batch() {
|
||||
|
||||
local run_day="${run_id#auto-}"
|
||||
run_day="${run_day%%T*}"
|
||||
local existing_for_day
|
||||
existing_for_day="$(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
|
||||
-name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print -quit)"
|
||||
[[ -z "$existing_for_day" ]] || fail "another automatic archive already exists for this UTC day"
|
||||
local existing_for_day="" replacement_rollback=""
|
||||
local -a existing_for_day_entries=()
|
||||
mapfile -t existing_for_day_entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
|
||||
-name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print)
|
||||
[[ "${#existing_for_day_entries[@]}" -le 1 ]] || fail "daily archive invariant is already violated"
|
||||
if [[ "${#existing_for_day_entries[@]}" -eq 1 ]]; then
|
||||
existing_for_day="${existing_for_day_entries[0]}"
|
||||
fi
|
||||
|
||||
if [[ "$already_archived" == "0" ]]; then
|
||||
validate_batch "$partial_dir" "$run_id" "$expected_manifest_sha"
|
||||
chmod 0400 "$partial_dir/database.dump" "$partial_dir/storage.tar.gz" "$partial_dir/manifest.json"
|
||||
mv -- "$partial_dir" "$archive_dir"
|
||||
if [[ -n "$existing_for_day" ]]; then
|
||||
[[ "$existing_for_day" =~ /hyb-oa-backup-auto-${run_day}T[0-9]{6}Z$ \
|
||||
&& -d "$existing_for_day" && ! -L "$existing_for_day" ]] \
|
||||
|| fail "existing daily archive is invalid"
|
||||
replacement_rollback="$BACKUP_ARCHIVE_DIR/.daily-replacement-${run_day}.$$"
|
||||
[[ ! -e "$replacement_rollback" ]] || fail "daily replacement rollback target already exists"
|
||||
mv -- "$existing_for_day" "$replacement_rollback"
|
||||
fi
|
||||
if ! mv -- "$partial_dir" "$archive_dir"; then
|
||||
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
||||
mv -- "$replacement_rollback" "$existing_for_day"
|
||||
fi
|
||||
fail "daily archive publication failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
local iso_week
|
||||
@@ -192,10 +243,17 @@ day = datetime.strptime(sys.argv[1], "%Y%m%d").date()
|
||||
iso = day.isocalendar()
|
||||
print(f"week-{iso.year:04d}-W{iso.week:02d}")
|
||||
PYTHON
|
||||
)"
|
||||
)"
|
||||
local weekly_dir="$BACKUP_WEEKLY_DIR/$iso_week"
|
||||
if [[ ! -e "$weekly_dir" ]]; then
|
||||
cp -al -- "$archive_dir" "$weekly_dir"
|
||||
if ! publish_weekly_checkpoint "$archive_dir" "$weekly_dir" "$([[ -n "$replacement_rollback" ]] && echo 1 || echo 0)"; then
|
||||
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
||||
rm -rf -- "$archive_dir"
|
||||
mv -- "$replacement_rollback" "$existing_for_day"
|
||||
fi
|
||||
fail "weekly checkpoint update failed"
|
||||
fi
|
||||
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
||||
rm -rf -- "$replacement_rollback"
|
||||
fi
|
||||
|
||||
prune_named_directories "$BACKUP_ARCHIVE_DIR" "hyb-oa-backup-auto-" "$DAILY_KEEP"
|
||||
|
||||
+55
-16
@@ -15,9 +15,10 @@ AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
|
||||
OFFHOST_TARGET="${OFFHOST_TARGET:-}"
|
||||
OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
|
||||
OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
|
||||
ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
|
||||
ALERT_HELPER="${ALERT_HELPER:-$ROOT_DIR/scripts/send-wecom-group-alert.py}"
|
||||
AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}"
|
||||
AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}"
|
||||
ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}"
|
||||
MODE="${1:-run}"
|
||||
|
||||
CURRENT_PHASE="preflight"
|
||||
@@ -79,7 +80,12 @@ validate_common() {
|
||||
|| fail "frontend build SHA must match the full build prefix"
|
||||
[[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive"
|
||||
[[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive"
|
||||
[[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid"
|
||||
validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE"
|
||||
validate_absolute_path ALERT_HELPER "$ALERT_HELPER"
|
||||
[[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
|
||||
&& "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" ]] \
|
||||
|| fail "alert webhook file must exist with mode 0600"
|
||||
[[ -f "$ALERT_HELPER" && ! -L "$ALERT_HELPER" ]] || fail "alert helper is missing"
|
||||
if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \
|
||||
|| "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \
|
||||
|| "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then
|
||||
@@ -218,10 +224,25 @@ PYTHON
|
||||
send_summary_alert() {
|
||||
local state_name="$1"
|
||||
[[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1
|
||||
[[ -n "$SERVER_CONTAINER" ]] || resolve_active_runtime
|
||||
compose exec -T server sh -lc \
|
||||
"cd /app/server && WARCH3_OPS_ALERT_SEND=1 npm run send:wecom-ops-alert -- --summary-file=- --recipient-employee-no=${ALERT_RECIPIENT_EMPLOYEE_NO} --send --state-file=/app/artifacts/qa-flow/${state_name} --format=json" \
|
||||
<"$SUMMARY_FILE" >/dev/null
|
||||
python3 "$ALERT_HELPER" \
|
||||
--webhook-file "$ALERT_WEBHOOK_FILE" \
|
||||
--summary-file "$SUMMARY_FILE" \
|
||||
--state-file "$AUTOMATIC_BACKUP_STATE_DIR/$state_name" >/dev/null
|
||||
}
|
||||
|
||||
capture_command() {
|
||||
local phase="$1"
|
||||
shift
|
||||
local temporary="$AUTOMATIC_BACKUP_STATE_DIR/.${phase}.diagnostic.$$"
|
||||
: >"$temporary"
|
||||
chmod 0600 "$temporary"
|
||||
if "$@" >"$temporary" 2>&1; then
|
||||
rm -f -- "$temporary"
|
||||
return 0
|
||||
fi
|
||||
mv -f -- "$temporary" "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
||||
chmod 0600 "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
||||
return 1
|
||||
}
|
||||
|
||||
on_error() {
|
||||
@@ -320,11 +341,12 @@ print(f"{unquote(value.username)}\t{database_name}")
|
||||
[[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid"
|
||||
|
||||
CURRENT_PHASE="paired_backup"
|
||||
if ! DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
|
||||
if ! capture_command paired_backup env \
|
||||
DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
|
||||
POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" POSTGRES_PASSWORD="$postgres_password" \
|
||||
STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" \
|
||||
BACKUP_RUN_ID="$run_id" BUILD_SHA="$BUILD_SHA" SOURCE_IMAGE_DIGEST="$source_image_digest" \
|
||||
"$ROOT_DIR/scripts/backup-all.sh" >/dev/null 2>&1; then
|
||||
"$ROOT_DIR/scripts/backup-all.sh"; then
|
||||
fail "paired backup command failed"
|
||||
fi
|
||||
[[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published"
|
||||
@@ -343,14 +365,14 @@ print(f"{unquote(value.username)}\t{database_name}")
|
||||
"$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS"
|
||||
|
||||
CURRENT_PHASE="offhost_transfer"
|
||||
if ! rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
|
||||
"$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/" >/dev/null 2>&1; then
|
||||
if ! capture_command offhost_transfer rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
|
||||
"$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/"; then
|
||||
fail "off-host transfer failed"
|
||||
fi
|
||||
|
||||
CURRENT_PHASE="offhost_finalize"
|
||||
if ! ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
|
||||
"hyb-backup-finalize $run_id $manifest_sha" >/dev/null 2>&1; then
|
||||
if ! capture_command offhost_finalize ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
|
||||
"hyb-backup-finalize $run_id $manifest_sha"; then
|
||||
fail "off-host finalize failed"
|
||||
fi
|
||||
|
||||
@@ -359,6 +381,7 @@ print(f"{unquote(value.username)}\t{database_name}")
|
||||
prune_local_backups "$batch_name"
|
||||
write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
rm -f -- "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
||||
trap - ERR
|
||||
echo "[automatic-backup][PASS] paired batch finalized off-host"
|
||||
}
|
||||
@@ -369,9 +392,11 @@ check_stale() {
|
||||
CURRENT_PHASE="stale_check"
|
||||
resolve_active_runtime
|
||||
local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
|
||||
if ! python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
||||
local stale_result
|
||||
if ! stale_result="$(python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -382,15 +407,29 @@ with open(file_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if payload.get("schemaVersion") != 1 or payload.get("status") != "complete":
|
||||
raise SystemExit(1)
|
||||
if payload.get("composeProject") != expected_project or payload.get("buildSha") != expected_build:
|
||||
if payload.get("composeProject") != expected_project:
|
||||
raise SystemExit(1)
|
||||
if not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", payload.get("runId", "")):
|
||||
raise SystemExit(1)
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", payload.get("manifestSha256", "")):
|
||||
raise SystemExit(1)
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{40,64}", payload.get("buildSha", "")):
|
||||
raise SystemExit(1)
|
||||
completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00"))
|
||||
age = (datetime.now(timezone.utc) - completed).total_seconds()
|
||||
if age < 0 or age > int(max_age):
|
||||
raise SystemExit(1)
|
||||
print("BUILD_MISMATCH" if payload.get("buildSha") != expected_build else "FRESH")
|
||||
PYTHON
|
||||
then
|
||||
fail "latest off-host success is missing, invalid, mismatched, or stale"
|
||||
)"; then
|
||||
fail "latest off-host success is missing, invalid, project-mismatched, or stale"
|
||||
fi
|
||||
if [[ "$stale_result" == "BUILD_MISMATCH" ]]; then
|
||||
write_summary "PASS_WITH_FINDINGS" 1 "recovery_point_build_differs"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
trap - ERR
|
||||
echo "[automatic-backup][PASS_WITH_FINDINGS] off-host recovery point is fresh but build differs"
|
||||
return 0
|
||||
fi
|
||||
write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point"
|
||||
send_summary_alert "automatic-backup-alert-state.json" || true
|
||||
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send a redacted, deduplicated OA operations alert to a WeCom group robot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SAFE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{1,160}$")
|
||||
ALLOWED_STATUSES = {
|
||||
"FAIL_AUTOMATIC_BACKUP",
|
||||
"FAIL_OFFHOST_DEADMAN",
|
||||
"PASS_AUTOMATIC_BACKUP_COMPLETE",
|
||||
"PASS_AUTOMATIC_BACKUP_FRESH",
|
||||
"PASS_OFFHOST_DEADMAN_FRESH",
|
||||
"PASS_WITH_FINDINGS",
|
||||
}
|
||||
|
||||
|
||||
def fail(message: str) -> "NoReturn":
|
||||
print(f"[wecom-group-alert][ERROR] {message}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def validate_private_file(path: Path, label: str) -> None:
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
fail(f"{label} is missing")
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
fail(f"{label} must be a regular non-symbolic file")
|
||||
if stat.S_IMODE(info.st_mode) != 0o600:
|
||||
fail(f"{label} mode must be 0600")
|
||||
if info.st_uid != os.geteuid():
|
||||
fail(f"{label} must be owned by the executing account")
|
||||
|
||||
|
||||
def read_webhook(path: Path) -> str:
|
||||
validate_private_file(path, "webhook file")
|
||||
try:
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
except OSError:
|
||||
fail("webhook file cannot be read")
|
||||
if "\n" in value or "\r" in value:
|
||||
fail("webhook file must contain one line")
|
||||
parsed = urllib.parse.urlparse(value)
|
||||
query = urllib.parse.parse_qs(parsed.query, strict_parsing=True)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != "qyapi.weixin.qq.com"
|
||||
or parsed.netloc != "qyapi.weixin.qq.com"
|
||||
or parsed.path != "/cgi-bin/webhook/send"
|
||||
or set(query) != {"key"}
|
||||
or len(query["key"]) != 1
|
||||
or not re.fullmatch(r"[A-Za-z0-9-]{16,128}", query["key"][0])
|
||||
or parsed.fragment
|
||||
):
|
||||
fail("webhook file format is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def load_summary(path: Path) -> tuple[str, str, int]:
|
||||
validate_private_file(path, "summary file")
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
fail("summary file is invalid")
|
||||
status = payload.get("status")
|
||||
findings = payload.get("findings") or []
|
||||
finding_count = payload.get("findingCount")
|
||||
if status not in ALLOWED_STATUSES or not isinstance(finding_count, int):
|
||||
fail("summary status is invalid")
|
||||
if finding_count != len(findings) or finding_count not in {0, 1}:
|
||||
fail("summary finding count is invalid")
|
||||
category = "healthy"
|
||||
if findings:
|
||||
category = findings[0].get("detail", "")
|
||||
if not isinstance(category, str) or not SAFE_TOKEN.fullmatch(category):
|
||||
fail("summary category is invalid")
|
||||
return status, category, finding_count
|
||||
|
||||
|
||||
def load_previous(path: Path) -> dict[str, object]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
validate_private_file(path, "alert state file")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
fail("alert state file is invalid")
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def atomic_write(path: Path, payload: dict[str, object]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(path.parent, 0o700)
|
||||
fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
os.replace(temp_name, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temp_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def should_send(previous: dict[str, object], status: str, category: str) -> bool:
|
||||
previous_status = previous.get("status")
|
||||
previous_category = previous.get("category")
|
||||
if status.startswith("FAIL_") or status == "PASS_WITH_FINDINGS":
|
||||
return previous_status != status or previous_category != category
|
||||
return isinstance(previous_status, str) and (
|
||||
previous_status.startswith("FAIL_") or previous_status == "PASS_WITH_FINDINGS"
|
||||
)
|
||||
|
||||
|
||||
def safe_text(status: str, category: str) -> str:
|
||||
if status.startswith("FAIL_"):
|
||||
headline = "内部 OA 运维告警"
|
||||
elif status == "PASS_WITH_FINDINGS":
|
||||
headline = "内部 OA 运维提醒"
|
||||
else:
|
||||
headline = "内部 OA 运维恢复通知"
|
||||
return f"{headline}\n状态：{status}\n类别：{category}\n请按恢复卡检查，不要在群内粘贴原始日志。"
|
||||
|
||||
|
||||
def send(webhook: str, text: str) -> None:
|
||||
body = json.dumps({"msgtype": "text", "text": {"content": text}}).encode("utf-8")
|
||||
capture = os.environ.get("HYB_WECOM_ALERT_TEST_CAPTURE")
|
||||
if capture:
|
||||
if os.environ.get("HYB_WECOM_ALERT_TEST_MODE") != "1":
|
||||
fail("test capture is disabled")
|
||||
capture_path = Path(capture)
|
||||
with capture_path.open("ab") as handle:
|
||||
handle.write(body + b"\n")
|
||||
os.chmod(capture_path, 0o600)
|
||||
return
|
||||
request = urllib.request.Request(
|
||||
webhook,
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
result = json.loads(response.read().decode("utf-8"))
|
||||
except Exception:
|
||||
fail("group robot request failed")
|
||||
if result.get("errcode") != 0:
|
||||
fail("group robot rejected the alert")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--webhook-file", required=True, type=Path)
|
||||
parser.add_argument("--summary-file", required=True, type=Path)
|
||||
parser.add_argument("--state-file", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
status, category, _ = load_summary(args.summary_file)
|
||||
previous = load_previous(args.state_file)
|
||||
send_now = should_send(previous, status, category)
|
||||
if send_now:
|
||||
webhook = read_webhook(args.webhook_file)
|
||||
send(webhook, safe_text(status, category))
|
||||
atomic_write(args.state_file, {"schemaVersion": 1, "status": status, "category": category})
|
||||
except OSError:
|
||||
fail("local alert file operation failed")
|
||||
print(f"[wecom-group-alert][PASS] evaluated sent={str(send_now).lower()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
在新工单中引用
屏蔽一个用户