wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 5 软件包 项目 发布 百科 活动

优化(08-28-production-arm-backup-transfer-incident): 收窄生产哈希读取

浏览代码
这个提交包含在：
wangsli
2026-08-28 14:14:11 +08:00
父节点 573c46fbe7
当前提交 d6a2b85dc2
共修改 5 个文件，包含 106 行新增和 18 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
scripts/qa/inspect-backup-transfer-incident.py
+68 -16
查看文件
@@ -33,6 +33,7 @@ SIMILAR_RELATIVE_PPM = 50_000
SIMILAR_ABSOLUTE_BYTES = 64 * 1024 * 1024
SAFE_KEY_RE = re.compile(r"^[a-z0-9_]+$")
SAFE_VALUE_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9_]*|-?[0-9]+)$")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
class InspectionError(Exception):
@@ -115,7 +116,13 @@ def run_epoch(stamp: str) -> int | None:
return int(parsed.timestamp())
def validate_batch(path: Path, expected_run_id: str, stamp: str) -> BatchResult:
def validate_batch(
path: Path,
expected_run_id: str,
stamp: str,
*,
verify_hashes: bool = True,
) -> BatchResult:
total = tree_bytes(path)
received_artifact_bytes = 0
expected_artifact_bytes: int | None = None
@@ -149,10 +156,15 @@ def validate_batch(path: Path, expected_run_id: str, stamp: str) -> BatchResult:
expected_size = item.get("bytes")
if not isinstance(expected_size, int) or expected_size < 0:
return BatchResult(False, total, created_epoch, None, 0)
expected_sha = item.get("sha256")
if not isinstance(expected_sha, str) or not SHA256_RE.fullmatch(expected_sha):
return BatchResult(False, total, created_epoch, None, 0)
expected_artifact_bytes += expected_size
actual_size = target.stat().st_size
received_artifact_bytes += actual_size
if actual_size != expected_size or file_sha256(target) != item.get("sha256"):
if actual_size != expected_size or (
verify_hashes and file_sha256(target) != expected_sha
):
return BatchResult(
False,
total,
@@ -329,14 +341,19 @@ def production(args: argparse.Namespace) -> None:
backup_root = safe_root(args.backup_root)
state_dir = safe_root(args.state_dir)
now = args.now_epoch
valid: list[BatchResult] = []
metadata_valid: list[tuple[Path, str, str, BatchResult]] = []
invalid_count = invalid_bytes = temp_count = temp_bytes = other_count = 0
for entry in backup_root.iterdir():
match = BATCH_RE.fullmatch(entry.name)
if match and entry.is_dir() and not entry.is_symlink():
result = validate_batch(entry, match.group(1), match.group(2))
result = validate_batch(
entry,
match.group(1),
match.group(2),
verify_hashes=False,
)
if result.valid and result.created_epoch is not None and result.created_epoch <= now:
valid.append(result)
metadata_valid.append((entry, match.group(1), match.group(2), result))
else:
invalid_count += 1
invalid_bytes += result.total_bytes
@@ -346,27 +363,62 @@ def production(args: argparse.Namespace) -> None:
else:
other_count += 1
valid.sort(key=lambda item: item.created_epoch or -1)
latest = valid[-1] if valid else None
metadata_valid.sort(key=lambda item: item[3].created_epoch or -1)
latest_metadata = metadata_valid[-1] if metadata_valid else None
latest_verified: BatchResult | None = None
if latest_metadata is not None:
latest_verified = validate_batch(
latest_metadata[0],
latest_metadata[1],
latest_metadata[2],
verify_hashes=True,
)
complete_local_recovery_point = bool(latest_verified and latest_verified.valid)
latest_result = latest_metadata[3] if latest_metadata else None
output: list[tuple[str, object]] = [
("production_probe_result", "PASS"),
("complete_local_recovery_point", bool(valid)),
("backup_risk_classification", "production_single_point" if valid else "no_proven_recovery_point"),
("complete_local_batch_count", len(valid)),
("complete_local_batch_total_bytes", sum(item.total_bytes for item in valid)),
("complete_local_recovery_point", complete_local_recovery_point),
(
"latest_complete_local_batch_age_seconds",
now - latest.created_epoch
if latest and latest.created_epoch is not None
"backup_risk_classification",
"production_single_point"
if complete_local_recovery_point
else "no_proven_recovery_point",
),
("fully_verified_local_batch_count", 1 if complete_local_recovery_point else 0),
("metadata_complete_local_batch_count", len(metadata_valid)),
(
"metadata_complete_local_batch_total_bytes",
sum(item[3].total_bytes for item in metadata_valid),
),
(
"older_metadata_batch_hash_status",
"NOT_RUN" if len(metadata_valid) > 1 else "NOT_APPLICABLE",
),
(
"latest_local_candidate_hash_status",
"PASS"
if complete_local_recovery_point
else ("FAIL" if latest_metadata is not None else "NOT_RUN"),
),
(
"latest_local_candidate_age_seconds",
now - latest_result.created_epoch
if latest_result and latest_result.created_epoch is not None
else "NOT_AVAILABLE",
),
("latest_complete_local_batch_bytes", latest.total_bytes if latest else "NOT_AVAILABLE"),
(
"latest_local_candidate_bytes",
latest_result.total_bytes if latest_result else "NOT_AVAILABLE",
),
("invalid_local_batch_count", invalid_count),
("invalid_local_batch_total_bytes", invalid_bytes),
("generation_temp_count", temp_count),
("generation_temp_total_bytes", temp_bytes),
("unclassified_local_entry_count", other_count),
("transition_recovery_point_action", "wait_natural" if valid else "request_manual_run"),
(
"transition_recovery_point_action",
"wait_natural" if complete_local_recovery_point else "request_manual_run",
),
]
output.extend(filesystem_fields("production_backup", backup_root))
output.extend(source_contract(args.wrapper_script, args.backup_script))
scripts/qa/run-backup-transfer-incident-contract.sh
+26
查看文件
@@ -95,6 +95,32 @@ assert_line "$TEST_ROOT/production-valid.out" "transfer_failure_category=partial
assert_line "$TEST_ROOT/production-valid.out" "transfer_duration_seconds=NOT_AVAILABLE"
pass "生产存在完整本地恢复点时识别为单点风险并保留未测时长"
PRODUCTION_MULTIPLE="$TEST_ROOT/production-multiple"
make_production_layout "$PRODUCTION_MULTIPLE"
make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf 'X' | dd of="$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none
run_production "$PRODUCTION_MULTIPLE" "$TEST_ROOT/production-multiple.out"
assert_line "$TEST_ROOT/production-multiple.out" "complete_local_recovery_point=yes"
assert_line "$TEST_ROOT/production-multiple.out" "fully_verified_local_batch_count=1"
assert_line "$TEST_ROOT/production-multiple.out" "metadata_complete_local_batch_count=2"
assert_line "$TEST_ROOT/production-multiple.out" "older_metadata_batch_hash_status=NOT_RUN"
assert_line "$TEST_ROOT/production-multiple.out" "latest_local_candidate_hash_status=PASS"
pass "生产多个批次时只对最新元数据合格候选执行全量 SHA-256"
PRODUCTION_LATEST_HASH_FAIL="$TEST_ROOT/production-latest-hash-fail"
make_production_layout "$PRODUCTION_LATEST_HASH_FAIL"
make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf 'X' | dd of="$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none
run_production "$PRODUCTION_LATEST_HASH_FAIL" "$TEST_ROOT/production-latest-hash-fail.out"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "complete_local_recovery_point=no"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "fully_verified_local_batch_count=0"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "metadata_complete_local_batch_count=2"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "older_metadata_batch_hash_status=NOT_RUN"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "latest_local_candidate_hash_status=FAIL"
pass "最新候选哈希失败时不回退扫描旧批次并保持无已证明恢复点"
PRODUCTION_EMPTY="$TEST_ROOT/production-empty"
make_production_layout "$PRODUCTION_EMPTY"
run_production "$PRODUCTION_EMPTY" "$TEST_ROOT/production-empty.out"