wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 5 软件包 项目 发布 百科 活动

增强(08-23-isolated-backup-restore-drill): 固化八阶段RTO合同

浏览代码
这个提交包含在：
wangsli
2026-08-25 20:57:15 +08:00
父节点 e0e2465e0c
当前提交 0b94b97a3a
共修改 5 个文件，包含 351 行新增和 18 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
scripts/qa/offhost-shadow-restore-runner.template.sh
+129 -5
查看文件
@@ -29,7 +29,18 @@ CONTAINER_NAME=""
RUNTIME_CLEANED=0
RESULT_STATUS="NO_GO"
RPO_SECONDS=""
RTO_SECONDS=""
RTO_SECONDS="NOT_RUN"
RTO_BATCH_COPY_SECONDS="NOT_RUN"
RTO_POSTGRES_CONTAINER_START_SECONDS="NOT_RUN"
RTO_DUMP_LIST_SECONDS="NOT_RUN"
RTO_TARGET_CONFIRMATION_SECONDS="NOT_RUN"
RTO_RESTORE_APPLY_SECONDS="NOT_RUN"
RTO_DATABASE_VERIFY_SECONDS="NOT_RUN"
RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS="NOT_RUN"
RTO_FINAL_INTEGRITY_SECONDS="NOT_RUN"
RTO_ACTIVE_STAGE=""
RTO_STAGE_STARTED_EPOCH=""
RTO_FINALIZED=0
MIGRATION_SUCCESS=""
MIGRATION_OTHER=""
MIGRATION_SET_MATCH=""
@@ -66,6 +77,94 @@ require_command() {
command -v "$1" >/dev/null 2>&1 || fail "required_command_missing"
}
rto_now() {
date -u +%s
}
rto_set_stage_seconds() {
local stage="$1"
local seconds="$2"
case "$stage" in
batch_copy) RTO_BATCH_COPY_SECONDS="$seconds" ;;
postgres_container_start) RTO_POSTGRES_CONTAINER_START_SECONDS="$seconds" ;;
dump_list) RTO_DUMP_LIST_SECONDS="$seconds" ;;
target_confirmation) RTO_TARGET_CONFIRMATION_SECONDS="$seconds" ;;
restore_apply) RTO_RESTORE_APPLY_SECONDS="$seconds" ;;
database_verify) RTO_DATABASE_VERIFY_SECONDS="$seconds" ;;
storage_attachment_verify) RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS="$seconds" ;;
final_integrity) RTO_FINAL_INTEGRITY_SECONDS="$seconds" ;;
*) return 1 ;;
esac
}
rto_begin() {
local stage="$1"
local now
[[ -z "$RTO_ACTIVE_STAGE" && "$RTO_FINALIZED" -eq 0 ]] || return 1
now="$(rto_now)"
[[ "$now" =~ ^[0-9]+$ ]] || return 1
rto_set_stage_seconds "$stage" "NOT_RUN" || return 1
RTO_ACTIVE_STAGE="$stage"
RTO_STAGE_STARTED_EPOCH="$now"
}
rto_close_active_at() {
local now="$1"
local elapsed
[[ -n "$RTO_ACTIVE_STAGE" && "$RTO_STAGE_STARTED_EPOCH" =~ ^[0-9]+$ \
&& "$now" =~ ^[0-9]+$ && "$now" -ge "$RTO_STAGE_STARTED_EPOCH" ]] \
|| return 1
elapsed="$((now - RTO_STAGE_STARTED_EPOCH))"
rto_set_stage_seconds "$RTO_ACTIVE_STAGE" "$elapsed" || return 1
}
rto_transition() {
local expected_stage="$1"
local next_stage="$2"
local now
[[ "$RTO_ACTIVE_STAGE" == "$expected_stage" && "$RTO_FINALIZED" -eq 0 ]] \
|| return 1
now="$(rto_now)"
rto_close_active_at "$now" || return 1
rto_set_stage_seconds "$next_stage" "NOT_RUN" || return 1
RTO_ACTIVE_STAGE="$next_stage"
RTO_STAGE_STARTED_EPOCH="$now"
}
rto_finalize() {
local now
local value
local total=0
local measured=0
if [[ "$RTO_FINALIZED" -eq 1 ]]; then
return 0
fi
if [[ -n "$RTO_ACTIVE_STAGE" ]]; then
now="$(rto_now)"
rto_close_active_at "$now" || return 1
RTO_ACTIVE_STAGE=""
RTO_STAGE_STARTED_EPOCH=""
fi
for value in \
"$RTO_BATCH_COPY_SECONDS" \
"$RTO_POSTGRES_CONTAINER_START_SECONDS" \
"$RTO_DUMP_LIST_SECONDS" \
"$RTO_TARGET_CONFIRMATION_SECONDS" \
"$RTO_RESTORE_APPLY_SECONDS" \
"$RTO_DATABASE_VERIFY_SECONDS" \
"$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS" \
"$RTO_FINAL_INTEGRITY_SECONDS"; do
if [[ "$value" =~ ^[0-9]+$ ]]; then
total="$((total + value))"
measured=1
fi
done
if [[ "$measured" -eq 1 ]]; then
RTO_SECONDS="$total"
fi
RTO_FINALIZED=1
}
confirm_postgres_target() {
local actual_database=""
local query_exit_code=0
@@ -182,6 +281,15 @@ emit_restore_diagnostics() {
echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"
echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"
echo "restore_error_category=$RESTORE_ERROR_CATEGORY"
echo "rto_batch_copy_seconds=$RTO_BATCH_COPY_SECONDS"
echo "rto_postgres_container_start_seconds=$RTO_POSTGRES_CONTAINER_START_SECONDS"
echo "rto_dump_list_seconds=$RTO_DUMP_LIST_SECONDS"
echo "rto_target_confirmation_seconds=$RTO_TARGET_CONFIRMATION_SECONDS"
echo "rto_restore_apply_seconds=$RTO_RESTORE_APPLY_SECONDS"
echo "rto_database_verify_seconds=$RTO_DATABASE_VERIFY_SECONDS"
echo "rto_storage_attachment_verify_seconds=$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS"
echo "rto_final_integrity_seconds=$RTO_FINAL_INTEGRITY_SECONDS"
echo "data_restore_rto_seconds=$RTO_SECONDS"
}
finish() {
@@ -191,6 +299,10 @@ finish() {
trap - EXIT HUP INT TERM
# Emit only allowlisted stage diagnostics while the private runtime still
# exists. Raw stderr remains mode 0600 and is removed by cleanup_runtime.
if ! rto_finalize; then
FAILURE_CATEGORY="rto_stage_finalize_failed"
exit_code=1
fi
emit_restore_diagnostics
if [[ "$RUNTIME_CLEANED" -ne 1 ]]; then
cleanup_runtime || runtime_cleanup="FAIL"
@@ -204,7 +316,6 @@ finish() {
if [[ "$exit_code" -eq 0 ]]; then
echo "result=$RESULT_STATUS"
echo "rpo_seconds=$RPO_SECONDS"
echo "data_restore_rto_seconds=$RTO_SECONDS"
echo "migration_success=$MIGRATION_SUCCESS"
echo "migration_other=$MIGRATION_OTHER"
echo "migration_set_match=$MIGRATION_SET_MATCH"
@@ -412,11 +523,13 @@ mkdir -m 0700 "$WORK_DIR"
private_log="$WORK_DIR/private.log"
: >"$private_log"
chmod 0600 "$private_log"
rto_started_epoch="$(date -u +%s)"
rto_begin batch_copy || fail "rto_stage_begin_failed"
cp -a -- "$selected_batch" "$WORK_DIR/batch"
copied_metadata="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \
|| fail "copied_batch_invalid"
[[ "$copied_metadata" == "$batch_metadata" ]] || fail "copied_batch_identity_mismatch"
rto_transition batch_copy postgres_container_start \
|| fail "rto_stage_transition_failed"
env_file="$WORK_DIR/postgres.env"
postgres_password="$(openssl rand -hex 24)"
@@ -429,6 +542,8 @@ chmod 0600 "$env_file"
printf '%s\n' '[stage=postgres_container_start]' >>"$private_log"
docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \
"$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed"
rto_transition postgres_container_start dump_list \
|| fail "rto_stage_transition_failed"
PHASE="restore_database"
DUMP_LIST_STATUS="FAIL"
@@ -444,6 +559,8 @@ if [[ "$DUMP_LIST_EXIT_CODE" -ne 0 ]]; then
fail "dump_list_failed"
fi
DUMP_LIST_STATUS="PASS"
rto_transition dump_list target_confirmation \
|| fail "rto_stage_transition_failed"
TARGET_CONFIRMATION_STATUS="FAIL"
printf '%s\n' '[stage=target_confirmation]' >>"$private_log"
@@ -452,6 +569,8 @@ if ! confirm_postgres_target; then
fail "restore_target_confirmation_failed"
fi
TARGET_CONFIRMATION_STATUS="PASS"
rto_transition target_confirmation restore_apply \
|| fail "rto_stage_transition_failed"
real_docker_path="$(command -v docker)"
docker_proxy_dir="$WORK_DIR/docker-proxy"
@@ -530,6 +649,8 @@ if [[ "$RESTORE_SCRIPT_EXIT_CODE" -ne 0 || "$PG_RESTORE_EXIT_CODE" -ne 0 ]]; the
fi
RESTORE_APPLY_STATUS="PASS"
RESTORE_ERROR_CATEGORY="NONE"
rto_transition restore_apply database_verify \
|| fail "rto_stage_transition_failed"
PHASE="verify_database"
migration_names="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \
@@ -581,6 +702,8 @@ SQL
)" || fail "table_count_query_failed"
[[ "$(printf '%s\n' "$TABLE_COUNTS" | sed '/^$/d' | wc -l | tr -d ' ')" == "12" ]] \
|| fail "table_count_key_mismatch"
rto_transition database_verify storage_attachment_verify \
|| fail "rto_stage_transition_failed"
PHASE="restore_storage"
restored_storage="$WORK_DIR/restored-storage"
@@ -613,6 +736,8 @@ SQL
read -r ATTACHMENT_TOTAL ATTACHMENT_MISMATCH attachment_s3_count <<<"$attachment_result"
[[ "$ATTACHMENT_TOTAL" =~ ^[0-9]+$ && "$ATTACHMENT_MISMATCH" == "0" \
&& "$attachment_s3_count" == "0" ]] || fail "attachment_reference_mismatch"
rto_transition storage_attachment_verify final_integrity \
|| fail "rto_stage_transition_failed"
PHASE="post_validation"
copied_metadata_after="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \
@@ -621,8 +746,7 @@ source_metadata_after="$(validate_batch "$selected_batch" source "")" \
|| fail "source_archive_postcheck_failed"
[[ "$copied_metadata_after" == "$batch_metadata" \
&& "$source_metadata_after" == "$batch_metadata" ]] || fail "archive_identity_changed"
rto_finished_epoch="$(date -u +%s)"
RTO_SECONDS="$((rto_finished_epoch - rto_started_epoch))"
rto_finalize || fail "rto_stage_finalize_failed"
deadman_enabled_after="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"
deadman_active_after="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"