修复(08-23-isolated-backup-restore-drill): 消除影子库就绪竞态

scripts/qa/offhost-shadow-restore-runner.template.sh

@@ -44,6 +44,11 @@ TARGET_CONFIRMATION_STATUS="NOT_RUN"
 RESTORE_APPLY_STATUS="NOT_RUN"
 RESTORE_EXECUTION_PATH="NOT_RUN"
 DUMP_LIST_EXIT_CODE="NOT_RUN"
+TARGET_CONFIRMATION_EXIT_CODE="NOT_RUN"
+TARGET_NAME_MATCH="NOT_RUN"
+TARGET_CONFIRMATION_ATTEMPT_COUNT="0"
+TARGET_CONFIRMATION_SUCCESS_STREAK="0"
+TARGET_CONFIRMATION_FAILURE_REASON="NOT_APPLICABLE"
 RESTORE_SCRIPT_EXIT_CODE="NOT_RUN"
 PG_RESTORE_EXIT_CODE="NOT_RUN"
 PG_RESTORE_ERROR_COUNT="NOT_RUN"
@@ -61,6 +66,47 @@ require_command() {
   command -v "$1" >/dev/null 2>&1 || fail "required_command_missing"
 }
 
+confirm_postgres_target() {
+  local actual_database=""
+  local query_exit_code=0
+  local attempt=0
+  TARGET_CONFIRMATION_EXIT_CODE="NOT_RUN"
+  TARGET_NAME_MATCH="NOT_RUN"
+  TARGET_CONFIRMATION_ATTEMPT_COUNT="0"
+  TARGET_CONFIRMATION_SUCCESS_STREAK="0"
+  TARGET_CONFIRMATION_FAILURE_REASON="TARGET_QUERY_NONZERO"
+  for attempt in $(seq 1 60); do
+    TARGET_CONFIRMATION_ATTEMPT_COUNT="$attempt"
+    set +e
+    actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \
+      "$CONTAINER_NAME" psql -h 127.0.0.1 -v ON_ERROR_STOP=1 \
+      -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
+      -Atqc 'SELECT current_database()' 2>>"$private_log")"
+    query_exit_code="$?"
+    set -e
+    TARGET_CONFIRMATION_EXIT_CODE="$query_exit_code"
+    if [[ "$query_exit_code" -ne 0 ]]; then
+      TARGET_NAME_MATCH="NOT_RUN"
+      TARGET_CONFIRMATION_SUCCESS_STREAK="0"
+      [[ "$attempt" -eq 60 ]] || sleep 1
+      continue
+    fi
+    if [[ "$actual_database" != "$SHADOW_DB_NAME" ]]; then
+      TARGET_NAME_MATCH="no"
+      TARGET_CONFIRMATION_FAILURE_REASON="TARGET_NAME_MISMATCH"
+      return 1
+    fi
+    TARGET_NAME_MATCH="yes"
+    TARGET_CONFIRMATION_SUCCESS_STREAK="$((TARGET_CONFIRMATION_SUCCESS_STREAK + 1))"
+    if [[ "$TARGET_CONFIRMATION_SUCCESS_STREAK" -ge 2 ]]; then
+      TARGET_CONFIRMATION_FAILURE_REASON="NONE"
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
+}
+
 cleanup_runtime() {
   local cleanup_ok=1
   local container_suffix=""
@@ -126,6 +172,10 @@ emit_restore_diagnostics() {
   echo "dump_list=$DUMP_LIST_STATUS"
   echo "dump_list_exit_code=$DUMP_LIST_EXIT_CODE"
   echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"
+  echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE"
+  echo "target_name_match=$TARGET_NAME_MATCH"
+  echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT"
+  echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK"
   echo "restore_execution_path=$RESTORE_EXECUTION_PATH"
   echo "restore_apply=$RESTORE_APPLY_STATUS"
   echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"
@@ -170,7 +220,7 @@ finish() {
 }
 trap finish EXIT HUP INT TERM
 
-for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl; do
+for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl seq; do
   require_command "$command_name"
 done
 
@@ -379,15 +429,6 @@ chmod 0600 "$env_file"
 printf '%s\n' '[stage=postgres_container_start]' >>"$private_log"
 docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \
   "$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed"
-for _ in $(seq 1 60); do
-  if docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
-    >/dev/null 2>&1; then
-    break
-  fi
-  sleep 1
-done
-docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
-  >/dev/null 2>&1 || fail "postgres_not_ready"
 
 PHASE="restore_database"
 DUMP_LIST_STATUS="FAIL"
@@ -406,15 +447,8 @@ DUMP_LIST_STATUS="PASS"
 
 TARGET_CONFIRMATION_STATUS="FAIL"
 printf '%s\n' '[stage=target_confirmation]' >>"$private_log"
-set +e
-actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \
-  "$CONTAINER_NAME" psql -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
-  -Atqc 'SELECT current_database()' 2>>"$private_log")"
-target_confirmation_exit_code="$?"
-set -e
-if [[ "$target_confirmation_exit_code" -ne 0 \
-  || "$actual_database" != "$SHADOW_DB_NAME" ]]; then
-  RESTORE_ERROR_CATEGORY="TARGET_CONFIRMATION_FAILED"
+if ! confirm_postgres_target; then
+  RESTORE_ERROR_CATEGORY="$TARGET_CONFIRMATION_FAILURE_REASON"
   fail "restore_target_confirmation_failed"
 fi
 TARGET_CONFIRMATION_STATUS="PASS"