wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 7 软件包 项目 发布 百科 活动

备份策略改为部署前手动快照：部署合同增加手动快照模式 [08-28-production-arm-backup-transfer-incident]

浏览代码 
- 用户决定仅用部署前手动轻量快照，不再推进对象存储、ARM/hyinda-20c 备份
- refresh-deployment-contract 新增 /etc/hyb-oa-backup-mode 显式模式：
  无文件保持原自动模式；manual_snapshot 要求三个备份 timer 停用且备份
  service 已退出，ops timer 仍须运行；内容/属主/符号链接异常 fail-closed
- probe 部署前与 refresh 部署后均校验模式与 timer/service 一致性
- 新增 44 backup_mode_invalid / 45 backup_timer_not_disabled 分类及合同测试
- 记录 10-05 ARM 与 hyinda-20c 只读核验结果、远端切换顺序与回滚
- 远端切换未执行，需另行批准

Co-Authored-By: Claude Opus 5.5 <[email protected]>
这个提交包含在：
wangsli
2026-10-05 17:28:45 +08:00
共同撰写人 Claude Opus 5.5
父节点 9cbefdf243
当前提交 f6d1025d85
共修改 9 个文件，包含 246 行新增和 7 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
scripts/qa/run-deployment-contract-refresh-contract.sh
+85 -1
查看文件
@@ -133,9 +133,42 @@ case "$action" in
esac
;;
daemon-reload) exit 0 ;;
is-enabled) echo enabled ;;
is-enabled)
unit="${2:-}"
if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then
case "${FIXTURE_BACKUP_TIMERS:-running}" in
disabled) echo disabled; exit 1 ;;
mixed)
if [[ "$unit" == *watchdog* ]]; then echo enabled; else echo disabled; exit 1; fi
;;
*) echo enabled ;;
esac
else
echo enabled
fi
;;
is-active)
unit="${2:-}"
if [[ "$unit" == *.service ]]; then
if [[ "$unit" == hyb-oa-automatic-backup-daily.service \
&& -n "${FIXTURE_DAILY_SERVICE_STATE:-}" ]]; then
echo "$FIXTURE_DAILY_SERVICE_STATE"
[[ "$FIXTURE_DAILY_SERVICE_STATE" == "active" ]] || exit 3
exit 0
fi
echo inactive; exit 3
fi
if [[ "${FIXTURE_TIMER_INACTIVE:-0}" == "1" ]]; then echo inactive; exit 3; fi
if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then
case "${FIXTURE_BACKUP_TIMERS:-running}" in
disabled) echo inactive; exit 3 ;;
mixed)
if [[ "$unit" == *watchdog* ]]; then echo active; else echo inactive; exit 3; fi
;;
esac
elif [[ "${FIXTURE_OPS_TIMER_INACTIVE:-0}" == "1" ]]; then
echo inactive; exit 3
fi
echo active
;;
*) exit 1 ;;
@@ -162,6 +195,57 @@ unsupported_status=$?
set -e
[[ "$unit_status" -eq 42 && "$timer_status" -eq 43 && "$unsupported_status" -eq 64 ]]
run_refresh_status() {
local command="$1"
shift
set +e
env HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" "$@" \
SSH_ORIGINAL_COMMAND="$command" bash "$HELPER" >/dev/null 2>&1
local status=$?
set -e
printf '%s' "$status"
}
mode_file="$FIXTURE_ROOT/etc/hyb-oa-backup-mode"
# Automatic mode (no mode file): disabled backup timers stay fail-closed,
# both before (probe) and after (refresh) a release.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]]
[[ "$(run_refresh_status probe FIXTURE_TIMER_INACTIVE=1)" -eq 43 ]]
printf 'manual_snapshot\n' >"$mode_file"
manual_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_BACKUP_TIMERS=disabled \
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")"
[[ "$manual_output" == '[deployment-contract-refresh][PASS] refreshed' ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 0 ]]
# Manual mode must not silently accept running or partially running backup timers,
# and the pre-deploy probe must reject the mismatch before any release write.
[[ "$(run_refresh_status refresh)" -eq 45 ]]
[[ "$(run_refresh_status probe)" -eq 45 ]]
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]]
# Disabled timers with an in-flight backup run are not a finished switch.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=active)" -eq 45 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=activating)" -eq 45 ]]
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=failed)" -eq 0 ]]
# Non-backup ops timer is still required in manual mode.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_OPS_TIMER_INACTIVE=1)" -eq 43 ]]
printf 'manual\n' >"$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
printf 'manual_snapshot\nautomatic\n' >"$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
rm -f -- "$mode_file"
printf 'manual_snapshot\n' >"$FIXTURE_ROOT/backup-mode-target"
ln -s "$FIXTURE_ROOT/backup-mode-target" "$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
rm -f -- "$mode_file"
[[ "$(run_refresh_status refresh)" -eq 0 ]]
[[ "$(deployment_contract_refresh_category 44 '')" == "backup_mode_invalid" ]]
[[ "$(deployment_contract_refresh_category 45 '')" == "backup_timer_not_disabled" ]]
public_key_file="$FIXTURE_ROOT/refresh.pub"
key_blob='AAAAC3NzaC1lZDI1NTE5AAAAIFixturerefreshkeymaterial000000000000'
printf 'ssh-ed25519 %s fixture-refresh\n' "$key_blob" >"$public_key_file"
scripts/refresh-deployment-contract.sh
+61 -5
查看文件
@@ -12,10 +12,17 @@ SERVICES=(
hyb-oa-automatic-backup-drill-reminder.service
hyb-oa-warch3-ops.service
)
TIMERS=(
BACKUP_TIMERS=(
hyb-oa-automatic-backup-daily.timer
hyb-oa-automatic-backup-watchdog.timer
hyb-oa-automatic-backup-drill-reminder.timer
)
BACKUP_SERVICES=(
hyb-oa-automatic-backup-daily.service
hyb-oa-automatic-backup-watchdog.service
hyb-oa-automatic-backup-drill-reminder.service
)
OPS_TIMERS=(
hyb-oa-warch3-ops.timer
)
CONTRACT_SERVICE="hyb-oa-automatic-backup-daily.service"
@@ -47,6 +54,11 @@ if [[ "$(id -u)" != "0" ]]; then
fi
UNIT_ROOT="${ROOT_PREFIX}/etc/systemd/system"
# Absent file = automatic backup mode (all backup timers must run). The only
# accepted content is "manual_snapshot": backup timers must then be disabled
# and inactive, because backups are manual Lighthouse snapshots before deploys.
BACKUP_MODE_FILE="${ROOT_PREFIX}/etc/hyb-oa-backup-mode"
BACKUP_MODE=""
LOCK_FILE="${ROOT_PREFIX}/run/lock/hyb-oa-deployment-contract-refresh.lock"
EXPECTED_TARGET="${ROOT_PREFIX}/usr/local/sbin/hyb-oa-deployment-contract-refresh"
@@ -160,25 +172,69 @@ verify_builds() {
done
}
resolve_backup_mode() {
local content
if [[ ! -e "$BACKUP_MODE_FILE" && ! -L "$BACKUP_MODE_FILE" ]]; then
BACKUP_MODE="automatic"
return 0
fi
[[ -f "$BACKUP_MODE_FILE" && ! -L "$BACKUP_MODE_FILE" ]] \
|| fail 44 backup_mode_invalid
if [[ "$TEST_MODE" == "0" ]]; then
[[ "$(stat -c %u:%g "$BACKUP_MODE_FILE")" == "0:0" ]] \
&& [[ "$(stat -c %a "$BACKUP_MODE_FILE")" =~ ^(644|600|444|400)$ ]] \
|| fail 44 backup_mode_invalid
fi
content="$(<"$BACKUP_MODE_FILE")"
[[ "$content" == "manual_snapshot" ]] || fail 44 backup_mode_invalid
BACKUP_MODE="manual_snapshot"
}
timer_state() {
local timer="$1"
printf '%s:%s' \
"$(systemctl is-enabled "$timer" 2>/dev/null || true)" \
"$(systemctl is-active "$timer" 2>/dev/null || true)"
}
verify_timers() {
local timer
for timer in "${TIMERS[@]}"; do
[[ "$(systemctl is-enabled "$timer")" == "enabled" \
&& "$(systemctl is-active "$timer")" == "active" ]] \
|| fail 43 timer_not_active
for timer in "${OPS_TIMERS[@]}"; do
[[ "$(timer_state "$timer")" == "enabled:active" ]] || fail 43 timer_not_active
done
for timer in "${BACKUP_TIMERS[@]}"; do
if [[ "$BACKUP_MODE" == "manual_snapshot" ]]; then
[[ "$(timer_state "$timer")" == "disabled:inactive" ]] \
|| fail 45 backup_timer_not_disabled
else
[[ "$(timer_state "$timer")" == "enabled:active" ]] || fail 43 timer_not_active
fi
done
[[ "$BACKUP_MODE" == "manual_snapshot" ]] || return 0
# A disabled timer does not stop an in-flight run; the switch is complete
# only after every backup service has exited (a failed last run still counts).
local service state
for service in "${BACKUP_SERVICES[@]}"; do
state="$(systemctl is-active "$service" 2>/dev/null || true)"
[[ "$state" == "inactive" || "$state" == "failed" ]] \
|| fail 45 backup_timer_not_disabled
done
}
probe() {
validate_self
validate_fixed_units
resolve_backup_mode
resolve_active_build
# Fail before any release write when timers disagree with the backup mode.
verify_timers
echo "[deployment-contract-refresh][PASS] probe"
}
refresh() {
validate_self
validate_fixed_units
resolve_backup_mode
resolve_active_build
mkdir -p "$(dirname "$LOCK_FILE")"
exec 9>"$LOCK_FILE"