完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）

2026-07-27 13:54:16 +08:00
@@ -7,6 +7,15 @@ PUBLIC_KEY_FILE="${PUBLIC_KEY_FILE:-}"
 RECEIVER_SOURCE="${RECEIVER_SOURCE:-}"
 RECEIVER_TARGET="${RECEIVER_TARGET:-/usr/local/sbin/hyb-oa-offhost-receiver}"
 CONFIG_FILE="${CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}"
+ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
+ALERT_HELPER_SOURCE="${ALERT_HELPER_SOURCE:-}"
+ALERT_HELPER_TARGET="${ALERT_HELPER_TARGET:-/usr/local/sbin/hyb-oa-wecom-group-alert}"
+DEADMAN_SOURCE="${DEADMAN_SOURCE:-}"
+DEADMAN_TARGET="${DEADMAN_TARGET:-/usr/local/sbin/hyb-oa-offhost-deadman}"
+DEADMAN_MAX_AGE_SECONDS="${DEADMAN_MAX_AGE_SECONDS:-93600}"
+DEADMAN_RUN_AT="${DEADMAN_RUN_AT:-06:30}"
+DEADMAN_SERVICE="${DEADMAN_SERVICE:-hyb-oa-offhost-deadman}"
+SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}"
 DAILY_KEEP="${DAILY_KEEP:-7}"
 WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
 MODE="${1:-install}"
@@ -45,7 +54,7 @@ validate_backup_root() {
  if [[ -d "$BACKUP_ROOT" ]]; then
    local unmanaged_entry
    unmanaged_entry="$(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 \
-      ! -name inbox ! -name archive ! -name weekly -print -quit)"
+      ! -name inbox ! -name archive ! -name weekly ! -name state -print -quit)"
    [[ -z "$unmanaged_entry" ]] || fail "backup root contains an unmanaged top-level entry"
  fi
 }
@@ -57,8 +66,22 @@ validate_inputs() {
  validate_absolute_path RECEIVER_SOURCE "$RECEIVER_SOURCE"
  validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
  validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
+  validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE"
+  validate_absolute_path ALERT_HELPER_SOURCE "$ALERT_HELPER_SOURCE"
+  validate_absolute_path ALERT_HELPER_TARGET "$ALERT_HELPER_TARGET"
+  validate_absolute_path DEADMAN_SOURCE "$DEADMAN_SOURCE"
+  validate_absolute_path DEADMAN_TARGET "$DEADMAN_TARGET"
  [[ -f "$PUBLIC_KEY_FILE" && ! -L "$PUBLIC_KEY_FILE" ]] || fail "public key file is missing"
  [[ -f "$RECEIVER_SOURCE" && ! -L "$RECEIVER_SOURCE" ]] || fail "receiver source is missing"
+  [[ -f "$ALERT_HELPER_SOURCE" && ! -L "$ALERT_HELPER_SOURCE" ]] || fail "alert helper source is missing"
+  [[ -f "$DEADMAN_SOURCE" && ! -L "$DEADMAN_SOURCE" ]] || fail "dead-man source is missing"
+  [[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
+    && "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" \
+    && "$(stat -c %u "$ALERT_WEBHOOK_FILE")" == "0" ]] \
+    || fail "alert webhook file must be root-owned with mode 0600"
+  [[ "$DEADMAN_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "dead-man threshold must be positive"
+  [[ "$DEADMAN_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "dead-man time is invalid"
+  [[ "$DEADMAN_SERVICE" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "dead-man service name is invalid"
  [[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ && "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] \
    || fail "retention values must be positive"
  command -v rsync >/dev/null 2>&1 || fail "rsync must be installed before receiver setup"
@@ -86,22 +109,28 @@ install_receiver() {
  fi
  passwd -l "$BACKUP_USER" >/dev/null 2>&1 || true

-  local home_dir inbox_dir archive_dir weekly_dir ssh_dir auth_file public_key rrsync_bin
+  local home_dir inbox_dir archive_dir weekly_dir state_dir ssh_dir auth_file public_key rrsync_bin
  home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
  validate_absolute_path BACKUP_USER_HOME "$home_dir"
  inbox_dir="$BACKUP_ROOT/inbox"
  archive_dir="$BACKUP_ROOT/archive"
  weekly_dir="$BACKUP_ROOT/weekly"
-  mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
+  state_dir="$BACKUP_ROOT/state"
+  mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$state_dir" "$home_dir/.ssh"
  [[ -d "$BACKUP_ROOT" && ! -L "$BACKUP_ROOT" \
    && -d "$inbox_dir" && ! -L "$inbox_dir" \
    && -d "$archive_dir" && ! -L "$archive_dir" \
    && -d "$weekly_dir" && ! -L "$weekly_dir" ]] \
    || fail "backup receiver directories must not be symbolic links"
-  chown -R "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT" "$home_dir/.ssh"
+  chown "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT"
+  chown -R "$BACKUP_USER:$BACKUP_USER" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
  chmod 0700 "$BACKUP_ROOT" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
+  chown -R root:root "$state_dir"
+  chmod 0700 "$state_dir"

  install -o root -g root -m 0755 "$RECEIVER_SOURCE" "$RECEIVER_TARGET"
+  install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET"
+  install -o root -g root -m 0755 "$DEADMAN_SOURCE" "$DEADMAN_TARGET"
  rrsync_bin="$(command -v rrsync)"
  cat >"$CONFIG_FILE" <<CONFIG
 BACKUP_INBOX_DIR=${inbox_dir}
@@ -130,7 +159,37 @@ AUTH

  runuser -u "$BACKUP_USER" -- env RECEIVER_CONFIG_FILE="$CONFIG_FILE" \
    "$RECEIVER_TARGET" validate-config >/dev/null
-  echo "[offhost-receiver-installer][PASS] restricted receiver installed"
+  cat >"$SYSTEMD_DIR/${DEADMAN_SERVICE}.service" <<SERVICE
+[Unit]
+Description=hyb-oa independent off-host backup dead-man
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+UMask=0077
+ExecStart=/usr/bin/python3 ${DEADMAN_TARGET} --archive-root ${archive_dir} --state-dir ${state_dir} --webhook-file ${ALERT_WEBHOOK_FILE} --alert-helper ${ALERT_HELPER_TARGET} --max-age-seconds ${DEADMAN_MAX_AGE_SECONDS}
+SERVICE
+  cat >"$SYSTEMD_DIR/${DEADMAN_SERVICE}.timer" <<TIMER
+[Unit]
+Description=Check hyb-oa off-host backup freshness independently
+
+[Timer]
+OnCalendar=*-*-* ${DEADMAN_RUN_AT}:00
+Persistent=true
+Unit=${DEADMAN_SERVICE}.service
+
+[Install]
+WantedBy=timers.target
+TIMER
+  systemctl daemon-reload
+  echo "[offhost-receiver-installer][PASS] restricted receiver and disabled dead-man installed"
+}
+
+enable_receiver() {
+  install_receiver
+  systemctl enable --now "${DEADMAN_SERVICE}.timer"
+  echo "[offhost-receiver-installer][PASS] dead-man timer enabled"
 }

 uninstall_receiver() {
@@ -138,6 +197,12 @@ uninstall_receiver() {
  [[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid"
  validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
  validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
+  validate_absolute_path ALERT_HELPER_TARGET "$ALERT_HELPER_TARGET"
+  validate_absolute_path DEADMAN_TARGET "$DEADMAN_TARGET"
+  [[ "$DEADMAN_SERVICE" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "dead-man service name is invalid"
+  systemctl disable --now "${DEADMAN_SERVICE}.timer" >/dev/null 2>&1 || true
+  systemctl stop "${DEADMAN_SERVICE}.service" >/dev/null 2>&1 || true
+  rm -f -- "$SYSTEMD_DIR/${DEADMAN_SERVICE}.timer" "$SYSTEMD_DIR/${DEADMAN_SERVICE}.service"
  if id "$BACKUP_USER" >/dev/null 2>&1; then
    local home_dir auth_file
    home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
@@ -146,19 +211,23 @@ uninstall_receiver() {
      rm -f -- "$auth_file"
    fi
  fi
-  rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE"
-  echo "[offhost-receiver-installer][PASS] receiver authorization removed; archives retained"
+  rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE" "$ALERT_HELPER_TARGET" "$DEADMAN_TARGET"
+  systemctl daemon-reload
+  echo "[offhost-receiver-installer][PASS] receiver authorization and dead-man removed; archives retained"
 }

 case "$MODE" in
  install)
    install_receiver
    ;;
+  enable)
+    enable_receiver
+    ;;
  uninstall|--uninstall)
    uninstall_receiver
    ;;
  --help|-h)
-    echo "Usage: BACKUP_ROOT=<absolute> PUBLIC_KEY_FILE=<absolute> RECEIVER_SOURCE=<absolute> bash install-offhost-backup-receiver.sh [install|uninstall]"
+    echo "Usage: <explicit backup/key/helper/dead-man/webhook paths> bash install-offhost-backup-receiver.sh [install|enable|uninstall]"
    ;;
  *)
    fail "unsupported mode"