559 行
22 KiB
Bash
可执行文件
559 行
22 KiB
Bash
可执行文件
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
CONFIG_FILE="${RECEIVER_CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}"
|
|
if [[ -f "$CONFIG_FILE" ]]; then
|
|
# Root-owned installer output with path-only assignments.
|
|
# shellcheck disable=SC1090
|
|
source "$CONFIG_FILE"
|
|
fi
|
|
|
|
BACKUP_INBOX_DIR="${BACKUP_INBOX_DIR:-}"
|
|
BACKUP_ARCHIVE_DIR="${BACKUP_ARCHIVE_DIR:-}"
|
|
BACKUP_WEEKLY_DIR="${BACKUP_WEEKLY_DIR:-}"
|
|
RRSYNC_BIN="${RRSYNC_BIN:-/usr/bin/rrsync}"
|
|
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
|
DAILY_KEEP="${DAILY_KEEP:-7}"
|
|
WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
|
|
declare -A RECONCILED_WEEK_SOURCE=()
|
|
declare -a DAILY_ORPHANS_PENDING_DELETE=()
|
|
|
|
fail() {
|
|
echo "[offhost-receiver][ERROR] $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
validate_absolute_path() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
|
|
fail "$name must be an explicit safe absolute path"
|
|
fi
|
|
if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
|
|
|| "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
|
|
fail "$name contains an unsafe path segment"
|
|
fi
|
|
}
|
|
|
|
validate_layout() {
|
|
local name value
|
|
for name in BACKUP_INBOX_DIR BACKUP_ARCHIVE_DIR BACKUP_WEEKLY_DIR; do
|
|
value="${!name:-}"
|
|
validate_absolute_path "$name" "$value"
|
|
[[ -d "$value" && ! -L "$value" ]] || fail "$name is not an existing regular directory"
|
|
done
|
|
if [[ "$BACKUP_INBOX_DIR" == "$BACKUP_ARCHIVE_DIR" \
|
|
|| "$BACKUP_INBOX_DIR" == "$BACKUP_WEEKLY_DIR" \
|
|
|| "$BACKUP_ARCHIVE_DIR" == "$BACKUP_WEEKLY_DIR" ]]; then
|
|
fail "backup receiver directories must be distinct"
|
|
fi
|
|
local first second
|
|
for first in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do
|
|
for second in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do
|
|
if [[ "$first" != "$second" && ( "$first" == "$second"/* || "$second" == "$first"/* ) ]]; then
|
|
fail "backup receiver directories must not be nested"
|
|
fi
|
|
done
|
|
done
|
|
if [[ "$(stat -c %d "$BACKUP_INBOX_DIR")" != "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" \
|
|
|| "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" != "$(stat -c %d "$BACKUP_WEEKLY_DIR")" ]]; then
|
|
fail "receiver directories must share a filesystem for atomic finalize and weekly hardlinks"
|
|
fi
|
|
[[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "DAILY_KEEP must be positive"
|
|
[[ "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "WEEKLY_KEEP must be positive"
|
|
command -v "$PYTHON_BIN" >/dev/null 2>&1 || fail "python3 is required"
|
|
command -v flock >/dev/null 2>&1 || fail "flock is required"
|
|
}
|
|
|
|
validate_run_id() {
|
|
local run_id="$1"
|
|
[[ "$run_id" =~ ^auto-[0-9]{8}T[0-9]{6}Z$ ]] \
|
|
|| fail "run id must use auto-YYYYMMDDTHHMMSSZ"
|
|
local run_day="${run_id#auto-}"
|
|
run_day="${run_day%%T*}"
|
|
[[ "$run_day" == "$(date -u +%Y%m%d)" || "$run_day" == "$(date -u -d yesterday +%Y%m%d)" ]] \
|
|
|| fail "run id UTC day is outside the accepted receiver window"
|
|
}
|
|
|
|
validate_batch() {
|
|
local batch_dir="$1"
|
|
local run_id="$2"
|
|
local expected_manifest_sha="$3"
|
|
|
|
[[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "partial batch is missing"
|
|
[[ "$expected_manifest_sha" =~ ^[0-9a-f]{64}$ ]] || fail "manifest SHA-256 is invalid"
|
|
|
|
local entry_count regular_count
|
|
entry_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -printf '.' | wc -c | tr -d ' ')"
|
|
regular_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -type f -printf '.' | wc -c | tr -d ' ')"
|
|
[[ "$entry_count" == "3" && "$regular_count" == "3" ]] \
|
|
|| fail "batch must contain exactly three regular files"
|
|
for file_name in database.dump storage.tar.gz manifest.json; do
|
|
[[ -f "$batch_dir/$file_name" && ! -L "$batch_dir/$file_name" ]] \
|
|
|| fail "batch required file is missing"
|
|
done
|
|
|
|
local actual_manifest_sha
|
|
actual_manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
|
[[ "$actual_manifest_sha" == "$expected_manifest_sha" ]] \
|
|
|| fail "manifest checksum mismatch"
|
|
|
|
"$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON' \
|
|
|| fail "batch manifest or artifact validation failed"
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
batch_dir, expected_run_id = sys.argv[1:]
|
|
manifest_path = os.path.join(batch_dir, "manifest.json")
|
|
with open(manifest_path, "r", encoding="utf-8") as handle:
|
|
manifest = json.load(handle)
|
|
|
|
if manifest.get("schemaVersion") != 1:
|
|
raise SystemExit("unsupported manifest schema")
|
|
if manifest.get("status") != "complete" or manifest.get("runId") != expected_run_id:
|
|
raise SystemExit("manifest identity mismatch")
|
|
if not manifest.get("buildSha") and not manifest.get("sourceImageDigest"):
|
|
raise SystemExit("manifest source identity missing")
|
|
|
|
artifacts = manifest.get("artifacts") or {}
|
|
expected = {
|
|
"database": "database.dump",
|
|
"storage": "storage.tar.gz",
|
|
}
|
|
for key, expected_name in expected.items():
|
|
artifact = artifacts.get(key) or {}
|
|
if artifact.get("file") != expected_name:
|
|
raise SystemExit("manifest artifact name mismatch")
|
|
file_path = os.path.join(batch_dir, expected_name)
|
|
if os.path.getsize(file_path) != artifact.get("bytes"):
|
|
raise SystemExit("manifest artifact size mismatch")
|
|
digest = hashlib.sha256()
|
|
with open(file_path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
if digest.hexdigest() != artifact.get("sha256"):
|
|
raise SystemExit("manifest artifact checksum mismatch")
|
|
PYTHON
|
|
}
|
|
|
|
validated_batch_identity() {
|
|
local batch_dir="$1"
|
|
[[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "recovery candidate is not a regular directory"
|
|
|
|
local run_id manifest_sha identity
|
|
run_id="$($PYTHON_BIN - "$batch_dir/manifest.json" <<'PYTHON'
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
|
manifest = json.load(handle)
|
|
run_id = manifest.get("runId")
|
|
if not isinstance(run_id, str) or not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", run_id):
|
|
raise SystemExit("manifest run id is invalid")
|
|
print(run_id)
|
|
PYTHON
|
|
)" || fail "recovery candidate manifest identity is invalid"
|
|
manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
|
validate_batch "$batch_dir" "$run_id" "$manifest_sha"
|
|
identity="$($PYTHON_BIN - "$run_id" <<'PYTHON'
|
|
from datetime import datetime
|
|
import sys
|
|
|
|
run_id = sys.argv[1]
|
|
day = run_id.removeprefix("auto-").split("T", 1)[0]
|
|
iso = datetime.strptime(day, "%Y%m%d").date().isocalendar()
|
|
print(f"{run_id}\t{day}\tweek-{iso.year:04d}-W{iso.week:02d}")
|
|
PYTHON
|
|
)" || fail "recovery candidate date identity is invalid"
|
|
printf '%s\t%s\n' "$identity" "$manifest_sha"
|
|
}
|
|
|
|
validate_temp_directory() {
|
|
local target="$1"
|
|
local root="$2"
|
|
local pattern="$3"
|
|
[[ "$(dirname -- "$target")" == "$root" ]] || fail "orphan target escaped its managed root"
|
|
[[ "$(basename -- "$target")" =~ $pattern ]] || fail "orphan target name is invalid"
|
|
[[ -d "$target" && ! -L "$target" ]] || fail "orphan target is not a regular directory"
|
|
}
|
|
|
|
remove_temp_directory() {
|
|
local target="$1"
|
|
local root="$2"
|
|
local pattern="$3"
|
|
validate_temp_directory "$target" "$root" "$pattern"
|
|
rm -rf -- "$target"
|
|
}
|
|
|
|
canonical_daily_for_day() {
|
|
local run_day="$1"
|
|
local -a entries=()
|
|
mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
|
-name "hyb-oa-backup-auto-${run_day}T*Z" -print | LC_ALL=C sort)
|
|
[[ "${#entries[@]}" -le 1 ]] || fail "daily archive invariant is ambiguous during recovery"
|
|
if [[ "${#entries[@]}" -eq 1 ]]; then
|
|
printf '%s\n' "${entries[0]}"
|
|
fi
|
|
}
|
|
|
|
reconcile_daily_orphans() {
|
|
local -a orphans=()
|
|
mapfile -t orphans < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
|
-name '.daily-replacement-*' -print | LC_ALL=C sort)
|
|
local orphan base run_day identity orphan_run_id orphan_day orphan_week orphan_sha
|
|
local canonical canonical_identity canonical_run_id canonical_day canonical_week canonical_sha
|
|
local -A seen_days=() identities=() canonicals=()
|
|
for orphan in "${orphans[@]}"; do
|
|
base="$(basename -- "$orphan")"
|
|
[[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
|
|
|| fail "daily replacement orphan name is invalid"
|
|
run_day="${BASH_REMATCH[1]}"
|
|
[[ -z "${seen_days[$run_day]:-}" ]] || fail "multiple daily replacement orphans exist for one day"
|
|
seen_days[$run_day]=1
|
|
validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
|
|
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
|
identity="$(validated_batch_identity "$orphan")" \
|
|
|| fail "daily replacement orphan validation failed"
|
|
IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
|
|
[[ "$orphan_day" == "$run_day" ]] || fail "daily replacement orphan day does not match its manifest"
|
|
identities[$orphan]="$identity"
|
|
|
|
canonical="$(canonical_daily_for_day "$run_day")" \
|
|
|| fail "daily archive lookup failed during recovery"
|
|
canonicals[$run_day]="$canonical"
|
|
if [[ -n "$canonical" ]]; then
|
|
[[ -d "$canonical" && ! -L "$canonical" ]] || fail "daily archive recovery target is invalid"
|
|
canonical_identity="$(validated_batch_identity "$canonical")" \
|
|
|| fail "daily archive validation failed during recovery"
|
|
IFS=$'\t' read -r canonical_run_id canonical_day canonical_week canonical_sha <<<"$canonical_identity"
|
|
[[ "$canonical_day" == "$run_day" \
|
|
&& "$(basename -- "$canonical")" == "hyb-oa-backup-$canonical_run_id" ]] \
|
|
|| fail "daily archive identity is inconsistent during recovery"
|
|
[[ "$canonical_week" == "$orphan_week" ]] || fail "daily replacement week identity is inconsistent"
|
|
else
|
|
canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
|
|
[[ ! -e "$canonical" ]] || fail "daily recovery target already exists"
|
|
fi
|
|
done
|
|
|
|
for orphan in "${orphans[@]}"; do
|
|
base="$(basename -- "$orphan")"
|
|
[[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
|
|
|| fail "daily replacement orphan name changed during recovery"
|
|
run_day="${BASH_REMATCH[1]}"
|
|
identity="${identities[$orphan]}"
|
|
IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
|
|
canonical="${canonicals[$run_day]}"
|
|
if [[ -z "$canonical" ]]; then
|
|
canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
|
|
validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
|
|
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
|
[[ ! -e "$canonical" ]] || fail "daily recovery target appeared during recovery"
|
|
mv -- "$orphan" "$canonical"
|
|
else
|
|
DAILY_ORPHANS_PENDING_DELETE+=("$orphan")
|
|
fi
|
|
RECONCILED_WEEK_SOURCE[$orphan_week]="$canonical"
|
|
done
|
|
}
|
|
|
|
prune_named_directories() {
|
|
local root="$1"
|
|
local prefix="$2"
|
|
local keep="$3"
|
|
local -a entries=()
|
|
mapfile -t entries < <(find "$root" -mindepth 1 -maxdepth 1 -type d \
|
|
-name "${prefix}*" -printf '%f\n' | LC_ALL=C sort)
|
|
local remove_count=$((${#entries[@]} - keep))
|
|
if (( remove_count <= 0 )); then
|
|
return 0
|
|
fi
|
|
local index entry target
|
|
for ((index = 0; index < remove_count; index += 1)); do
|
|
entry="${entries[$index]}"
|
|
target="$root/$entry"
|
|
[[ "$entry" == "$prefix"* && -d "$target" && ! -L "$target" ]] \
|
|
|| fail "retention target failed safety validation"
|
|
rm -rf -- "$target"
|
|
done
|
|
}
|
|
|
|
publish_weekly_checkpoint() {
|
|
local archive_dir="$1"
|
|
local weekly_dir="$2"
|
|
local replace_existing="$3"
|
|
if [[ -e "$weekly_dir" && "$replace_existing" != "1" ]]; then
|
|
return 0
|
|
fi
|
|
local weekly_temp="${weekly_dir}.new.$$"
|
|
local weekly_rollback="${weekly_dir}.rollback.$$"
|
|
[[ ! -e "$weekly_temp" && ! -e "$weekly_rollback" ]] || return 1
|
|
if ! cp -al -- "$archive_dir" "$weekly_temp"; then
|
|
rm -rf -- "$weekly_temp"
|
|
return 1
|
|
fi
|
|
if [[ -e "$weekly_dir" ]]; then
|
|
[[ -d "$weekly_dir" && ! -L "$weekly_dir" ]] || return 1
|
|
mv -- "$weekly_dir" "$weekly_rollback" || return 1
|
|
fi
|
|
if ! mv -- "$weekly_temp" "$weekly_dir"; then
|
|
rm -rf -- "$weekly_temp"
|
|
if [[ -e "$weekly_rollback" ]]; then
|
|
mv -- "$weekly_rollback" "$weekly_dir"
|
|
fi
|
|
return 1
|
|
fi
|
|
if [[ -e "$weekly_rollback" ]]; then
|
|
rm -rf -- "$weekly_rollback"
|
|
fi
|
|
}
|
|
|
|
unique_daily_source_for_week() {
|
|
local expected_week="$1"
|
|
local -a candidates=() entries=()
|
|
mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
|
-name 'hyb-oa-backup-auto-*' -print | LC_ALL=C sort)
|
|
local entry identity run_id run_day iso_week manifest_sha
|
|
for entry in "${entries[@]}"; do
|
|
[[ -d "$entry" && ! -L "$entry" ]] || fail "daily archive candidate is invalid during weekly recovery"
|
|
identity="$(validated_batch_identity "$entry")" \
|
|
|| fail "daily archive validation failed during weekly recovery"
|
|
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
|
[[ "$(basename -- "$entry")" == "hyb-oa-backup-$run_id" ]] \
|
|
|| fail "daily archive name does not match its manifest"
|
|
if [[ "$iso_week" == "$expected_week" ]]; then
|
|
candidates+=("$entry")
|
|
fi
|
|
done
|
|
[[ "${#candidates[@]}" -eq 1 ]] || fail "weekly recovery source is not unique"
|
|
printf '%s\n' "${candidates[0]}"
|
|
}
|
|
|
|
reconcile_weekly_orphans() {
|
|
local -a entries=()
|
|
mapfile -t entries < <(find "$BACKUP_WEEKLY_DIR" -mindepth 1 -maxdepth 1 \
|
|
\( -name 'week-*-W*.new.*' -o -name 'week-*-W*.rollback.*' \) -print | LC_ALL=C sort)
|
|
local -A new_by_week=() rollback_by_week=() weeks=()
|
|
local entry base week kind
|
|
for entry in "${entries[@]}"; do
|
|
base="$(basename -- "$entry")"
|
|
if [[ "$base" =~ ^(week-[0-9]{4}-W[0-9]{2})\.(new|rollback)\.([0-9]+)$ ]]; then
|
|
week="${BASH_REMATCH[1]}"
|
|
kind="${BASH_REMATCH[2]}"
|
|
else
|
|
fail "weekly orphan name is invalid"
|
|
fi
|
|
validate_temp_directory "$entry" "$BACKUP_WEEKLY_DIR" \
|
|
'^week-[0-9]{4}-W[0-9]{2}\.(new|rollback)\.[0-9]+$'
|
|
weeks[$week]=1
|
|
if [[ "$kind" == "new" ]]; then
|
|
[[ -z "${new_by_week[$week]:-}" ]] || fail "multiple weekly staging orphans exist"
|
|
new_by_week[$week]="$entry"
|
|
else
|
|
[[ -z "${rollback_by_week[$week]:-}" ]] || fail "multiple weekly rollback orphans exist"
|
|
rollback_by_week[$week]="$entry"
|
|
fi
|
|
done
|
|
for week in "${!RECONCILED_WEEK_SOURCE[@]}"; do
|
|
weeks[$week]=1
|
|
done
|
|
|
|
local -A sources=()
|
|
local canonical new_dir rollback_dir identity run_id run_day iso_week manifest_sha source
|
|
for week in "${!weeks[@]}"; do
|
|
canonical="$BACKUP_WEEKLY_DIR/$week"
|
|
new_dir="${new_by_week[$week]:-}"
|
|
rollback_dir="${rollback_by_week[$week]:-}"
|
|
|
|
if [[ -e "$canonical" ]]; then
|
|
[[ -d "$canonical" && ! -L "$canonical" ]] || fail "weekly checkpoint is not a regular directory"
|
|
identity="$(validated_batch_identity "$canonical")" \
|
|
|| fail "weekly checkpoint validation failed during recovery"
|
|
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
|
[[ "$iso_week" == "$week" ]] || fail "weekly checkpoint manifest belongs to another week"
|
|
fi
|
|
if [[ -n "$rollback_dir" ]]; then
|
|
identity="$(validated_batch_identity "$rollback_dir")" \
|
|
|| fail "weekly rollback validation failed"
|
|
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
|
[[ "$iso_week" == "$week" ]] || fail "weekly rollback manifest belongs to another week"
|
|
fi
|
|
|
|
source="${RECONCILED_WEEK_SOURCE[$week]:-}"
|
|
if [[ -z "$source" && ! -e "$canonical" && -z "$rollback_dir" && -n "$new_dir" ]]; then
|
|
source="$(unique_daily_source_for_week "$week")" \
|
|
|| fail "weekly staging recovery source could not be resolved"
|
|
fi
|
|
if [[ -n "$source" ]]; then
|
|
[[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
|
|
identity="$(validated_batch_identity "$source")" \
|
|
|| fail "weekly rebuild source validation failed"
|
|
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
|
[[ "$iso_week" == "$week" ]] || fail "weekly rebuild source belongs to another week"
|
|
sources[$week]="$source"
|
|
fi
|
|
[[ -e "$canonical" || -n "$rollback_dir" || -n "$source" ]] \
|
|
|| fail "weekly checkpoint has no authoritative recovery source"
|
|
done
|
|
|
|
for week in "${!weeks[@]}"; do
|
|
canonical="$BACKUP_WEEKLY_DIR/$week"
|
|
new_dir="${new_by_week[$week]:-}"
|
|
rollback_dir="${rollback_by_week[$week]:-}"
|
|
|
|
if [[ ! -e "$canonical" && -n "$rollback_dir" ]]; then
|
|
validate_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
|
|
'^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
|
|
mv -- "$rollback_dir" "$canonical"
|
|
rollback_dir=""
|
|
fi
|
|
|
|
source="${sources[$week]:-}"
|
|
if [[ -n "$source" ]]; then
|
|
[[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
|
|
if ! publish_weekly_checkpoint "$source" "$canonical" "$([[ -e "$canonical" ]] && echo 1 || echo 0)"; then
|
|
fail "weekly checkpoint reconciliation failed"
|
|
fi
|
|
fi
|
|
|
|
[[ -e "$canonical" ]] || fail "weekly checkpoint is missing after reconciliation"
|
|
if [[ -n "$rollback_dir" ]]; then
|
|
remove_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
|
|
'^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
|
|
fi
|
|
if [[ -n "$new_dir" ]]; then
|
|
remove_temp_directory "$new_dir" "$BACKUP_WEEKLY_DIR" \
|
|
'^week-[0-9]{4}-W[0-9]{2}\.new\.[0-9]+$'
|
|
fi
|
|
done
|
|
local pending_daily_orphan
|
|
for pending_daily_orphan in "${DAILY_ORPHANS_PENDING_DELETE[@]}"; do
|
|
remove_temp_directory "$pending_daily_orphan" "$BACKUP_ARCHIVE_DIR" \
|
|
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
|
done
|
|
DAILY_ORPHANS_PENDING_DELETE=()
|
|
RECONCILED_WEEK_SOURCE=()
|
|
}
|
|
|
|
reconcile_interrupted_finalize() {
|
|
reconcile_daily_orphans
|
|
reconcile_weekly_orphans
|
|
}
|
|
|
|
finalize_batch() {
|
|
local run_id="$1"
|
|
local expected_manifest_sha="$2"
|
|
validate_run_id "$run_id"
|
|
validate_layout
|
|
exec 9>"$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
|
chmod 0600 "$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
|
flock -n 9 || fail "another finalize operation is already running"
|
|
reconcile_interrupted_finalize
|
|
|
|
local batch_name="hyb-oa-backup-$run_id"
|
|
local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"
|
|
local archive_dir="$BACKUP_ARCHIVE_DIR/$batch_name"
|
|
local already_archived=0
|
|
if [[ -e "$archive_dir" ]]; then
|
|
[[ -d "$archive_dir" && ! -L "$archive_dir" ]] || fail "archive target is invalid"
|
|
validate_batch "$archive_dir" "$run_id" "$expected_manifest_sha"
|
|
already_archived=1
|
|
fi
|
|
|
|
local run_day="${run_id#auto-}"
|
|
run_day="${run_day%%T*}"
|
|
local existing_for_day="" replacement_rollback=""
|
|
local -a existing_for_day_entries=()
|
|
mapfile -t existing_for_day_entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
|
|
-name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print)
|
|
[[ "${#existing_for_day_entries[@]}" -le 1 ]] || fail "daily archive invariant is already violated"
|
|
if [[ "${#existing_for_day_entries[@]}" -eq 1 ]]; then
|
|
existing_for_day="${existing_for_day_entries[0]}"
|
|
fi
|
|
|
|
if [[ "$already_archived" == "0" ]]; then
|
|
validate_batch "$partial_dir" "$run_id" "$expected_manifest_sha"
|
|
chmod 0400 "$partial_dir/database.dump" "$partial_dir/storage.tar.gz" "$partial_dir/manifest.json"
|
|
if [[ -n "$existing_for_day" ]]; then
|
|
[[ "$existing_for_day" =~ /hyb-oa-backup-auto-${run_day}T[0-9]{6}Z$ \
|
|
&& -d "$existing_for_day" && ! -L "$existing_for_day" ]] \
|
|
|| fail "existing daily archive is invalid"
|
|
replacement_rollback="$BACKUP_ARCHIVE_DIR/.daily-replacement-${run_day}.$$"
|
|
[[ ! -e "$replacement_rollback" ]] || fail "daily replacement rollback target already exists"
|
|
mv -- "$existing_for_day" "$replacement_rollback"
|
|
fi
|
|
if ! mv -- "$partial_dir" "$archive_dir"; then
|
|
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
|
mv -- "$replacement_rollback" "$existing_for_day"
|
|
fi
|
|
fail "daily archive publication failed"
|
|
fi
|
|
fi
|
|
|
|
local iso_week
|
|
iso_week="$($PYTHON_BIN - "$run_day" <<'PYTHON'
|
|
from datetime import datetime
|
|
import sys
|
|
day = datetime.strptime(sys.argv[1], "%Y%m%d").date()
|
|
iso = day.isocalendar()
|
|
print(f"week-{iso.year:04d}-W{iso.week:02d}")
|
|
PYTHON
|
|
)"
|
|
local weekly_dir="$BACKUP_WEEKLY_DIR/$iso_week"
|
|
if ! publish_weekly_checkpoint "$archive_dir" "$weekly_dir" "$([[ -n "$replacement_rollback" ]] && echo 1 || echo 0)"; then
|
|
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
|
rm -rf -- "$archive_dir"
|
|
mv -- "$replacement_rollback" "$existing_for_day"
|
|
fi
|
|
fail "weekly checkpoint update failed"
|
|
fi
|
|
if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
|
|
rm -rf -- "$replacement_rollback"
|
|
fi
|
|
|
|
prune_named_directories "$BACKUP_ARCHIVE_DIR" "hyb-oa-backup-auto-" "$DAILY_KEEP"
|
|
prune_named_directories "$BACKUP_WEEKLY_DIR" "week-" "$WEEKLY_KEEP"
|
|
echo "[offhost-receiver][PASS] finalized batch with verified manifest"
|
|
}
|
|
|
|
run_forced_command() {
|
|
validate_layout
|
|
local original_command="${SSH_ORIGINAL_COMMAND:-}"
|
|
if [[ "$original_command" == rsync\ --server\ * ]]; then
|
|
if [[ "$original_command" =~ (^|[[:space:]])--sender([[:space:]]|$) ]]; then
|
|
fail "read access is forbidden"
|
|
fi
|
|
if [[ "$original_command" =~ (^|[[:space:]])--(delete|delete-before|delete-after|delete-during|delete-delay|delete-excluded|remove-source-files|remove-sent-files|inplace)(=|[[:space:]]|$) ]]; then
|
|
fail "destructive rsync option is forbidden"
|
|
fi
|
|
[[ -x "$RRSYNC_BIN" ]] || fail "rrsync is required"
|
|
exec "$RRSYNC_BIN" -wo "$BACKUP_INBOX_DIR"
|
|
fi
|
|
|
|
if [[ "$original_command" =~ ^hyb-backup-finalize[[:space:]](auto-[0-9]{8}T[0-9]{6}Z)[[:space:]]([0-9a-f]{64})$ ]]; then
|
|
finalize_batch "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
|
|
return 0
|
|
fi
|
|
fail "unsupported forced command"
|
|
}
|
|
|
|
case "${1:-forced}" in
|
|
forced)
|
|
run_forced_command
|
|
;;
|
|
finalize-test)
|
|
[[ "${RECEIVER_TEST_MODE:-0}" == "1" ]] || fail "test mode is disabled"
|
|
finalize_batch "${2:-}" "${3:-}"
|
|
;;
|
|
validate-config)
|
|
validate_layout
|
|
echo "[offhost-receiver][PASS] configuration valid"
|
|
;;
|
|
*)
|
|
fail "unsupported mode"
|
|
;;
|
|
esac
|