固化部署体检与备份合同刷新（08-02-deployment-preflight-backup-contract-refresh）
这个提交包含在：
@@ -0,0 +1,181 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
CHECKER="$ROOT_DIR/deploy/trialrun/check-existing-deployment.sh"
|
||||
HELPER="$ROOT_DIR/scripts/refresh-deployment-contract.sh"
|
||||
LIBRARY="$ROOT_DIR/deploy/trialrun/deployment-contract-lib.sh"
|
||||
INSTALLER="$ROOT_DIR/deploy/trialrun/install-deployment-contract-refresh.sh"
|
||||
FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hyb-oa-contract-refresh.XXXXXX")"
|
||||
trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT
|
||||
mkdir -p "$FIXTURE_ROOT/bin" "$FIXTURE_ROOT/etc/systemd/system" "$FIXTURE_ROOT/run/lock"
|
||||
|
||||
cat >"$FIXTURE_ROOT/bin/ssh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
target="${1:-}"
|
||||
shift || true
|
||||
mode="${FIXTURE_SSH_MODE:-pass}"
|
||||
if [[ "$target" == "deploy" && "${1:-}" == "true" ]]; then
|
||||
[[ "$mode" != "deploy_transport_fail" ]] || exit 255
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$target" == "deploy" && "$*" == *"systemctl show"* ]]; then
|
||||
[[ "$mode" != "contract_fail" ]] || exit 1
|
||||
printf '%s\n' 'REMOTE_DIR=/srv/app REMOTE_ENV_FILE=/srv/app/deploy/trialrun/.env COMPOSE_FILE=deploy/trialrun/docker-compose.yml COMPOSE_PROJECT_NAME=fixture POSTGRES_DATA_DIR=/srv/data/postgres STORAGE_HOST_DIR=/srv/data/storage BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$target" == "deploy" ]]; then
|
||||
if [[ "$mode" == "multi_fail" ]]; then
|
||||
cat <<'OUT'
|
||||
repository_root=FAIL
|
||||
repository_writable=PASS
|
||||
env_readable=FAIL
|
||||
data_directories=PASS
|
||||
runtime_unique=PASS
|
||||
runtime_build=PASS
|
||||
storage_mount=FAIL
|
||||
pgdata_mount=PASS
|
||||
backup_build=PASS
|
||||
OUT
|
||||
else
|
||||
for item in repository_root repository_writable env_readable data_directories runtime_unique runtime_build storage_mount pgdata_mount backup_build; do
|
||||
printf '%s=PASS\n' "$item"
|
||||
done
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$target" == "refresh" && "${1:-}" == "probe" ]]; then
|
||||
case "$mode" in
|
||||
root_transport_fail) exit 255 ;;
|
||||
program_missing|multi_fail) exit 127 ;;
|
||||
*) echo '[deployment-contract-refresh][PASS] probe'; exit 0 ;;
|
||||
esac
|
||||
fi
|
||||
exit 1
|
||||
SH
|
||||
chmod 0755 "$FIXTURE_ROOT/bin/ssh"
|
||||
|
||||
run_checker() {
|
||||
PATH="$FIXTURE_ROOT/bin:$PATH" DEPLOY_SERVER=deploy DEPLOY_REFRESH_SERVER=refresh \
|
||||
FIXTURE_SSH_MODE="${1:-pass}" bash "$CHECKER"
|
||||
}
|
||||
|
||||
pass_output="$(run_checker pass)"
|
||||
[[ "$pass_output" == *"summary PASS=13 FAIL=0"* ]]
|
||||
|
||||
set +e
|
||||
multi_output="$(run_checker multi_fail 2>&1)"
|
||||
multi_status=$?
|
||||
set -e
|
||||
[[ "$multi_status" -ne 0 ]]
|
||||
[[ "$multi_output" == *"repository_root FAIL"* \
|
||||
&& "$multi_output" == *"env_readable FAIL"* \
|
||||
&& "$multi_output" == *"storage_mount FAIL"* \
|
||||
&& "$multi_output" == *"refresh_program FAIL"* \
|
||||
&& "$multi_output" == *"summary PASS=9 FAIL=4"* ]]
|
||||
[[ "$multi_output" != *"/srv/"* && "$multi_output" != *"aaaaaaaaaaaaaaaa"* ]]
|
||||
|
||||
set +e
|
||||
root_output="$(run_checker root_transport_fail 2>&1)"
|
||||
root_status=$?
|
||||
set -e
|
||||
[[ "$root_status" -ne 0 && "$root_output" == *"root_ssh_identity FAIL"* \
|
||||
&& "$root_output" == *"refresh_program FAIL"* ]]
|
||||
|
||||
source "$LIBRARY"
|
||||
[[ "$(deployment_contract_refresh_category 255 '')" == "root_identity_unavailable" ]]
|
||||
[[ "$(deployment_contract_refresh_category 127 '')" == "refresh_program_unavailable" ]]
|
||||
[[ "$(deployment_contract_refresh_category 42 '')" == "unit_build_not_refreshed" ]]
|
||||
[[ "$(deployment_contract_refresh_category 43 '')" == "timer_not_active" ]]
|
||||
[[ "$(deployment_contract_refresh_category 0 '[deployment-contract-refresh][PASS] refreshed')" == "success" ]]
|
||||
|
||||
for service in \
|
||||
hyb-oa-automatic-backup-daily.service \
|
||||
hyb-oa-automatic-backup-watchdog.service \
|
||||
hyb-oa-automatic-backup-drill-reminder.service \
|
||||
hyb-oa-warch3-ops.service; do
|
||||
: >"$FIXTURE_ROOT/etc/systemd/system/$service"
|
||||
done
|
||||
|
||||
cat >"$FIXTURE_ROOT/bin/docker" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
if [[ "${1:-}" == "ps" ]]; then
|
||||
echo server-fixture
|
||||
elif [[ "${1:-}" == "inspect" ]]; then
|
||||
echo 'BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
SH
|
||||
chmod 0755 "$FIXTURE_ROOT/bin/docker"
|
||||
|
||||
cat >"$FIXTURE_ROOT/bin/systemctl" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root="${HYB_CONTRACT_REFRESH_TEST_ROOT:?}"
|
||||
action="${1:-}"
|
||||
case "$action" in
|
||||
show)
|
||||
unit="${2:-}"
|
||||
property="${4:-}"
|
||||
case "$property" in
|
||||
LoadState) echo loaded ;;
|
||||
FragmentPath) echo "$root/etc/systemd/system/$unit" ;;
|
||||
Environment)
|
||||
echo 'COMPOSE_PROJECT_NAME=fixture BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa VITE_BUILD_SHA=aaaaaaaaaaaa OLD_BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb OLD_VITE_BUILD_SHA=bbbbbbbbbbbb'
|
||||
if [[ "${FIXTURE_IGNORE_DROPINS:-0}" != "1" ]]; then
|
||||
dropin="$root/etc/systemd/system/${unit}.d/20-active-build.conf"
|
||||
[[ ! -f "$dropin" ]] || sed -n 's/^Environment=//p' "$dropin" | paste -sd' ' -
|
||||
fi
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
daemon-reload) exit 0 ;;
|
||||
is-enabled) echo enabled ;;
|
||||
is-active)
|
||||
if [[ "${FIXTURE_TIMER_INACTIVE:-0}" == "1" ]]; then echo inactive; exit 3; fi
|
||||
echo active
|
||||
;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
SH
|
||||
chmod 0755 "$FIXTURE_ROOT/bin/systemctl"
|
||||
|
||||
probe_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=probe bash "$HELPER")"
|
||||
[[ "$probe_output" == '[deployment-contract-refresh][PASS] probe' ]]
|
||||
refresh_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")"
|
||||
[[ "$refresh_output" == '[deployment-contract-refresh][PASS] refreshed' ]]
|
||||
[[ "$(find "$FIXTURE_ROOT/etc/systemd/system" -name 20-active-build.conf -type f | wc -l)" -eq 4 ]]
|
||||
|
||||
set +e
|
||||
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_IGNORE_DROPINS=1 \
|
||||
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1
|
||||
unit_status=$?
|
||||
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_TIMER_INACTIVE=1 \
|
||||
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1
|
||||
timer_status=$?
|
||||
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" \
|
||||
SSH_ORIGINAL_COMMAND='refresh extra' bash "$HELPER" >/dev/null 2>&1
|
||||
unsupported_status=$?
|
||||
set -e
|
||||
[[ "$unit_status" -eq 42 && "$timer_status" -eq 43 && "$unsupported_status" -eq 64 ]]
|
||||
|
||||
public_key_file="$FIXTURE_ROOT/refresh.pub"
|
||||
key_blob='AAAAC3NzaC1lZDI1NTE5AAAAIFixturerefreshkeymaterial000000000000'
|
||||
printf 'ssh-ed25519 %s fixture-refresh\n' "$key_blob" >"$public_key_file"
|
||||
installer_output="$(HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \
|
||||
DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install)"
|
||||
[[ "$installer_output" == '[deployment-contract-refresh-installer][PASS] forced command installed' ]]
|
||||
authorized_keys="$FIXTURE_ROOT/root/.ssh/authorized_keys"
|
||||
[[ "$(grep -Fc '# hyb-oa-managed-deployment-contract-refresh' "$authorized_keys")" -eq 1 ]]
|
||||
printf 'ssh-ed25519 %s unrestricted-duplicate\n' "$key_blob" >>"$authorized_keys"
|
||||
set +e
|
||||
HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \
|
||||
DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install >/dev/null 2>&1
|
||||
duplicate_status=$?
|
||||
set -e
|
||||
[[ "$duplicate_status" -ne 0 ]]
|
||||
|
||||
echo '[deployment-contract-refresh-contract][PASS] aggregate and forced-command contracts verified'
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PATH="/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
export PATH
|
||||
umask 077
|
||||
unset BASH_ENV ENV CDPATH GLOBIGNORE LD_PRELOAD LD_LIBRARY_PATH PYTHONPATH NODE_OPTIONS
|
||||
|
||||
SERVICES=(
|
||||
hyb-oa-automatic-backup-daily.service
|
||||
hyb-oa-automatic-backup-watchdog.service
|
||||
hyb-oa-automatic-backup-drill-reminder.service
|
||||
hyb-oa-warch3-ops.service
|
||||
)
|
||||
TIMERS=(
|
||||
hyb-oa-automatic-backup-daily.timer
|
||||
hyb-oa-automatic-backup-watchdog.timer
|
||||
hyb-oa-automatic-backup-drill-reminder.timer
|
||||
hyb-oa-warch3-ops.timer
|
||||
)
|
||||
CONTRACT_SERVICE="hyb-oa-automatic-backup-daily.service"
|
||||
DROPIN_NAME="20-active-build.conf"
|
||||
ROOT_PREFIX=""
|
||||
TEST_MODE=0
|
||||
|
||||
fail() {
|
||||
local code="$1"
|
||||
local category="$2"
|
||||
echo "[deployment-contract-refresh][FAIL] category=${category}" >&2
|
||||
exit "$code"
|
||||
}
|
||||
|
||||
if [[ "$(id -u)" != "0" ]]; then
|
||||
test_root="${HYB_CONTRACT_REFRESH_TEST_ROOT:-}"
|
||||
if [[ "$test_root" =~ ^/tmp/[A-Za-z0-9._/-]+$ && "$test_root" != */ \
|
||||
&& "$test_root" != *"//"* && "/$test_root/" != *"/../"* ]]; then
|
||||
TEST_MODE=1
|
||||
ROOT_PREFIX="$test_root"
|
||||
PATH="$test_root/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
export PATH
|
||||
else
|
||||
fail 70 root_required
|
||||
fi
|
||||
fi
|
||||
|
||||
UNIT_ROOT="${ROOT_PREFIX}/etc/systemd/system"
|
||||
LOCK_FILE="${ROOT_PREFIX}/run/lock/hyb-oa-deployment-contract-refresh.lock"
|
||||
EXPECTED_TARGET="${ROOT_PREFIX}/usr/local/sbin/hyb-oa-deployment-contract-refresh"
|
||||
|
||||
need_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail 70 runtime_prerequisite_missing
|
||||
}
|
||||
|
||||
for required_command in systemctl docker install mktemp mv mkdir chmod flock stat; do
|
||||
need_command "$required_command"
|
||||
done
|
||||
|
||||
validate_self() {
|
||||
if [[ "$TEST_MODE" == "0" ]]; then
|
||||
[[ "$0" == "$EXPECTED_TARGET" && -f "$EXPECTED_TARGET" && ! -L "$EXPECTED_TARGET" \
|
||||
&& "$(stat -c %u:%g:%a "$EXPECTED_TARGET")" == "0:0:755" ]] \
|
||||
|| fail 70 refresh_program_integrity_invalid
|
||||
else
|
||||
[[ -f "$0" && ! -L "$0" ]] || fail 70 refresh_program_integrity_invalid
|
||||
fi
|
||||
}
|
||||
|
||||
systemctl_value() {
|
||||
local unit="$1"
|
||||
local property="$2"
|
||||
systemctl show "$unit" -p "$property" --value
|
||||
}
|
||||
|
||||
unit_environment_value() {
|
||||
local unit="$1"
|
||||
local wanted="$2"
|
||||
local entry value=""
|
||||
for entry in $(systemctl_value "$unit" Environment); do
|
||||
if [[ "${entry%%=*}" == "$wanted" ]]; then
|
||||
value="${entry#*=}"
|
||||
fi
|
||||
done
|
||||
[[ -n "$value" ]] || return 1
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
validate_fixed_units() {
|
||||
local unit fragment owner
|
||||
for unit in "${SERVICES[@]}"; do
|
||||
[[ "$(systemctl_value "$unit" LoadState)" == "loaded" ]] \
|
||||
|| fail 70 service_not_loaded
|
||||
fragment="$(systemctl_value "$unit" FragmentPath)"
|
||||
[[ -n "$fragment" ]] || fail 70 service_fragment_missing
|
||||
if [[ "$TEST_MODE" == "0" ]]; then
|
||||
[[ "$fragment" == /etc/systemd/system/* && -f "$fragment" && ! -L "$fragment" ]] \
|
||||
|| fail 70 service_fragment_invalid
|
||||
owner="$(stat -c %u "$fragment")"
|
||||
[[ "$owner" == "0" ]] || fail 70 service_fragment_not_root_owned
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
resolve_active_build() {
|
||||
local project
|
||||
local -a servers=()
|
||||
project="$(unit_environment_value "$CONTRACT_SERVICE" COMPOSE_PROJECT_NAME)" \
|
||||
|| fail 70 compose_project_missing
|
||||
[[ "$project" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail 70 compose_project_invalid
|
||||
mapfile -t servers < <(docker ps \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=server" \
|
||||
--format '{{.ID}}')
|
||||
[[ "${#servers[@]}" -eq 1 && -n "${servers[0]}" ]] \
|
||||
|| fail 70 active_server_not_unique
|
||||
ACTIVE_BUILD="$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "${servers[0]}" \
|
||||
| sed -n 's/^BUILD_SHA=//p' | tail -n 1)"
|
||||
ACTIVE_BUILD="${ACTIVE_BUILD,,}"
|
||||
[[ "$ACTIVE_BUILD" =~ ^[0-9a-f]{40,64}$ ]] || fail 70 active_build_invalid
|
||||
ACTIVE_FRONTEND_BUILD="${ACTIVE_BUILD:0:12}"
|
||||
}
|
||||
|
||||
write_build_dropin() {
|
||||
local service="$1"
|
||||
local dropin_dir="$UNIT_ROOT/${service}.d"
|
||||
local target="$dropin_dir/$DROPIN_NAME"
|
||||
local temporary
|
||||
|
||||
if [[ "$TEST_MODE" == "0" ]]; then
|
||||
install -d -o root -g root -m 0755 "$dropin_dir"
|
||||
else
|
||||
mkdir -p "$dropin_dir"
|
||||
chmod 0755 "$dropin_dir"
|
||||
fi
|
||||
temporary="$(mktemp "$dropin_dir/.${DROPIN_NAME}.XXXXXX")"
|
||||
{
|
||||
printf '[Service]\n'
|
||||
printf 'Environment=BUILD_SHA=%s\n' "$ACTIVE_BUILD"
|
||||
printf 'Environment=VITE_BUILD_SHA=%s\n' "$ACTIVE_FRONTEND_BUILD"
|
||||
} >"$temporary"
|
||||
chmod 0644 "$temporary"
|
||||
if [[ "$TEST_MODE" == "0" ]]; then
|
||||
chown root:root "$temporary"
|
||||
fi
|
||||
mv -f -- "$temporary" "$target"
|
||||
}
|
||||
|
||||
verify_builds() {
|
||||
local service service_build service_frontend_build
|
||||
for service in "${SERVICES[@]}"; do
|
||||
service_build="$(unit_environment_value "$service" BUILD_SHA)" \
|
||||
|| fail 42 unit_build_not_refreshed
|
||||
service_frontend_build="$(unit_environment_value "$service" VITE_BUILD_SHA)" \
|
||||
|| fail 42 unit_build_not_refreshed
|
||||
[[ "$service_build" == "$ACTIVE_BUILD" \
|
||||
&& "$service_frontend_build" == "$ACTIVE_FRONTEND_BUILD" ]] \
|
||||
|| fail 42 unit_build_not_refreshed
|
||||
done
|
||||
}
|
||||
|
||||
verify_timers() {
|
||||
local timer
|
||||
for timer in "${TIMERS[@]}"; do
|
||||
[[ "$(systemctl is-enabled "$timer")" == "enabled" \
|
||||
&& "$(systemctl is-active "$timer")" == "active" ]] \
|
||||
|| fail 43 timer_not_active
|
||||
done
|
||||
}
|
||||
|
||||
probe() {
|
||||
validate_self
|
||||
validate_fixed_units
|
||||
resolve_active_build
|
||||
echo "[deployment-contract-refresh][PASS] probe"
|
||||
}
|
||||
|
||||
refresh() {
|
||||
validate_self
|
||||
validate_fixed_units
|
||||
resolve_active_build
|
||||
mkdir -p "$(dirname "$LOCK_FILE")"
|
||||
exec 9>"$LOCK_FILE"
|
||||
flock -n 9 || fail 42 refresh_already_running
|
||||
local service
|
||||
for service in "${SERVICES[@]}"; do
|
||||
write_build_dropin "$service"
|
||||
done
|
||||
systemctl daemon-reload
|
||||
verify_builds
|
||||
verify_timers
|
||||
echo "[deployment-contract-refresh][PASS] refreshed"
|
||||
}
|
||||
|
||||
[[ "$#" -eq 0 ]] || fail 64 unsupported_forced_command
|
||||
case "${SSH_ORIGINAL_COMMAND:-}" in
|
||||
probe) probe ;;
|
||||
refresh) refresh ;;
|
||||
*) fail 64 unsupported_forced_command ;;
|
||||
esac
|
||||
在新工单中引用
屏蔽一个用户