#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" CHECKER="$ROOT_DIR/deploy/trialrun/check-existing-deployment.sh" HELPER="$ROOT_DIR/scripts/refresh-deployment-contract.sh" LIBRARY="$ROOT_DIR/deploy/trialrun/deployment-contract-lib.sh" INSTALLER="$ROOT_DIR/deploy/trialrun/install-deployment-contract-refresh.sh" FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hyb-oa-contract-refresh.XXXXXX")" trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT mkdir -p "$FIXTURE_ROOT/bin" "$FIXTURE_ROOT/etc/systemd/system" "$FIXTURE_ROOT/run/lock" cat >"$FIXTURE_ROOT/bin/ssh" <<'SH' #!/usr/bin/env bash set -uo pipefail target="${1:-}" shift || true mode="${FIXTURE_SSH_MODE:-pass}" if [[ "$target" == "deploy" && "${1:-}" == "true" ]]; then [[ "$mode" != "deploy_transport_fail" ]] || exit 255 exit 0 fi if [[ "$target" == "deploy" && "$*" == *"systemctl show"* ]]; then [[ "$mode" != "contract_fail" ]] || exit 1 printf '%s\n' 'REMOTE_DIR=/srv/app REMOTE_ENV_FILE=/srv/app/deploy/trialrun/.env COMPOSE_FILE=deploy/trialrun/docker-compose.yml COMPOSE_PROJECT_NAME=fixture POSTGRES_DATA_DIR=/srv/data/postgres STORAGE_HOST_DIR=/srv/data/storage BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' exit 0 fi if [[ "$target" == "deploy" ]]; then if [[ "$mode" == "multi_fail" ]]; then cat <<'OUT' repository_root=FAIL repository_writable=PASS env_readable=FAIL data_directories=PASS runtime_unique=PASS runtime_build=PASS storage_mount=FAIL pgdata_mount=PASS backup_build=PASS OUT else for item in repository_root repository_writable env_readable data_directories runtime_unique runtime_build storage_mount pgdata_mount backup_build; do printf '%s=PASS\n' "$item" done fi exit 0 fi if [[ "$target" == "refresh" && "${1:-}" == "probe" ]]; then case "$mode" in root_transport_fail) exit 255 ;; program_missing|multi_fail) exit 127 ;; *) echo '[deployment-contract-refresh][PASS] probe'; exit 0 ;; esac fi exit 1 SH chmod 0755 "$FIXTURE_ROOT/bin/ssh" run_checker() { PATH="$FIXTURE_ROOT/bin:$PATH" DEPLOY_SERVER=deploy DEPLOY_REFRESH_SERVER=refresh \ FIXTURE_SSH_MODE="${1:-pass}" bash "$CHECKER" } pass_output="$(run_checker pass)" [[ "$pass_output" == *"summary PASS=13 FAIL=0"* ]] set +e multi_output="$(run_checker multi_fail 2>&1)" multi_status=$? set -e [[ "$multi_status" -ne 0 ]] [[ "$multi_output" == *"repository_root FAIL"* \ && "$multi_output" == *"env_readable FAIL"* \ && "$multi_output" == *"storage_mount FAIL"* \ && "$multi_output" == *"refresh_program FAIL"* \ && "$multi_output" == *"summary PASS=9 FAIL=4"* ]] [[ "$multi_output" != *"/srv/"* && "$multi_output" != *"aaaaaaaaaaaaaaaa"* ]] set +e root_output="$(run_checker root_transport_fail 2>&1)" root_status=$? set -e [[ "$root_status" -ne 0 && "$root_output" == *"root_ssh_identity FAIL"* \ && "$root_output" == *"refresh_program FAIL"* ]] source "$LIBRARY" [[ "$(deployment_contract_refresh_category 255 '')" == "root_identity_unavailable" ]] [[ "$(deployment_contract_refresh_category 127 '')" == "refresh_program_unavailable" ]] [[ "$(deployment_contract_refresh_category 42 '')" == "unit_build_not_refreshed" ]] [[ "$(deployment_contract_refresh_category 43 '')" == "timer_not_active" ]] [[ "$(deployment_contract_refresh_category 0 '[deployment-contract-refresh][PASS] refreshed')" == "success" ]] for service in \ hyb-oa-automatic-backup-daily.service \ hyb-oa-automatic-backup-watchdog.service \ hyb-oa-automatic-backup-drill-reminder.service \ hyb-oa-warch3-ops.service; do : >"$FIXTURE_ROOT/etc/systemd/system/$service" done cat >"$FIXTURE_ROOT/bin/docker" <<'SH' #!/usr/bin/env bash if [[ "${1:-}" == "ps" ]]; then echo server-fixture elif [[ "${1:-}" == "inspect" ]]; then echo 'BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' else exit 1 fi SH chmod 0755 "$FIXTURE_ROOT/bin/docker" cat >"$FIXTURE_ROOT/bin/systemctl" <<'SH' #!/usr/bin/env bash set -euo pipefail root="${HYB_CONTRACT_REFRESH_TEST_ROOT:?}" action="${1:-}" case "$action" in show) unit="${2:-}" property="${4:-}" case "$property" in LoadState) echo loaded ;; FragmentPath) echo "$root/etc/systemd/system/$unit" ;; Environment) echo 'COMPOSE_PROJECT_NAME=fixture BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa VITE_BUILD_SHA=aaaaaaaaaaaa OLD_BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb OLD_VITE_BUILD_SHA=bbbbbbbbbbbb' if [[ "${FIXTURE_IGNORE_DROPINS:-0}" != "1" ]]; then dropin="$root/etc/systemd/system/${unit}.d/20-active-build.conf" [[ ! -f "$dropin" ]] || sed -n 's/^Environment=//p' "$dropin" | paste -sd' ' - fi ;; *) exit 1 ;; esac ;; daemon-reload) exit 0 ;; is-enabled) unit="${2:-}" if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then case "${FIXTURE_BACKUP_TIMERS:-running}" in disabled|disabled_active) echo disabled; exit 1 ;; mixed) if [[ "$unit" == *watchdog* ]]; then echo enabled; else echo disabled; exit 1; fi ;; *) echo enabled ;; esac else echo enabled fi ;; is-active) unit="${2:-}" if [[ "$unit" == *.service ]]; then case "$unit" in hyb-oa-automatic-backup-daily.service) state="${FIXTURE_DAILY_SERVICE_STATE:-inactive}" ;; hyb-oa-automatic-backup-watchdog.service) state="${FIXTURE_WATCHDOG_SERVICE_STATE:-inactive}" ;; hyb-oa-automatic-backup-drill-reminder.service) state="${FIXTURE_DRILL_SERVICE_STATE:-inactive}" ;; *) state=inactive ;; esac echo "$state" [[ "$state" == "active" ]] || exit 3 exit 0 fi if [[ "${FIXTURE_TIMER_INACTIVE:-0}" == "1" ]]; then echo inactive; exit 3; fi if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then case "${FIXTURE_BACKUP_TIMERS:-running}" in disabled) echo inactive; exit 3 ;; disabled_active) echo active ;; mixed) if [[ "$unit" == *watchdog* ]]; then echo active; else echo inactive; exit 3; fi ;; esac elif [[ "${FIXTURE_OPS_TIMER_INACTIVE:-0}" == "1" ]]; then echo inactive; exit 3 fi echo active ;; *) exit 1 ;; esac SH chmod 0755 "$FIXTURE_ROOT/bin/systemctl" probe_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=probe bash "$HELPER")" [[ "$probe_output" == '[deployment-contract-refresh][PASS] probe' ]] refresh_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")" [[ "$refresh_output" == '[deployment-contract-refresh][PASS] refreshed' ]] [[ "$(find "$FIXTURE_ROOT/etc/systemd/system" -name 20-active-build.conf -type f | wc -l)" -eq 4 ]] set +e HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_IGNORE_DROPINS=1 \ SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1 unit_status=$? HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_TIMER_INACTIVE=1 \ SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1 timer_status=$? HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" \ SSH_ORIGINAL_COMMAND='refresh extra' bash "$HELPER" >/dev/null 2>&1 unsupported_status=$? set -e [[ "$unit_status" -eq 42 && "$timer_status" -eq 43 && "$unsupported_status" -eq 64 ]] run_refresh_status() { local command="$1" shift set +e env HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" "$@" \ SSH_ORIGINAL_COMMAND="$command" bash "$HELPER" >/dev/null 2>&1 local status=$? set -e printf '%s' "$status" } mode_file="$FIXTURE_ROOT/etc/hyb-oa-backup-mode" # Automatic mode (no mode file): disabled backup timers stay fail-closed, # both before (probe) and after (refresh) a release. [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]] [[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]] [[ "$(run_refresh_status probe FIXTURE_TIMER_INACTIVE=1)" -eq 43 ]] printf 'manual_snapshot\n' >"$mode_file" manual_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_BACKUP_TIMERS=disabled \ SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")" [[ "$manual_output" == '[deployment-contract-refresh][PASS] refreshed' ]] [[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 0 ]] # Manual mode must not silently accept running or partially running backup timers, # and the pre-deploy probe must reject the mismatch before any release write. [[ "$(run_refresh_status refresh)" -eq 45 ]] [[ "$(run_refresh_status probe)" -eq 45 ]] [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]] [[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]] # Disabled timers with an in-flight backup run are not a finished switch. [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=active)" -eq 45 ]] [[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=activating)" -eq 45 ]] [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=failed)" -eq 0 ]] # Each non-daily service can still be running independently of the daily job. # disable without --now also leaves a timer active despite its disabled state. for phase in probe refresh; do for service_state in FIXTURE_WATCHDOG_SERVICE_STATE FIXTURE_DRILL_SERVICE_STATE; do [[ "$(run_refresh_status "$phase" FIXTURE_BACKUP_TIMERS=disabled "$service_state=active")" -eq 45 ]] done [[ "$(run_refresh_status "$phase" FIXTURE_BACKUP_TIMERS=disabled_active)" -eq 45 ]] done # Non-backup ops timer is still required in manual mode. [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_OPS_TIMER_INACTIVE=1)" -eq 43 ]] printf 'manual\n' >"$mode_file" [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]] [[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]] printf 'manual_snapshot\nautomatic\n' >"$mode_file" [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]] rm -f -- "$mode_file" printf 'manual_snapshot\n' >"$FIXTURE_ROOT/backup-mode-target" ln -s "$FIXTURE_ROOT/backup-mode-target" "$mode_file" [[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]] # A dangling link is invalid, not an absent file selecting automatic mode. rm -f -- "$FIXTURE_ROOT/backup-mode-target" [[ -L "$mode_file" && ! -e "$mode_file" ]] [[ "$(run_refresh_status probe)" -eq 44 ]] [[ "$(run_refresh_status refresh)" -eq 44 ]] rm -f -- "$mode_file" [[ "$(run_refresh_status refresh)" -eq 0 ]] [[ "$(deployment_contract_refresh_category 44 '')" == "backup_mode_invalid" ]] [[ "$(deployment_contract_refresh_category 45 '')" == "backup_timer_not_disabled" ]] public_key_file="$FIXTURE_ROOT/refresh.pub" key_blob='AAAAC3NzaC1lZDI1NTE5AAAAIFixturerefreshkeymaterial000000000000' printf 'ssh-ed25519 %s fixture-refresh\n' "$key_blob" >"$public_key_file" installer_output="$(HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \ DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install)" [[ "$installer_output" == '[deployment-contract-refresh-installer][PASS] forced command installed' ]] authorized_keys="$FIXTURE_ROOT/root/.ssh/authorized_keys" [[ "$(grep -Fc '# hyb-oa-managed-deployment-contract-refresh' "$authorized_keys")" -eq 1 ]] printf 'ssh-ed25519 %s unrestricted-duplicate\n' "$key_blob" >>"$authorized_keys" set +e HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \ DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install >/dev/null 2>&1 duplicate_status=$? set -e [[ "$duplicate_status" -ne 0 ]] echo '[deployment-contract-refresh-contract][PASS] aggregate and forced-command contracts verified'