#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" TEST_DIR="$(mktemp -d)" STUB_PID="" PASS_COUNT=0 cleanup() { if [[ -n "$STUB_PID" ]]; then kill "$STUB_PID" >/dev/null 2>&1 || true wait "$STUB_PID" >/dev/null 2>&1 || true fi rm -rf -- "$TEST_DIR" } trap cleanup EXIT pass() { PASS_COUNT=$((PASS_COUNT + 1)) echo "[PASS] $1" } expect_failure() { local label="$1" shift if "$@"; then echo "[FAIL] expected failure: $label" >&2 exit 1 fi pass "$label" } capture_lines() { local capture="$1" if [[ -f "$capture" ]]; then wc -l <"$capture" | tr -d ' ' else echo 0 fi } new_state_dir() { local target="$1" mkdir -p "$target" chmod 0700 "$target" } set_mode() { printf '%s\n' "$1" >"$TEST_DIR/mode" } probe() { local state_dir="$1" local capture="$2" shift 2 env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$capture" \ python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ --ready-url "https://localhost:${STUB_PORT}/ready" \ --state-dir "$state_dir" \ --webhook-file "$TEST_DIR/wecom-webhook" \ --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ --failure-threshold 2 \ --timeout-seconds 1 \ --ca-file "$TEST_DIR/cert.pem" \ "$@" } openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 1 \ -subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost' \ -keyout "$TEST_DIR/key.pem" -out "$TEST_DIR/cert.pem" >/dev/null 2>&1 chmod 0600 "$TEST_DIR/key.pem" "$TEST_DIR/cert.pem" WECOM_TEST_HOST='qyapi.weixin.qq.com' WECOM_TEST_PATH='/cgi-bin/webhook/send' printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \ '01234567-89ab-cdef-0123-456789abcdef' >"$TEST_DIR/wecom-webhook" chmod 0600 "$TEST_DIR/wecom-webhook" printf '%s\n' healthy >"$TEST_DIR/mode" python3 - "$TEST_DIR/cert.pem" "$TEST_DIR/key.pem" "$TEST_DIR/mode" "$TEST_DIR/port" <<'PY' >/dev/null 2>&1 & import http.server import json import ssl import sys import time from pathlib import Path cert_file, key_file, mode_file, port_file = map(Path, sys.argv[1:]) class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): mode = mode_file.read_text(encoding="utf-8").strip() if self.path != "/ready": self.send_response(404) self.end_headers() return if mode == "timeout": time.sleep(2) if mode == "unavailable": status, body = 503, b'{"error":{"details":{"db":"do-not-leak"}}}' elif mode == "invalid-json": status, body = 200, b'not-json-private-payload' elif mode == "bad-contract": status, body = 200, json.dumps({"data": {"ok": False}}).encode() elif mode == "oversized": status, body = 200, b"x" * (64 * 1024 + 1) else: status, body = 200, json.dumps({"data": {"ok": True, "buildSha": "abcdef123456"}}).encode() self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *_args): pass server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) context.load_cert_chain(certfile=cert_file, keyfile=key_file) server.socket = context.wrap_socket(server.socket, server_side=True) port_file.write_text(str(server.server_port), encoding="utf-8") server.serve_forever() PY STUB_PID=$! for _ in $(seq 1 50); do [[ -s "$TEST_DIR/port" ]] && break sleep 0.1 done [[ -s "$TEST_DIR/port" ]] STUB_PORT="$(cat "$TEST_DIR/port")" MAIN_STATE="$TEST_DIR/main-state" MAIN_CAPTURE="$TEST_DIR/main-capture.jsonl" new_state_dir "$MAIN_STATE" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null [[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]] pass "healthy baseline writes state without sending an alert" set_mode unavailable expect_failure "first failure stays pending" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 [[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]] expect_failure "second consecutive failure triggers alert evaluation" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 [[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]] expect_failure "identical continued failure remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 [[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]] pass "identical application failure is deduplicated" set_mode invalid-json expect_failure "changed failure category remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 [[ "$(capture_lines "$MAIN_CAPTURE")" == "2" ]] pass "changed failure category sends one updated alert" set_mode healthy probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null [[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]] probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null [[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]] pass "recovery sends exactly one notification" TRANSIENT_STATE="$TEST_DIR/transient-state" TRANSIENT_CAPTURE="$TEST_DIR/transient-capture.jsonl" new_state_dir "$TRANSIENT_STATE" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null set_mode unavailable expect_failure "single transient outage is recorded" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null 2>&1 set_mode healthy probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null [[ "$(capture_lines "$TRANSIENT_CAPTURE")" == "0" ]] pass "single transient outage does not send failure or recovery noise" for mode in bad-contract oversized timeout; do CASE_STATE="$TEST_DIR/${mode}-state" new_state_dir "$CASE_STATE" set_mode "$mode" expect_failure "$mode is rejected" probe "$CASE_STATE" "$TEST_DIR/${mode}-capture.jsonl" >/dev/null 2>&1 python3 - "$CASE_STATE/app-health-probe-state.json" "$mode" <<'PY' import json, sys expected = { "bad-contract": "response_contract_invalid", "oversized": "response_too_large", "timeout": "request_timeout", }[sys.argv[2]] payload = json.load(open(sys.argv[1], encoding="utf-8")) assert payload["lastCategory"] == expected, payload PY done UNUSED_PORT="$(python3 - <<'PY' import socket sock = socket.socket() sock.bind(("127.0.0.1", 0)) print(sock.getsockname()[1]) sock.close() PY )" CONNECT_STATE="$TEST_DIR/connect-state" new_state_dir "$CONNECT_STATE" expect_failure "connection failure is rejected" env \ HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/connect-capture.jsonl" \ python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ --ready-url "https://localhost:${UNUSED_PORT}/ready" --state-dir "$CONNECT_STATE" \ --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ --failure-threshold 2 --timeout-seconds 1 --ca-file "$TEST_DIR/cert.pem" >/dev/null 2>&1 python3 - "$CONNECT_STATE/app-health-probe-state.json" <<'PY' import json, sys payload = json.load(open(sys.argv[1], encoding="utf-8")) assert payload["lastCategory"] == "dns_or_connect_failed", payload PY pass "connection failure is stored as a stable category" TLS_STATE="$TEST_DIR/tls-state" new_state_dir "$TLS_STATE" expect_failure "untrusted TLS certificate is rejected" env \ HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/tls-capture.jsonl" \ python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ --ready-url "https://localhost:${STUB_PORT}/ready" --state-dir "$TLS_STATE" \ --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ --failure-threshold 2 --timeout-seconds 1 >/dev/null 2>&1 python3 - "$TLS_STATE/app-health-probe-state.json" <<'PY' import json, sys payload = json.load(open(sys.argv[1], encoding="utf-8")) assert payload["lastCategory"] == "tls_validation_failed", payload PY pass "TLS failure is stored as a stable category" CORRUPT_STATE="$TEST_DIR/corrupt-state" new_state_dir "$CORRUPT_STATE" printf '%s\n' 'private-payload-must-not-appear' >"$CORRUPT_STATE/app-health-probe-state.json" chmod 0600 "$CORRUPT_STATE/app-health-probe-state.json" set_mode healthy expect_failure "corrupt local state fails closed" probe "$CORRUPT_STATE" "$TEST_DIR/corrupt-capture.jsonl" \ >"$TEST_DIR/corrupt-stdout" 2>"$TEST_DIR/corrupt-stderr" grep -Fxq '[app-health][ERROR] local_state_invalid' "$TEST_DIR/corrupt-stderr" ! grep -Fq 'private-payload' "$TEST_DIR/corrupt-stderr" ! grep -Fq "$TEST_DIR" "$TEST_DIR/corrupt-stderr" pass "local state error output is stable and redacted" python3 - "$MAIN_CAPTURE" <<'PY' import json, sys messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")] assert len(messages) == 3, messages assert "FAIL_APP_HEALTH" in messages[0] assert "http_unavailable" in messages[0] assert "response_contract_invalid" in messages[1] assert "PASS_APP_HEALTH_READY" in messages[2] for forbidden in ("do-not-leak", "localhost", "https://", "/tmp/", "not-json-private-payload"): assert all(forbidden not in message for message in messages), (forbidden, messages) PY pass "captured alerts contain only stable status and category text" expect_failure "plain HTTP readiness URL is rejected" env \ HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/http-capture.jsonl" \ python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ --ready-url "http://localhost:${STUB_PORT}/ready" --state-dir "$TEST_DIR/http-state" \ --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ >/dev/null 2>&1 PYTHONPYCACHEPREFIX="$TEST_DIR/pycache" python3 -m py_compile \ "$ROOT_DIR/scripts/check-public-app-health.py" "$ROOT_DIR/scripts/send-wecom-group-alert.py" bash -n "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'OnCalendar=*:0/5' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" systemd-analyze calendar '*:0/5' >/dev/null grep -Fq 'disabled application health monitor installed' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET"' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'state and shared alert files retained' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'PROBE_TARGET must remain /usr/local/sbin/hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'ALERT_HELPER_TARGET must remain /usr/local/sbin/hyb-oa-wecom-group-alert' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'STATE_DIR must remain /var/lib/hyb-oa-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'SERVICE_NAME must remain hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" grep -Fq 'SYSTEMD_DIR must remain /etc/systemd/system' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" pass "installer is five-minute, disabled-by-default, exact-target restricted, and preserves shared state" printf '[PASS] public application health contract completed (%d checks)\n' "$PASS_COUNT"