#!/usr/bin/env node import { spawn, spawnSync } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import { existsSync, readFileSync, readlinkSync, readdirSync, mkdirSync, mkdtempSync, chmodSync, writeFileSync, renameSync, rmSync } from "node:fs"; import os from "node:os"; import { createServer, createConnection } from "node:net"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { sourceSnapshot, compareSnapshots } from "./full-local-diagnostic.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const policyFiles = ["scripts/qa/full-local-environment.mjs", "scripts/qa/full-local-environment-inner.sh", "scripts/qa/local-postgres-proxy.py", "scripts/qa/browser-environment-probe.mjs", "scripts/qa/run-web-gates.sh", "scripts/qa/prepare-local-e2e-accounts.mjs", "scripts/qa/full-local-diagnostic.mjs", "client/playwright.config.ts"]; const sha = value => createHash("sha256").update(value).digest("hex"); const readJson = file => JSON.parse(readFileSync(file, "utf8")); export function atomicEnvironmentJson(file, value) { writeFileSync(`${file}.tmp`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); renameSync(`${file}.tmp`, file); } function command(program, args) { const result = spawnSync(program, args, { encoding: "utf8", timeout: 30000 }); if (result.status !== 0) throw new Error(`environment_command_failed:${path.basename(program)}`); return result.stdout.trim(); } export function environmentPolicyDigest() { return sha(JSON.stringify(policyFiles.map(file => [file, sha(readFileSync(path.join(root, file)))]))); } export function namespaceFingerprint(pid = "self") { try { return sha(readlinkSync(`/proc/${pid}/ns/net`)); } catch (error) { throw new Error(`environment_namespace_unreadable:${pid === "self" ? "self" : "other"}:${error.code ?? error.name}`); } } function parentPid(pid) { const value = readFileSync(`/proc/${pid}/stat`, "utf8"); return Number(value.slice(value.lastIndexOf(")") + 2).split(" ")[1]); } function descendsFrom(pid, ancestor) { const visited = new Set(); while (pid > 1 && !visited.has(pid)) { if (pid === ancestor) return true; visited.add(pid); pid = parentPid(pid); } return false; } export async function requestSupervisorObservation(context) { return await new Promise((resolve, reject) => { const socket = createConnection(path.join(context.scratch, "supervisor.sock")); let response = ""; socket.setTimeout(5000, () => socket.destroy(new Error("environment_supervisor_timeout"))); socket.on("error", () => reject(new Error("environment_supervisor_unavailable"))); socket.on("data", data => { response += data; if (response.length > 4096) socket.destroy(new Error("environment_supervisor_response_invalid")); }); socket.on("end", () => { try { const observation = JSON.parse(response); if (observation.runId !== context.runId || observation.pid !== context.supervisorPid || typeof observation.namespace !== "string" || observation.policyDigest !== context.policyDigest) throw new Error(); resolve(observation); } catch { reject(new Error("environment_supervisor_response_invalid")); } }); }); } export function topologyMatches(addresses, routes, ipv6Routes, links) { const names = addresses.map(entry => entry.ifname).sort(); if (JSON.stringify(names) !== JSON.stringify(["lo", "qaonline", "qapeer"])) return false; for (const [name, address] of [["qaonline", "192.0.2.1"], ["qapeer", "192.0.2.2"]]) { const entry = addresses.find(item => item.ifname === name); if (!entry.flags?.includes("UP") || entry.operstate !== "UP" || !entry.addr_info?.some(item => item.local === address && item.prefixlen === 30)) return false; } if (!Array.isArray(links) || JSON.stringify(links.map(entry => entry.ifname).sort()) !== JSON.stringify(names)) return false; const left = links.find(entry => entry.ifname === "qaonline"), right = links.find(entry => entry.ifname === "qapeer"); if (left.ifindex === right.ifindex) return false; for (const [entry, peer] of [[left, right], [right, left]]) { if (!Number.isInteger(entry.ifindex) || entry.ifindex <= 0 || entry.linkinfo?.info_kind !== "veth" || entry.link_netnsid !== undefined || entry.master !== undefined) return false; if (entry.link !== peer.ifname && entry.link_index !== peer.ifindex) return false; } const defaults = routes.filter(entry => entry.dst === "default"); if (defaults.length !== 1 || defaults[0].dev !== "qaonline" || defaults[0].scope !== "link" || (defaults[0].table !== undefined && defaults[0].table !== "main")) return false; if (ipv6Routes.some(entry => entry.dst === "default")) return false; return ![...routes, ...ipv6Routes].some(entry => entry.gateway || entry.via || entry.multipath || entry.nexthops || entry.encap || !names.includes(entry.dev)); } function topology() { return topologyMatches(JSON.parse(command("ip", ["-j", "address", "show"])), JSON.parse(command("ip", ["-j", "route", "show", "table", "all"])), JSON.parse(command("ip", ["-j", "-6", "route", "show", "table", "all"])), JSON.parse(command("ip", ["-j", "-d", "link", "show"]))); } export function validateEnvironmentFacts(context, facts) { const failures = []; if (context.schemaVersion !== 1 || context.profile !== "isolated-online-v1") failures.push("profile_invalid"); if (context.policyDigest !== facts.policyDigest) failures.push("policy_changed"); if (!facts.supervisorRecognized || !facts.descendantOfSupervisor) failures.push("supervisor_unverified"); if (context.inheritedNamespace !== facts.supervisorNamespace) failures.push("supervisor_namespace_changed"); if (!context.ownedNamespace || context.ownedNamespace === context.inheritedNamespace || context.ownedNamespace !== facts.currentNamespace) failures.push("namespace_unowned"); if (!facts.topologyMatched) failures.push("topology_invalid"); return failures; } export function verifyManagedEnvironment(contextFile = process.env.QA_ENVIRONMENT_CONTEXT, initialize = false) { if (!contextFile || !existsSync(contextFile)) throw new Error("environment_context_required"); const context = readJson(contextFile); const actualFile = path.resolve(contextFile); if (actualFile !== path.join(context.environmentDir, "context.private.json")) throw new Error("environment_context_path_invalid"); const supervisorArgs = readFileSync(`/proc/${context.supervisorPid}/cmdline`, "utf8").split("\0"); const supervisor = JSON.parse(command(process.execPath, [path.join(root, "scripts/qa/full-local-environment.mjs"), "observe-supervisor", actualFile])); const facts = { policyDigest: environmentPolicyDigest(), supervisorRecognized: supervisorArgs.includes(path.join(root, "scripts/qa/full-local-environment.mjs")), descendantOfSupervisor: descendsFrom(process.pid, context.supervisorPid), supervisorNamespace: supervisor.namespace, supervisorObservation: supervisor, currentNamespace: namespaceFingerprint(), topologyMatched: topology(), }; if (initialize && !context.ownedNamespace) context.ownedNamespace = facts.currentNamespace; const failures = validateEnvironmentFacts(context, facts); if (failures.length) throw new Error(`environment_invalid:${failures.join(",")}`); if (initialize) atomicEnvironmentJson(actualFile, context); return { context, facts }; } export function validateEnvironmentReceipt(receipt, digest, requireGuest = true) { const failures = []; if (!receipt || receipt.schemaVersion !== 1 || receipt.profile !== "isolated-online-v1" || receipt.policyDigest !== digest) failures.push("environment_receipt_policy_invalid"); const namespace = receipt?.namespace; if (receipt?.supervisorObservation?.source !== "live-supervisor-unix-ipc" || !Number.isFinite(Date.parse(receipt?.supervisorObservation?.sampledAt)) || receipt?.supervisorObservation?.namespace !== namespace?.inheritedRunner) failures.push("supervisor_observation_invalid"); if (!namespace?.inheritedRunner || !namespace.ownedChild || namespace.inheritedRunner === namespace.ownedChild || namespace.runnerCreated !== true || namespace.currentMatchesOwnedChild !== true || namespace.topologyMatched !== true) failures.push("environment_receipt_namespace_invalid"); if (receipt?.nativeOnlineProbe?.browserNamespace !== namespace?.ownedChild || receipt?.nativeOnlineProbe?.noNavigatorOverride !== true) failures.push("native_probe_namespace_invalid"); if (!receipt?.nativeOnlineProbe?.valid || receipt.nativeOnlineProbe.online !== true || !receipt.nativeOnlineProbe.browserMatchesOwnedChild) failures.push("native_probe_invalid"); const engines = receipt?.nativeOnlineProbe?.engines; if (!Array.isArray(engines) || JSON.stringify(engines.map(engine => engine.name)) !== JSON.stringify(["chromium", "webkit"]) || engines.some(engine => engine.online !== true || engine.valid !== true || engine.noNavigatorOverride !== true || engine.browserNamespace !== namespace?.ownedChild || engine.browserMatchesOwnedChild !== true || !engine.version)) failures.push("native_engine_probe_invalid"); if (requireGuest) { const observed = receipt?.bootstrapObservation; if (observed?.browserNamespace !== namespace?.ownedChild || observed?.noNavigatorOverride !== true || observed?.loginRequestCount !== 0 || observed?.documentStatus !== 200 || observed?.stayedOnLogin !== true) failures.push("guest_probe_evidence_invalid"); if (!observed || observed.observationPoint !== "guest-entry-black-box" || !observed.freshContext || observed.initialSessionHint !== false || !observed.listenerBeforeNavigation || !observed.noRouteMocks || !observed.loginFormVisible || observed.refreshRequestCount !== 0 || observed.nativeOnlineSamples?.beforeNavigation !== true || observed.nativeOnlineSamples?.documentStart !== true || observed.nativeOnlineSamples?.afterForm !== true || !observed.browserMatchesOwnedChild || !observed.valid) failures.push("guest_probe_invalid"); } return failures; } export function environmentReceipt(contextFile = process.env.QA_ENVIRONMENT_CONTEXT, requireGuest = false) { const { context, facts } = verifyManagedEnvironment(contextFile); const nativeFile = path.join(context.environmentDir, "native-online.json"); const guestFile = path.join(context.environmentDir, "guest-entry.json"); const receipt = { schemaVersion: 1, profile: context.profile, policyDigest: context.policyDigest, platform: process.platform, runnerDigest: sha(readFileSync(path.join(root, "scripts/qa/full-local-environment.mjs"))), probeDigest: sha(readFileSync(path.join(root, "scripts/qa/browser-environment-probe.mjs"))), supervisorObservation: { source: "live-supervisor-unix-ipc", sampledAt: facts.supervisorObservation.sampledAt, namespace: facts.supervisorNamespace }, namespace: { inheritedRunner: context.inheritedNamespace, ownedChild: context.ownedNamespace, currentMatchesOwnedChild: facts.currentNamespace === context.ownedNamespace, topologyMatched: facts.topologyMatched, runnerCreated: true }, nativeOnlineProbe: existsSync(nativeFile) ? readJson(nativeFile) : null, bootstrapObservation: existsSync(guestFile) ? readJson(guestFile) : null, }; const failures = validateEnvironmentReceipt(receipt, context.policyDigest, requireGuest); if (failures.length) throw new Error(`environment_invalid:${failures.join(",")}`); return receipt; } async function launch(mode) { if (!["full-local", "full-local-diagnostic", "retained-local", "retained-server-local"].includes(mode)) throw new Error("environment_mode_invalid"); if (process.platform !== "linux") throw new Error("environment_platform_unsupported"); if (mode === "full-local-diagnostic") for (const name of ["REASON", "OPERATOR", "APPROVAL_REF"]) { const value = process.env[`QA_DIAGNOSTIC_${name}`]?.trim(); if (!value || value.length > 500) throw new Error(`diagnostic_${name.toLowerCase()}_required`); } const runId = `${Date.now()}-${randomBytes(5).toString("hex")}`; const base = path.resolve(process.env.QA_DIAGNOSTIC_REPORT_DIR ?? process.env.QA_FULL_REPORT_DIR ?? path.join(root, "artifacts/qa-flow", mode, runId)); if (!base.startsWith(`${path.join(root, "artifacts/qa-flow")}${path.sep}`)) throw new Error("environment_report_outside_artifacts"); if (existsSync(base) && readdirSync(base).length) throw new Error("environment_report_not_empty"); mkdirSync(base, { recursive: true, mode: 0o700 }); const environmentDir = path.join(base, "environment"); mkdirSync(environmentDir); const scratch = mkdtempSync(path.join(os.tmpdir(), "hyb-qa-managed-")); const socket = path.join(scratch, "socket"); mkdirSync(socket); chmodSync(scratch, 0o755); chmodSync(socket, 0o777); const owner = `hyb-qa-${runId}`; const record = { schemaVersion: 1, profile: "isolated-online-v1", policyDigest: environmentPolicyDigest(), platform: process.platform, mode, startedAt: new Date().toISOString(), status: "RUNNING", normalReleaseEligible: false }; const context = { schemaVersion: 1, profile: record.profile, policyDigest: record.policyDigest, runId, supervisorPid: process.pid, inheritedNamespace: namespaceFingerprint(), ownedNamespace: null, environmentDir, scratch, socket, mode }; const contextFile = path.join(environmentDir, "context.private.json"); atomicEnvironmentJson(contextFile, context); let containerId = null; let child = null; let exitCode = 1; let interrupted = null; let supervisorServer = null; const forward = signal => { interrupted = signal; if (child?.pid) { try { process.kill(-child.pid, signal); } catch {} } }; const onInt = () => forward("SIGINT"); const onTerm = () => forward("SIGTERM"); process.on("SIGINT", onInt); process.on("SIGTERM", onTerm); try { record.sourceBefore = sourceSnapshot(); const image = command("docker", ["image", "inspect", "--format", "{{.Id}}", "postgres:15-alpine"]); containerId = command("docker", ["run", "--detach", "--pull=never", "--name", owner, "--label", `hyb.qa.owner=${owner}`, "--network", "none", "--tmpfs", "/var/lib/postgresql/data:rw,size=512m", "--mount", `type=bind,src=${socket},dst=/var/run/postgresql`, "-e", "POSTGRES_HOST_AUTH_METHOD=trust", image]); if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error("environment_container_id_invalid"); record.databaseImageId = image; writeFileSync(path.join(environmentDir, "container.private"), containerId, { mode: 0o600 }); let ready = false; for (let attempt = 0; attempt < 60 && !interrupted; attempt += 1) { if (spawnSync("docker", ["exec", containerId, "pg_isready", "-U", "postgres"], { stdio: "ignore", timeout: 10000 }).status === 0) { ready = true; break; } await new Promise(resolve => setTimeout(resolve, 500)); } if (!ready) throw new Error("environment_database_not_ready"); supervisorServer = createServer(connection => { connection.on("error", () => connection.destroy()); try { connection.end(JSON.stringify({ runId, pid: process.pid, namespace: namespaceFingerprint(), policyDigest: environmentPolicyDigest(), sampledAt: new Date().toISOString() })); } catch { connection.destroy(); } }); await new Promise((resolve, reject) => { supervisorServer.once("error", reject); supervisorServer.listen(path.join(scratch, "supervisor.sock"), resolve); }); chmodSync(path.join(scratch, "supervisor.sock"), 0o600); const env = {}; for (const name of ["PATH", "HOME", "LANG", "TERM", "CI", "NODE_OPTIONS", "PAYMENT_ORDER_PYTHON_BIN", "QA_DIAGNOSTIC_REASON", "QA_DIAGNOSTIC_OPERATOR", "QA_DIAGNOSTIC_APPROVAL_REF", "QA_RETAINED_SPECS", "QA_RETAINED_GREP", "QA_RETAINED_SERVER_SUITE", "DOCKER_CONFIG", "PLAYWRIGHT_BROWSERS_PATH", "npm_config_userconfig", "npm_config_globalconfig"]) if (process.env[name] !== undefined) env[name] = process.env[name]; Object.assign(env, { QA_ENVIRONMENT_CONTEXT: contextFile, QA_FULL_REPORT_DIR: path.join(base, "results"), QA_DIAGNOSTIC_REPORT_DIR: path.join(base, "results"), QA_MANAGED_SCRATCH: scratch, QA_MANAGED_SOCKET: socket, QA_HOST: "127.0.0.1", QA_PORT: "23000", QA_DB_HOST: "127.0.0.1", QA_DB_PORT: "25432", QA_DB_CLIENT: "host", QA_DB_USER: "postgres", QA_DB_PASSWORD: "local_qa_only", QA_KEEP_DB: "0", QA_SKIP_CLIENT_BUILD: "0", E2E_MODE: "local", WECOM_ENABLED: "false", EMAIL_ENABLED: "false", OIDC_ENABLED: "false", STORAGE_DRIVER: "local", NODE_ENV: "test", npm_config_offline: "true", NO_COLOR: "1" }); const outcome = await new Promise(resolve => { child = spawn("unshare", ["-Urn", "bash", path.join(root, "scripts/qa/full-local-environment-inner.sh"), mode], { cwd: root, env, stdio: "inherit", detached: true }); child.once("error", () => resolve({ code: 1, signal: null })); child.once("close", (code, signal) => resolve({ code, signal })); }); child = null; exitCode = outcome.code ?? 1; record.childExit = outcome.code; record.childSignal = outcome.signal; const completionFile = path.join(environmentDir, "completed.json"); if (existsSync(completionFile)) { const completion = readJson(completionFile); record.environment = completion; record.status = validateEnvironmentReceipt(completion, environmentPolicyDigest(), true).length ? "ENVIRONMENT_INVALID" : "PASS"; } else record.status = "ENVIRONMENT_INVALID"; if (record.status !== "PASS") exitCode = mode === "full-local-diagnostic" ? 2 : 1; } catch (error) { record.status = "ENVIRONMENT_INVALID"; record.error = error.message.startsWith("environment_") ? error.message : "environment_internal_error"; exitCode = mode === "full-local-diagnostic" ? 2 : 1; } finally { if (supervisorServer) await new Promise(resolve => supervisorServer.close(resolve)); record.cleanup = { containerOwnerVerified: false, containerRemoved: containerId === null, scratchRemoved: false }; if (containerId) { try { const identity = command("docker", ["inspect", "--format", '{{.Id}}|{{index .Config.Labels "hyb.qa.owner"}}|{{.HostConfig.NetworkMode}}', containerId]); if (identity !== `${containerId}|${owner}|none`) throw new Error("environment_cleanup_owner_mismatch"); record.cleanup.containerOwnerVerified = true; spawnSync("docker", ["exec", containerId, "chmod", "0777", "/var/run/postgresql"], { stdio: "ignore", timeout: 10000 }); command("docker", ["rm", "-f", containerId]); record.cleanup.containerRemoved = true; } catch { record.cleanup.error = "owned_container_cleanup_failed"; } } if (record.cleanup.containerRemoved) { try { rmSync(scratch, { recursive: true }); record.cleanup.scratchRemoved = true; } catch { record.cleanup.error = "owned_scratch_cleanup_failed"; } } if (!record.cleanup.containerRemoved || !record.cleanup.scratchRemoved) { record.status = "ENVIRONMENT_INVALID"; exitCode = mode === "full-local-diagnostic" ? 2 : 1; } try { record.sourceAfter = sourceSnapshot(); Object.assign(record, compareSnapshots(record.sourceBefore, record.sourceAfter)); if (record.candidateIntegrityStatus !== "UNCHANGED") exitCode = mode === "full-local-diagnostic" ? 2 : 1; } catch { record.candidateIntegrityStatus = "UNKNOWN"; record.repositorySnapshotError = "post_run_snapshot_unavailable"; exitCode = mode === "full-local-diagnostic" ? 2 : 1; } record.interrupted = interrupted; record.endedAt = new Date().toISOString(); record.exitCode = exitCode; atomicEnvironmentJson(path.join(environmentDir, "result.json"), record); process.off("SIGINT", onInt); process.off("SIGTERM", onTerm); } console.log(`[environment] ${record.status}; ${mode === "full-local-diagnostic" ? "PARTIAL; " : ""}exit=${exitCode}; no release activation`); process.exitCode = exitCode; } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { try { const action = process.argv[2]; if (action === "observe-supervisor") console.log(JSON.stringify(await requestSupervisorObservation(readJson(process.argv[3])))); else if (action === "initialize") verifyManagedEnvironment(undefined, true); else if (action === "verify") environmentReceipt(undefined, false); else if (action === "report" || action === "report-guest") console.log(JSON.stringify(environmentReceipt(undefined, action === "report-guest"))); else if (action === "complete") { const receipt = environmentReceipt(undefined, false); const { context } = verifyManagedEnvironment(); atomicEnvironmentJson(path.join(context.environmentDir, "completed.json"), receipt); } else await launch(action); } catch (error) { console.error(`[environment] ${error.message.startsWith("environment_") ? error.message : `environment_verification_failed:${error.code ?? error.name}`}`); process.exitCode = 1; } }