hyb-oa/scripts/qa/verify-local-candidate-results.mjs

#!/usr/bin/env node
import { readFileSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";

const WITNESS_TAG_PATTERN = /@local-candidate:[a-z0-9-]+/g;
const FAILED_STATUSES = new Set(["failed", "timedOut", "interrupted"]);

function collectSpecs(suites, output = [], inheritedFile = "") {
  for (const suite of Array.isArray(suites) ? suites : []) {
    const suiteFile = suite?.file ?? inheritedFile;
    for (const spec of Array.isArray(suite?.specs) ? suite.specs : []) {
      output.push({ ...spec, __sourceFile: spec?.file ?? suiteFile });
    }
    collectSpecs(suite?.suites, output, suiteFile);
  }
  return output;
}

function tagsForSpec(spec) {
  const candidates = [
    ...(String(spec?.title ?? "").match(WITNESS_TAG_PATTERN) ?? []),
    ...(Array.isArray(spec?.tags) ? spec.tags : []).flatMap((tag) => String(tag).match(WITNESS_TAG_PATTERN) ?? []),
  ];
  return [...new Set(candidates)];
}

function finalStatus(test) {
  const results = Array.isArray(test?.results) ? test.results : [];
  return results.at(-1)?.status ?? test?.status ?? "missing";
}

function safeDiagnosticText(value, maxLength = 160) {
  return String(value ?? "")
    .replace(/[\r\n\t]+/gu, " ")
    .trim()
    .slice(0, maxLength);
}

function safeSpecDiagnostic(spec, status) {
  const line = Number(spec?.line);
  return {
    file: path.basename(safeDiagnosticText(spec?.__sourceFile ?? spec?.file, 240)) || "unknown-spec",
    line: Number.isSafeInteger(line) && line > 0 ? line : null,
    title: safeDiagnosticText(spec?.title) || "unknown-test",
    status,
  };
}

function normalizeWitnessSpecFile(file) {
  const normalized = String(file ?? "").replaceAll("\\", "/");
  if (normalized.startsWith("client/tests/e2e/")) return normalized;
  if (normalized.startsWith("tests/e2e/")) return `client/${normalized}`;
  const marker = "/client/tests/e2e/";
  const markerIndex = normalized.lastIndexOf(marker);
  if (markerIndex >= 0) return normalized.slice(markerIndex + 1);
  if (/^[^/]+\.spec\.[cm]?[jt]sx?$/u.test(normalized)) return normalized;
  return "";
}

function witnessSpecMatches(observedFile, expectedFile) {
  if (!observedFile || !expectedFile) return false;
  if (observedFile.includes("/")) return observedFile === expectedFile;
  return observedFile === path.basename(expectedFile);
}

function duplicates(values) {
  const counts = new Map();
  for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
  return [...counts.entries()].filter(([, count]) => count > 1).map(([value]) => value).sort();
}

export function verifyLocalCandidateResults(manifest, results, { expectedTagCount = 10 } = {}) {
  const flows = Array.isArray(manifest?.localCandidateFlows) ? manifest.localCandidateFlows : [];
  const expectedTags = flows.map((flow) => flow?.witnessTag).filter((tag) => typeof tag === "string");
  const expectedTagSet = new Set(expectedTags);
  const expectedSpecByTag = new Map(
    flows.map((flow) => [flow?.witnessTag, normalizeWitnessSpecFile(flow?.witnessSpec)]),
  );
  const manifestDuplicateTags = duplicates(expectedTags);
  const manifestInvalidTags = flows
    .filter((flow) => flow?.witnessTag !== `@local-candidate:${flow?.bundle ?? ""}`)
    .map((flow) => String(flow?.witnessTag ?? ""))
    .sort();

  const observed = [];
  const allTests = [];
  const multiTagTests = [];
  const wrongWitnessSpecs = [];
  for (const spec of collectSpecs(results?.suites)) {
    const tags = tagsForSpec(spec);
    const specFile = normalizeWitnessSpecFile(spec?.__sourceFile ?? spec?.file);
    const tests = Array.isArray(spec?.tests) ? spec.tests : [];
    for (const test of tests) {
      const status = finalStatus(test);
      allTests.push(safeSpecDiagnostic(spec, status));
      if (tags.length > 1) multiTagTests.push(safeSpecDiagnostic(spec, status));
      for (const tag of tags) {
        observed.push({ tag, status });
        const expectedSpec = expectedSpecByTag.get(tag);
        if (!witnessSpecMatches(specFile, expectedSpec)) {
          wrongWitnessSpecs.push({
            tag,
            expectedFile: path.basename(expectedSpec || "unknown-spec"),
            observedFile: path.basename(specFile || "unknown-spec"),
          });
        }
      }
    }
  }

  const observedTags = observed.map((item) => item.tag);
  const duplicateTags = duplicates(observedTags);
  const unknownTags = [...new Set(observedTags.filter((tag) => !expectedTagSet.has(tag)))].sort();
  const missingTags = expectedTags.filter((tag) => !observedTags.includes(tag)).sort();
  const skippedTags = [...new Set(observed.filter((item) => item.status === "skipped").map((item) => item.tag))].sort();
  const failedTags = [...new Set(observed.filter((item) => item.status !== "passed" && item.status !== "skipped").map((item) => item.tag))].sort();
  const passedTags = [...new Set(observed.filter((item) => item.status === "passed").map((item) => item.tag))].sort();
  const suiteFailedTestCount = allTests.filter((item) => FAILED_STATUSES.has(item.status)).length;
  const suiteSkippedTestCount = allTests.filter((item) => item.status === "skipped").length;
  const suiteUnexpectedCount = Number.isFinite(results?.stats?.unexpected) ? Number(results.stats.unexpected) : 0;
  const suiteReportedSkippedCount = Number.isFinite(results?.stats?.skipped) ? Number(results.stats.skipped) : 0;
  const resultErrorCount = Array.isArray(results?.errors) ? results.errors.length : 0;
  const failedTests = allTests.filter((item) => FAILED_STATUSES.has(item.status)).slice(0, 20);
  const skippedTests = allTests.filter((item) => item.status === "skipped").slice(0, 20);

  const issues = [];
  if (expectedTags.length !== expectedTagCount) issues.push("unexpected_expected_tag_count");
  if (manifestDuplicateTags.length) issues.push("manifest_duplicate_tag");
  if (manifestInvalidTags.length) issues.push("manifest_invalid_tag");
  if (missingTags.length) issues.push("missing_tag_result");
  if (duplicateTags.length) issues.push("duplicate_tag_result");
  if (unknownTags.length) issues.push("unknown_tag_result");
  if (multiTagTests.length) issues.push("multi_tag_test_result");
  if (wrongWitnessSpecs.length) issues.push("wrong_witness_spec_result");
  if (skippedTags.length) issues.push("skipped_tag_result");
  if (failedTags.length) issues.push("failed_tag_result");
  if (passedTags.length !== expectedTags.length) issues.push("incomplete_passed_tag_set");
  if (suiteFailedTestCount > 0 || suiteUnexpectedCount > 0 || resultErrorCount > 0) issues.push("playwright_suite_failed");
  if (suiteSkippedTestCount > 0 || suiteReportedSkippedCount > 0) issues.push("playwright_suite_skipped");

  return {
    ok: issues.length === 0,
    issues,
    expectedTagCount: expectedTags.length,
    passedTagCount: passedTags.length,
    skippedTagCount: skippedTags.length,
    failedTagCount: failedTags.length,
    missingTags,
    duplicateTags,
    unknownTags,
    skippedTags,
    failedTags,
    manifestDuplicateTags,
    manifestInvalidTags,
    multiTagTests,
    wrongWitnessSpecs,
    totalTestResultCount: allTests.length,
    suiteFailedTestCount,
    suiteSkippedTestCount,
    suiteUnexpectedCount,
    suiteReportedSkippedCount,
    resultErrorCount,
    failedTests,
    skippedTests,
  };
}

function readJson(file) {
  return JSON.parse(readFileSync(file, "utf8"));
}

function parseArgs(argv = process.argv.slice(2)) {
  const manifestIndex = argv.indexOf("--manifest");
  const resultsIndex = argv.indexOf("--results");
  if (manifestIndex < 0 || resultsIndex < 0 || !argv[manifestIndex + 1] || !argv[resultsIndex + 1]) {
    throw new Error("missing_arguments");
  }
  return {
    manifestPath: path.resolve(argv[manifestIndex + 1]),
    resultsPath: path.resolve(argv[resultsIndex + 1]),
  };
}

function main() {
  try {
    const args = parseArgs();
    const summary = verifyLocalCandidateResults(readJson(args.manifestPath), readJson(args.resultsPath));
    console.log(JSON.stringify(summary, null, 2));
    if (!summary.ok) process.exitCode = 1;
  } catch {
    console.log(JSON.stringify({ ok: false, issues: ["invalid_or_missing_result_json"] }, null, 2));
    process.exitCode = 1;
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
  main();
}