test(llm-security): v8 Phase 5 step 1 - golden baseline before any table swap

Records the reference artifacts Phase 5 swaps will be measured against, and
the gate that reads them. No extraction yet: this is the "before" picture,
and it had to land first or every later comparison would be confounded.
Typed test/ rather than feat/ deliberately - the only production change is
one widened export; the rest is gate, artifacts and generator.

Written failing-first (7 red on missing artifacts), then generated.

Three layers, because the plan's "assert .source/.flags of every regex" is
necessary but not sufficient:

  1. regex records - .source/.flags off the COMPILED object. After a swap a
     pattern is new RegExp(jsonString, flags), so the plan's named hazard
     (JSON backslash-doubling on 83+18 regexes) is visible here and nowhere
     else. Source-text comparison cannot see it.
  2. table records - key/value digests. Most of what Phase 4 moves is not a
     regex at all: HOMOGLYPH_MAP (x3, AS-IS), the typosquat tokens and the
     four OWASP maps are char->char and string->string data. A regex-only
     dump is blind to a broken homoglyph swap, i.e. to the bulk of the
     payload. Operator decision: widen the dump.
  3. file records - sha256 of the five moving-set sources. This dissolves
     STATE's open question (how to enumerate every regex): it is complete by
     construction, covering inline regexes in function bodies that no export
     walk reaches, with no JS parser in a zero-dep repo. A lexical count
     would have pinned a lie - severity.mjs scores 4 "regexes" that way and
     exports none.

Both layers were proven to fire, not assumed to: mutating one HOMOGLYPH_MAP
entry reddens the table layer, and widening an inline regex inside
decodeHexEscapes (unreachable by any export walk) reddens the file layer.

HOMOGLYPH_MAP is now exported from string-utils.mjs. That export is a source
change the plan already flags as a surface hazard ("private tables become
loaded"), so it is pre-paid here rather than confounding the before/after.

Reference run: the 61 showcase payloads through the real hook entry points,
sequentially - array order is semantic and the plan forbids key-sorting, so
concurrency is removed rather than sorted away. 61/61 match expectation,
which also settles the plan's open assumption that payloads.json expectations
match current behaviour. Coverage is recorded, not assumed: 47/83 patterns,
with the other 36 listed by key so the gate never implies coverage it lacks.

Suite counts are per-file, each file run alone - a total is unattributable,
and the three known timing-sensitive files flake only under concurrency.
2045 pass / 0 fail across 91 files, matching the pre-existing count exactly.
The gate's own file is excluded (it reads the artifact the run produces) and
that exclusion is named in the artifact.

Suite: 2053/2053, 0 fail.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BB4vXvwvtW4dxbPRd6vsez

scripts/lib/golden-dump.mjs

@@ -0,0 +1,384 @@
+// golden-dump.mjs — builds the v8 Phase 5 golden baseline artifacts.
+//
+// Two builders, both pure and deterministic, both used by BOTH the generator
+// (`scripts/golden-baseline.mjs --write`) and the gate
+// (`tests/lib/golden-baseline.test.mjs`). One code path, so the gate cannot
+// pass against a stale generator.
+//
+// Determinism rules observed here:
+//   - no timestamps, no absolute paths, no host details in the output;
+//   - every collection is emitted in a deterministic order — export-walk order
+//     comes from sorted key names, not from V8 enumeration luck;
+//   - the reference run executes hooks SEQUENTIALLY. Array order is semantic
+//     in this codebase (dedup + output order) and the plan forbids
+//     key-sorting tooling, so the fix for concurrency-dependent ordering is to
+//     remove the concurrency, not to sort the result.
+
+import { createHash } from 'node:crypto';
+import { execFile, execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { resolve, relative } from 'node:path';
+
+// ---------------------------------------------------------------------------
+// Shared
+// ---------------------------------------------------------------------------
+
+/** Canonical serialization for every artifact: 2-space JSON + trailing NL. */
+export function serialize(obj) {
+  return JSON.stringify(obj, null, 2) + '\n';
+}
+
+function sha256(s) {
+  return createHash('sha256').update(s, 'utf8').digest('hex');
+}
+
+/** Modules whose exports are walked for regex + table records. */
+const WALKED_MODULES = [
+  ['injection-patterns', 'scanners/lib/injection-patterns.mjs'],
+  ['string-utils', 'scanners/lib/string-utils.mjs'],
+  ['severity', 'scanners/lib/severity.mjs'],
+];
+
+/**
+ * Source files pinned by digest. This is the completeness layer: it covers
+ * every regex inlined in a function body that no export walk can reach.
+ */
+const PINNED_FILES = [
+  'scanners/lib/injection-patterns.mjs',
+  'scanners/lib/string-utils.mjs',
+  'scanners/lib/severity.mjs',
+  'knowledge/signatures.json',
+  'knowledge/attack-mutations.json',
+];
+
+// ---------------------------------------------------------------------------
+// Layer 1+2: export walk -> regex records and table records
+// ---------------------------------------------------------------------------
+
+/**
+ * Depth-first walk of an exported value, collecting RegExp instances with a
+ * stable path key (`CRITICAL_PATTERNS[3].pattern`).
+ *
+ * Reads `.source`/`.flags` off the compiled object. This is the whole point:
+ * after a Phase 5 swap the pattern is `new RegExp(jsonString, flags)`, and a
+ * backslash lost to JSON escaping is visible ONLY in the compiled source.
+ */
+function collectRegexes(value, path, out, seen) {
+  if (value instanceof RegExp) {
+    out.push({ kind: 'regex', key: path, source: value.source, flags: value.flags });
+    return;
+  }
+  if (value === null || typeof value !== 'object') return;
+  if (seen.has(value)) return;
+  seen.add(value);
+
+  if (Array.isArray(value)) {
+    value.forEach((v, i) => collectRegexes(v, `${path}[${i}]`, out, seen));
+    return;
+  }
+  for (const k of Object.keys(value).sort()) {
+    collectRegexes(value[k], `${path}.${k}`, out, seen);
+  }
+}
+
+/**
+ * A data table is an exported plain object or array of scalars — the shape
+ * Phase 4 moves into commons as JSON. Arrays of pattern objects are NOT
+ * tables; their regexes are already pinned individually above, and digesting
+ * them again would double-count without adding signal.
+ */
+function isDataTable(value) {
+  if (value === null || typeof value !== 'object') return false;
+  if (value instanceof RegExp) return false;
+  const values = Array.isArray(value) ? value : Object.values(value);
+  if (values.length === 0) return false;
+  // A value is either a scalar (HOMOGLYPH_MAP: char -> char, SEVERITY: name ->
+  // name) or an array of scalars (the four OWASP maps: prefix -> ['LLM01']).
+  const scalar = (v) => typeof v === 'string' || typeof v === 'number';
+  return values.every((v) => scalar(v) || (Array.isArray(v) && v.every(scalar)));
+}
+
+/**
+ * Stable digest of a table's contents. Keys are sorted for the digest input
+ * only — the artifact never reorders anything the runtime reads.
+ */
+function digestTable(value) {
+  const canonical = Array.isArray(value)
+    ? JSON.stringify(value)
+    : JSON.stringify(
+        Object.keys(value)
+          .sort()
+          .map((k) => [k, value[k]])
+      );
+  return {
+    entries: Array.isArray(value) ? value.length : Object.keys(value).length,
+    digest: `sha256:${sha256(canonical)}`,
+  };
+}
+
+export async function buildPatternDump(root) {
+  const records = [];
+
+  for (const [alias, relPath] of WALKED_MODULES) {
+    const mod = await import(resolve(root, relPath));
+    for (const exportName of Object.keys(mod).sort()) {
+      const value = mod[exportName];
+      if (typeof value === 'function') continue;
+
+      collectRegexes(value, `${alias}:${exportName}`, records, new WeakSet());
+
+      if (isDataTable(value)) {
+        records.push({ kind: 'table', key: `${alias}:${exportName}`, ...digestTable(value) });
+      }
+    }
+  }
+
+  for (const relPath of PINNED_FILES) {
+    records.push({
+      kind: 'file',
+      key: relPath,
+      sha256: sha256(readFileSync(resolve(root, relPath), 'utf8')),
+    });
+  }
+
+  // Stable ordering across kinds so a re-run of the walk cannot reshuffle the
+  // artifact for reasons unrelated to the code under test.
+  const kindRank = { regex: 0, table: 1, file: 2 };
+  records.sort(
+    (a, b) => kindRank[a.kind] - kindRank[b.kind] || a.key.localeCompare(b.key)
+  );
+
+  return {
+    artifact: 'golden-pattern-dump',
+    schema: 1,
+    counts: {
+      regex: records.filter((r) => r.kind === 'regex').length,
+      table: records.filter((r) => r.kind === 'table').length,
+      file: records.filter((r) => r.kind === 'file').length,
+    },
+    records,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Layer 4: reference run — 61 showcase payloads through real hook entry points
+// ---------------------------------------------------------------------------
+
+const HOOK_SCRIPTS = {
+  'pre-prompt-inject-scan': 'hooks/scripts/pre-prompt-inject-scan.mjs',
+  'post-mcp-verify': 'hooks/scripts/post-mcp-verify.mjs',
+  'pre-bash-destructive': 'hooks/scripts/pre-bash-destructive.mjs',
+};
+
+/** Mirrors the stdin protocol in examples/prompt-injection-showcase/run-showcase.mjs. */
+function buildInput(payload) {
+  switch (payload.hook) {
+    case 'pre-prompt-inject-scan':
+      return { session_id: 'golden', message: { role: 'user', content: payload.payload } };
+    case 'post-mcp-verify':
+      return {
+        tool_name: payload.inputTool || 'mcp__server__tool',
+        tool_input: {},
+        tool_output: payload.payload,
+      };
+    case 'pre-bash-destructive':
+      return { tool_name: 'Bash', tool_input: { command: payload.payload } };
+    default:
+      throw new Error(`Unknown hook: ${payload.hook}`);
+  }
+}
+
+function runHook(scriptPath, input, cwd) {
+  return new Promise((res) => {
+    const child = execFile(
+      process.execPath,
+      [scriptPath],
+      { timeout: 15000, cwd, env: { ...process.env, LLM_SECURITY_UPDATE_CHECK: 'off' } },
+      (_err, stdout, stderr) => {
+        res({ code: child.exitCode ?? 1, stdout: stdout || '', stderr: stderr || '' });
+      }
+    );
+    child.stdin.end(JSON.stringify(input));
+  });
+}
+
+/**
+ * Exit code + stdout -> verdict. Mirrors `classify()` in
+ * examples/prompt-injection-showcase/run-showcase.mjs, including the
+ * `advisory` state (exit 0 WITH output): a hook that warns is not a hook that
+ * allowed, and collapsing the two would understate detection across the whole
+ * MEDIUM tier of the corpus.
+ */
+function classify(code, stdout) {
+  if (code === 2) return 'block';
+  if (code === 0) return stdout.trim() ? 'advisory' : 'allow';
+  return `error(${code})`;
+}
+
+/**
+ * Strips everything host- or run-specific from hook output so the artifact is
+ * byte-stable: absolute paths, PIDs, durations, and the session file path.
+ */
+function normalizeOutput(s, root) {
+  return s
+    .split(root)
+    .join('<ROOT>')
+    .replace(/llm-security-session-\d+\.jsonl/g, 'llm-security-session-<PID>.jsonl')
+    .replace(/\b\d+(\.\d+)?\s?ms\b/g, '<MS>')
+    .replace(/\b\d{4}-\d{2}-\d{2}T[\d:.]+Z\b/g, '<TS>')
+    .trimEnd();
+}
+
+export async function buildReferenceRun(root) {
+  const payloads = JSON.parse(
+    readFileSync(resolve(root, 'examples/prompt-injection-showcase/payloads.json'), 'utf8')
+  );
+
+  const cases = [];
+  // Sequential on purpose — see the determinism note at the top of this file.
+  for (const p of payloads) {
+    const { code, stdout, stderr } = await runHook(
+      resolve(root, HOOK_SCRIPTS[p.hook]),
+      buildInput(p),
+      root
+    );
+    const verdict = classify(code, stdout);
+    cases.push({
+      id: p.id,
+      hook: p.hook,
+      category: p.category,
+      expected: p.expected,
+      exitCode: code,
+      verdict,
+      matchesExpectation: verdict === p.expected,
+      stdout: normalizeOutput(stdout, root),
+      stderr: normalizeOutput(stderr, root),
+    });
+  }
+
+  return {
+    artifact: 'golden-reference-run',
+    schema: 1,
+    coverage: await buildCoverage(root, payloads),
+    summary: {
+      total: cases.length,
+      matchingExpectation: cases.filter((c) => c.matchesExpectation).length,
+    },
+    cases,
+  };
+}
+
+/**
+ * Which injection patterns the corpus actually reaches.
+ *
+ * Byte-identity over a corpus that trips 5 of 90 patterns would prove almost
+ * nothing, so the uncovered ones are listed by key rather than silently
+ * dropped — a shrinking number is a gate failure, and a large uncovered list
+ * is an honest statement of what this gate does NOT cover.
+ */
+async function buildCoverage(root, payloads) {
+  const ip = await import(resolve(root, 'scanners/lib/injection-patterns.mjs'));
+  const su = await import(resolve(root, 'scanners/lib/string-utils.mjs'));
+
+  const groups = ['CRITICAL_PATTERNS', 'HIGH_PATTERNS', 'MEDIUM_PATTERNS', 'HYBRID_PATTERNS'];
+  const entries = [];
+  for (const g of groups) {
+    (ip[g] || []).forEach((entry, i) => {
+      if (entry && entry.pattern instanceof RegExp) entries.push({ key: `${g}[${i}]`, re: entry.pattern });
+    });
+  }
+
+  const texts = payloads.flatMap((p) => [p.payload, su.normalizeForScan(p.payload)]);
+  const exercised = [];
+  const uncovered = [];
+  for (const { key, re } of entries) {
+    const probe = new RegExp(re.source, re.flags.replace('g', ''));
+    (texts.some((t) => probe.test(t)) ? exercised : uncovered).push(key);
+  }
+
+  const homoglyphHit = texts.some((t) => su.foldHomoglyphs(t) !== t);
+
+  return {
+    patternsTotal: entries.length,
+    patternsExercised: exercised.length,
+    uncoveredPatterns: uncovered,
+    tablesExercised: {
+      HOMOGLYPH_MAP: homoglyphHit,
+      normalizeForScan: texts.some((t, i) => i % 2 === 1 && t !== payloads[(i - 1) / 2].payload),
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Artifact (a): per-file suite counts
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-file pass/fail counts, collected by running each test file in its OWN
+ * process, sequentially.
+ *
+ * Two reasons it is per-file rather than one total:
+ *
+ *   1. A total is unattributable. `2045` -> `2044` tells you something broke;
+ *      it does not tell you where, and during a table-by-table swap that is
+ *      most of the information.
+ *   2. `npm test` runs files concurrently, and three timing-sensitive files
+ *      in this suite (pre-compact-scan size cap, attack-simulator
+ *      --benchmark, pre-install-supply-chain F-3) flake under that load while
+ *      passing in isolation. Running each file alone is exactly the
+ *      already-documented procedure for trusting a red result, so the
+ *      baseline is recorded the only way it is reproducible.
+ *
+ * Slow by construction (one node process per file) — hence opt-in via
+ * `--suite`, not part of the default regeneration.
+ */
+/**
+ * The gate's own file cannot be in its own floor: it READS
+ * `suite-counts.json`, which is produced by the run that would include it, so
+ * the first recording always captures it mid-bootstrap as a failure. Excluded
+ * by name and reported in the artifact rather than dropped silently — `npm
+ * test` still covers it.
+ */
+const SUITE_SELF_REFERENCE = 'tests/lib/golden-baseline.test.mjs';
+
+export function buildSuiteCounts(root, files) {
+  const entries = [];
+  const excluded = [];
+  for (const file of files) {
+    const rel = relative(root, file);
+    if (rel === SUITE_SELF_REFERENCE) {
+      excluded.push({ file: rel, reason: 'self-reference: this file reads suite-counts.json' });
+      continue;
+    }
+    let out = '';
+    try {
+      out = execFileSync(process.execPath, ['--test', '--test-reporter=tap', rel], {
+        cwd: root,
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+        timeout: 300000,
+        env: { ...process.env, LLM_SECURITY_UPDATE_CHECK: 'off' },
+      });
+    } catch (err) {
+      // Non-zero exit is a failing file, not a harness error — its TAP
+      // summary is still on stdout and is what we want to record.
+      out = (err.stdout || '') + (err.stderr || '');
+    }
+    const pass = Number((out.match(/^# pass (\d+)$/m) || [])[1] ?? -1);
+    const fail = Number((out.match(/^# fail (\d+)$/m) || [])[1] ?? -1);
+    entries.push({ file: rel, pass, fail });
+  }
+
+  entries.sort((a, b) => a.file.localeCompare(b.file));
+  return {
+    artifact: 'golden-suite-counts',
+    schema: 1,
+    totals: {
+      files: entries.length,
+      pass: entries.reduce((n, e) => n + Math.max(e.pass, 0), 0),
+      fail: entries.reduce((n, e) => n + Math.max(e.fail, 0), 0),
+    },
+    excluded,
+    files: entries,
+  };
+}