// golden-baseline.test.mjs — Phase 5 step 1: the golden gate. // // Phase 5 swaps four data tables (injection patterns, codepoint/homoglyph // tables, severity maps, signatures) out of `scanners/lib/*.mjs` and into a // vendored `llm-security-commons` copy, table by table. The plan does not // assume that extraction is behaviour-preserving — this gate is what proves // it, per swap, with rollback if it fails. // // What it pins, and why each layer exists: // // 1. REGEX records — `.source` + `.flags` of every RegExp reachable from a // module's exports. After a swap a pattern is `new RegExp(jsonString, // flags)`, so the plan's named hazard ("JSON backslash-doubling on 83+18 // regexes") shows up here and nowhere else. Source-text comparison cannot // see it; only the COMPILED object can. // // 2. TABLE records — a stable key/value digest of the data tables. Most of // what Phase 4 moves is not a regex at all: HOMOGLYPH_MAP (x3, moving // AS-IS), the typosquat token list, and the four OWASP maps are char->char // and string->string data. A regex-only dump is blind to a broken // homoglyph swap, i.e. blind to the bulk of the payload. // // 3. FILE records — sha256 of each source file in the moving set. This is // the completeness layer, and it is complete BY CONSTRUCTION: it covers // regexes inlined in function bodies (`NAMED` at string-utils.mjs:291, // the BIDI/tag/PUA codepoint ranges at 357-404) that no export walk can // reach. It is why this gate needs no JS parser and no lexer — a lexical // regex count over these files is contaminated by comments and division // anyway, so pinning one would pin a lie. // // 4. REFERENCE RUN — the 61 showcase payloads through the real hook entry // points (subprocess, stdin protocol), plus a coverage assertion. Byte // identity over a corpus that exercises 5 of 90 patterns would prove // almost nothing, so the artifact records WHICH patterns and tables the // run actually reaches, and the gate fails if that coverage shrinks. // // Failure mode this file is written against: a gate that reports success // without running (v7.8.2 shipped four of those). `JSON.stringify(/a/g)` is // `{}` — a dump built by stringifying RegExp objects is byte-stable and // permanently empty. Hence the non-emptiness assertions below; they are not // paranoia, they are the specific bug. import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { existsSync, readFileSync } from 'node:fs'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(__dirname, '../..'); const GOLDEN_DIR = resolve(ROOT, 'tests/golden'); const PATTERNS_ARTIFACT = resolve(GOLDEN_DIR, 'patterns.json'); const REFERENCE_ARTIFACT = resolve(GOLDEN_DIR, 'reference-run.json'); function readArtifact(path) { assert.ok( existsSync(path), `missing golden artifact: ${path}\n` + `Regenerate with: node scripts/golden-baseline.mjs --write` ); return readFileSync(path, 'utf8'); } describe('golden baseline — pattern + table dump', () => { it('regenerates byte-identically from the current source tree', async () => { const { buildPatternDump, serialize } = await import( '../../scripts/lib/golden-dump.mjs' ); const committed = readArtifact(PATTERNS_ARTIFACT); const fresh = serialize(await buildPatternDump(ROOT)); assert.equal( fresh, committed, 'pattern dump drifted from the committed baseline.\n' + 'If a Phase 5 swap caused this, the swap is NOT behaviour-preserving — roll it back.\n' + 'If the change is intentional, re-bless with: node scripts/golden-baseline.mjs --write' ); }); it('contains regex records with a non-empty source (JSON.stringify(/a/g) === "{}" guard)', async () => { const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT)); const regexes = dump.records.filter((r) => r.kind === 'regex'); assert.ok( regexes.length >= 80, `expected >=80 regex records, got ${regexes.length} — an empty or ` + `near-empty dump is byte-stable and proves nothing` ); for (const r of regexes) { assert.ok( typeof r.source === 'string' && r.source.length > 0, `regex record ${r.key} has an empty source — the dump serialized ` + `RegExp objects instead of reading .source` ); assert.ok(typeof r.flags === 'string', `regex record ${r.key} lost its flags`); } }); it('covers every data table Phase 4 moves, including the private HOMOGLYPH_MAP', async () => { const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT)); const tableKeys = dump.records.filter((r) => r.kind === 'table').map((r) => r.key); // HOMOGLYPH_MAP is module-private at string-utils.mjs:448. It must be // exported for the walk to reach it, and that export has to land BEFORE // the baseline is recorded — otherwise the before/after comparison is // confounded by the very surface change the plan flags as a hazard // ("private tables become loaded"). for (const required of [ 'string-utils:HOMOGLYPH_MAP', 'string-utils:TYPOSQUAT_SUSPICIOUS_TOKENS', 'severity:OWASP_MAP', 'severity:OWASP_AGENTIC_MAP', 'severity:OWASP_SKILLS_MAP', 'severity:OWASP_MCP_MAP', 'severity:SEVERITY', ]) { assert.ok( tableKeys.includes(required), `table record missing: ${required} (have: ${tableKeys.join(', ')})` ); } for (const r of dump.records.filter((x) => x.kind === 'table')) { assert.ok(r.entries > 0, `table ${r.key} is empty`); assert.match(r.digest, /^sha256:[0-9a-f]{64}$/, `table ${r.key} has no digest`); } }); it('pins a file digest for every source file in the moving set', async () => { const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT)); const files = dump.records.filter((r) => r.kind === 'file'); const keys = files.map((r) => r.key); for (const required of [ 'scanners/lib/injection-patterns.mjs', 'scanners/lib/string-utils.mjs', 'scanners/lib/severity.mjs', 'knowledge/signatures.json', 'knowledge/attack-mutations.json', ]) { assert.ok(keys.includes(required), `file record missing: ${required}`); } for (const r of files) { assert.match(r.sha256, /^[0-9a-f]{64}$/, `file ${r.key} has no digest`); } }); }); describe('golden baseline — suite counts', () => { // Deliberately NOT a byte-identity gate. Adding a test is normal work and // must not turn the baseline red; what Phase 5 actually asks is narrower — // "did a swap make a file that used to pass stop passing?". So this asserts // no REGRESSION, and re-blessing is only needed to raise the floor. const SUITE_ARTIFACT = resolve(GOLDEN_DIR, 'suite-counts.json'); it('records a per-file floor with no failures in it', () => { const suite = JSON.parse(readArtifact(SUITE_ARTIFACT)); assert.ok(suite.files.length >= 90, `expected >=90 test files, got ${suite.files.length}`); assert.equal(suite.totals.fail, 0, 'the baseline was recorded with failing files'); // Whatever is left out is named, so "covered everything" is never implied // by omission. assert.ok(Array.isArray(suite.excluded), 'exclusions must be listed, not silent'); for (const e of suite.excluded) { assert.ok(e.reason, `excluded ${e.file} without a reason`); } for (const f of suite.files) { assert.ok( f.pass >= 0, `${f.file} produced no TAP summary — it did not run, and a baseline ` + `that silently records a non-run file is the v7.8.2 defect class` ); } }); }); describe('golden baseline — reference run (61 payloads, real entry points)', () => { it('regenerates byte-identically through the real hook entry points', async () => { const { buildReferenceRun, serialize } = await import( '../../scripts/lib/golden-dump.mjs' ); const committed = readArtifact(REFERENCE_ARTIFACT); const fresh = serialize(await buildReferenceRun(ROOT)); assert.equal( fresh, committed, 'reference run drifted from the committed baseline.\n' + 'Re-bless with: node scripts/golden-baseline.mjs --write' ); }); it('ran all 61 showcase payloads and none of them silently no-opped', async () => { const run = JSON.parse(readArtifact(REFERENCE_ARTIFACT)); assert.equal(run.cases.length, 61, 'the conformance seed is 61 cases'); for (const c of run.cases) { assert.ok(typeof c.exitCode === 'number', `case ${c.id} has no exit code`); assert.ok(typeof c.verdict === 'string' && c.verdict.length > 0, `case ${c.id} produced no verdict`); } }); it('records static pattern reachability, labelled as such, and fails if it shrinks', async () => { const run = JSON.parse(readArtifact(REFERENCE_ARTIFACT)); assert.ok(run.coverage, 'reference run carries no coverage block'); // The label is load-bearing. This block probes payload strings against // patterns in-process; it does NOT measure what the 61 hook invocations // evaluated. A future session quoting it as observed coverage would be // quoting a number that measures something else — pinned here so the // honest name cannot be quietly dropped. assert.equal(run.coverage.kind, 'static-reachability'); assert.match(run.coverage.note, /NOT a measurement/); assert.ok( run.coverage.patternsReachable > 0, 'zero patterns reachable — a byte-identical gate over a corpus that ' + 'reaches nothing proves nothing' ); assert.ok( Array.isArray(run.coverage.unreachablePatterns), 'unreachable patterns must be listed explicitly, not silently dropped' ); assert.equal( run.coverage.patternsReachable + run.coverage.unreachablePatterns.length, run.coverage.patternsTotal, 'reachability accounting does not add up' ); }); });