llm-security/tests/lib/golden-baseline.test.mjs

// golden-baseline.test.mjs — Phase 5 step 1: the golden gate.
//
// Phase 5 swaps four data tables (injection patterns, codepoint/homoglyph
// tables, severity maps, signatures) out of `scanners/lib/*.mjs` and into a
// vendored `llm-security-commons` copy, table by table. The plan does not
// assume that extraction is behaviour-preserving — this gate is what proves
// it, per swap, with rollback if it fails.
//
// What it pins, and why each layer exists:
//
//   1. REGEX records — `.source` + `.flags` of every RegExp reachable from a
//      module's exports. After a swap a pattern is `new RegExp(jsonString,
//      flags)`, so the plan's named hazard ("JSON backslash-doubling on 83+18
//      regexes") shows up here and nowhere else. Source-text comparison cannot
//      see it; only the COMPILED object can.
//
//   2. TABLE records — a stable key/value digest of the data tables. Most of
//      what Phase 4 moves is not a regex at all: HOMOGLYPH_MAP (x3, moving
//      AS-IS), the typosquat token list, and the four OWASP maps are char->char
//      and string->string data. A regex-only dump is blind to a broken
//      homoglyph swap, i.e. blind to the bulk of the payload.
//
//   3. FILE records — sha256 of each source file in the moving set. This is
//      the completeness layer, and it is complete BY CONSTRUCTION: it covers
//      regexes inlined in function bodies (`NAMED` at string-utils.mjs:291,
//      the BIDI/tag/PUA codepoint ranges at 357-404) that no export walk can
//      reach. It is why this gate needs no JS parser and no lexer — a lexical
//      regex count over these files is contaminated by comments and division
//      anyway, so pinning one would pin a lie.
//
//   4. REFERENCE RUN — the 61 showcase payloads through the real hook entry
//      points (subprocess, stdin protocol), plus a coverage assertion. Byte
//      identity over a corpus that exercises 5 of 90 patterns would prove
//      almost nothing, so the artifact records WHICH patterns and tables the
//      run actually reaches, and the gate fails if that coverage shrinks.
//
// Failure mode this file is written against: a gate that reports success
// without running (v7.8.2 shipped four of those). `JSON.stringify(/a/g)` is
// `{}` — a dump built by stringifying RegExp objects is byte-stable and
// permanently empty. Hence the non-emptiness assertions below; they are not
// paranoia, they are the specific bug.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, readFileSync } from 'node:fs';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, '../..');
const GOLDEN_DIR = resolve(ROOT, 'tests/golden');

const PATTERNS_ARTIFACT = resolve(GOLDEN_DIR, 'patterns.json');
const REFERENCE_ARTIFACT = resolve(GOLDEN_DIR, 'reference-run.json');

function readArtifact(path) {
  assert.ok(
    existsSync(path),
    `missing golden artifact: ${path}\n` +
      `Regenerate with: node scripts/golden-baseline.mjs --write`
  );
  return readFileSync(path, 'utf8');
}

describe('golden baseline — pattern + table dump', () => {
  it('regenerates byte-identically from the current source tree', async () => {
    const { buildPatternDump, serialize } = await import(
      '../../scripts/lib/golden-dump.mjs'
    );
    const committed = readArtifact(PATTERNS_ARTIFACT);
    const fresh = serialize(await buildPatternDump(ROOT));

    assert.equal(
      fresh,
      committed,
      'pattern dump drifted from the committed baseline.\n' +
        'If a Phase 5 swap caused this, the swap is NOT behaviour-preserving — roll it back.\n' +
        'If the change is intentional, re-bless with: node scripts/golden-baseline.mjs --write'
    );
  });

  it('contains regex records with a non-empty source (JSON.stringify(/a/g) === "{}" guard)', async () => {
    const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT));
    const regexes = dump.records.filter((r) => r.kind === 'regex');

    assert.ok(
      regexes.length >= 80,
      `expected >=80 regex records, got ${regexes.length} — an empty or ` +
        `near-empty dump is byte-stable and proves nothing`
    );
    for (const r of regexes) {
      assert.ok(
        typeof r.source === 'string' && r.source.length > 0,
        `regex record ${r.key} has an empty source — the dump serialized ` +
          `RegExp objects instead of reading .source`
      );
      assert.ok(typeof r.flags === 'string', `regex record ${r.key} lost its flags`);
    }
  });

  it('covers every data table Phase 4 moves, including the private HOMOGLYPH_MAP', async () => {
    const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT));
    const tableKeys = dump.records.filter((r) => r.kind === 'table').map((r) => r.key);

    // HOMOGLYPH_MAP is module-private at string-utils.mjs:448. It must be
    // exported for the walk to reach it, and that export has to land BEFORE
    // the baseline is recorded — otherwise the before/after comparison is
    // confounded by the very surface change the plan flags as a hazard
    // ("private tables become loaded").
    for (const required of [
      'string-utils:HOMOGLYPH_MAP',
      'string-utils:TYPOSQUAT_SUSPICIOUS_TOKENS',
      'severity:OWASP_MAP',
      'severity:OWASP_AGENTIC_MAP',
      'severity:OWASP_SKILLS_MAP',
      'severity:OWASP_MCP_MAP',
      'severity:SEVERITY',
    ]) {
      assert.ok(
        tableKeys.includes(required),
        `table record missing: ${required} (have: ${tableKeys.join(', ')})`
      );
    }

    for (const r of dump.records.filter((x) => x.kind === 'table')) {
      assert.ok(r.entries > 0, `table ${r.key} is empty`);
      assert.match(r.digest, /^sha256:[0-9a-f]{64}$/, `table ${r.key} has no digest`);
    }
  });

  it('pins a file digest for every source file in the moving set', async () => {
    const dump = JSON.parse(readArtifact(PATTERNS_ARTIFACT));
    const files = dump.records.filter((r) => r.kind === 'file');
    const keys = files.map((r) => r.key);

    for (const required of [
      'scanners/lib/injection-patterns.mjs',
      'scanners/lib/string-utils.mjs',
      'scanners/lib/severity.mjs',
      // knowledge/signatures.json is deliberately absent: the SIG ruleset moved
      // to vendored commons in v8 Phase 5 step 4 and is anchored by the
      // `malware-signatures` walked module instead of a file digest.
      'knowledge/attack-mutations.json',
    ]) {
      assert.ok(keys.includes(required), `file record missing: ${required}`);
    }

    for (const r of files) {
      assert.match(r.sha256, /^[0-9a-f]{64}$/, `file ${r.key} has no digest`);
    }
  });
});

describe('golden baseline — suite counts', () => {
  // Deliberately NOT a byte-identity gate. Adding a test is normal work and
  // must not turn the baseline red; what Phase 5 actually asks is narrower —
  // "did a swap make a file that used to pass stop passing?". So this asserts
  // no REGRESSION, and re-blessing is only needed to raise the floor.
  const SUITE_ARTIFACT = resolve(GOLDEN_DIR, 'suite-counts.json');

  it('records a per-file floor with no failures in it', () => {
    const suite = JSON.parse(readArtifact(SUITE_ARTIFACT));
    assert.ok(suite.files.length >= 90, `expected >=90 test files, got ${suite.files.length}`);
    assert.equal(suite.totals.fail, 0, 'the baseline was recorded with failing files');

    // Whatever is left out is named, so "covered everything" is never implied
    // by omission.
    assert.ok(Array.isArray(suite.excluded), 'exclusions must be listed, not silent');
    for (const e of suite.excluded) {
      assert.ok(e.reason, `excluded ${e.file} without a reason`);
    }

    for (const f of suite.files) {
      assert.ok(
        f.pass >= 0,
        `${f.file} produced no TAP summary — it did not run, and a baseline ` +
          `that silently records a non-run file is the v7.8.2 defect class`
      );
    }
  });
});

describe('golden baseline — reference run (61 payloads, real entry points)', () => {
  it('regenerates byte-identically through the real hook entry points', async () => {
    const { buildReferenceRun, serialize } = await import(
      '../../scripts/lib/golden-dump.mjs'
    );
    const committed = readArtifact(REFERENCE_ARTIFACT);
    const fresh = serialize(await buildReferenceRun(ROOT));

    assert.equal(
      fresh,
      committed,
      'reference run drifted from the committed baseline.\n' +
        'Re-bless with: node scripts/golden-baseline.mjs --write'
    );
  });

  it('ran all 61 showcase payloads and none of them silently no-opped', async () => {
    const run = JSON.parse(readArtifact(REFERENCE_ARTIFACT));
    assert.equal(run.cases.length, 61, 'the conformance seed is 61 cases');
    for (const c of run.cases) {
      assert.ok(typeof c.exitCode === 'number', `case ${c.id} has no exit code`);
      assert.ok(typeof c.verdict === 'string' && c.verdict.length > 0,
        `case ${c.id} produced no verdict`);
    }
  });

  it('records static pattern reachability, labelled as such, and fails if it shrinks', async () => {
    const run = JSON.parse(readArtifact(REFERENCE_ARTIFACT));
    assert.ok(run.coverage, 'reference run carries no coverage block');

    // The label is load-bearing. This block probes payload strings against
    // patterns in-process; it does NOT measure what the 61 hook invocations
    // evaluated. A future session quoting it as observed coverage would be
    // quoting a number that measures something else — pinned here so the
    // honest name cannot be quietly dropped.
    assert.equal(run.coverage.kind, 'static-reachability');
    assert.match(run.coverage.note, /NOT a measurement/);

    assert.ok(
      run.coverage.patternsReachable > 0,
      'zero patterns reachable — a byte-identical gate over a corpus that ' +
        'reaches nothing proves nothing'
    );
    assert.ok(
      Array.isArray(run.coverage.unreachablePatterns),
      'unreachable patterns must be listed explicitly, not silently dropped'
    );
    assert.equal(
      run.coverage.patternsReachable + run.coverage.unreachablePatterns.length,
      run.coverage.patternsTotal,
      'reachability accounting does not add up'
    );
  });
});