llm-security/tests/lib/conformance-corpus.test.mjs

// conformance-corpus.test.mjs — runs the vendored llm-security-commons
// conformance corpus through the real scanForInjection() entry point.
//
// v8 Phase 5 step 3. The corpus is a cross-runtime contract: the same 84 cases
// are run by llm-security and by llm-ingestion-pipeline-security, so a table
// swap in step 4 that quietly changes what we detect shows up here as a named
// failing case rather than as nothing at all. Note that 84 cases exercise 83
// patterns: commons v0.3.0 gave `hybrid-xss:script-tag` a second case
// (`--src-no-close`) when it converged on our open-tag-only form, so the
// case-to-pattern relation is many-to-one and only the alias map is a
// bijection. Normative semantics live in
// scanners/commons/spec/conformance-corpus.md; where this file and that
// document disagree, the document wins.
//
// Three spec rules shape the structure below and are easy to violate by accident:
//
//   §1  A runtime MUST run every case, and a case that cannot be run MUST be
//       reported as an error, not as a pass. So case discovery is driven by
//       the manifest's declared list, cross-checked against the directories on
//       disk — a dropped case dir must fail loudly, not shrink the gate.
//
//   §1.1 A runtime MUST declare the set of commons data files it implements and
//       publish that set with its result. A case scoped to a file outside the
//       declared set is `not-applicable` — a third verdict, distinct from §1's
//       error: §1 means *we tried and could not*, not-applicable means *this
//       question was never addressed to us*. Such a case MUST still be
//       enumerated, MUST NOT count as a pass, and MUST NOT leave the
//       denominator. `84/84 passed, 6 not-applicable` is the shape.
//
//   §4  A runtime MUST reject a `match` it does not implement rather than
//       degrade to a weaker comparison. Hence the explicit throw.
//
// §1.1 has a gate of its own: a runtime MUST NOT narrow its declared set to
// turn failures into not-applicable. That is enforced here by construction
// rather than remembered — DECLARED_TABLES is the *same* constant the runner
// uses to accept a scope, so narrowing the declaration narrows what this suite
// will run at all, and shows up as a code change. A hand-maintained
// declaration would be a claim; a derived one is a measurement.
//
// Findings are named by commons `pattern_id`, which scanForInjection() does not
// carry: it returns our own labels. The lexicon publishes the mapping per
// pattern in `aliases.llm_security`, and spec §3.1 permits a runtime that is
// registered there — as we are — to compare through it. Measured before this
// file was written: the map is a total bijection, 83 labels to 83 ids, no
// duplicates, family membership agreeing on every entry. It is therefore a
// lookup, not a private translation table: nothing here restates a pattern's
// id, severity or label, so nothing here can drift from the lexicon.

import { describe, it, before } from 'node:test';
import assert from 'node:assert';
import { readFileSync, readdirSync, statSync, existsSync, mkdirSync, writeFileSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createHash } from 'node:crypto';
import { scanForInjection } from '../../scanners/lib/injection-patterns.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = join(__dirname, '..', '..');
const COMMONS_ROOT = join(REPO_ROOT, 'scanners', 'commons');
const CORPUS_ROOT = join(COMMONS_ROOT, 'conformance');

/**
 * Where the §1.1 declaration is published, per run.
 *
 * Gitignored on purpose. A committed declaration is a claim frozen at commit
 * time: its `measured_date` and both commit fields keep saying what was true
 * once, and nothing makes them wrong out loud when they stop being true. A
 * declaration regenerated by the run that measured it cannot rot, because it
 * does not outlive the measurement.
 */
const DECLARATION_PATH = join(REPO_ROOT, 'reports', 'conformance-declaration.json');

const SUPPORTED_MATCH = 'exact-within-scope';

/**
 * Spec §1.1: the set of commons data files this runtime implements.
 *
 * This is the ONLY scope authority in the file. It is what gets published as
 * our declaration, and it is what decides whether a case is run — one constant,
 * two uses, so the two cannot disagree. `signatures/active-content.json` is
 * absent because we do not implement that table, not because its cases fail;
 * see the note on the not-applicable verdict below.
 */
const DECLARED_TABLES = Object.freeze(['lexicon/injection-lexicon.json']);

const manifest = JSON.parse(readFileSync(join(CORPUS_ROOT, 'manifest.json'), 'utf8'));

/**
 * case_id -> parsed expected.json, read once at load.
 *
 * Eager on purpose: a case the manifest declares but which has no directory is
 * a §1 error, and erroring here makes the whole file fail loudly rather than
 * letting the suite shrink to the cases that happen to still exist.
 */
const caseSpecs = new Map(manifest.cases.map((caseId) => {
  const specPath = join(CORPUS_ROOT, caseId, 'expected.json');
  try {
    return [caseId, JSON.parse(readFileSync(specPath, 'utf8'))];
  } catch (err) {
    throw new Error(`case ${caseId}: declared by the manifest but unreadable at ${specPath} — spec §1 requires an error, not a shrunken gate (${err.message})`);
  }
}));

/**
 * §1.1 verdict for one case, decided by its scope alone.
 *
 * Note what this deliberately cannot express: there is no per-case opt-out.
 * The verdict attaches to the TABLE, so a case scoped to a table we declare is
 * run even when it fails. That is the point — per-case opt-out is exactly the
 * silent skip §1 forbids.
 */
function isApplicable(caseId) {
  const scope = caseSpecs.get(caseId).scope || [];
  if (scope.length === 0) {
    throw new Error(`case ${caseId}: empty scope — spec §3 requires a scope, and an unscoped case cannot be compared`);
  }
  return scope.every((table) => DECLARED_TABLES.includes(table));
}

const applicableCases = manifest.cases.filter(isApplicable);
const notApplicableCases = manifest.cases.filter((c) => !isApplicable(c));

/**
 * §1.1's anti-narrowing constraint, as a gate rather than an intention.
 *
 * Measured, not assumed: narrowing DECLARED_TABLES to `[]` turns all 90 cases
 * not-applicable and leaves this suite GREEN with zero cases run — precisely
 * the exit §1.1 forbids. "Narrowing is visible as a code change" describes a
 * reviewer, not a gate, so the floor below is derived and asserted instead.
 *
 * The floor is deliberately NOT a second hand-maintained list of tables; that
 * would be the parallel declaration this runtime promised commons it would not
 * keep, and it would rot against DECLARED_TABLES the first time one moved. It
 * is read out of the vendored tables themselves: a table whose `aliases` name
 * `llm_security` has registered this runtime as a consumer (spec §3.1), and a
 * registered consumer that stops declaring the table is withdrawing a
 * published claim rather than describing what it implements.
 */

/** Every data file the corpus can scope a case to — from the manifest alone. */
function corpusTables() {
  return [...new Set([
    ...(manifest.scope_covered || []),
    ...Object.keys(manifest.scope_planned || {}),
  ])].filter((key) => key.includes('/') && key.endsWith('.json'));
}

/** Does this vendored table register `llm_security` as a consumer? */
function registersUs(table) {
  const tablePath = join(COMMONS_ROOT, table);
  if (!existsSync(tablePath)) return false;
  let found = false;
  (function walk(node) {
    if (found || node === null || typeof node !== 'object') return;
    if (!Array.isArray(node) && node.aliases !== null && typeof node.aliases === 'object'
        && Object.hasOwn(node.aliases, 'llm_security')) {
      found = true;
      return;
    }
    for (const child of Array.isArray(node) ? node : Object.values(node)) walk(child);
  })(JSON.parse(readFileSync(tablePath, 'utf8')));
  return found;
}

const REGISTERED_TABLES = corpusTables().filter(registersUs);

/** label -> pattern_id, read from the lexicon's published aliases. */
function buildAliasMap() {
  const lexicon = JSON.parse(readFileSync(join(COMMONS_ROOT, 'lexicon', 'injection-lexicon.json'), 'utf8'));
  const map = new Map();
  for (const family of lexicon.families) {
    for (const pattern of family.patterns) {
      const alias = pattern.aliases?.llm_security;
      if (alias !== undefined) map.set(alias, pattern.id);
    }
  }
  return map;
}

const aliasMap = buildAliasMap();

/** case_id -> 'passed' | 'failed' | 'error', filled in by the cases as they run. */
const verdicts = new Map();

/**
 * The commons commit this corpus came from, read out of our own history.
 *
 * The schema requires a COMMIT and notes that a declaration citing a tag
 * should give the commit the tag resolved to, "because a tag can be moved and
 * a commit cannot". `git subtree pull --squash` records the upstream range in
 * its own commit subject, so the coordinate is already in this repository and
 * does not have to be transcribed by hand into a constant that would drift
 * from the vendored tree at the next pull.
 *
 * Deliberately not fallback-tolerant: a declaration naming a commons commit it
 * could not determine would be a fabricated coordinate, which is worse than no
 * declaration. Tests do not ship (`package.json` `files` covers bin/, scanners/,
 * knowledge/), so this only ever runs inside a checkout.
 */
function commonsCommit() {
  const subject = execFileSync(
    'git', ['log', '-1', '--format=%s', "--grep=Squashed 'scanners/commons/'"],
    { cwd: REPO_ROOT, encoding: 'utf8' },
  ).trim();
  const m = /\.\.([0-9a-f]{7,40})$/.exec(subject);
  if (m === null) {
    throw new Error(`cannot determine the vendored commons commit from git history (last subtree subject: "${subject}") — refusing to publish a declaration with a coordinate nobody can check`);
  }
  return m[1];
}

/**
 * Build the spec §1.1 declaration from what the run actually produced.
 *
 * Every number here is counted from `verdicts`, and `declared_tables` is the
 * same constant the runner accepts scopes against — which is what lets this
 * declaration claim `declaration_source: derived-from-runner` honestly. The
 * five counts are emitted including the zeros: commons' schema is explicit
 * that an absent count is indistinguishable from one the runtime never
 * tracked, so `0` is information and a missing key is not.
 */
function buildDeclaration() {
  const tally = (name) => [...verdicts.values()].filter((v) => v === name).length;
  const idsWith = (name) => [...verdicts.entries()].filter(([, v]) => v === name).map(([id]) => id).sort();

  const declaration = {
    runtime: 'llm_security',
    runtime_commit: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT, encoding: 'utf8' }).trim(),
    commons_commit: commonsCommit(),
    commons_version: `v${manifest.version}`,
    declared_tables: [...DECLARED_TABLES],
    declaration_source: 'derived-from-runner',
    measured_date: new Date().toISOString().slice(0, 10),
    result: {
      total: applicableCases.length + notApplicableCases.length,
      passed: tally('passed'),
      failed: tally('failed'),
      not_applicable: notApplicableCases.length,
      error: tally('error'),
      match: manifest.match_semantics,
    },
    not_applicable_cases: [...notApplicableCases].sort(),
  };

  // The schema requires these arrays whenever their count is non-zero: an
  // unnamed failure is one nobody can reproduce. Omitted when empty rather
  // than published as `[]`, so their presence always means something happened.
  const failed = idsWith('failed');
  const errored = idsWith('error');
  if (failed.length > 0) declaration.failed_cases = failed;
  if (errored.length > 0) declaration.error_cases = errored;

  return declaration;
}

/**
 * Run one case and return { missing, extra } within the case's scope.
 * Anything the runtime raises that is not a lexicon pattern is out of scope and
 * ignored by the comparison — spec §4: neither required nor forbidden.
 */
function runCase(caseId) {
  const caseDir = join(CORPUS_ROOT, caseId);
  const expected = caseSpecs.get(caseId);

  assert.equal(expected.case_id, caseId, 'expected.json case_id must equal its directory name');
  if (expected.match !== SUPPORTED_MATCH) {
    throw new Error(`case ${caseId}: unimplemented match "${expected.match}" — spec §4 forbids degrading to a weaker comparison`);
  }
  // Scope is not re-checked here: §1.1 decided it before the case was routed to
  // this function, and re-deciding it would be the second authority the single
  // DECLARED_TABLES constant exists to prevent.
  assert.ok(isApplicable(caseId), `case ${caseId}: routed to the runner despite an undeclared scope`);

  // §2: bytes, not text. Both integrity fields are checked because two cases
  // carry characters that are invisible on screen — a zero-width space and a
  // Cyrillic homoglyph — so an editor that "tidies" a fixture can change what
  // it tests without changing how it looks.
  const raw = readFileSync(join(caseDir, expected.input.file));
  assert.equal(raw.length, expected.input.bytes, `case ${caseId}: input byte length differs — fixture altered`);
  assert.equal(createHash('sha256').update(raw).digest('hex'), expected.input.sha256,
    `case ${caseId}: input sha256 differs — fixture altered, the scan result means nothing`);

  const result = scanForInjection(raw.toString('utf8'));
  const observed = new Set();
  for (const label of [...result.critical, ...result.high, ...result.medium]) {
    const patternId = aliasMap.get(label);
    if (patternId !== undefined) observed.add(patternId);
  }
  const expectedIds = new Set((expected.findings || []).map(f => f.pattern_id));

  return {
    missing: [...expectedIds].filter(id => !observed.has(id)),
    extra: [...observed].filter(id => !expectedIds.has(id)),
  };
}

describe('commons conformance corpus — exact-within-scope through scanForInjection()', () => {
  before(() => {
    assert.ok(existsSync(CORPUS_ROOT), 'scanners/commons/conformance is not vendored — run the subtree add');
  });

  // Self-coverage. Without this, deleting case directories would silently
  // shrink the gate to nothing while every remaining case still passed.
  it('runs every case the manifest declares, and no case the manifest omits', () => {
    const onDisk = readdirSync(CORPUS_ROOT)
      .filter(entry => statSync(join(CORPUS_ROOT, entry)).isDirectory())
      .sort();
    assert.deepEqual(onDisk, [...manifest.cases].sort(),
      'case directories on disk disagree with manifest.cases');
    assert.equal(onDisk.length, manifest.count, 'manifest.count disagrees with its own case list');
    assert.equal(manifest.count, 90, 'corpus size changed — re-verify before adjusting this number');
  });

  it('maps every one of our injection labels onto a lexicon pattern_id', () => {
    // The comparison is only as exact as the mapping. A label the lexicon does
    // not name would silently drop out of scope and turn a real extra finding
    // into a pass, so the bijection is asserted rather than assumed.
    assert.equal(aliasMap.size, 83, 'lexicon no longer publishes 83 llm_security aliases');
    assert.equal(manifest.match_semantics, SUPPORTED_MATCH);
  });

  it('cannot withdraw a table it is registered against (spec §1.1 anti-narrowing)', () => {
    // Without this, DECLARED_TABLES = [] is a GREEN suite that runs zero cases.
    // Measured — it was, before this test existed.
    assert.ok(REGISTERED_TABLES.length > 0,
      'no vendored table names llm_security in its aliases — the floor has nothing to stand on, so narrowing the declaration would go undetected again');
    for (const table of REGISTERED_TABLES) {
      assert.ok(DECLARED_TABLES.includes(table),
        `${table} registers llm_security as a consumer but is not in the declared set — §1.1 forbids withdrawing a table to convert failures into not-applicable; if this runtime genuinely dropped the table, the registration in commons must go first`);
    }
  });

  it('declares its table set, and the declaration partitions the corpus (spec §1.1)', () => {
    // 1. The declaration is meaningful only against tables the corpus actually
    //    covers. Declaring a table the corpus does not scope any case to would
    //    be an unfalsifiable claim.
    for (const table of DECLARED_TABLES) {
      assert.ok(manifest.scope_covered.includes(table),
        `declared table ${table} is not in the corpus's scope_covered — the declaration claims something the corpus cannot check`);
    }

    // 2. The partition is total and disjoint: every declared case gets exactly
    //    one verdict, so nothing leaves the denominator.
    assert.deepEqual(
      [...applicableCases, ...notApplicableCases].sort(),
      [...manifest.cases].sort(),
      'applicable + not-applicable is not the whole corpus — a case has escaped the denominator');
    assert.equal(applicableCases.length + notApplicableCases.length, manifest.count);

    // 3. The counts are DERIVED, not restated. manifest.count_by_scope is the
    //    corpus's own tally per table; our applicable count must equal the sum
    //    over the tables we declare. This is what makes narrowing the
    //    declaration visible: drop a table and this number drops with it.
    const expectedApplicable = DECLARED_TABLES
      .reduce((sum, table) => sum + (manifest.count_by_scope[table] ?? 0), 0);
    assert.equal(applicableCases.length, expectedApplicable,
      'applicable count disagrees with manifest.count_by_scope for the declared tables');

    // 4. Every not-applicable case is scoped ONLY to undeclared tables. Without
    //    this, a case scoped to a table we declare could slip into the
    //    not-applicable bucket — the per-case opt-out §1.1 forbids.
    for (const caseId of notApplicableCases) {
      const scope = caseSpecs.get(caseId).scope;
      assert.ok(scope.some((t) => !DECLARED_TABLES.includes(t)),
        `case ${caseId} is marked not-applicable but every table it scopes is declared`);
    }
  });

  for (const caseId of applicableCases) {
    it(`${caseId}`, () => {
      // The verdict is recorded here, from the case that produced it, because
      // §1.1's declaration must be a measurement rather than a summary written
      // afterwards. `failed` and `error` are kept apart on the same distinction
      // the spec turns on: an AssertionError means the case ran and disagreed;
      // anything else means the runtime could not run it at all (§1).
      let verdict = 'error';
      try {
        const { missing, extra } = runCase(caseId);
        assert.deepEqual(missing, [], `case ${caseId}: expected findings the runtime did not raise`);
        assert.deepEqual(extra, [], `case ${caseId}: lexicon findings the runtime raised but the case does not list`);
        verdict = 'passed';
      } catch (err) {
        verdict = err instanceof assert.AssertionError ? 'failed' : 'error';
        throw err;
      } finally {
        verdicts.set(caseId, verdict);
      }
    });
  }

  // §1.1: enumerated by name, reported as skipped rather than passed. The
  // runner's own `# skipped` tally is the third verdict — these cases are
  // neither counted as passes nor dropped from the corpus total.
  for (const caseId of notApplicableCases) {
    const scope = caseSpecs.get(caseId).scope.join(', ');
    it.skip(`${caseId} — not-applicable: scoped to ${scope}, which this runtime does not declare`, () => {});
  }

  // Declared LAST on purpose, and a test rather than an `after` hook.
  //
  // Last, because it can only publish what the cases above have already
  // decided; node:test runs subtests in declaration order unless concurrency
  // is opted into, and the `verdicts.size` assertion below fails loudly if
  // that ever stops holding.
  //
  // A test and not a hook, because MEASURED on Node 25.8.2: an assertion that
  // fails inside `after()` prints under "failing tests" and marks the suite ✖,
  // but leaves `fail 0` and an exit code of ZERO. `npm test` and CI would read
  // that run as green. This gate spent its first draft in that hook, which
  // made it exactly the "reports success without running" failure it exists to
  // catch. Verified by mutation: falsifying a count now exits non-zero.
  it('publishes a spec §1.1 declaration derived from this run', () => {
    // The declaration §1.1 requires us to publish alongside the result, in the
    // shape commons' schema/conformance-declaration.schema.json 0.1.0 defines.
    // Until this landed, what we published was a console.log summary — the
    // right facts in a format only its author could parse, which is most of
    // what §1.1 exists to prevent.
    const declaration = buildDeclaration();

    // The schema states two invariants it cannot express in JSON Schema, and
    // notes that "a validator that checks the schema and not these has checked
    // the shape of a claim without checking the claim". They are asserted
    // before the write, so an unsound declaration is never published.
    const { total, passed, failed, not_applicable: notApplicable, error } = declaration.result;
    assert.equal(passed + failed + notApplicable + error, total,
      'denominator_intact: the four verdict counts do not sum to total — cases were dropped between the run and the publication');
    assert.equal(total, manifest.count,
      'total_matches_corpus: fewer cases were enumerated than the corpus holds at this commons commit — §1 forbids skipping');
    // Every applicable case must have reached a verdict. Without this, a case
    // whose body never ran would leave `total` intact while quietly reducing
    // the numerator, which is the same shrunken gate §1 forbids.
    assert.equal(verdicts.size, applicableCases.length,
      'an applicable case produced no verdict — the declaration would understate what was measured');

    // `declaration_source: derived-from-runner` is a claim about where the
    // array above came from, and commons is explicit that the distinction is
    // not cosmetic. Found by mutation: substituting a literal list for
    // DECLARED_TABLES in the builder left every other assertion green, because
    // they all read the constant rather than what was published. So the
    // published array is tied back to the constant the runner routes cases
    // with — which is what makes the field a measurement instead of a label.
    assert.deepEqual(declaration.declared_tables, [...DECLARED_TABLES],
      'declared_tables was published as something other than the runner\'s own constant — declaration_source claims derived-from-runner, so these cannot differ');
    assert.equal(declaration.declaration_source, 'derived-from-runner');

    // The schema closes `additionalProperties` on both objects and requires all
    // five counts including the zeros. Validating that properly needs a JSON
    // Schema implementation, and this suite has no external dependencies — the
    // full 2020-12 validation was run once against the vendored schema when
    // this landed (valid, with six negative controls rejected). What is worth
    // holding continuously is the cheap half that actually drifts: a field
    // added to the builder without commons publishing it, or a count dropped.
    assert.deepEqual(Object.keys(declaration).sort(), [
      'commons_commit', 'commons_version', 'declaration_source', 'declared_tables',
      'measured_date', 'not_applicable_cases', 'result', 'runtime', 'runtime_commit',
    ], 'declaration key set drifted from the schema — additionalProperties is closed, and failed_cases/error_cases appear only when non-zero');
    assert.deepEqual(Object.keys(declaration.result).sort(),
      ['error', 'failed', 'match', 'not_applicable', 'passed', 'total'],
      'result key set drifted from the schema — all five counts are required, zeros included');

    mkdirSync(dirname(DECLARATION_PATH), { recursive: true });
    writeFileSync(DECLARATION_PATH, `${JSON.stringify(declaration, null, 2)}\n`, 'utf8');

    const lines = [
      '',
      `  commons conformance — ${manifest.id} v${manifest.version} (${manifest.count} cases)`,
      `  declared tables (spec §1.1): ${DECLARED_TABLES.join(', ')}`,
      `  ${passed} passed, ${failed} failed, ${error} error, ${notApplicable} not-applicable  (of ${total})`,
      ...notApplicableCases.map((c) => `    not-applicable  ${c}  [${caseSpecs.get(c).scope.join(', ')}]`),
      `  declaration: ${DECLARATION_PATH}`,
      '',
    ];
    console.log(lines.join('\n'));
  });
});