llm-security/scripts/lib/golden-dump.mjs

// golden-dump.mjs — builds the v8 Phase 5 golden baseline artifacts.
//
// Two builders, both pure and deterministic, both used by BOTH the generator
// (`scripts/golden-baseline.mjs --write`) and the gate
// (`tests/lib/golden-baseline.test.mjs`). One code path, so the gate cannot
// pass against a stale generator.
//
// Determinism rules observed here:
//   - no timestamps, no absolute paths, no host details in the output;
//   - every collection is emitted in a deterministic order — export-walk order
//     comes from sorted key names, not from V8 enumeration luck;
//   - the reference run executes hooks SEQUENTIALLY. Array order is semantic
//     in this codebase (dedup + output order) and the plan forbids
//     key-sorting tooling, so the fix for concurrency-dependent ordering is to
//     remove the concurrency, not to sort the result.

import { createHash } from 'node:crypto';
import { execFile, execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { resolve, relative } from 'node:path';

// ---------------------------------------------------------------------------
// Shared
// ---------------------------------------------------------------------------

/** Canonical serialization for every artifact: 2-space JSON + trailing NL. */
export function serialize(obj) {
  return JSON.stringify(obj, null, 2) + '\n';
}

function sha256(s) {
  return createHash('sha256').update(s, 'utf8').digest('hex');
}

/** Modules whose exports are walked for regex + table records. */
const WALKED_MODULES = [
  ['injection-patterns', 'scanners/lib/injection-patterns.mjs'],
  ['string-utils', 'scanners/lib/string-utils.mjs'],
  ['severity', 'scanners/lib/severity.mjs'],
];

/**
 * Source files pinned by digest. This is the completeness layer: it covers
 * every regex inlined in a function body that no export walk can reach.
 */
const PINNED_FILES = [
  'scanners/lib/injection-patterns.mjs',
  'scanners/lib/string-utils.mjs',
  'scanners/lib/severity.mjs',
  'knowledge/signatures.json',
  'knowledge/attack-mutations.json',
];

// ---------------------------------------------------------------------------
// Layer 1+2: export walk -> regex records and table records
// ---------------------------------------------------------------------------

/**
 * Depth-first walk of an exported value, collecting RegExp instances with a
 * stable path key (`CRITICAL_PATTERNS[3].pattern`).
 *
 * Reads `.source`/`.flags` off the compiled object. This is the whole point:
 * after a Phase 5 swap the pattern is `new RegExp(jsonString, flags)`, and a
 * backslash lost to JSON escaping is visible ONLY in the compiled source.
 */
function collectRegexes(value, path, out, seen) {
  if (value instanceof RegExp) {
    out.push({ kind: 'regex', key: path, source: value.source, flags: value.flags });
    return;
  }
  if (value === null || typeof value !== 'object') return;
  if (seen.has(value)) return;
  seen.add(value);

  if (Array.isArray(value)) {
    value.forEach((v, i) => collectRegexes(v, `${path}[${i}]`, out, seen));
    return;
  }
  for (const k of Object.keys(value).sort()) {
    collectRegexes(value[k], `${path}.${k}`, out, seen);
  }
}

/**
 * A data table is an exported plain object or array of scalars — the shape
 * Phase 4 moves into commons as JSON. Arrays of pattern objects are NOT
 * tables; their regexes are already pinned individually above, and digesting
 * them again would double-count without adding signal.
 */
function isDataTable(value) {
  if (value === null || typeof value !== 'object') return false;
  if (value instanceof RegExp) return false;
  const values = Array.isArray(value) ? value : Object.values(value);
  if (values.length === 0) return false;
  // A value is either a scalar (HOMOGLYPH_MAP: char -> char, SEVERITY: name ->
  // name) or an array of scalars (the four OWASP maps: prefix -> ['LLM01']).
  const scalar = (v) => typeof v === 'string' || typeof v === 'number';
  return values.every((v) => scalar(v) || (Array.isArray(v) && v.every(scalar)));
}

/**
 * Stable digest of a table's contents. Keys are sorted for the digest input
 * only — the artifact never reorders anything the runtime reads.
 */
function digestTable(value) {
  const canonical = Array.isArray(value)
    ? JSON.stringify(value)
    : JSON.stringify(
        Object.keys(value)
          .sort()
          .map((k) => [k, value[k]])
      );
  return {
    entries: Array.isArray(value) ? value.length : Object.keys(value).length,
    digest: `sha256:${sha256(canonical)}`,
  };
}

export async function buildPatternDump(root) {
  const records = [];

  for (const [alias, relPath] of WALKED_MODULES) {
    const mod = await import(resolve(root, relPath));
    for (const exportName of Object.keys(mod).sort()) {
      const value = mod[exportName];
      if (typeof value === 'function') continue;

      collectRegexes(value, `${alias}:${exportName}`, records, new WeakSet());

      if (isDataTable(value)) {
        records.push({ kind: 'table', key: `${alias}:${exportName}`, ...digestTable(value) });
      }
    }
  }

  for (const relPath of PINNED_FILES) {
    records.push({
      kind: 'file',
      key: relPath,
      sha256: sha256(readFileSync(resolve(root, relPath), 'utf8')),
    });
  }

  // Stable ordering across kinds so a re-run of the walk cannot reshuffle the
  // artifact for reasons unrelated to the code under test.
  const kindRank = { regex: 0, table: 1, file: 2 };
  records.sort(
    (a, b) => kindRank[a.kind] - kindRank[b.kind] || a.key.localeCompare(b.key)
  );

  return {
    artifact: 'golden-pattern-dump',
    schema: 1,
    counts: {
      regex: records.filter((r) => r.kind === 'regex').length,
      table: records.filter((r) => r.kind === 'table').length,
      file: records.filter((r) => r.kind === 'file').length,
    },
    records,
  };
}

// ---------------------------------------------------------------------------
// Layer 4: reference run — 61 showcase payloads through real hook entry points
// ---------------------------------------------------------------------------

const HOOK_SCRIPTS = {
  'pre-prompt-inject-scan': 'hooks/scripts/pre-prompt-inject-scan.mjs',
  'post-mcp-verify': 'hooks/scripts/post-mcp-verify.mjs',
  'pre-bash-destructive': 'hooks/scripts/pre-bash-destructive.mjs',
};

/** Mirrors the stdin protocol in examples/prompt-injection-showcase/run-showcase.mjs. */
function buildInput(payload) {
  switch (payload.hook) {
    case 'pre-prompt-inject-scan':
      return { session_id: 'golden', message: { role: 'user', content: payload.payload } };
    case 'post-mcp-verify':
      return {
        tool_name: payload.inputTool || 'mcp__server__tool',
        tool_input: {},
        tool_output: payload.payload,
      };
    case 'pre-bash-destructive':
      return { tool_name: 'Bash', tool_input: { command: payload.payload } };
    default:
      throw new Error(`Unknown hook: ${payload.hook}`);
  }
}

function runHook(scriptPath, input, cwd) {
  return new Promise((res) => {
    const child = execFile(
      process.execPath,
      [scriptPath],
      { timeout: 15000, cwd, env: { ...process.env, LLM_SECURITY_UPDATE_CHECK: 'off' } },
      (_err, stdout, stderr) => {
        res({ code: child.exitCode ?? 1, stdout: stdout || '', stderr: stderr || '' });
      }
    );
    child.stdin.end(JSON.stringify(input));
  });
}

/**
 * Exit code + stdout -> verdict. Mirrors `classify()` in
 * examples/prompt-injection-showcase/run-showcase.mjs, including the
 * `advisory` state (exit 0 WITH output): a hook that warns is not a hook that
 * allowed, and collapsing the two would understate detection across the whole
 * MEDIUM tier of the corpus.
 */
function classify(code, stdout) {
  if (code === 2) return 'block';
  if (code === 0) return stdout.trim() ? 'advisory' : 'allow';
  return `error(${code})`;
}

/**
 * Strips everything host- or run-specific from hook output so the artifact is
 * byte-stable: absolute paths, PIDs, durations, and the session file path.
 */
function normalizeOutput(s, root) {
  return s
    .split(root)
    .join('<ROOT>')
    .replace(/llm-security-session-\d+\.jsonl/g, 'llm-security-session-<PID>.jsonl')
    .replace(/\b\d+(\.\d+)?\s?ms\b/g, '<MS>')
    .replace(/\b\d{4}-\d{2}-\d{2}T[\d:.]+Z\b/g, '<TS>')
    .trimEnd();
}

export async function buildReferenceRun(root) {
  const payloads = JSON.parse(
    readFileSync(resolve(root, 'examples/prompt-injection-showcase/payloads.json'), 'utf8')
  );

  const cases = [];
  // Sequential on purpose — see the determinism note at the top of this file.
  for (const p of payloads) {
    const { code, stdout, stderr } = await runHook(
      resolve(root, HOOK_SCRIPTS[p.hook]),
      buildInput(p),
      root
    );
    const verdict = classify(code, stdout);
    cases.push({
      id: p.id,
      hook: p.hook,
      category: p.category,
      expected: p.expected,
      exitCode: code,
      verdict,
      matchesExpectation: verdict === p.expected,
      stdout: normalizeOutput(stdout, root),
      stderr: normalizeOutput(stderr, root),
    });
  }

  return {
    artifact: 'golden-reference-run',
    schema: 1,
    coverage: await buildCoverage(root, payloads),
    summary: {
      total: cases.length,
      matchingExpectation: cases.filter((c) => c.matchesExpectation).length,
    },
    cases,
  };
}

/**
 * STATIC REACHABILITY — not observed coverage. Read the distinction before
 * quoting the number anywhere.
 *
 * This probes every injection pattern against every payload string
 * in-process. It answers "if you threw all 61 payload strings at all 83
 * regexes, how many would match?" It does NOT answer "how many patterns did
 * the 61 hook invocations actually evaluate": the `pre-bash-destructive`
 * payloads never reach injection-patterns at all, yet their strings are in
 * the probe set and can mark a pattern reachable.
 *
 * It is kept because it still bounds the gate honestly — an unreachable
 * pattern is one the corpus cannot protect under ANY attribution — but it is
 * named for what it measures. A number labelled as measuring one thing while
 * measuring another is this repo's v7.8.2 defect class.
 *
 * Not covered here at all: the four OWASP maps are scanner-side and no hook
 * in this corpus reaches them. Their only protection is the table digest.
 */
async function buildCoverage(root, payloads) {
  const ip = await import(resolve(root, 'scanners/lib/injection-patterns.mjs'));
  const su = await import(resolve(root, 'scanners/lib/string-utils.mjs'));

  const groups = ['CRITICAL_PATTERNS', 'HIGH_PATTERNS', 'MEDIUM_PATTERNS', 'HYBRID_PATTERNS'];
  const entries = [];
  for (const g of groups) {
    (ip[g] || []).forEach((entry, i) => {
      if (entry && entry.pattern instanceof RegExp) entries.push({ key: `${g}[${i}]`, re: entry.pattern });
    });
  }

  const texts = payloads.flatMap((p) => [p.payload, su.normalizeForScan(p.payload)]);
  const reachable = [];
  const unreachable = [];
  for (const { key, re } of entries) {
    const probe = new RegExp(re.source, re.flags.replace('g', ''));
    (texts.some((t) => probe.test(t)) ? reachable : unreachable).push(key);
  }

  return {
    kind: 'static-reachability',
    note:
      'Payload strings probed against patterns in-process. NOT a measurement ' +
      'of what the 61 hook invocations evaluated — pre-bash-destructive ' +
      'payloads never reach injection-patterns yet are in the probe set. ' +
      'The four OWASP maps are scanner-side and have no behavioural coverage ' +
      'here at all; their only protection is the table digest.',
    patternsTotal: entries.length,
    patternsReachable: reachable.length,
    unreachablePatterns: unreachable,
    corpusContains: {
      // "a payload contains a homoglyph", not "the run folded one".
      homoglyphChars: texts.some((t) => su.foldHomoglyphs(t) !== t),
      payloadsAlteredByNormalize: payloads.filter(
        (p) => su.normalizeForScan(p.payload) !== p.payload
      ).length,
    },
  };
}

// ---------------------------------------------------------------------------
// Artifact (a): per-file suite counts
// ---------------------------------------------------------------------------

/**
 * Per-file pass/fail counts, collected by running each test file in its OWN
 * process, sequentially.
 *
 * Two reasons it is per-file rather than one total:
 *
 *   1. A total is unattributable. `2045` -> `2044` tells you something broke;
 *      it does not tell you where, and during a table-by-table swap that is
 *      most of the information.
 *   2. `npm test` runs files concurrently, and three timing-sensitive files
 *      in this suite (pre-compact-scan size cap, attack-simulator
 *      --benchmark, pre-install-supply-chain F-3) flake under that load while
 *      passing in isolation. Running each file alone is exactly the
 *      already-documented procedure for trusting a red result, so the
 *      baseline is recorded the only way it is reproducible.
 *
 * Slow by construction (one node process per file) — hence opt-in via
 * `--suite`, not part of the default regeneration.
 */
/**
 * The gate's own file cannot be in its own floor: it READS
 * `suite-counts.json`, which is produced by the run that would include it, so
 * the first recording always captures it mid-bootstrap as a failure. Excluded
 * by name and reported in the artifact rather than dropped silently — `npm
 * test` still covers it.
 */
const SUITE_SELF_REFERENCE = 'tests/lib/golden-baseline.test.mjs';

export function buildSuiteCounts(root, files) {
  const entries = [];
  const excluded = [];
  for (const file of files) {
    const rel = relative(root, file);
    if (rel === SUITE_SELF_REFERENCE) {
      excluded.push({ file: rel, reason: 'self-reference: this file reads suite-counts.json' });
      continue;
    }
    let out = '';
    try {
      out = execFileSync(process.execPath, ['--test', '--test-reporter=tap', rel], {
        cwd: root,
        encoding: 'utf8',
        stdio: ['ignore', 'pipe', 'pipe'],
        timeout: 300000,
        env: { ...process.env, LLM_SECURITY_UPDATE_CHECK: 'off' },
      });
    } catch (err) {
      // Non-zero exit is a failing file, not a harness error — its TAP
      // summary is still on stdout and is what we want to record.
      out = (err.stdout || '') + (err.stderr || '');
    }
    const pass = Number((out.match(/^# pass (\d+)$/m) || [])[1] ?? -1);
    const fail = Number((out.match(/^# fail (\d+)$/m) || [])[1] ?? -1);
    entries.push({ file: rel, pass, fail });
  }

  entries.sort((a, b) => a.file.localeCompare(b.file));
  return {
    artifact: 'golden-suite-counts',
    schema: 1,
    totals: {
      files: entries.length,
      pass: entries.reduce((n, e) => n + Math.max(e.pass, 0), 0),
      fail: entries.reduce((n, e) => n + Math.max(e.fail, 0), 0),
    },
    excluded,
    files: entries,
  };
}