llm-security/tests/lib/doc-consistency.test.mjs

// doc-consistency.test.mjs — Static asserts that prose documentation
// stays aligned with the v2 risk-scoring model in scanners/lib/severity.mjs.
//
// Background: v7.0.0 introduced the severity-dominated v2 risk-score model
// (BLOCK ≥65, WARNING ≥15) but several prose surfaces (commands/, agents/)
// continued to emit the v1 formula (`critical*25 + ...`, BLOCK ≥61,
// WARNING ≥21). v7.1.1 fixed two of them (agents/skill-scanner-agent.md,
// templates/unified-report.md). Batch B → v7.2.0 closes the trifecta:
// commands/scan.md, commands/audit.md, agents/mcp-scanner-agent.md.
//
// This test pins the closure. If any future edit re-introduces v1 formula
// tokens in commands/ or agents/, this test fails fast.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { readdirSync, readFileSync, statSync } from 'node:fs';
import { join, dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const PLUGIN_ROOT = resolve(dirname(__filename), '..', '..');

// v1 formula tokens that must NOT appear in commands/ or agents/.
// These are the patterns the brief's verification step 4 grep checks.
const V1_TOKENS = [
  /\bscore\s*[><]?=\s*61\b/,                        // verdict cutoff
  /\bscore\s*[><]?=\s*21\b/,                        // verdict cutoff
  /score\s*≥\s*61/,                                 // unicode variant
  /score\s*≥\s*21/,                                 // unicode variant
  /critical\s*\*\s*25/,                             // formula multiplier
  /Critical\s*[×x]\s*25/,                           // formula multiplier (table form)
  /min\(\s*100\s*,\s*critical\s*\*\s*25/i,          // full v1 formula prefix
];

function* walkMarkdown(dir) {
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    const stat = statSync(full);
    if (stat.isDirectory()) {
      yield* walkMarkdown(full);
    } else if (entry.endsWith('.md')) {
      yield full;
    }
  }
}

describe('doc-consistency — v1 risk-formula tokens are absent from prose', () => {
  const COMMANDS_DIR = join(PLUGIN_ROOT, 'commands');
  const AGENTS_DIR = join(PLUGIN_ROOT, 'agents');

  for (const dir of [COMMANDS_DIR, AGENTS_DIR]) {
    for (const file of walkMarkdown(dir)) {
      const rel = file.replace(PLUGIN_ROOT + '/', '');
      it(`${rel} contains no v1 formula tokens`, () => {
        const content = readFileSync(file, 'utf-8');
        for (const token of V1_TOKENS) {
          assert.equal(
            token.test(content),
            false,
            `${rel} still contains v1 formula token matching ${token}. ` +
              `v7.2.0 unified all command/agent prose to v2 (BLOCK ≥65, WARNING ≥15). ` +
              `If a new file legitimately needs to reference v1 (e.g. CHANGELOG history), ` +
              `move that file out of commands/ or agents/.`,
          );
        }
      });
    }
  }
});

describe('doc-consistency — v2 cutoffs are documented in unified prose', () => {
  it('commands/scan.md mentions the v2 BLOCK cutoff (≥ 65)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'scan.md'), 'utf-8');
    assert.match(content, /score\s*[≥>=]+\s*65/);
  });

  it('commands/audit.md references riskScore() (v2 helper)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'audit.md'), 'utf-8');
    assert.match(content, /riskScore/);
  });

  it('agents/mcp-scanner-agent.md mentions the v2 BLOCK cutoff (≥ 65)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'agents', 'mcp-scanner-agent.md'), 'utf-8');
    assert.match(content, /score\s*[≥>=]+\s*65/);
  });
});

// ---------------------------------------------------------------------------
// D4 (Batch C, Wave D): Hooks count must stay synchronized across three
// surfaces — the CLAUDE.md `## Hooks (N)` header, the markdown table directly
// underneath that header, and the canonical hooks/hooks.json definition.
// Drift previously masked a missing `pre-compact-scan.mjs` row in CLAUDE.md.
// This block fails fast if any of the three surfaces drift.
// ---------------------------------------------------------------------------

describe('doc-consistency — Hooks count consistency (D4)', () => {
  const CLAUDE_MD = join(PLUGIN_ROOT, 'CLAUDE.md');
  const HOOKS_JSON = join(PLUGIN_ROOT, 'hooks', 'hooks.json');

  function readHeaderNumber(text) {
    const match = text.match(/^##\s+Hooks\s*\((\d+)\)\s*$/m);
    if (!match) throw new Error('No `## Hooks (N)` header found in CLAUDE.md');
    return parseInt(match[1], 10);
  }

  function readTableRowCount(text) {
    // Section spans from `## Hooks (N)` to next `^## ` heading.
    const startIdx = text.search(/^##\s+Hooks\s*\(\d+\)\s*$/m);
    if (startIdx < 0) throw new Error('Hooks header not found');
    const tail = text.slice(startIdx);
    const nextHeader = tail.search(/\n##\s+\S/);
    const section = nextHeader > 0 ? tail.slice(0, nextHeader) : tail;
    // Count rows that look like `| \`<name>.mjs\` | ...`.
    // Excludes the header row (which uses bare `Script` not a backtick).
    const rows = section.match(/^\|\s*`[^`|]+\.mjs`\s*\|/gm) || [];
    return rows.length;
  }

  function readJsonHookCount(jsonText) {
    const parsed = JSON.parse(jsonText);
    const seen = new Set();
    for (const eventArr of Object.values(parsed.hooks || {})) {
      for (const entry of eventArr) {
        for (const h of entry.hooks || []) {
          // Dedupe by command path — a hook registered to multiple events
          // counts as one script.
          if (h.command) seen.add(h.command);
        }
      }
    }
    return seen.size;
  }

  it('header count, table row count, and hooks.json count agree', () => {
    const claudeText = readFileSync(CLAUDE_MD, 'utf-8');
    const hooksJsonText = readFileSync(HOOKS_JSON, 'utf-8');

    const headerNumber = readHeaderNumber(claudeText);
    const tableRowCount = readTableRowCount(claudeText);
    const jsonHookCount = readJsonHookCount(hooksJsonText);

    assert.equal(
      headerNumber,
      jsonHookCount,
      `CLAUDE.md '## Hooks (${headerNumber})' header disagrees with hooks/hooks.json (${jsonHookCount} hooks). ` +
        `Update the header to match.`,
    );
    assert.equal(
      tableRowCount,
      jsonHookCount,
      `CLAUDE.md hooks table has ${tableRowCount} rows but hooks/hooks.json defines ${jsonHookCount} hooks. ` +
        `Add/remove rows in the table to match.`,
    );
    assert.equal(
      headerNumber,
      tableRowCount,
      `CLAUDE.md header (${headerNumber}) and table row count (${tableRowCount}) disagree. ` +
        `These two surfaces must stay in sync.`,
    );
  });
});

// ---------------------------------------------------------------------------
// B10 (v8.0.0 Phase 2): inventory counts stated in prose must be derived from
// source, not hand-maintained. Before this block, four independent counts had
// drifted — orchestrated scanners (docs said 10, source had 14), total
// scanners (README said 23, the documented rule yields 22), knowledge files
// (README said 22, the directory holds 23), and the `finding()` prefix list in
// output.mjs (documented 10 of the 17 prefixes actually passed to it).
//
// Every count below has ONE derivation from source, stated next to the helper.
// A doc surface that disagrees fails here rather than misleading a reader.
// ---------------------------------------------------------------------------

describe('doc-consistency — inventory counts are derived from source (B10)', () => {
  const SCANNERS_DIR = join(PLUGIN_ROOT, 'scanners');
  const KNOWLEDGE_DIR = join(PLUGIN_ROOT, 'knowledge');

  // Counting rule (scanner total): every `scanners/*.mjs` IS a scanner except
  // these five, which are a runner, a remediator, an extraction library, a
  // cache utility and a CLI wrapper around an already-counted scanner.
  const NON_SCANNER_MODULES = [
    'scan-orchestrator.mjs',
    'auto-cleaner.mjs',
    'content-extractor.mjs',
    'mcp-baseline-reset.mjs',
    'supply-chain-recheck-cli.mjs',
  ];

  function scannerModules() {
    return readdirSync(SCANNERS_DIR).filter(f => f.endsWith('.mjs'));
  }

  // Ground truth: the SCANNERS array literal in scan-orchestrator.mjs.
  function orchestratedCount() {
    const src = readFileSync(join(SCANNERS_DIR, 'scan-orchestrator.mjs'), 'utf-8');
    const start = src.indexOf('const SCANNERS = [');
    if (start < 0) throw new Error('scan-orchestrator.mjs: `const SCANNERS = [` not found');
    const end = src.indexOf('];', start);
    const block = src.slice(start, end);
    return (block.match(/^\s*\{\s*name:/gm) || []).length;
  }

  // Ground truth: scanner modules minus the five non-scanner modules above.
  function totalScannerCount() {
    return scannerModules().filter(f => !NON_SCANNER_MODULES.includes(f)).length;
  }

  // Ground truth: every entry in knowledge/.
  function knowledgeFileCount() {
    return readdirSync(KNOWLEDGE_DIR).length;
  }

  // Ground truth: the CATEGORIES array literal in posture-scanner.mjs.
  function postureCategoryCount() {
    const src = readFileSync(join(SCANNERS_DIR, 'posture-scanner.mjs'), 'utf-8');
    const start = src.indexOf('const CATEGORIES = [');
    if (start < 0) throw new Error('posture-scanner.mjs: `const CATEGORIES = [` not found');
    const end = src.indexOf('];', start);
    return (src.slice(start, end).match(/^\s*\{\s*id:\s*\d+/gm) || []).length;
  }

  // Ground truth: the prefix every module that calls `finding()` passes as
  // `opts.scanner`. Two idioms are in use — an inline `scanner: 'XXX'` literal
  // and a module-level `const SCANNER`/`SCANNER_PREFIX`.
  function findingPrefixes() {
    const prefixes = new Set();
    for (const file of scannerModules()) {
      const src = readFileSync(join(SCANNERS_DIR, file), 'utf-8');
      if (!/import\s*\{[^}]*\bfinding\b[^}]*\}\s*from\s*'\.\/lib\/output\.mjs'/.test(src)) continue;
      for (const m of src.matchAll(/scanner:\s*'([A-Z]{2,4})'/g)) prefixes.add(m[1]);
      for (const m of src.matchAll(/^const SCANNER(?:_PREFIX)?\s*=\s*'([A-Z]{2,4})'/gm)) prefixes.add(m[1]);
    }
    return prefixes;
  }

  it('NON_SCANNER_MODULES all exist (the exclusion list cannot silently rot)', () => {
    const present = new Set(scannerModules());
    for (const name of NON_SCANNER_MODULES) {
      assert.equal(
        present.has(name),
        true,
        `NON_SCANNER_MODULES lists scanners/${name}, which no longer exists. ` +
          `A rename here silently inflates the total scanner count — update the list.`,
      );
    }
  });

  // -- Orchestrated scanner count -------------------------------------------
  // Every "<N> deterministic|orchestrated scanners" claim across prose and the
  // standalone CLI must equal the SCANNERS array length.
  const ORCHESTRATED_CLAIM_FILES = [
    'README.md',
    'CLAUDE.md',
    join('docs', 'ci-cd-guide.md'),
    join('docs', 'scanner-reference.md'),
    join('bin', 'llm-security.mjs'),
    join('knowledge', 'norwegian-context.md'),
    join('agents', 'deep-scan-synthesizer-agent.md'),
  ];

  for (const rel of ORCHESTRATED_CLAIM_FILES) {
    it(`${rel} states the orchestrated scanner count correctly`, () => {
      const expected = orchestratedCount();
      const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8');
      const patterns = [
        /(\d+)\s+deterministic(?:\s+Node\.js)?\s+scanners/gi,
        /(\d+)\s+orchestrated\s+deterministic\s+scanners/gi,
        /\*\*Orchestrated\s*\((\d+)\):?\*\*/g,
        /^(\d+)\s+scanners:/gm,
        /\((\d+)\s+scanners\)/g,
        /findings\s+from\s+(\d+)\s+scanners/gi,
        /scanner\s+output\s+\((\d+)\s+scanners/gi,
        /hooks,\s*(\d+)\s+scanners/gi,
      ];
      for (const pattern of patterns) {
        for (const m of content.matchAll(pattern)) {
          assert.equal(
            Number(m[1]),
            expected,
            `${rel} claims ${m[1]} orchestrated scanners in "${m[0].trim()}", but ` +
              `scan-orchestrator.mjs registers ${expected}. Update the prose.`,
          );
        }
      }
    });
  }

  // -- Total scanner count (orchestrated + standalone) -----------------------
  it('README scanner badge matches the total scanner count', () => {
    const expected = totalScannerCount();
    const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
    const m = content.match(/badge\/scanners-(\d+)-/);
    assert.notEqual(m, null, 'README.md has no `scanners-<N>` badge');
    assert.equal(
      Number(m[1]),
      expected,
      `README scanner badge says ${m[1]} but scanners/ holds ${expected} scanner modules ` +
        `(${scannerModules().length} .mjs files minus ${NON_SCANNER_MODULES.length} non-scanner modules).`,
    );
  });

  it('docs/scanner-reference.md orchestrated + standalone equals the total', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'scanner-reference.md'), 'utf-8');
    const orch = content.match(/\*\*Orchestrated\s*\((\d+)\):?\*\*/);
    const standalone = content.match(/\*\*Standalone\s*\((\d+)\):?\*\*/);
    assert.notEqual(orch, null, 'scanner-reference.md has no `**Orchestrated (N)**` marker');
    assert.notEqual(standalone, null, 'scanner-reference.md has no `**Standalone (N)**` marker');
    assert.equal(
      Number(orch[1]) + Number(standalone[1]),
      totalScannerCount(),
      `scanner-reference.md declares ${orch[1]} orchestrated + ${standalone[1]} standalone ` +
        `= ${Number(orch[1]) + Number(standalone[1])}, but scanners/ holds ${totalScannerCount()}. ` +
        `This split is the counting rule the README badge depends on.`,
    );
  });

  it('README prose scanner totals match the badge', () => {
    const expected = totalScannerCount();
    const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
    const patterns = [
      /Deterministic analysis\s*[—-]\s*(\d+)\s+scanners/g,
      /^(\d+)\s+scanners\.\s+Zero external dependencies/gm,
      /frameworks,\s*(\d+)\s+scanners/g,
    ];
    for (const pattern of patterns) {
      for (const m of content.matchAll(pattern)) {
        assert.equal(
          Number(m[1]),
          expected,
          `README claims ${m[1]} scanners in "${m[0].trim()}" but the total is ${expected}.`,
        );
      }
    }
  });

  // -- Knowledge-file count --------------------------------------------------
  it('README knowledge badge and prose match knowledge/ contents', () => {
    const expected = knowledgeFileCount();
    const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
    const badge = content.match(/badge\/knowledge_docs-(\d+)-/);
    assert.notEqual(badge, null, 'README.md has no `knowledge_docs-<N>` badge');
    assert.equal(
      Number(badge[1]),
      expected,
      `README knowledge badge says ${badge[1]} but knowledge/ holds ${expected} files.`,
    );
    for (const m of content.matchAll(/(\d+)\s+knowledge files/g)) {
      assert.equal(
        Number(m[1]),
        expected,
        `README claims ${m[1]} in "${m[0]}" but knowledge/ holds ${expected} files.`,
      );
    }
  });

  // -- output.mjs finding() prefix list --------------------------------------
  it('output.mjs JSDoc lists every prefix passed to finding()', () => {
    const actual = findingPrefixes();
    const src = readFileSync(join(SCANNERS_DIR, 'lib', 'output.mjs'), 'utf-8');
    const m = src.match(/@param\s+\{string\}\s+opts\.scanner\s*-\s*Scanner prefix\s*\(([^)]*)\)/);
    assert.notEqual(m, null, 'output.mjs has no `opts.scanner - Scanner prefix (...)` JSDoc line');
    const documented = new Set(m[1].split(',').map(s => s.trim()).filter(Boolean));

    const missing = [...actual].filter(p => !documented.has(p)).sort();
    const extra = [...documented].filter(p => !actual.has(p)).sort();
    assert.deepEqual(
      { missing, extra },
      { missing: [], extra: [] },
      `output.mjs JSDoc prefix list is out of sync with the scanners that call finding(). ` +
        `Missing from JSDoc: [${missing.join(', ')}]. Documented but unused: [${extra.join(', ')}].`,
    );
  });

  // -- Posture category count ------------------------------------------------
  it('posture category count agrees across CLAUDE.md, README and scanner-reference', () => {
    const expected = postureCategoryCount();
    for (const rel of ['CLAUDE.md', 'README.md', join('docs', 'scanner-reference.md')]) {
      const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8');
      for (const m of content.matchAll(/(\d+)\s+posture categories/gi)) {
        assert.equal(Number(m[1]), expected, `${rel}: "${m[0]}" but posture-scanner has ${expected}.`);
      }
    }
  });

  it('compliance-mapping.md maps its stated number of posture categories', () => {
    const content = readFileSync(join(KNOWLEDGE_DIR, 'compliance-mapping.md'), 'utf-8');
    // The matrix deliberately covers only the code-level categories; the
    // governance categories (14-16) are consumers of this file, not rows in it.
    const start = content.indexOf('## Mapping Matrix');
    assert.notEqual(start, -1, 'compliance-mapping.md has no `## Mapping Matrix` section');
    const tail = content.slice(start);
    const end = tail.indexOf('\n---');
    const section = end > 0 ? tail.slice(0, end) : tail;
    // Data rows only: skip the header row and the `|---|` separator.
    const rows = (section.match(/^\|(?!\s*Plugin Control)(?!-)[^|]+\|/gm) || [])
      .filter(r => !/^\|\s*-+/.test(r));
    const claimed = content.match(/(\d+)\s+code-level posture categories/);
    assert.notEqual(
      claimed,
      null,
      'compliance-mapping.md must state "<N> code-level posture categories" so the ' +
        'claim is distinguishable from the plugin\'s full posture category count.',
    );
    assert.equal(
      rows.length,
      Number(claimed[1]),
      `compliance-mapping.md claims ${claimed[1]} code-level posture categories but the ` +
        `Mapping Matrix has ${rows.length} data rows.`,
    );
    assert.equal(
      rows.length < postureCategoryCount(),
      true,
      `The matrix is documented as a subset of the ${postureCategoryCount()} posture categories; ` +
        `it now has ${rows.length} rows. If it grew to cover all of them, update the prose too.`,
    );
  });
});