// doc-consistency.test.mjs — Static asserts that prose documentation // stays aligned with the v2 risk-scoring model in scanners/lib/severity.mjs. // // Background: v7.0.0 introduced the severity-dominated v2 risk-score model // (BLOCK ≥65, WARNING ≥15) but several prose surfaces (commands/, agents/) // continued to emit the v1 formula (`critical*25 + ...`, BLOCK ≥61, // WARNING ≥21). v7.1.1 fixed two of them (agents/skill-scanner-agent.md, // templates/unified-report.md). Batch B → v7.2.0 closes the trifecta: // commands/scan.md, commands/audit.md, agents/mcp-scanner-agent.md. // // This test pins the closure. If any future edit re-introduces v1 formula // tokens in commands/ or agents/, this test fails fast. import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { readdirSync, readFileSync, statSync } from 'node:fs'; import { join, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const PLUGIN_ROOT = resolve(dirname(__filename), '..', '..'); // v1 formula tokens that must NOT appear in commands/ or agents/. // These are the patterns the brief's verification step 4 grep checks. const V1_TOKENS = [ /\bscore\s*[><]?=\s*61\b/, // verdict cutoff /\bscore\s*[><]?=\s*21\b/, // verdict cutoff /score\s*≥\s*61/, // unicode variant /score\s*≥\s*21/, // unicode variant /critical\s*\*\s*25/, // formula multiplier /Critical\s*[×x]\s*25/, // formula multiplier (table form) /min\(\s*100\s*,\s*critical\s*\*\s*25/i, // full v1 formula prefix ]; function* walkMarkdown(dir) { for (const entry of readdirSync(dir)) { const full = join(dir, entry); const stat = statSync(full); if (stat.isDirectory()) { yield* walkMarkdown(full); } else if (entry.endsWith('.md')) { yield full; } } } describe('doc-consistency — v1 risk-formula tokens are absent from prose', () => { const COMMANDS_DIR = join(PLUGIN_ROOT, 'commands'); const AGENTS_DIR = join(PLUGIN_ROOT, 'agents'); for (const dir of [COMMANDS_DIR, AGENTS_DIR]) { for (const file of walkMarkdown(dir)) { const rel = file.replace(PLUGIN_ROOT + '/', ''); it(`${rel} contains no v1 formula tokens`, () => { const content = readFileSync(file, 'utf-8'); for (const token of V1_TOKENS) { assert.equal( token.test(content), false, `${rel} still contains v1 formula token matching ${token}. ` + `v7.2.0 unified all command/agent prose to v2 (BLOCK ≥65, WARNING ≥15). ` + `If a new file legitimately needs to reference v1 (e.g. CHANGELOG history), ` + `move that file out of commands/ or agents/.`, ); } }); } } }); describe('doc-consistency — v2 cutoffs are documented in unified prose', () => { it('commands/scan.md mentions the v2 BLOCK cutoff (≥ 65)', () => { const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'scan.md'), 'utf-8'); assert.match(content, /score\s*[≥>=]+\s*65/); }); it('commands/audit.md references riskScore() (v2 helper)', () => { const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'audit.md'), 'utf-8'); assert.match(content, /riskScore/); }); it('agents/mcp-scanner-agent.md mentions the v2 BLOCK cutoff (≥ 65)', () => { const content = readFileSync(join(PLUGIN_ROOT, 'agents', 'mcp-scanner-agent.md'), 'utf-8'); assert.match(content, /score\s*[≥>=]+\s*65/); }); }); // --------------------------------------------------------------------------- // D4 (Batch C, Wave D): Hooks count must stay synchronized across three // surfaces — the CLAUDE.md `## Hooks (N)` header, the markdown table directly // underneath that header, and the canonical hooks/hooks.json definition. // Drift previously masked a missing `pre-compact-scan.mjs` row in CLAUDE.md. // This block fails fast if any of the three surfaces drift. // --------------------------------------------------------------------------- describe('doc-consistency — Hooks count consistency (D4)', () => { const CLAUDE_MD = join(PLUGIN_ROOT, 'CLAUDE.md'); const HOOKS_JSON = join(PLUGIN_ROOT, 'hooks', 'hooks.json'); function readHeaderNumber(text) { const match = text.match(/^##\s+Hooks\s*\((\d+)\)\s*$/m); if (!match) throw new Error('No `## Hooks (N)` header found in CLAUDE.md'); return parseInt(match[1], 10); } function readTableRowCount(text) { // Section spans from `## Hooks (N)` to next `^## ` heading. const startIdx = text.search(/^##\s+Hooks\s*\(\d+\)\s*$/m); if (startIdx < 0) throw new Error('Hooks header not found'); const tail = text.slice(startIdx); const nextHeader = tail.search(/\n##\s+\S/); const section = nextHeader > 0 ? tail.slice(0, nextHeader) : tail; // Count rows that look like `| \`.mjs\` | ...`. // Excludes the header row (which uses bare `Script` not a backtick). const rows = section.match(/^\|\s*`[^`|]+\.mjs`\s*\|/gm) || []; return rows.length; } function readJsonHookCount(jsonText) { const parsed = JSON.parse(jsonText); const seen = new Set(); for (const eventArr of Object.values(parsed.hooks || {})) { for (const entry of eventArr) { for (const h of entry.hooks || []) { // Dedupe by command path — a hook registered to multiple events // counts as one script. if (h.command) seen.add(h.command); } } } return seen.size; } it('header count, table row count, and hooks.json count agree', () => { const claudeText = readFileSync(CLAUDE_MD, 'utf-8'); const hooksJsonText = readFileSync(HOOKS_JSON, 'utf-8'); const headerNumber = readHeaderNumber(claudeText); const tableRowCount = readTableRowCount(claudeText); const jsonHookCount = readJsonHookCount(hooksJsonText); assert.equal( headerNumber, jsonHookCount, `CLAUDE.md '## Hooks (${headerNumber})' header disagrees with hooks/hooks.json (${jsonHookCount} hooks). ` + `Update the header to match.`, ); assert.equal( tableRowCount, jsonHookCount, `CLAUDE.md hooks table has ${tableRowCount} rows but hooks/hooks.json defines ${jsonHookCount} hooks. ` + `Add/remove rows in the table to match.`, ); assert.equal( headerNumber, tableRowCount, `CLAUDE.md header (${headerNumber}) and table row count (${tableRowCount}) disagree. ` + `These two surfaces must stay in sync.`, ); }); }); // --------------------------------------------------------------------------- // B10 (v8.0.0 Phase 2): inventory counts stated in prose must be derived from // source, not hand-maintained. Before this block, four independent counts had // drifted — orchestrated scanners (docs said 10, source had 14), total // scanners (README said 23, the documented rule yields 22), knowledge files // (README said 22, the directory holds 23), and the `finding()` prefix list in // output.mjs (documented 10 of the 17 prefixes actually passed to it). // // Every count below has ONE derivation from source, stated next to the helper. // A doc surface that disagrees fails here rather than misleading a reader. // --------------------------------------------------------------------------- describe('doc-consistency — inventory counts are derived from source (B10)', () => { const SCANNERS_DIR = join(PLUGIN_ROOT, 'scanners'); const KNOWLEDGE_DIR = join(PLUGIN_ROOT, 'knowledge'); // Counting rule (scanner total): every `scanners/*.mjs` IS a scanner except // these five, which are a runner, a remediator, an extraction library, a // cache utility and a CLI wrapper around an already-counted scanner. const NON_SCANNER_MODULES = [ 'scan-orchestrator.mjs', 'auto-cleaner.mjs', 'content-extractor.mjs', 'mcp-baseline-reset.mjs', 'supply-chain-recheck-cli.mjs', ]; function scannerModules() { return readdirSync(SCANNERS_DIR).filter(f => f.endsWith('.mjs')); } // Ground truth: the SCANNERS array literal in scan-orchestrator.mjs. function orchestratedCount() { const src = readFileSync(join(SCANNERS_DIR, 'scan-orchestrator.mjs'), 'utf-8'); const start = src.indexOf('const SCANNERS = ['); if (start < 0) throw new Error('scan-orchestrator.mjs: `const SCANNERS = [` not found'); const end = src.indexOf('];', start); const block = src.slice(start, end); return (block.match(/^\s*\{\s*name:/gm) || []).length; } // Ground truth: scanner modules minus the five non-scanner modules above. function totalScannerCount() { return scannerModules().filter(f => !NON_SCANNER_MODULES.includes(f)).length; } // Ground truth: every entry in knowledge/. function knowledgeFileCount() { return readdirSync(KNOWLEDGE_DIR).length; } // Ground truth: the CATEGORIES array literal in posture-scanner.mjs. function postureCategoryCount() { const src = readFileSync(join(SCANNERS_DIR, 'posture-scanner.mjs'), 'utf-8'); const start = src.indexOf('const CATEGORIES = ['); if (start < 0) throw new Error('posture-scanner.mjs: `const CATEGORIES = [` not found'); const end = src.indexOf('];', start); return (src.slice(start, end).match(/^\s*\{\s*id:\s*\d+/gm) || []).length; } // Ground truth: the prefix every module that calls `finding()` passes as // `opts.scanner`. Two idioms are in use — an inline `scanner: 'XXX'` literal // and a module-level `const SCANNER`/`SCANNER_PREFIX`. function findingPrefixes() { const prefixes = new Set(); for (const file of scannerModules()) { const src = readFileSync(join(SCANNERS_DIR, file), 'utf-8'); if (!/import\s*\{[^}]*\bfinding\b[^}]*\}\s*from\s*'\.\/lib\/output\.mjs'/.test(src)) continue; for (const m of src.matchAll(/scanner:\s*'([A-Z]{2,4})'/g)) prefixes.add(m[1]); for (const m of src.matchAll(/^const SCANNER(?:_PREFIX)?\s*=\s*'([A-Z]{2,4})'/gm)) prefixes.add(m[1]); } return prefixes; } it('NON_SCANNER_MODULES all exist (the exclusion list cannot silently rot)', () => { const present = new Set(scannerModules()); for (const name of NON_SCANNER_MODULES) { assert.equal( present.has(name), true, `NON_SCANNER_MODULES lists scanners/${name}, which no longer exists. ` + `A rename here silently inflates the total scanner count — update the list.`, ); } }); // -- Orchestrated scanner count ------------------------------------------- // Every " deterministic|orchestrated scanners" claim across prose and the // standalone CLI must equal the SCANNERS array length. const ORCHESTRATED_CLAIM_FILES = [ 'README.md', 'CLAUDE.md', join('docs', 'ci-cd-guide.md'), join('docs', 'scanner-reference.md'), join('bin', 'llm-security.mjs'), join('knowledge', 'norwegian-context.md'), join('agents', 'deep-scan-synthesizer-agent.md'), ]; for (const rel of ORCHESTRATED_CLAIM_FILES) { it(`${rel} states the orchestrated scanner count correctly`, () => { const expected = orchestratedCount(); const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8'); const patterns = [ /(\d+)\s+deterministic(?:\s+Node\.js)?\s+scanners/gi, /(\d+)\s+orchestrated\s+deterministic\s+scanners/gi, /\*\*Orchestrated\s*\((\d+)\):?\*\*/g, /^(\d+)\s+scanners:/gm, /\((\d+)\s+scanners\)/g, /findings\s+from\s+(\d+)\s+scanners/gi, /scanner\s+output\s+\((\d+)\s+scanners/gi, /hooks,\s*(\d+)\s+scanners/gi, ]; for (const pattern of patterns) { for (const m of content.matchAll(pattern)) { assert.equal( Number(m[1]), expected, `${rel} claims ${m[1]} orchestrated scanners in "${m[0].trim()}", but ` + `scan-orchestrator.mjs registers ${expected}. Update the prose.`, ); } } }); } // -- Total scanner count (orchestrated + standalone) ----------------------- it('README scanner badge matches the total scanner count', () => { const expected = totalScannerCount(); const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8'); const m = content.match(/badge\/scanners-(\d+)-/); assert.notEqual(m, null, 'README.md has no `scanners-` badge'); assert.equal( Number(m[1]), expected, `README scanner badge says ${m[1]} but scanners/ holds ${expected} scanner modules ` + `(${scannerModules().length} .mjs files minus ${NON_SCANNER_MODULES.length} non-scanner modules).`, ); }); it('docs/scanner-reference.md orchestrated + standalone equals the total', () => { const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'scanner-reference.md'), 'utf-8'); const orch = content.match(/\*\*Orchestrated\s*\((\d+)\):?\*\*/); const standalone = content.match(/\*\*Standalone\s*\((\d+)\):?\*\*/); assert.notEqual(orch, null, 'scanner-reference.md has no `**Orchestrated (N)**` marker'); assert.notEqual(standalone, null, 'scanner-reference.md has no `**Standalone (N)**` marker'); assert.equal( Number(orch[1]) + Number(standalone[1]), totalScannerCount(), `scanner-reference.md declares ${orch[1]} orchestrated + ${standalone[1]} standalone ` + `= ${Number(orch[1]) + Number(standalone[1])}, but scanners/ holds ${totalScannerCount()}. ` + `This split is the counting rule the README badge depends on.`, ); }); it('README prose scanner totals match the badge', () => { const expected = totalScannerCount(); const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8'); const patterns = [ /Deterministic analysis\s*[—-]\s*(\d+)\s+scanners/g, /^(\d+)\s+scanners\.\s+Zero external dependencies/gm, /frameworks,\s*(\d+)\s+scanners/g, ]; for (const pattern of patterns) { for (const m of content.matchAll(pattern)) { assert.equal( Number(m[1]), expected, `README claims ${m[1]} scanners in "${m[0].trim()}" but the total is ${expected}.`, ); } } }); // -- Knowledge-file count -------------------------------------------------- it('README knowledge badge and prose match knowledge/ contents', () => { const expected = knowledgeFileCount(); const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8'); const badge = content.match(/badge\/knowledge_docs-(\d+)-/); assert.notEqual(badge, null, 'README.md has no `knowledge_docs-` badge'); assert.equal( Number(badge[1]), expected, `README knowledge badge says ${badge[1]} but knowledge/ holds ${expected} files.`, ); for (const m of content.matchAll(/(\d+)\s+knowledge files/g)) { assert.equal( Number(m[1]), expected, `README claims ${m[1]} in "${m[0]}" but knowledge/ holds ${expected} files.`, ); } }); // -- output.mjs finding() prefix list -------------------------------------- it('output.mjs JSDoc lists every prefix passed to finding()', () => { const actual = findingPrefixes(); const src = readFileSync(join(SCANNERS_DIR, 'lib', 'output.mjs'), 'utf-8'); const m = src.match(/@param\s+\{string\}\s+opts\.scanner\s*-\s*Scanner prefix\s*\(([^)]*)\)/); assert.notEqual(m, null, 'output.mjs has no `opts.scanner - Scanner prefix (...)` JSDoc line'); const documented = new Set(m[1].split(',').map(s => s.trim()).filter(Boolean)); const missing = [...actual].filter(p => !documented.has(p)).sort(); const extra = [...documented].filter(p => !actual.has(p)).sort(); assert.deepEqual( { missing, extra }, { missing: [], extra: [] }, `output.mjs JSDoc prefix list is out of sync with the scanners that call finding(). ` + `Missing from JSDoc: [${missing.join(', ')}]. Documented but unused: [${extra.join(', ')}].`, ); }); // -- Posture category count ------------------------------------------------ it('posture category count agrees across CLAUDE.md, README and scanner-reference', () => { const expected = postureCategoryCount(); for (const rel of ['CLAUDE.md', 'README.md', join('docs', 'scanner-reference.md')]) { const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8'); for (const m of content.matchAll(/(\d+)\s+posture categories/gi)) { assert.equal(Number(m[1]), expected, `${rel}: "${m[0]}" but posture-scanner has ${expected}.`); } } }); it('compliance-mapping.md maps its stated number of posture categories', () => { const content = readFileSync(join(KNOWLEDGE_DIR, 'compliance-mapping.md'), 'utf-8'); // The matrix deliberately covers only the code-level categories; the // governance categories (14-16) are consumers of this file, not rows in it. const start = content.indexOf('## Mapping Matrix'); assert.notEqual(start, -1, 'compliance-mapping.md has no `## Mapping Matrix` section'); const tail = content.slice(start); const end = tail.indexOf('\n---'); const section = end > 0 ? tail.slice(0, end) : tail; // Data rows only: skip the header row and the `|---|` separator. const rows = (section.match(/^\|(?!\s*Plugin Control)(?!-)[^|]+\|/gm) || []) .filter(r => !/^\|\s*-+/.test(r)); const claimed = content.match(/(\d+)\s+code-level posture categories/); assert.notEqual( claimed, null, 'compliance-mapping.md must state " code-level posture categories" so the ' + 'claim is distinguishable from the plugin\'s full posture category count.', ); assert.equal( rows.length, Number(claimed[1]), `compliance-mapping.md claims ${claimed[1]} code-level posture categories but the ` + `Mapping Matrix has ${rows.length} data rows.`, ); assert.equal( rows.length < postureCategoryCount(), true, `The matrix is documented as a subset of the ${postureCategoryCount()} posture categories; ` + `it now has ${rows.length} rows. If it grew to cover all of them, update the prose too.`, ); }); });