docs(llm-security): v8 Phase 2 — B10 docs consistency, counts pinned by test
Extends tests/lib/doc-consistency.test.mjs with 15 cases that derive every inventory count from source instead of trusting prose. Each count has one stated derivation; a doc surface that disagrees now fails the suite. Counts corrected (all were wrong before the test existed): - orchestrated scanners: docs said 10 (README, ci-cd-guide), CLAUDE.md said 12, the synthesizer agent said 9 — scan-orchestrator registers 14 - total scanners: README badge + 3 prose sites said 23; the counting rule in docs/scanner-reference.md (14 orchestrated + 8 standalone) yields 22 - knowledge files: README badge + prose said 22; knowledge/ holds 23 - output.mjs finding() prefix JSDoc listed 10 of the 17 prefixes actually passed to it (missing IDE, MCI, MEM, PST, SCR, TFA, WFL) - norwegian-context.md said "8 hooks, 10 scanners" -> 9 and 14 - ci-cd-guide "what gets scanned" table listed 10 of 14 rows; adds workflow, trigger abuse, signature, AST taint Two plan items changed after verifying against ground truth: - CLAUDE.md's synthesizer "(12 scanners)" was not a deliberate subset; the agent file itself claimed 9. Both bumped to 14. - compliance-mapping.md's "13 posture categories" is substantively correct — its matrix has exactly 13 data rows, and categories 14-16 are governance consumers of the file, not rows in it. The planned 13->16 bump would have made the document false. Wording clarified to "code-level" instead, and the test now pins row count against the stated claim. Framework currency (both verified against primary reporting): - EU AI Act: Digital Omnibus (EP 2026-06-16, Council 2026-06-29) deferred the high-risk obligations behind Art. 9/15 to 2027-12-02 (Annex III) and 2028-08-02 (Annex I); transparency still applies from 2026-08-02 - OWASP Agentic AI Top 10 labelled as the 2026 edition Also: CLAUDE.md Distribution section rewritten monorepo -> polyrepo (each plugin is its own repo; the catalog pins url + ref per plugin), and current-state test counts synced 2013 -> 2034. Release-note paragraphs keep their historical numbers. No scanner, hook, or command behaviour changes. Suite 2034/2034. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wt4YQGoXwRja5K2Zmv8RZE
This commit is contained in:
parent
ff4d8e8a31
commit
0f1be986d0
8 changed files with 287 additions and 22 deletions
|
|
@ -159,3 +159,260 @@ describe('doc-consistency — Hooks count consistency (D4)', () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// B10 (v8.0.0 Phase 2): inventory counts stated in prose must be derived from
|
||||
// source, not hand-maintained. Before this block, four independent counts had
|
||||
// drifted — orchestrated scanners (docs said 10, source had 14), total
|
||||
// scanners (README said 23, the documented rule yields 22), knowledge files
|
||||
// (README said 22, the directory holds 23), and the `finding()` prefix list in
|
||||
// output.mjs (documented 10 of the 17 prefixes actually passed to it).
|
||||
//
|
||||
// Every count below has ONE derivation from source, stated next to the helper.
|
||||
// A doc surface that disagrees fails here rather than misleading a reader.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('doc-consistency — inventory counts are derived from source (B10)', () => {
|
||||
const SCANNERS_DIR = join(PLUGIN_ROOT, 'scanners');
|
||||
const KNOWLEDGE_DIR = join(PLUGIN_ROOT, 'knowledge');
|
||||
|
||||
// Counting rule (scanner total): every `scanners/*.mjs` IS a scanner except
|
||||
// these five, which are a runner, a remediator, an extraction library, a
|
||||
// cache utility and a CLI wrapper around an already-counted scanner.
|
||||
const NON_SCANNER_MODULES = [
|
||||
'scan-orchestrator.mjs',
|
||||
'auto-cleaner.mjs',
|
||||
'content-extractor.mjs',
|
||||
'mcp-baseline-reset.mjs',
|
||||
'supply-chain-recheck-cli.mjs',
|
||||
];
|
||||
|
||||
function scannerModules() {
|
||||
return readdirSync(SCANNERS_DIR).filter(f => f.endsWith('.mjs'));
|
||||
}
|
||||
|
||||
// Ground truth: the SCANNERS array literal in scan-orchestrator.mjs.
|
||||
function orchestratedCount() {
|
||||
const src = readFileSync(join(SCANNERS_DIR, 'scan-orchestrator.mjs'), 'utf-8');
|
||||
const start = src.indexOf('const SCANNERS = [');
|
||||
if (start < 0) throw new Error('scan-orchestrator.mjs: `const SCANNERS = [` not found');
|
||||
const end = src.indexOf('];', start);
|
||||
const block = src.slice(start, end);
|
||||
return (block.match(/^\s*\{\s*name:/gm) || []).length;
|
||||
}
|
||||
|
||||
// Ground truth: scanner modules minus the five non-scanner modules above.
|
||||
function totalScannerCount() {
|
||||
return scannerModules().filter(f => !NON_SCANNER_MODULES.includes(f)).length;
|
||||
}
|
||||
|
||||
// Ground truth: every entry in knowledge/.
|
||||
function knowledgeFileCount() {
|
||||
return readdirSync(KNOWLEDGE_DIR).length;
|
||||
}
|
||||
|
||||
// Ground truth: the CATEGORIES array literal in posture-scanner.mjs.
|
||||
function postureCategoryCount() {
|
||||
const src = readFileSync(join(SCANNERS_DIR, 'posture-scanner.mjs'), 'utf-8');
|
||||
const start = src.indexOf('const CATEGORIES = [');
|
||||
if (start < 0) throw new Error('posture-scanner.mjs: `const CATEGORIES = [` not found');
|
||||
const end = src.indexOf('];', start);
|
||||
return (src.slice(start, end).match(/^\s*\{\s*id:\s*\d+/gm) || []).length;
|
||||
}
|
||||
|
||||
// Ground truth: the prefix every module that calls `finding()` passes as
|
||||
// `opts.scanner`. Two idioms are in use — an inline `scanner: 'XXX'` literal
|
||||
// and a module-level `const SCANNER`/`SCANNER_PREFIX`.
|
||||
function findingPrefixes() {
|
||||
const prefixes = new Set();
|
||||
for (const file of scannerModules()) {
|
||||
const src = readFileSync(join(SCANNERS_DIR, file), 'utf-8');
|
||||
if (!/import\s*\{[^}]*\bfinding\b[^}]*\}\s*from\s*'\.\/lib\/output\.mjs'/.test(src)) continue;
|
||||
for (const m of src.matchAll(/scanner:\s*'([A-Z]{2,4})'/g)) prefixes.add(m[1]);
|
||||
for (const m of src.matchAll(/^const SCANNER(?:_PREFIX)?\s*=\s*'([A-Z]{2,4})'/gm)) prefixes.add(m[1]);
|
||||
}
|
||||
return prefixes;
|
||||
}
|
||||
|
||||
it('NON_SCANNER_MODULES all exist (the exclusion list cannot silently rot)', () => {
|
||||
const present = new Set(scannerModules());
|
||||
for (const name of NON_SCANNER_MODULES) {
|
||||
assert.equal(
|
||||
present.has(name),
|
||||
true,
|
||||
`NON_SCANNER_MODULES lists scanners/${name}, which no longer exists. ` +
|
||||
`A rename here silently inflates the total scanner count — update the list.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// -- Orchestrated scanner count -------------------------------------------
|
||||
// Every "<N> deterministic|orchestrated scanners" claim across prose and the
|
||||
// standalone CLI must equal the SCANNERS array length.
|
||||
const ORCHESTRATED_CLAIM_FILES = [
|
||||
'README.md',
|
||||
'CLAUDE.md',
|
||||
join('docs', 'ci-cd-guide.md'),
|
||||
join('docs', 'scanner-reference.md'),
|
||||
join('bin', 'llm-security.mjs'),
|
||||
join('knowledge', 'norwegian-context.md'),
|
||||
join('agents', 'deep-scan-synthesizer-agent.md'),
|
||||
];
|
||||
|
||||
for (const rel of ORCHESTRATED_CLAIM_FILES) {
|
||||
it(`${rel} states the orchestrated scanner count correctly`, () => {
|
||||
const expected = orchestratedCount();
|
||||
const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8');
|
||||
const patterns = [
|
||||
/(\d+)\s+deterministic(?:\s+Node\.js)?\s+scanners/gi,
|
||||
/(\d+)\s+orchestrated\s+deterministic\s+scanners/gi,
|
||||
/\*\*Orchestrated\s*\((\d+)\):?\*\*/g,
|
||||
/^(\d+)\s+scanners:/gm,
|
||||
/\((\d+)\s+scanners\)/g,
|
||||
/findings\s+from\s+(\d+)\s+scanners/gi,
|
||||
/scanner\s+output\s+\((\d+)\s+scanners/gi,
|
||||
/hooks,\s*(\d+)\s+scanners/gi,
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
for (const m of content.matchAll(pattern)) {
|
||||
assert.equal(
|
||||
Number(m[1]),
|
||||
expected,
|
||||
`${rel} claims ${m[1]} orchestrated scanners in "${m[0].trim()}", but ` +
|
||||
`scan-orchestrator.mjs registers ${expected}. Update the prose.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// -- Total scanner count (orchestrated + standalone) -----------------------
|
||||
it('README scanner badge matches the total scanner count', () => {
|
||||
const expected = totalScannerCount();
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
|
||||
const m = content.match(/badge\/scanners-(\d+)-/);
|
||||
assert.notEqual(m, null, 'README.md has no `scanners-<N>` badge');
|
||||
assert.equal(
|
||||
Number(m[1]),
|
||||
expected,
|
||||
`README scanner badge says ${m[1]} but scanners/ holds ${expected} scanner modules ` +
|
||||
`(${scannerModules().length} .mjs files minus ${NON_SCANNER_MODULES.length} non-scanner modules).`,
|
||||
);
|
||||
});
|
||||
|
||||
it('docs/scanner-reference.md orchestrated + standalone equals the total', () => {
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'scanner-reference.md'), 'utf-8');
|
||||
const orch = content.match(/\*\*Orchestrated\s*\((\d+)\):?\*\*/);
|
||||
const standalone = content.match(/\*\*Standalone\s*\((\d+)\):?\*\*/);
|
||||
assert.notEqual(orch, null, 'scanner-reference.md has no `**Orchestrated (N)**` marker');
|
||||
assert.notEqual(standalone, null, 'scanner-reference.md has no `**Standalone (N)**` marker');
|
||||
assert.equal(
|
||||
Number(orch[1]) + Number(standalone[1]),
|
||||
totalScannerCount(),
|
||||
`scanner-reference.md declares ${orch[1]} orchestrated + ${standalone[1]} standalone ` +
|
||||
`= ${Number(orch[1]) + Number(standalone[1])}, but scanners/ holds ${totalScannerCount()}. ` +
|
||||
`This split is the counting rule the README badge depends on.`,
|
||||
);
|
||||
});
|
||||
|
||||
it('README prose scanner totals match the badge', () => {
|
||||
const expected = totalScannerCount();
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
|
||||
const patterns = [
|
||||
/Deterministic analysis\s*[—-]\s*(\d+)\s+scanners/g,
|
||||
/^(\d+)\s+scanners\.\s+Zero external dependencies/gm,
|
||||
/frameworks,\s*(\d+)\s+scanners/g,
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
for (const m of content.matchAll(pattern)) {
|
||||
assert.equal(
|
||||
Number(m[1]),
|
||||
expected,
|
||||
`README claims ${m[1]} scanners in "${m[0].trim()}" but the total is ${expected}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// -- Knowledge-file count --------------------------------------------------
|
||||
it('README knowledge badge and prose match knowledge/ contents', () => {
|
||||
const expected = knowledgeFileCount();
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'README.md'), 'utf-8');
|
||||
const badge = content.match(/badge\/knowledge_docs-(\d+)-/);
|
||||
assert.notEqual(badge, null, 'README.md has no `knowledge_docs-<N>` badge');
|
||||
assert.equal(
|
||||
Number(badge[1]),
|
||||
expected,
|
||||
`README knowledge badge says ${badge[1]} but knowledge/ holds ${expected} files.`,
|
||||
);
|
||||
for (const m of content.matchAll(/(\d+)\s+knowledge files/g)) {
|
||||
assert.equal(
|
||||
Number(m[1]),
|
||||
expected,
|
||||
`README claims ${m[1]} in "${m[0]}" but knowledge/ holds ${expected} files.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// -- output.mjs finding() prefix list --------------------------------------
|
||||
it('output.mjs JSDoc lists every prefix passed to finding()', () => {
|
||||
const actual = findingPrefixes();
|
||||
const src = readFileSync(join(SCANNERS_DIR, 'lib', 'output.mjs'), 'utf-8');
|
||||
const m = src.match(/@param\s+\{string\}\s+opts\.scanner\s*-\s*Scanner prefix\s*\(([^)]*)\)/);
|
||||
assert.notEqual(m, null, 'output.mjs has no `opts.scanner - Scanner prefix (...)` JSDoc line');
|
||||
const documented = new Set(m[1].split(',').map(s => s.trim()).filter(Boolean));
|
||||
|
||||
const missing = [...actual].filter(p => !documented.has(p)).sort();
|
||||
const extra = [...documented].filter(p => !actual.has(p)).sort();
|
||||
assert.deepEqual(
|
||||
{ missing, extra },
|
||||
{ missing: [], extra: [] },
|
||||
`output.mjs JSDoc prefix list is out of sync with the scanners that call finding(). ` +
|
||||
`Missing from JSDoc: [${missing.join(', ')}]. Documented but unused: [${extra.join(', ')}].`,
|
||||
);
|
||||
});
|
||||
|
||||
// -- Posture category count ------------------------------------------------
|
||||
it('posture category count agrees across CLAUDE.md, README and scanner-reference', () => {
|
||||
const expected = postureCategoryCount();
|
||||
for (const rel of ['CLAUDE.md', 'README.md', join('docs', 'scanner-reference.md')]) {
|
||||
const content = readFileSync(join(PLUGIN_ROOT, rel), 'utf-8');
|
||||
for (const m of content.matchAll(/(\d+)\s+posture categories/gi)) {
|
||||
assert.equal(Number(m[1]), expected, `${rel}: "${m[0]}" but posture-scanner has ${expected}.`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('compliance-mapping.md maps its stated number of posture categories', () => {
|
||||
const content = readFileSync(join(KNOWLEDGE_DIR, 'compliance-mapping.md'), 'utf-8');
|
||||
// The matrix deliberately covers only the code-level categories; the
|
||||
// governance categories (14-16) are consumers of this file, not rows in it.
|
||||
const start = content.indexOf('## Mapping Matrix');
|
||||
assert.notEqual(start, -1, 'compliance-mapping.md has no `## Mapping Matrix` section');
|
||||
const tail = content.slice(start);
|
||||
const end = tail.indexOf('\n---');
|
||||
const section = end > 0 ? tail.slice(0, end) : tail;
|
||||
// Data rows only: skip the header row and the `|---|` separator.
|
||||
const rows = (section.match(/^\|(?!\s*Plugin Control)(?!-)[^|]+\|/gm) || [])
|
||||
.filter(r => !/^\|\s*-+/.test(r));
|
||||
const claimed = content.match(/(\d+)\s+code-level posture categories/);
|
||||
assert.notEqual(
|
||||
claimed,
|
||||
null,
|
||||
'compliance-mapping.md must state "<N> code-level posture categories" so the ' +
|
||||
'claim is distinguishable from the plugin\'s full posture category count.',
|
||||
);
|
||||
assert.equal(
|
||||
rows.length,
|
||||
Number(claimed[1]),
|
||||
`compliance-mapping.md claims ${claimed[1]} code-level posture categories but the ` +
|
||||
`Mapping Matrix has ${rows.length} data rows.`,
|
||||
);
|
||||
assert.equal(
|
||||
rows.length < postureCategoryCount(),
|
||||
true,
|
||||
`The matrix is documented as a subset of the ${postureCategoryCount()} posture categories; ` +
|
||||
`it now has ${rows.length} rows. If it grew to cover all of them, update the prose too.`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue