test(scanners): assert the published IDs off real output, by title
The three guards landed with M-BUG-28 verify key->number (registry), that every emitted code is declared (sweep), and that every declared code is claimed by a call site (orphan check). None of them verifies title->number: that the call site at PLH source position 3 passes `plugin-json-shadows-default` and not its neighbour. Transposing two keys that are both valid satisfies all three. Measured, not assumed: with the two PLH keys swapped, finding-codes.test.mjs, finding-code-coverage.test.mjs and the orphan check all stayed GREEN. Only this test goes red. It reads the ID off a real scan and keys on the finding TITLE -- the assertion README actually makes. Covers the three places where numbering is deliberately not source order (CA-PLH-015 and CA-PLH-016 at source positions 3 and 4, CA-TOK-006 at position 8) plus CA-CML-001. The PLH-016 case also pins the documented non-uniqueness: four entry problems, one check, one ID. Suite 1573 -> 1577, 0 failing. Frozen v5.0.0 untouched. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MyqCQKK2ornJ1jFWwqx17E
This commit is contained in:
parent
7a794b47eb
commit
542f983178
1 changed files with 59 additions and 0 deletions
59
tests/scanners/published-ids-on-real-output.test.mjs
Normal file
59
tests/scanners/published-ids-on-real-output.test.mjs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* The published IDs, asserted the way README states them: by TITLE, off real
|
||||
* scanner output (M-BUG-28).
|
||||
*
|
||||
* `finding-codes.test.mjs` proves key→number. The coverage sweep proves every
|
||||
* emitted code is declared. Neither proves the right call site passes the right
|
||||
* key: transposing two keys that are both valid passes both tests. That gap
|
||||
* matters most where the numbering is deliberately NOT source order —
|
||||
* `CA-PLH-015`/`016` sit at source positions 3 and 4, and `CA-TOK-006` at
|
||||
* position 8 — which is exactly where a transposition would hide.
|
||||
*/
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { scan as scanPluginHealth } from '../../scanners/plugin-health-scanner.mjs';
|
||||
import { scan as scanTokens } from '../../scanners/token-hotspots.mjs';
|
||||
import { scan as scanClaudeMd } from '../../scanners/claude-md-linter.mjs';
|
||||
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
|
||||
const FIXTURES = resolve(fileURLToPath(new URL('.', import.meta.url)), '../fixtures');
|
||||
|
||||
function idFor(findings, titleFragment) {
|
||||
const match = findings.filter((f) => f.title.includes(titleFragment));
|
||||
assert.ok(match.length > 0, `fixture no longer produces a finding matching "${titleFragment}"`);
|
||||
const ids = new Set(match.map((f) => f.id));
|
||||
assert.equal(ids.size, 1, `one check must yield one ID, got ${[...ids].join(', ')}`);
|
||||
return [...ids][0];
|
||||
}
|
||||
|
||||
async function withDiscovery(scan, fixture) {
|
||||
const target = resolve(FIXTURES, fixture);
|
||||
return scan(target, await discoverConfigFiles(target));
|
||||
}
|
||||
|
||||
describe('published finding IDs, read off real scans', () => {
|
||||
it('CA-PLH-015 is the plugin-folder shadowing check', async () => {
|
||||
const { findings } = await scanPluginHealth(resolve(FIXTURES, 'plugin-shadow-folder'));
|
||||
assert.equal(idFor(findings, 'shadows the default'), 'CA-PLH-015');
|
||||
});
|
||||
|
||||
it('CA-PLH-016 is the skills-array entry check', async () => {
|
||||
const { findings } = await scanPluginHealth(resolve(FIXTURES, 'plugin-skills-array'));
|
||||
// Four different entry problems, one check — the documented non-uniqueness.
|
||||
assert.equal(idFor(findings, '"skills" entry'), 'CA-PLH-016');
|
||||
assert.ok(findings.filter((f) => f.id === 'CA-PLH-016').length > 1, 'expected several instances');
|
||||
});
|
||||
|
||||
it('CA-TOK-006 is the MCP schema-deferral check', async () => {
|
||||
const { findings } = await withDiscovery(scanTokens, 'mcp-deferral/alwaysload');
|
||||
assert.equal(idFor(findings, 'forced into the always-loaded prefix'), 'CA-TOK-006');
|
||||
});
|
||||
|
||||
it('CA-CML-001 is the missing-CLAUDE.md check', async () => {
|
||||
const { findings } = await withDiscovery(scanClaudeMd, 'empty-project');
|
||||
assert.equal(idFor(findings, 'No CLAUDE.md found'), 'CA-CML-001');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue