test(humanizer): replace title-string assertions with ID-based checks
Wave 2 / Step 4 of v5.1.0 plain-language UX humanizer rollout. Re-anchors 34 title-string assertions across 4 test files so they survive Wave 3's title/description/recommendation rewriting at the CLI layer. Anchoring strategy per scanner: - GAP findings: scanner + category + recommendation substring (humanizer preserves stable identifiers like CLAUDE.md, .mcp.json, hook in rec). Hardcoded CA-GAP-NNN IDs for positive checks. - HKV findings: scanner + evidence regex (evidence preserved verbatim). - SET findings: scanner + evidence regex (evidence preserved verbatim). - PLH findings: scanner + hardcoded CA-PLH-NNN IDs (no evidence on most PLH findings, so ID is the only stable anchor for specific cases; negative checks use scanner + title-substring spanning raw + humanized). Per docs/v5.1.0-test-audit.md classification: only (b) WILL BREAK assertions modified. (a) shape-only assertions (error-message formatting, pure existence checks) untouched. tests/lib/output.test.mjs and tests/lib/diff-engine.test.mjs and tests/scanners/fix-engine.test.mjs unchanged (synthetic test inputs, not scanner output). Test count unchanged: 689/689 pass. IDs harvested via deterministic runtime dump per fixture (resetCounter + scan).
This commit is contained in:
parent
b69fdea883
commit
dff278f02a
4 changed files with 73 additions and 33 deletions
|
|
@ -49,14 +49,21 @@ describe('scan on valid test-plugin', () => {
|
|||
it('no findings for missing plugin.json fields', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(TEST_PLUGIN);
|
||||
const missingFields = result.findings.filter(f => f.title.includes('Missing required field'));
|
||||
// Anchor on PLH + a title-substring stable across humanizer rewrites.
|
||||
// Raw: "Missing required field in plugin.json: <field>". Humanized: "A plugin's manifest is missing a required field".
|
||||
const missingFields = result.findings.filter(f =>
|
||||
f.scanner === 'PLH' && /(missing.{0,40}(field|manifest))|(manifest.{0,40}missing)/i.test(f.title || '')
|
||||
);
|
||||
assert.equal(missingFields.length, 0, 'All required fields present in test-plugin');
|
||||
});
|
||||
|
||||
it('no findings for missing CLAUDE.md sections', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(TEST_PLUGIN);
|
||||
const missingSections = result.findings.filter(f => f.title.includes('missing') && f.title.includes('section'));
|
||||
// Raw: "CLAUDE.md missing '<name>' section". Humanized: "A plugin's instructions file is missing a recommended section".
|
||||
const missingSections = result.findings.filter(f =>
|
||||
f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '')
|
||||
);
|
||||
assert.equal(missingSections.length, 0, 'All sections present in test-plugin CLAUDE.md');
|
||||
});
|
||||
});
|
||||
|
|
@ -65,32 +72,38 @@ describe('scan on broken-plugin', () => {
|
|||
it('detects missing plugin.json fields', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(BROKEN_PLUGIN);
|
||||
const missingFields = result.findings.filter(f => f.title.includes('Missing required field'));
|
||||
// CA-PLH-001 (description) and CA-PLH-002 (version) in broken-plugin.
|
||||
const missingFields = result.findings.filter(f =>
|
||||
f.scanner === 'PLH' && (f.id === 'CA-PLH-001' || f.id === 'CA-PLH-002')
|
||||
);
|
||||
assert.ok(missingFields.length >= 2, 'Should detect missing description and version');
|
||||
});
|
||||
|
||||
it('detects missing CLAUDE.md', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(BROKEN_PLUGIN);
|
||||
const missingMd = result.findings.filter(f => f.title === 'Missing CLAUDE.md');
|
||||
// CA-PLH-003 in broken-plugin = Missing CLAUDE.md.
|
||||
const missingMd = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-003');
|
||||
assert.equal(missingMd.length, 1, 'Should detect missing CLAUDE.md');
|
||||
});
|
||||
|
||||
it('detects command without frontmatter', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(BROKEN_PLUGIN);
|
||||
const noFrontmatter = result.findings.filter(f => f.title === 'Command missing frontmatter');
|
||||
// CA-PLH-004 in broken-plugin = Command missing frontmatter.
|
||||
const noFrontmatter = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-004');
|
||||
assert.equal(noFrontmatter.length, 1, 'Should detect command without frontmatter');
|
||||
});
|
||||
|
||||
it('detects agent missing required frontmatter fields', async () => {
|
||||
resetCounter();
|
||||
const result = await scan(BROKEN_PLUGIN);
|
||||
// CA-PLH-005 (missing model) and CA-PLH-006 (missing tools) in broken-plugin.
|
||||
const missingAgent = result.findings.filter(f =>
|
||||
f.title.startsWith('Agent missing frontmatter field:')
|
||||
f.scanner === 'PLH' && (f.id === 'CA-PLH-005' || f.id === 'CA-PLH-006')
|
||||
);
|
||||
// bad-agent.md has name+description but missing model and tools
|
||||
assert.ok(missingAgent.length >= 2, `Should detect missing model and tools, got ${missingAgent.length}: ${missingAgent.map(f => f.title).join(', ')}`);
|
||||
assert.ok(missingAgent.length >= 2, `Should detect missing model and tools, got ${missingAgent.length}: ${missingAgent.map(f => f.id).join(', ')}`);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -99,7 +112,9 @@ describe('scan with no plugins', () => {
|
|||
resetCounter();
|
||||
const result = await scan(resolve(FIXTURES, 'empty-project'));
|
||||
assert.equal(result.findings.length, 1);
|
||||
assert.equal(result.findings[0].title, 'No plugins found');
|
||||
// CA-PLH-001 in empty-project = No plugins found.
|
||||
assert.equal(result.findings[0].id, 'CA-PLH-001');
|
||||
assert.equal(result.findings[0].scanner, 'PLH');
|
||||
assert.equal(result.findings[0].severity, 'info');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue