From dff278f02ab9e95bde865137259ec99d15df319d Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 17:22:55 +0200 Subject: [PATCH] test(humanizer): replace title-string assertions with ID-based checks Wave 2 / Step 4 of v5.1.0 plain-language UX humanizer rollout. Re-anchors 34 title-string assertions across 4 test files so they survive Wave 3's title/description/recommendation rewriting at the CLI layer. Anchoring strategy per scanner: - GAP findings: scanner + category + recommendation substring (humanizer preserves stable identifiers like CLAUDE.md, .mcp.json, hook in rec). Hardcoded CA-GAP-NNN IDs for positive checks. - HKV findings: scanner + evidence regex (evidence preserved verbatim). - SET findings: scanner + evidence regex (evidence preserved verbatim). - PLH findings: scanner + hardcoded CA-PLH-NNN IDs (no evidence on most PLH findings, so ID is the only stable anchor for specific cases; negative checks use scanner + title-substring spanning raw + humanized). Per docs/v5.1.0-test-audit.md classification: only (b) WILL BREAK assertions modified. (a) shape-only assertions (error-message formatting, pure existence checks) untouched. tests/lib/output.test.mjs and tests/lib/diff-engine.test.mjs and tests/scanners/fix-engine.test.mjs unchanged (synthetic test inputs, not scanner output). Test count unchanged: 689/689 pass. IDs harvested via deterministic runtime dump per fixture (resetCounter + scan). --- .../scanners/feature-gap-scanner.test.mjs | 21 ++++++++---- .../tests/scanners/hook-validator.test.mjs | 22 ++++++++----- .../scanners/plugin-health-scanner.test.mjs | 31 +++++++++++++----- .../scanners/settings-validator.test.mjs | 32 ++++++++++++------- 4 files changed, 73 insertions(+), 33 deletions(-) diff --git a/plugins/config-audit/tests/scanners/feature-gap-scanner.test.mjs b/plugins/config-audit/tests/scanners/feature-gap-scanner.test.mjs index 3ce9062..a33aad6 100644 --- a/plugins/config-audit/tests/scanners/feature-gap-scanner.test.mjs +++ b/plugins/config-audit/tests/scanners/feature-gap-scanner.test.mjs @@ -42,15 +42,21 @@ describe('GAP scanner — healthy project', () => { }); it('does NOT report missing CLAUDE.md', () => { - assert.ok(!result.findings.some(f => f.title === 'No CLAUDE.md file')); + assert.ok(!result.findings.some(f => + f.scanner === 'GAP' && f.category === 't1' && /CLAUDE\.md/.test(f.recommendation || '') + )); }); it('does NOT report missing MCP', () => { - assert.ok(!result.findings.some(f => f.title === 'No MCP servers configured')); + assert.ok(!result.findings.some(f => + f.scanner === 'GAP' && f.category === 't1' && /\.mcp\.json/.test(f.recommendation || '') + )); }); it('does NOT report missing hooks', () => { - assert.ok(!result.findings.some(f => f.title === 'No hooks configured')); + assert.ok(!result.findings.some(f => + f.scanner === 'GAP' && f.category === 't1' && /hook/i.test(f.recommendation || '') + )); }); it('has counts object with all severity levels', () => { @@ -93,11 +99,13 @@ describe('GAP scanner — minimal project', () => { }); it('reports missing hooks', () => { - assert.ok(result.findings.some(f => f.title === 'No hooks configured')); + // CA-GAP-002 in minimal-project = t1_3 (No hooks configured); see docs/v5.1.0-test-audit.md. + assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-002')); }); it('reports missing MCP', () => { - assert.ok(result.findings.some(f => f.title === 'No MCP servers configured')); + // CA-GAP-004 in minimal-project = t1_5 (No MCP servers configured). + assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-004')); }); it('T1 gaps are medium severity', () => { @@ -147,7 +155,8 @@ describe('GAP scanner — empty project', () => { }); it('reports T1 gaps including missing CLAUDE.md', () => { - assert.ok(result.findings.some(f => f.title === 'No CLAUDE.md file')); + // CA-GAP-001 in empty-project = t1_1 (No CLAUDE.md file). + assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-001')); }); }); diff --git a/plugins/config-audit/tests/scanners/hook-validator.test.mjs b/plugins/config-audit/tests/scanners/hook-validator.test.mjs index 624e907..4ed8270 100644 --- a/plugins/config-audit/tests/scanners/hook-validator.test.mjs +++ b/plugins/config-audit/tests/scanners/hook-validator.test.mjs @@ -46,27 +46,32 @@ describe('HKV scanner — broken project', () => { }); it('detects unknown hook event', () => { - const found = result.findings.some(f => f.title === 'Unknown hook event'); + // CA-HKV-001 in broken-project, evidence='InvalidEvent'. + const found = result.findings.some(f => f.scanner === 'HKV' && /InvalidEvent/.test(f.evidence || '')); assert.ok(found, 'Should detect InvalidEvent'); }); it('detects object matcher (should be string)', () => { - const found = result.findings.some(f => f.title.includes('Matcher must be a string')); + // CA-HKV-002 in broken-project, evidence contains the object matcher snippet. + const found = result.findings.some(f => f.scanner === 'HKV' && f.id === 'CA-HKV-002'); assert.ok(found, 'Should detect nested object matcher'); }); it('detects invalid handler type', () => { - const found = result.findings.some(f => f.title === 'Invalid hook handler type'); + // CA-HKV-003 in broken-project, evidence='type: "invalid_type"'. + const found = result.findings.some(f => f.scanner === 'HKV' && /invalid_type/.test(f.evidence || '')); assert.ok(found, 'Should detect invalid_type'); }); it('detects timeout below minimum', () => { - const found = result.findings.some(f => f.title.includes('timeout')); + // CA-HKV-004 in broken-project, evidence='timeout: 500'. + const found = result.findings.some(f => f.scanner === 'HKV' && /timeout:\s*500/.test(f.evidence || '')); assert.ok(found, 'Should detect timeout of 500ms'); }); it('marks unknown event as high severity', () => { - const f = result.findings.find(f => f.title === 'Unknown hook event'); + // CA-HKV-001 in broken-project = unknown-event finding (evidence='InvalidEvent'). + const f = result.findings.find(x => x.scanner === 'HKV' && /InvalidEvent/.test(x.evidence || '')); assert.strictEqual(f?.severity, 'high'); }); }); @@ -77,7 +82,8 @@ describe('HKV scanner — verbose hook output (v5 M5)', () => { const path = resolve(FIXTURES, 'hooks-verbose'); const discovery = await discoverConfigFiles(path); const result = await scan(path, discovery); - const f = result.findings.find(x => /verbose hook output/i.test(x.title || '')); + // Verbose-hook finding in hooks-verbose; evidence carries the line-count metric. + const f = result.findings.find(x => x.scanner === 'HKV' && /console_log_or_stdout_lines=/.test(x.evidence || '')); assert.ok(f, `expected verbose-hook finding; got: ${result.findings.map(x => x.title).join(' | ')}`); assert.equal(f.severity, 'low', `expected low, got ${f.severity}`); assert.match(f.evidence || '', /console_log_or_stdout_lines=6\d/); @@ -88,8 +94,8 @@ describe('HKV scanner — verbose hook output (v5 M5)', () => { const path = resolve(FIXTURES, 'hooks-quiet'); const discovery = await discoverConfigFiles(path); const result = await scan(path, discovery); - const f = result.findings.find(x => /verbose hook output/i.test(x.title || '')); - assert.equal(f, undefined, `expected no verbose-hook finding; got: ${f?.title}`); + const f = result.findings.find(x => x.scanner === 'HKV' && /console_log_or_stdout_lines=/.test(x.evidence || '')); + assert.equal(f, undefined, `expected no verbose-hook finding; got id=${f?.id}`); }); }); diff --git a/plugins/config-audit/tests/scanners/plugin-health-scanner.test.mjs b/plugins/config-audit/tests/scanners/plugin-health-scanner.test.mjs index c5083fa..5b39615 100644 --- a/plugins/config-audit/tests/scanners/plugin-health-scanner.test.mjs +++ b/plugins/config-audit/tests/scanners/plugin-health-scanner.test.mjs @@ -49,14 +49,21 @@ describe('scan on valid test-plugin', () => { it('no findings for missing plugin.json fields', async () => { resetCounter(); const result = await scan(TEST_PLUGIN); - const missingFields = result.findings.filter(f => f.title.includes('Missing required field')); + // Anchor on PLH + a title-substring stable across humanizer rewrites. + // Raw: "Missing required field in plugin.json: ". Humanized: "A plugin's manifest is missing a required field". + const missingFields = result.findings.filter(f => + f.scanner === 'PLH' && /(missing.{0,40}(field|manifest))|(manifest.{0,40}missing)/i.test(f.title || '') + ); assert.equal(missingFields.length, 0, 'All required fields present in test-plugin'); }); it('no findings for missing CLAUDE.md sections', async () => { resetCounter(); const result = await scan(TEST_PLUGIN); - const missingSections = result.findings.filter(f => f.title.includes('missing') && f.title.includes('section')); + // Raw: "CLAUDE.md missing '' section". Humanized: "A plugin's instructions file is missing a recommended section". + const missingSections = result.findings.filter(f => + f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '') + ); assert.equal(missingSections.length, 0, 'All sections present in test-plugin CLAUDE.md'); }); }); @@ -65,32 +72,38 @@ describe('scan on broken-plugin', () => { it('detects missing plugin.json fields', async () => { resetCounter(); const result = await scan(BROKEN_PLUGIN); - const missingFields = result.findings.filter(f => f.title.includes('Missing required field')); + // CA-PLH-001 (description) and CA-PLH-002 (version) in broken-plugin. + const missingFields = result.findings.filter(f => + f.scanner === 'PLH' && (f.id === 'CA-PLH-001' || f.id === 'CA-PLH-002') + ); assert.ok(missingFields.length >= 2, 'Should detect missing description and version'); }); it('detects missing CLAUDE.md', async () => { resetCounter(); const result = await scan(BROKEN_PLUGIN); - const missingMd = result.findings.filter(f => f.title === 'Missing CLAUDE.md'); + // CA-PLH-003 in broken-plugin = Missing CLAUDE.md. + const missingMd = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-003'); assert.equal(missingMd.length, 1, 'Should detect missing CLAUDE.md'); }); it('detects command without frontmatter', async () => { resetCounter(); const result = await scan(BROKEN_PLUGIN); - const noFrontmatter = result.findings.filter(f => f.title === 'Command missing frontmatter'); + // CA-PLH-004 in broken-plugin = Command missing frontmatter. + const noFrontmatter = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-004'); assert.equal(noFrontmatter.length, 1, 'Should detect command without frontmatter'); }); it('detects agent missing required frontmatter fields', async () => { resetCounter(); const result = await scan(BROKEN_PLUGIN); + // CA-PLH-005 (missing model) and CA-PLH-006 (missing tools) in broken-plugin. const missingAgent = result.findings.filter(f => - f.title.startsWith('Agent missing frontmatter field:') + f.scanner === 'PLH' && (f.id === 'CA-PLH-005' || f.id === 'CA-PLH-006') ); // bad-agent.md has name+description but missing model and tools - assert.ok(missingAgent.length >= 2, `Should detect missing model and tools, got ${missingAgent.length}: ${missingAgent.map(f => f.title).join(', ')}`); + assert.ok(missingAgent.length >= 2, `Should detect missing model and tools, got ${missingAgent.length}: ${missingAgent.map(f => f.id).join(', ')}`); }); }); @@ -99,7 +112,9 @@ describe('scan with no plugins', () => { resetCounter(); const result = await scan(resolve(FIXTURES, 'empty-project')); assert.equal(result.findings.length, 1); - assert.equal(result.findings[0].title, 'No plugins found'); + // CA-PLH-001 in empty-project = No plugins found. + assert.equal(result.findings[0].id, 'CA-PLH-001'); + assert.equal(result.findings[0].scanner, 'PLH'); assert.equal(result.findings[0].severity, 'info'); }); }); diff --git a/plugins/config-audit/tests/scanners/settings-validator.test.mjs b/plugins/config-audit/tests/scanners/settings-validator.test.mjs index 2c2d611..3084328 100644 --- a/plugins/config-audit/tests/scanners/settings-validator.test.mjs +++ b/plugins/config-audit/tests/scanners/settings-validator.test.mjs @@ -46,32 +46,37 @@ describe('SET scanner — broken project', () => { }); it('detects unknown settings key', () => { - const found = result.findings.some(f => f.title === 'Unknown settings key'); + // CA-SET-001 in broken-project, evidence='unknownKey123'. + const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || '')); assert.ok(found, 'Should detect unknownKey123'); }); it('detects deprecated key (includeCoAuthoredBy)', () => { - const found = result.findings.some(f => f.title === 'Deprecated settings key'); + // CA-SET-002 in broken-project, evidence='includeCoAuthoredBy: true'. + const found = result.findings.some(f => f.scanner === 'SET' && /includeCoAuthoredBy/.test(f.evidence || '')); assert.ok(found, 'Should detect includeCoAuthoredBy'); }); it('detects type mismatch (alwaysThinkingEnabled as string)', () => { - const found = result.findings.some(f => f.title === 'Type mismatch in settings'); + // CA-SET-003 in broken-project, evidence='alwaysThinkingEnabled: "yes" (string)'. + const found = result.findings.some(f => f.scanner === 'SET' && /alwaysThinkingEnabled/.test(f.evidence || '')); assert.ok(found, 'Should detect boolean/string mismatch'); }); it('detects invalid effortLevel value', () => { - const found = result.findings.some(f => f.title === 'Invalid effortLevel value'); + // CA-SET-004 in broken-project, evidence='effortLevel: "turbo"'. + const found = result.findings.some(f => f.scanner === 'SET' && /effortLevel:\s*"turbo"/.test(f.evidence || '')); assert.ok(found, 'Should detect effortLevel "turbo"'); }); it('detects hooks as array', () => { - const found = result.findings.some(f => f.title.includes('array instead of object')); + // CA-SET-006 in broken-project, evidence='"hooks": [...]'. + const found = result.findings.some(f => f.scanner === 'SET' && /"hooks":\s*\[/.test(f.evidence || '')); assert.ok(found, 'Should detect hooks array format'); }); it('marks hooks-as-array as critical', () => { - const f = result.findings.find(f => f.title.includes('array instead of object')); + const f = result.findings.find(x => x.scanner === 'SET' && /"hooks":\s*\[/.test(x.evidence || '')); assert.strictEqual(f?.severity, 'critical'); }); }); @@ -82,8 +87,10 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => { const path = resolve(FIXTURES, 'additional-dirs-ok'); const discovery = await discoverConfigFiles(path); const result = await scan(path, discovery); + // SET findings preserve evidence verbatim; an unknown-key finding for additionalDirectories + // would carry "additionalDirectories" in evidence regardless of humanizer rewriting the title. const unknown = result.findings.find(f => - f.title === 'Unknown settings key' && /additionalDirectories/.test(f.evidence || '')); + f.scanner === 'SET' && /additionalDirectories/.test(f.evidence || '')); assert.equal(unknown, undefined, 'additionalDirectories should be in KNOWN_KEYS'); }); @@ -93,9 +100,11 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => { const path = resolve(FIXTURES, 'additional-dirs-ok'); const discovery = await discoverConfigFiles(path); const result = await scan(path, discovery); - const f = result.findings.find(x => /additionalDirectories/i.test(x.title || '')); + // The additionalDirectories threshold finding writes paths into evidence (e.g., "~/work/repo-a", ...). + // additional-dirs-ok is below threshold, so no SET finding fires at all. + const f = result.findings.find(x => x.scanner === 'SET'); assert.equal(f, undefined, - `expected no additionalDirectories threshold finding for 2 entries, got: ${f?.title}`); + `expected no SET findings for 2 entries, got id=${f?.id}`); }); it('flags > 2 entries as low finding', async () => { @@ -103,8 +112,9 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => { const path = resolve(FIXTURES, 'additional-dirs-many'); const discovery = await discoverConfigFiles(path); const result = await scan(path, discovery); - const f = result.findings.find(x => /additionalDirectories/i.test(x.title || '')); - assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + // CA-SET-001 in additional-dirs-many = the additionalDirectories threshold finding. + const f = result.findings.find(x => x.scanner === 'SET' && x.id === 'CA-SET-001'); + assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.id).join(' | ')}`); assert.equal(f.severity, 'low', `expected low severity, got ${f.severity}`); }); });