open/ktg-plugin-marketplace
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity Actions

test(humanizer): replace title-string assertions with ID-based checks

Browse source 
Wave 2 / Step 4 of v5.1.0 plain-language UX humanizer rollout. Re-anchors
34 title-string assertions across 4 test files so they survive Wave 3's
title/description/recommendation rewriting at the CLI layer.

Anchoring strategy per scanner:
- GAP findings: scanner + category + recommendation substring (humanizer
  preserves stable identifiers like CLAUDE.md, .mcp.json, hook in rec).
  Hardcoded CA-GAP-NNN IDs for positive checks.
- HKV findings: scanner + evidence regex (evidence preserved verbatim).
- SET findings: scanner + evidence regex (evidence preserved verbatim).
- PLH findings: scanner + hardcoded CA-PLH-NNN IDs (no evidence on most
  PLH findings, so ID is the only stable anchor for specific cases;
  negative checks use scanner + title-substring spanning raw + humanized).

Per docs/v5.1.0-test-audit.md classification: only (b) WILL BREAK
assertions modified. (a) shape-only assertions (error-message formatting,
pure existence checks) untouched. tests/lib/output.test.mjs and
tests/lib/diff-engine.test.mjs and tests/scanners/fix-engine.test.mjs
unchanged (synthetic test inputs, not scanner output).

Test count unchanged: 689/689 pass. IDs harvested via deterministic
runtime dump per fixture (resetCounter + scan).
This commit is contained in:
Kjell Tore Guttormsen 2026-05-01 17:22:55 +02:00
commit dff278f02a
4 changed files with 73 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

32
plugins/config-audit/tests/scanners/settings-validator.test.mjs
View file

@ -46,32 +46,37 @@ describe('SET scanner — broken project', () => {
});
it('detects unknown settings key', () => {
const found = result.findings.some(f => f.title === 'Unknown settings key');
// CA-SET-001 in broken-project, evidence='unknownKey123'.
const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || ''));
assert.ok(found, 'Should detect unknownKey123');
});
it('detects deprecated key (includeCoAuthoredBy)', () => {
const found = result.findings.some(f => f.title === 'Deprecated settings key');
// CA-SET-002 in broken-project, evidence='includeCoAuthoredBy: true'.
const found = result.findings.some(f => f.scanner === 'SET' && /includeCoAuthoredBy/.test(f.evidence || ''));
assert.ok(found, 'Should detect includeCoAuthoredBy');
});
it('detects type mismatch (alwaysThinkingEnabled as string)', () => {
const found = result.findings.some(f => f.title === 'Type mismatch in settings');
// CA-SET-003 in broken-project, evidence='alwaysThinkingEnabled: "yes" (string)'.
const found = result.findings.some(f => f.scanner === 'SET' && /alwaysThinkingEnabled/.test(f.evidence || ''));
assert.ok(found, 'Should detect boolean/string mismatch');
});
it('detects invalid effortLevel value', () => {
const found = result.findings.some(f => f.title === 'Invalid effortLevel value');
// CA-SET-004 in broken-project, evidence='effortLevel: "turbo"'.
const found = result.findings.some(f => f.scanner === 'SET' && /effortLevel:\s*"turbo"/.test(f.evidence || ''));
assert.ok(found, 'Should detect effortLevel "turbo"');
});
it('detects hooks as array', () => {
const found = result.findings.some(f => f.title.includes('array instead of object'));
// CA-SET-006 in broken-project, evidence='"hooks": [...]'.
const found = result.findings.some(f => f.scanner === 'SET' && /"hooks":\s*\[/.test(f.evidence || ''));
assert.ok(found, 'Should detect hooks array format');
});
it('marks hooks-as-array as critical', () => {
const f = result.findings.find(f => f.title.includes('array instead of object'));
const f = result.findings.find(x => x.scanner === 'SET' && /"hooks":\s*\[/.test(x.evidence || ''));
assert.strictEqual(f?.severity, 'critical');
});
});
@ -82,8 +87,10 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => {
const path = resolve(FIXTURES, 'additional-dirs-ok');
const discovery = await discoverConfigFiles(path);
const result = await scan(path, discovery);
// SET findings preserve evidence verbatim; an unknown-key finding for additionalDirectories
// would carry "additionalDirectories" in evidence regardless of humanizer rewriting the title.
const unknown = result.findings.find(f =>
f.title === 'Unknown settings key' && /additionalDirectories/.test(f.evidence || ''));
f.scanner === 'SET' && /additionalDirectories/.test(f.evidence || ''));
assert.equal(unknown, undefined,
'additionalDirectories should be in KNOWN_KEYS');
});
@ -93,9 +100,11 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => {
const path = resolve(FIXTURES, 'additional-dirs-ok');
const discovery = await discoverConfigFiles(path);
const result = await scan(path, discovery);
const f = result.findings.find(x => /additionalDirectories/i.test(x.title || ''));
// The additionalDirectories threshold finding writes paths into evidence (e.g., "~/work/repo-a", ...).
// additional-dirs-ok is below threshold, so no SET finding fires at all.
const f = result.findings.find(x => x.scanner === 'SET');
assert.equal(f, undefined,
`expected no additionalDirectories threshold finding for 2 entries, got: ${f?.title}`);
`expected no SET findings for 2 entries, got id=${f?.id}`);
});
it('flags > 2 entries as low finding', async () => {
@ -103,8 +112,9 @@ describe('SET scanner — additionalDirectories (v5 M6)', () => {
const path = resolve(FIXTURES, 'additional-dirs-many');
const discovery = await discoverConfigFiles(path);
const result = await scan(path, discovery);
const f = result.findings.find(x => /additionalDirectories/i.test(x.title || ''));
assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
// CA-SET-001 in additional-dirs-many = the additionalDirectories threshold finding.
const f = result.findings.find(x => x.scanner === 'SET' && x.id === 'CA-SET-001');
assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.id).join(' | ')}`);
assert.equal(f.severity, 'low', `expected low severity, got ${f.severity}`);
});
});