config-audit/tests/lib/humanizer.test.mjs
Kjell Tore Guttormsen 7bb254780a feat(skill-listing): add SKL scanner for the skill-listing token budget
Fase 4 Items 2+3 (CC 2.1.114→181 gap-review). New orchestrated scanner
`skill-listing-scanner.mjs` (prefix SKL) flags every active skill whose
description exceeds the verified 1,536-char listing cap (CC 2.1.105, changelog
L1502). Past the cap, Claude Code silently truncates the description the model
reads to route skill invocation — dropping the trigger phrases at the tail.
HOME-scoped over all user + plugin skills via enumerateSkills (COL is the model).

- CA-SKL-001 (medium): description > 1,536 chars. Remediation folds in Item
  2(b) — recommends disableBundledSkills + skillOverrides + trimming
  (designvalg A: no standalone GAP-check, which would fire for nearly everyone).
- Designvalg B: v1 ships the verified cap ONLY. The aggregate 2%-of-context
  listing budget is deferred — it needs a context-window assumption that would
  turn a verified fact into a guess (would carry a CALIBRATION_NOTE if added).
- Choice C: recognize the skillOverrides settings key (CC 2.1.129) in
  KNOWN_KEYS. Left OUT of TYPE_CHECKS — the value is a per-skill object
  (off/user-invocable-only/name-only), not a string; a 'string' check (as the
  plan sketched) would create a NEW false positive. Verify-first deviation.

Registration: scan-orchestrator (13th scanner), humanizer (SKL → 'Wasted
tokens' + static/_default translations), scoring SCANNER_AREA_MAP (→ Token
Efficiency; no 11th area), README badge 12→13, CLAUDE.md (finding-id +
test-count), docs/scanner-internals.md, gap-matrix + plan status notes.

Snapshots reseeded hermetically (SEED_SNAPSHOT/UPDATE_SNAPSHOT): SKL entry with
0 findings in empty HOME, scanners_ok 11→12, claudeMdEstimatedTokens bump from
the CLAUDE.md edits flowing through the cascade. Contamination grep clean.

Suite 868/868 (856 baseline + 11 SKL + 1 skillOverrides). RED→GREEN logged
per cycle.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
2026-06-18 17:36:28 +02:00

302 lines
12 KiB
JavaScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
humanizeFinding,
humanizeFindings,
humanizeEnvelope,
computeRelevanceContext,
} from '../../scanners/lib/humanizer.mjs';
// ─── helpers ────────────────────────────────────────────────────────────
function makeFinding(overrides = {}) {
return {
id: 'CA-CML-001',
scanner: 'CML',
severity: 'medium',
title: 'No CLAUDE.md found',
description: 'No CLAUDE.md file at the project root.',
file: '/Users/test/project/CLAUDE.md',
line: null,
evidence: 'evidence text',
category: 'config',
recommendation: 'Create one.',
autoFixable: false,
...overrides,
};
}
// ─── purity ─────────────────────────────────────────────────────────────
test('humanizeFinding does not mutate its input', () => {
const input = makeFinding();
const before = JSON.parse(JSON.stringify(input));
humanizeFinding(input);
assert.deepEqual(input, before, 'input was mutated');
});
test('humanizeFindings does not mutate its input array', () => {
const input = [makeFinding(), makeFinding({ id: 'CA-CML-002', title: 'Repeated content detected' })];
const before = JSON.parse(JSON.stringify(input));
humanizeFindings(input);
assert.deepEqual(input, before, 'input array was mutated');
});
test('humanizeEnvelope does not mutate its input', () => {
const env = {
target_path: '/tmp',
scanners: [
{ scanner: 'CML', status: 'ok', findings: [makeFinding()], counts: {} },
],
};
const before = JSON.parse(JSON.stringify(env));
humanizeEnvelope(env);
assert.deepEqual(env, before, 'envelope was mutated');
});
// ─── field preservation ────────────────────────────────────────────────
test('humanizeFinding preserves id, scanner, severity, file, line, evidence, category, autoFixable', () => {
const input = makeFinding({
id: 'CA-CML-042',
scanner: 'CML',
severity: 'high',
file: '/tmp/x.md',
line: 17,
evidence: 'specific snippet',
category: 'cml',
autoFixable: true,
});
const out = humanizeFinding(input);
assert.equal(out.id, 'CA-CML-042');
assert.equal(out.scanner, 'CML');
assert.equal(out.severity, 'high');
assert.equal(out.file, '/tmp/x.md');
assert.equal(out.line, 17);
assert.equal(out.evidence, 'specific snippet');
assert.equal(out.category, 'cml');
assert.equal(out.autoFixable, true);
});
test('humanizeFinding preserves optional details payload', () => {
const input = makeFinding();
input.details = { foo: 'bar', count: 7 };
const out = humanizeFinding(input);
assert.deepEqual(out.details, { foo: 'bar', count: 7 });
});
// ─── translation lookup ────────────────────────────────────────────────
test('humanizeFinding rewrites title for known static title', () => {
const input = makeFinding({ scanner: 'CML', title: 'No CLAUDE.md found' });
const out = humanizeFinding(input);
assert.notEqual(out.title, input.title, 'title should be translated');
assert.ok(out.title.toLowerCase().includes('instructions') || out.title.toLowerCase().includes('claude'),
`humanized title should mention instructions or claude, got: ${out.title}`);
});
test('humanizeFinding falls back to _default when title unknown', () => {
const input = makeFinding({ scanner: 'CML', title: 'Unrecognized brand-new finding title' });
const out = humanizeFinding(input);
assert.notEqual(out.title, input.title, '_default should kick in');
// CML _default mentions "instructions file"
assert.ok(/instructions file/i.test(out.title), `expected CML _default title, got: ${out.title}`);
});
test('humanizeFinding passes through original strings when scanner prefix unknown', () => {
const input = makeFinding({ scanner: 'XXX', title: 'whatever' });
const out = humanizeFinding(input);
assert.equal(out.title, 'whatever');
assert.equal(out.description, input.description);
assert.equal(out.recommendation, input.recommendation);
});
test('humanizeFinding matches pattern entries (template-literal titles)', () => {
const input = makeFinding({
scanner: 'COL',
title: 'Skill name "okr-helper" used by multiple plugins',
});
const out = humanizeFinding(input);
assert.notEqual(out.title, input.title);
assert.ok(/two plugins|same name|multiple/i.test(out.title) ||
/two plugins|same name|multiple/i.test(out.description),
`expected pattern match for COL multiple-plugins case, got title: ${out.title}`);
});
// ─── userActionLanguage ────────────────────────────────────────────────
test('humanizeFinding maps severity=critical -> "Fix this now"', () => {
const out = humanizeFinding(makeFinding({ severity: 'critical' }));
assert.equal(out.userActionLanguage, 'Fix this now');
});
test('humanizeFinding maps severity=high -> "Fix soon"', () => {
const out = humanizeFinding(makeFinding({ severity: 'high' }));
assert.equal(out.userActionLanguage, 'Fix soon');
});
test('humanizeFinding maps severity=medium -> "Fix when convenient"', () => {
const out = humanizeFinding(makeFinding({ severity: 'medium' }));
assert.equal(out.userActionLanguage, 'Fix when convenient');
});
test('humanizeFinding maps severity=low -> "Optional cleanup"', () => {
const out = humanizeFinding(makeFinding({ severity: 'low' }));
assert.equal(out.userActionLanguage, 'Optional cleanup');
});
test('humanizeFinding maps severity=info -> "FYI"', () => {
const out = humanizeFinding(makeFinding({ severity: 'info' }));
assert.equal(out.userActionLanguage, 'FYI');
});
test('humanizeFinding falls back to "FYI" for unknown severity', () => {
const out = humanizeFinding(makeFinding({ severity: 'mystery' }));
assert.equal(out.userActionLanguage, 'FYI');
});
// ─── userImpactCategory ────────────────────────────────────────────────
test('humanizeFinding sets category Configuration mistake for CML/SET/HKV/RUL/MCP/IMP/PLH', () => {
for (const s of ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'PLH']) {
const out = humanizeFinding(makeFinding({ scanner: s }));
assert.equal(out.userImpactCategory, 'Configuration mistake', `${s} should map to Configuration mistake`);
}
});
test('humanizeFinding sets category Conflict for CNF/COL', () => {
for (const s of ['CNF', 'COL']) {
const out = humanizeFinding(makeFinding({ scanner: s }));
assert.equal(out.userImpactCategory, 'Conflict');
}
});
test('humanizeFinding sets category Wasted tokens for TOK/CPS/SKL', () => {
for (const s of ['TOK', 'CPS', 'SKL']) {
const out = humanizeFinding(makeFinding({ scanner: s }));
assert.equal(out.userImpactCategory, 'Wasted tokens');
}
});
test('humanizeFinding sets category Dead config for DIS', () => {
const out = humanizeFinding(makeFinding({ scanner: 'DIS' }));
assert.equal(out.userImpactCategory, 'Dead config');
});
test('humanizeFinding sets category Missed opportunity for GAP', () => {
const out = humanizeFinding(makeFinding({ scanner: 'GAP', title: 'No CLAUDE.md file' }));
assert.equal(out.userImpactCategory, 'Missed opportunity');
});
test('humanizeFinding sets category Other for unknown scanner', () => {
const out = humanizeFinding(makeFinding({ scanner: 'XXX' }));
assert.equal(out.userImpactCategory, 'Other');
});
// ─── relevanceContext ──────────────────────────────────────────────────
test('computeRelevanceContext detects test-fixture paths', () => {
assert.equal(computeRelevanceContext('/repo/tests/fixtures/foo/CLAUDE.md'), 'test-fixture-no-impact');
assert.equal(computeRelevanceContext('/repo/test/fixtures/bar.json'), 'test-fixture-no-impact');
});
test('computeRelevanceContext detects local-only paths via .local. infix', () => {
assert.equal(computeRelevanceContext('/repo/.claude/settings.local.json'), 'affects-this-machine-only');
assert.equal(computeRelevanceContext('/repo/CLAUDE.local.md'), 'affects-this-machine-only');
});
test('computeRelevanceContext defaults to affects-everyone for normal paths', () => {
assert.equal(computeRelevanceContext('/repo/CLAUDE.md'), 'affects-everyone');
assert.equal(computeRelevanceContext('/repo/.claude/settings.json'), 'affects-everyone');
});
test('computeRelevanceContext defaults to affects-everyone for null/empty paths', () => {
assert.equal(computeRelevanceContext(null), 'affects-everyone');
assert.equal(computeRelevanceContext(undefined), 'affects-everyone');
assert.equal(computeRelevanceContext(''), 'affects-everyone');
});
test('humanizeFinding sets relevanceContext from file', () => {
const f = makeFinding({ file: '/repo/tests/fixtures/x.json' });
assert.equal(humanizeFinding(f).relevanceContext, 'test-fixture-no-impact');
const g = makeFinding({ file: '/repo/.claude/settings.local.json' });
assert.equal(humanizeFinding(g).relevanceContext, 'affects-this-machine-only');
const h = makeFinding({ file: '/repo/CLAUDE.md' });
assert.equal(humanizeFinding(h).relevanceContext, 'affects-everyone');
});
// ─── humanizeFindings & humanizeEnvelope ──────────────────────────────
test('humanizeFindings translates each finding in the array', () => {
const findings = [
makeFinding({ scanner: 'CML', title: 'No CLAUDE.md found' }),
makeFinding({ id: 'CA-CML-002', scanner: 'CML', title: 'Uses HTML comments' }),
];
const out = humanizeFindings(findings);
assert.equal(out.length, 2);
assert.notEqual(out[0].title, findings[0].title);
assert.notEqual(out[1].title, findings[1].title);
});
test('humanizeFindings returns input unchanged if not an array', () => {
assert.equal(humanizeFindings(null), null);
assert.equal(humanizeFindings(undefined), undefined);
});
test('humanizeEnvelope walks scanners[].findings and humanizes each', () => {
const env = {
target_path: '/tmp',
scanners: [
{
scanner: 'CML',
status: 'ok',
findings: [makeFinding({ scanner: 'CML', title: 'No CLAUDE.md found' })],
counts: {},
},
{
scanner: 'TOK',
status: 'ok',
findings: [makeFinding({
id: 'CA-TOK-001',
scanner: 'TOK',
severity: 'low',
title: 'Cache-breaking volatile content at top of CLAUDE.md',
file: '/tmp/CLAUDE.md',
})],
counts: {},
},
],
};
const out = humanizeEnvelope(env);
assert.equal(out.target_path, '/tmp');
assert.equal(out.scanners.length, 2);
assert.notEqual(out.scanners[0].findings[0].title, env.scanners[0].findings[0].title);
assert.notEqual(out.scanners[1].findings[0].title, env.scanners[1].findings[0].title);
assert.equal(out.scanners[1].findings[0].userImpactCategory, 'Wasted tokens');
});
test('humanizeEnvelope returns input unchanged if shape is wrong', () => {
assert.equal(humanizeEnvelope(null), null);
assert.equal(humanizeEnvelope({}).scanners, undefined); // unchanged object
assert.equal(humanizeEnvelope({ scanners: 'not-an-array' }).scanners, 'not-an-array');
});
// ─── new fields presence ───────────────────────────────────────────────
test('humanizeFinding always sets the three new fields', () => {
const out = humanizeFinding(makeFinding());
assert.equal(typeof out.userImpactCategory, 'string');
assert.equal(typeof out.userActionLanguage, 'string');
assert.equal(typeof out.relevanceContext, 'string');
});
// ─── ANSI-free guarantee ───────────────────────────────────────────────
test('humanized output contains no ANSI escape sequences', () => {
const out = humanizeFinding(makeFinding({ scanner: 'CML', title: 'No CLAUDE.md found' }));
const allText = `${out.title} ${out.description} ${out.recommendation}`;
// eslint-disable-next-line no-control-regex
assert.equal(/\[/.test(allText), false, 'ANSI escape detected in humanized output');
});