config-audit/tests/lib/humanizer-data.test.mjs
Kjell Tore Guttormsen 7bb254780a feat(skill-listing): add SKL scanner for the skill-listing token budget
Fase 4 Items 2+3 (CC 2.1.114→181 gap-review). New orchestrated scanner
`skill-listing-scanner.mjs` (prefix SKL) flags every active skill whose
description exceeds the verified 1,536-char listing cap (CC 2.1.105, changelog
L1502). Past the cap, Claude Code silently truncates the description the model
reads to route skill invocation — dropping the trigger phrases at the tail.
HOME-scoped over all user + plugin skills via enumerateSkills (COL is the model).

- CA-SKL-001 (medium): description > 1,536 chars. Remediation folds in Item
  2(b) — recommends disableBundledSkills + skillOverrides + trimming
  (designvalg A: no standalone GAP-check, which would fire for nearly everyone).
- Designvalg B: v1 ships the verified cap ONLY. The aggregate 2%-of-context
  listing budget is deferred — it needs a context-window assumption that would
  turn a verified fact into a guess (would carry a CALIBRATION_NOTE if added).
- Choice C: recognize the skillOverrides settings key (CC 2.1.129) in
  KNOWN_KEYS. Left OUT of TYPE_CHECKS — the value is a per-skill object
  (off/user-invocable-only/name-only), not a string; a 'string' check (as the
  plan sketched) would create a NEW false positive. Verify-first deviation.

Registration: scan-orchestrator (13th scanner), humanizer (SKL → 'Wasted
tokens' + static/_default translations), scoring SCANNER_AREA_MAP (→ Token
Efficiency; no 11th area), README badge 12→13, CLAUDE.md (finding-id +
test-count), docs/scanner-internals.md, gap-matrix + plan status notes.

Snapshots reseeded hermetically (SEED_SNAPSHOT/UPDATE_SNAPSHOT): SKL entry with
0 findings in empty HOME, scanners_ok 11→12, claudeMdEstimatedTokens bump from
the CLAUDE.md edits flowing through the cascade. Contamination grep clean.

Suite 868/868 (856 baseline + 11 SKL + 1 skillOverrides). RED→GREEN logged
per cycle.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
2026-06-18 17:36:28 +02:00

177 lines
7.1 KiB
JavaScript

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import { TRANSLATIONS } from '../../scanners/lib/humanizer-data.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const FORBIDDEN_PATH = resolve(__dirname, '..', 'lint-forbidden-words.json');
const EXPECTED_SCANNERS = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK', 'CPS', 'DIS', 'COL', 'PLH', 'SKL'];
function stripBacktickSpans(s) {
return s.replace(/`[^`]*`/g, '');
}
async function loadForbidden() {
const raw = await readFile(FORBIDDEN_PATH, 'utf8');
return JSON.parse(raw);
}
test('TRANSLATIONS exports an object', () => {
assert.equal(typeof TRANSLATIONS, 'object');
assert.ok(TRANSLATIONS !== null);
});
test('TRANSLATIONS covers all 14 expected scanner prefixes', () => {
for (const prefix of EXPECTED_SCANNERS) {
assert.ok(TRANSLATIONS[prefix], `missing scanner prefix: ${prefix}`);
}
});
test('every scanner has a _default fallback with all 3 fields', () => {
for (const prefix of EXPECTED_SCANNERS) {
const scanner = TRANSLATIONS[prefix];
assert.ok(scanner._default, `${prefix} missing _default`);
assert.ok(typeof scanner._default.title === 'string' && scanner._default.title.length > 0,
`${prefix} _default missing title`);
assert.ok(typeof scanner._default.description === 'string' && scanner._default.description.length > 0,
`${prefix} _default missing description`);
assert.ok(typeof scanner._default.recommendation === 'string' && scanner._default.recommendation.length > 0,
`${prefix} _default missing recommendation`);
}
});
test('every scanner has a static map (may be empty)', () => {
for (const prefix of EXPECTED_SCANNERS) {
assert.equal(typeof TRANSLATIONS[prefix].static, 'object',
`${prefix} missing static map`);
assert.ok(TRANSLATIONS[prefix].static !== null);
}
});
test('every scanner has a patterns array (may be empty)', () => {
for (const prefix of EXPECTED_SCANNERS) {
assert.ok(Array.isArray(TRANSLATIONS[prefix].patterns),
`${prefix} patterns must be an array`);
}
});
test('every static-title entry has all 3 fields', () => {
for (const prefix of EXPECTED_SCANNERS) {
const staticMap = TRANSLATIONS[prefix].static;
for (const [title, t] of Object.entries(staticMap)) {
assert.ok(typeof t.title === 'string' && t.title.length > 0,
`${prefix} static["${title}"] missing title`);
assert.ok(typeof t.description === 'string' && t.description.length > 0,
`${prefix} static["${title}"] missing description`);
assert.ok(typeof t.recommendation === 'string' && t.recommendation.length > 0,
`${prefix} static["${title}"] missing recommendation`);
}
}
});
test('every pattern entry has regex + translation with all 3 fields', () => {
for (const prefix of EXPECTED_SCANNERS) {
for (const p of TRANSLATIONS[prefix].patterns) {
assert.ok(p.regex instanceof RegExp,
`${prefix} pattern missing regex`);
assert.ok(typeof p.translation.title === 'string' && p.translation.title.length > 0,
`${prefix} pattern translation missing title`);
assert.ok(typeof p.translation.description === 'string' && p.translation.description.length > 0,
`${prefix} pattern translation missing description`);
assert.ok(typeof p.translation.recommendation === 'string' && p.translation.recommendation.length > 0,
`${prefix} pattern translation missing recommendation`);
}
}
});
test('no translated string contains tier1 forbidden words (outside backtick spans)', async () => {
const data = await loadForbidden();
const tier1Words = data.tier1.map((e) => e.word);
const violations = [];
for (const prefix of EXPECTED_SCANNERS) {
const scanner = TRANSLATIONS[prefix];
const allTranslations = [
scanner._default,
...Object.values(scanner.static),
...scanner.patterns.map((p) => p.translation),
];
for (const t of allTranslations) {
for (const field of ['title', 'description', 'recommendation']) {
const text = stripBacktickSpans(t[field]).toLowerCase();
for (const word of tier1Words) {
const lower = word.toLowerCase();
// word-boundary match for single words, plain substring for multi-word phrases
const re = lower.includes(' ')
? new RegExp(lower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
: new RegExp(`\\b${lower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`);
if (re.test(text)) {
violations.push(`${prefix} ${field}: "${word}" in "${t[field]}"`);
}
}
}
}
}
assert.equal(violations.length, 0,
`tier1 violations:\n ${violations.slice(0, 20).join('\n ')}`);
});
test('no translated string contains tier3 jargon (outside backtick spans)', async () => {
const data = await loadForbidden();
const tier3Words = data.tier3.map((e) => e.word);
const violations = [];
for (const prefix of EXPECTED_SCANNERS) {
const scanner = TRANSLATIONS[prefix];
const allTranslations = [
scanner._default,
...Object.values(scanner.static),
...scanner.patterns.map((p) => p.translation),
];
for (const t of allTranslations) {
for (const field of ['title', 'description', 'recommendation']) {
const text = stripBacktickSpans(t[field]);
for (const word of tier3Words) {
const lower = word.toLowerCase();
const re = lower.includes(' ') || lower.includes('/') || lower.includes('-') || lower.includes('.')
? new RegExp(lower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'i')
: new RegExp(`\\b${lower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i');
if (re.test(text)) {
violations.push(`${prefix} ${field}: "${word}" in "${t[field]}"`);
}
}
}
}
}
assert.equal(violations.length, 0,
`tier3 violations (jargon outside backticks):\n ${violations.slice(0, 20).join('\n ')}`);
});
test('CML, SET, HKV, RUL, MCP, IMP, GAP, TOK, PLH have non-empty static maps', () => {
// These scanners produce findings with titles we documented. Empty static map suggests missed coverage.
for (const prefix of ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'GAP', 'TOK', 'PLH']) {
const count = Object.keys(TRANSLATIONS[prefix].static).length;
assert.ok(count > 0, `${prefix}.static is empty — expected at least 1 translated title`);
}
});
test('CNF, COL, PLH have at least one pattern entry (template-literal titles)', () => {
// These scanners use template-literal titles for some findings.
for (const prefix of ['CNF', 'COL', 'PLH']) {
assert.ok(TRANSLATIONS[prefix].patterns.length > 0,
`${prefix} expected ≥1 pattern entry for template-literal titles`);
}
});
test('TRANSLATIONS does not mutate when re-imported (deep-frozen-ish)', async () => {
// Quick sanity — translate object reference equality between imports
const { TRANSLATIONS: t2 } = await import('../../scanners/lib/humanizer-data.mjs');
assert.equal(t2, TRANSLATIONS, 'TRANSLATIONS reference should be stable across imports');
});