feat(skill-listing): add SKL scanner for the skill-listing token budget

Fase 4 Items 2+3 (CC 2.1.114→181 gap-review). New orchestrated scanner
`skill-listing-scanner.mjs` (prefix SKL) flags every active skill whose
description exceeds the verified 1,536-char listing cap (CC 2.1.105, changelog
L1502). Past the cap, Claude Code silently truncates the description the model
reads to route skill invocation — dropping the trigger phrases at the tail.
HOME-scoped over all user + plugin skills via enumerateSkills (COL is the model).

- CA-SKL-001 (medium): description > 1,536 chars. Remediation folds in Item
  2(b) — recommends disableBundledSkills + skillOverrides + trimming
  (designvalg A: no standalone GAP-check, which would fire for nearly everyone).
- Designvalg B: v1 ships the verified cap ONLY. The aggregate 2%-of-context
  listing budget is deferred — it needs a context-window assumption that would
  turn a verified fact into a guess (would carry a CALIBRATION_NOTE if added).
- Choice C: recognize the skillOverrides settings key (CC 2.1.129) in
  KNOWN_KEYS. Left OUT of TYPE_CHECKS — the value is a per-skill object
  (off/user-invocable-only/name-only), not a string; a 'string' check (as the
  plan sketched) would create a NEW false positive. Verify-first deviation.

Registration: scan-orchestrator (13th scanner), humanizer (SKL → 'Wasted
tokens' + static/_default translations), scoring SCANNER_AREA_MAP (→ Token
Efficiency; no 11th area), README badge 12→13, CLAUDE.md (finding-id +
test-count), docs/scanner-internals.md, gap-matrix + plan status notes.

Snapshots reseeded hermetically (SEED_SNAPSHOT/UPDATE_SNAPSHOT): SKL entry with
0 findings in empty HOME, scanners_ok 11→12, claudeMdEstimatedTokens bump from
the CLAUDE.md edits flowing through the cascade. Contamination grep clean.

Suite 868/868 (856 baseline + 11 SKL + 1 skillOverrides). RED→GREEN logged
per cycle.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
This commit is contained in:
Kjell Tore Guttormsen 2026-06-18 17:36:28 +02:00
commit 7bb254780a
22 changed files with 390 additions and 37 deletions

View file

@ -8,7 +8,7 @@ import { TRANSLATIONS } from '../../scanners/lib/humanizer-data.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const FORBIDDEN_PATH = resolve(__dirname, '..', 'lint-forbidden-words.json');
const EXPECTED_SCANNERS = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK', 'CPS', 'DIS', 'COL', 'PLH'];
const EXPECTED_SCANNERS = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK', 'CPS', 'DIS', 'COL', 'PLH', 'SKL'];
function stripBacktickSpans(s) {
return s.replace(/`[^`]*`/g, '');
@ -24,7 +24,7 @@ test('TRANSLATIONS exports an object', () => {
assert.ok(TRANSLATIONS !== null);
});
test('TRANSLATIONS covers all 13 expected scanner prefixes', () => {
test('TRANSLATIONS covers all 14 expected scanner prefixes', () => {
for (const prefix of EXPECTED_SCANNERS) {
assert.ok(TRANSLATIONS[prefix], `missing scanner prefix: ${prefix}`);
}

View file

@ -171,8 +171,8 @@ test('humanizeFinding sets category Conflict for CNF/COL', () => {
}
});
test('humanizeFinding sets category Wasted tokens for TOK/CPS', () => {
for (const s of ['TOK', 'CPS']) {
test('humanizeFinding sets category Wasted tokens for TOK/CPS/SKL', () => {
for (const s of ['TOK', 'CPS', 'SKL']) {
const out = humanizeFinding(makeFinding({ scanner: s }));
assert.equal(out.userImpactCategory, 'Wasted tokens');
}

View file

@ -101,7 +101,7 @@ describe('scan-orchestrator humanizer wiring (Step 5)', () => {
assert.ok(actual.meta, 'meta present');
assert.ok(Array.isArray(actual.scanners), 'scanners array present');
assert.ok(actual.aggregate, 'aggregate present');
assert.equal(actual.scanners.length, 12, 'all 12 scanners present');
assert.equal(actual.scanners.length, 13, 'all 13 scanners present');
});
it('preserves scanner shape (scanner/status/findings/counts)', async () => {

View file

@ -131,7 +131,7 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi
'sandbox', 'fallbackModel', 'enforceAvailableModels', 'disableBundledSkills',
'pluginSuggestionMarketplaces', 'requiredMinimumVersion', 'requiredMaximumVersion',
'allowAllClaudeAiMcps', 'footerLinksRegexes', 'wheelScrollAccelerationEnabled',
'parentSettingsBehavior',
'parentSettingsBehavior', 'skillOverrides',
];
beforeEach(async () => {
@ -152,6 +152,7 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi
footerLinksRegexes: ['^https://internal\\.'], // CC 2.1.181
wheelScrollAccelerationEnabled: true, // CC 2.1.174
parentSettingsBehavior: 'merge', // CC 2.1.133
skillOverrides: { 'example-skill': 'name-only' }, // CC 2.1.129 (per-skill object)
permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] },
};
await writeFile(

View file

@ -0,0 +1,189 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { join } from 'node:path';
import { mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/skill-listing-scanner.mjs';
const CAP = 1536; // verified per-description listing cap (CC 2.1.105, changelog L1502)
function uniqueDir(suffix) {
return join(tmpdir(), `config-audit-skl-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
}
/**
* The SKL scanner enumerates active skills via process.env.HOME
* (enumeratePlugins/enumerateSkills). Tests must override HOME, run, restore
* never rely on the developer's real ~/.claude.
*/
async function runScannerWithHome(home) {
resetCounter();
const original = process.env.HOME;
process.env.HOME = home;
try {
return await scan('/unused', { files: [] });
} finally {
process.env.HOME = original;
}
}
/** Build a fake HOME with one user skill whose description has `len` chars. */
async function homeWithUserSkill(name, descLen) {
const home = uniqueDir(name);
const dir = join(home, '.claude', 'skills', name);
await mkdir(dir, { recursive: true });
const desc = 'a'.repeat(descLen);
await writeFile(
join(dir, 'SKILL.md'),
`---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`,
);
return home;
}
describe('SKL scanner — basic structure', () => {
it('reports scanner prefix SKL', async () => {
const home = uniqueDir('empty');
try {
await mkdir(join(home, '.claude'), { recursive: true });
const result = await runScannerWithHome(home);
assert.equal(result.scanner, 'SKL');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('finding IDs match CA-SKL-NNN pattern', async () => {
const home = await homeWithUserSkill('longdesc', CAP + 64);
try {
const result = await runScannerWithHome(home);
for (const f of result.findings) {
assert.match(f.id, /^CA-SKL-\d{3}$/);
}
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — per-description 1536-char cap (CA-SKL-001)', () => {
it('flags a skill whose description exceeds the cap', async () => {
const home = await homeWithUserSkill('toolong', CAP + 100);
try {
const result = await runScannerWithHome(home);
const f = result.findings.find(x => /toolong/.test(x.evidence || x.description || ''));
assert.ok(f, `expected a cap finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`);
assert.match(f.id, /^CA-SKL-001$/);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('evidence carries the measured char count and the 1536 cap', async () => {
const home = await homeWithUserSkill('measured', CAP + 200);
try {
const result = await runScannerWithHome(home);
const f = result.findings.find(x => /measured/.test(x.evidence || ''));
assert.ok(f, 'expected a finding for the oversized skill');
assert.match(String(f.evidence), new RegExp(String(CAP + 200)));
assert.match(String(f.evidence), new RegExp(String(CAP)));
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('points file at the offending SKILL.md', async () => {
const home = await homeWithUserSkill('filepath', CAP + 1);
try {
const result = await runScannerWithHome(home);
const f = result.findings.find(x => /filepath/.test(x.evidence || ''));
assert.ok(f);
assert.match(String(f.file), /filepath[\\/]SKILL\.md$/);
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — boundary and negative cases', () => {
it('a description exactly at the cap (1536) yields no finding', async () => {
const home = await homeWithUserSkill('exact', CAP);
try {
const result = await runScannerWithHome(home);
assert.equal(result.findings.length, 0,
`expected 0 findings at the cap; got: ${result.findings.map(f => f.title).join(' | ')}`);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('one char over the cap (1537) yields a finding', async () => {
const home = await homeWithUserSkill('over', CAP + 1);
try {
const result = await runScannerWithHome(home);
assert.equal(result.findings.length, 1,
`expected 1 finding at cap+1; got: ${result.findings.map(f => f.title).join(' | ')}`);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('a short description yields no finding', async () => {
const home = await homeWithUserSkill('short', 80);
try {
const result = await runScannerWithHome(home);
assert.equal(result.findings.length, 0,
`expected 0 findings; got: ${result.findings.map(f => f.title).join(' | ')}`);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('an empty HOME (no skills) yields zero findings', async () => {
const home = uniqueDir('noskills');
try {
await mkdir(join(home, '.claude'), { recursive: true });
const result = await runScannerWithHome(home);
assert.equal(result.findings.length, 0);
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — remediation levers (Item 2b folded in)', () => {
it('recommendation lists disableBundledSkills, skillOverrides, and trimming', async () => {
const home = await homeWithUserSkill('levers', CAP + 300);
try {
const result = await runScannerWithHome(home);
const f = result.findings.find(x => /levers/.test(x.evidence || ''));
assert.ok(f, 'expected a finding to carry remediation');
const rec = String(f.recommendation);
assert.match(rec, /disableBundledSkills/);
assert.match(rec, /skillOverrides/);
assert.match(rec, /trim/i);
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — suppression compatibility', () => {
it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');
const home = await homeWithUserSkill('suppress', CAP + 50);
try {
const result = await runScannerWithHome(home);
assert.ok(result.findings.length > 0, 'precondition: at least one SKL finding');
const { active, suppressed } = applySuppressions(result.findings, [
{ pattern: 'CA-TOK-*', source: 'test', sourceLine: 1 },
]);
assert.equal(active.length, result.findings.length,
'CA-TOK-* glob should not match CA-SKL-* findings');
assert.equal(suppressed.length, 0);
} finally {
await rm(home, { recursive: true, force: true });
}
});
});

View file

@ -1,4 +1,4 @@
{
"kind": "text",
"payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
"payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
}

View file

@ -539,6 +539,20 @@
"low": 0,
"info": 0
}
},
{
"scanner": "SKL",
"status": "ok",
"files_scanned": 0,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
"high": 0,
"medium": 0,
"low": 0,
"info": 0
}
}
],
"aggregate": {
@ -553,7 +567,7 @@
"risk_score": 11,
"risk_band": "Medium",
"verdict": "PASS",
"scanners_ok": 11,
"scanners_ok": 12,
"scanners_error": 0,
"scanners_skipped": 1
}

View file

@ -1,15 +1,16 @@
[CML] CLAUDE.md Linter: 1 finding(s) (14ms)
[SET] Settings Validator: 0 finding(s) (2ms)
[HKV] Hook Validator: 0 finding(s) (1ms)
[SET] Settings Validator: 0 finding(s) (1ms)
[HKV] Hook Validator: 0 finding(s) (0ms)
[RUL] Rules Validator: 0 finding(s) (0ms)
[MCP] MCP Config Validator: 0 finding(s) (0ms)
[IMP] Import Resolver: 0 finding(s) (2ms)
[IMP] Import Resolver: 0 finding(s) (1ms)
[CNF] Conflict Detector: 0 finding(s) (1ms)
[GAP] Feature Gap Scanner: 17 finding(s) (2ms)
[TOK] Token Hotspots: 1 finding(s) (8ms)
[CPS] Cache-Prefix Stability: 0 finding(s) (0ms)
[DIS] Disabled-In-Schema: 0 finding(s) (1ms)
[COL] Plugin Skill Collision: 0 finding(s) (0ms)
[CPS] Cache-Prefix Stability: 0 finding(s) (1ms)
[DIS] Disabled-In-Schema: 0 finding(s) (0ms)
[COL] Plugin Skill Collision: 0 finding(s) (1ms)
[SKL] Skill-Listing Budget: 0 finding(s) (0ms)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
Config-Audit Health Score

View file

@ -94,7 +94,7 @@
"scannerEnvelope": {
"meta": {
"target": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium",
"timestamp": "2026-06-18T12:19:30.125Z",
"timestamp": "2026-06-18T15:31:48.663Z",
"version": "2.2.0",
"tool": "config-audit"
},
@ -103,7 +103,7 @@
"scanner": "CML",
"status": "ok",
"files_scanned": 1,
"duration_ms": 2,
"duration_ms": 3,
"findings": [
{
"id": "CA-CML-001",
@ -145,7 +145,7 @@
"scanner": "HKV",
"status": "ok",
"files_scanned": 1,
"duration_ms": 2,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -173,7 +173,7 @@
"scanner": "MCP",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -187,7 +187,7 @@
"scanner": "IMP",
"status": "ok",
"files_scanned": 1,
"duration_ms": 2,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -526,7 +526,7 @@
],
"total_estimated_tokens": 801,
"activeConfig": {
"claudeMdEstimatedTokens": 1639,
"claudeMdEstimatedTokens": 1647,
"mcpServerCount": 1,
"pluginCount": 0,
"skillCount": 0
@ -536,7 +536,7 @@
"scanner": "CPS",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -573,6 +573,20 @@
"low": 0,
"info": 0
}
},
{
"scanner": "SKL",
"status": "ok",
"files_scanned": 0,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
"high": 0,
"medium": 0,
"low": 0,
"info": 0
}
}
],
"aggregate": {
@ -587,7 +601,7 @@
"risk_score": 11,
"risk_band": "Medium",
"verdict": "PASS",
"scanners_ok": 11,
"scanners_ok": 12,
"scanners_error": 0,
"scanners_skipped": 1
}

View file

@ -1,7 +1,7 @@
{
"meta": {
"target": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium",
"timestamp": "2026-06-18T12:19:29.934Z",
"timestamp": "2026-06-18T15:31:48.479Z",
"version": "2.2.0",
"tool": "config-audit"
},
@ -80,7 +80,7 @@
"scanner": "MCP",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -94,7 +94,7 @@
"scanner": "IMP",
"status": "ok",
"files_scanned": 1,
"duration_ms": 2,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -108,7 +108,7 @@
"scanner": "CNF",
"status": "ok",
"files_scanned": 2,
"duration_ms": 1,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
@ -433,7 +433,7 @@
],
"total_estimated_tokens": 801,
"activeConfig": {
"claudeMdEstimatedTokens": 1639,
"claudeMdEstimatedTokens": 1647,
"mcpServerCount": 1,
"pluginCount": 0,
"skillCount": 0
@ -480,6 +480,20 @@
"low": 0,
"info": 0
}
},
{
"scanner": "SKL",
"status": "ok",
"files_scanned": 0,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
"high": 0,
"medium": 0,
"low": 0,
"info": 0
}
}
],
"aggregate": {
@ -494,7 +508,7 @@
"risk_score": 11,
"risk_band": "Medium",
"verdict": "PASS",
"scanners_ok": 11,
"scanners_ok": 12,
"scanners_error": 0,
"scanners_skipped": 1
}