From 9a44df22accc645cd9da7ff55fded0822a197734 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 06:58:42 +0200 Subject: [PATCH] feat(config-audit): TOK flags skill description > 500 chars (v5 M2) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New Pattern F in TOK: low-severity finding when SKILL.md description > 500 chars - Scoped to discovery.files (project-local) — activeConfig.skills walk would pull in user/plugin skills out of project scope - New fixtures: skill-bloated (594-char desc) + skill-tight (46-char baseline) 574 → 576 tests, all green. --- .../config-audit/scanners/token-hotspots.mjs | 40 ++++++++++++++++++- .../skill-bloated/skills/bloated/SKILL.md | 8 ++++ .../skill-tight/skills/tight/SKILL.md | 8 ++++ .../tests/scanners/token-hotspots.test.mjs | 16 ++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 plugins/config-audit/tests/fixtures/skill-bloated/skills/bloated/SKILL.md create mode 100644 plugins/config-audit/tests/fixtures/skill-tight/skills/tight/SKILL.md diff --git a/plugins/config-audit/scanners/token-hotspots.mjs b/plugins/config-audit/scanners/token-hotspots.mjs index 889378b..3d64f80 100644 --- a/plugins/config-audit/scanners/token-hotspots.mjs +++ b/plugins/config-audit/scanners/token-hotspots.mjs @@ -26,7 +26,7 @@ import { stat } from 'node:fs/promises'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; -import { findImports, parseJson } from './lib/yaml-parser.mjs'; +import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { estimateTokens, readActiveConfig } from './lib/active-config-reader.mjs'; const SCANNER = 'TOK'; @@ -48,6 +48,10 @@ const MAX_IMPORT_DEPTH = 2; // any tool description loads. Heuristic for "context budget under pressure". const CASCADE_TOKEN_THRESHOLD = 10_000; +// v5 M2: SKILL.md `description` loads on every turn even when the body does +// not. Anything past this hints the description is doing the body's job. +const SKILL_DESCRIPTION_THRESHOLD = 500; + const HOTSPOTS_MAX = 10; // v5 F7: shared evidence note appended to every TOK pattern finding. @@ -334,6 +338,40 @@ export async function scan(targetPath, discovery) { } } + // ── Pattern F: SKILL.md description > 500 chars (v5 M2) ── + // Scoped to discovery.files (project-local skill-md). The plan mentioned + // walking activeConfig.skills, but that pulls in user's ~/.claude/skills + // and installed plugin skills which are out-of-scope for a project audit + // and add noise the user can't act on. Project-local discovery is what + // /config-audit on a path is actually asking about. + for (const f of discovery.files) { + if (f.type !== 'skill-md') continue; + const content = await readTextFile(f.absPath); + if (!content) continue; + filesScanned++; + const fm = parseFrontmatter(content)?.frontmatter || null; + const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + if (desc.length <= SKILL_DESCRIPTION_THRESHOLD) continue; + const skillName = (fm && fm.name) || f.absPath.split('/').slice(-2, -1)[0] || f.absPath; + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Bloated skill description (loads on every turn)', + description: + `Skill "${skillName}" has a description of ${desc.length} characters ` + + `(>${SKILL_DESCRIPTION_THRESHOLD}). The description block loads on every turn ` + + 'even when the skill body does not — long descriptions inflate per-turn cost.', + file: f.absPath, + evidence: + `description_chars=${desc.length}; threshold=${SKILL_DESCRIPTION_THRESHOLD}; ` + + `skill="${skillName}" — ${CALIBRATION_NOTE}`, + recommendation: + 'Tighten the description to a single sentence (≤500 chars) covering trigger phrases ' + + 'only. Move detailed usage / examples into the SKILL.md body.', + category: 'token-efficiency', + })); + } + // ── Pattern E: CLAUDE.md cascade > CASCADE_TOKEN_THRESHOLD (v5 M4) ── if (activeConfig?.claudeMd?.estimatedTokens > CASCADE_TOKEN_THRESHOLD) { const cascadeTokens = activeConfig.claudeMd.estimatedTokens; diff --git a/plugins/config-audit/tests/fixtures/skill-bloated/skills/bloated/SKILL.md b/plugins/config-audit/tests/fixtures/skill-bloated/skills/bloated/SKILL.md new file mode 100644 index 0000000..32ab4a3 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/skill-bloated/skills/bloated/SKILL.md @@ -0,0 +1,8 @@ +--- +name: bloated +description: Bloated skill description used to trip the v5 M2 check. Repeats verbose framing about when, how, and why this skill should be used; lists every conceivable trigger phrase, every adjacent skill it composes with, and every alias and synonym a user might type, then explains in detail what the skill produces, what it does not produce, and what the user should run instead in edge cases. By design this description is comfortably over 500 characters so the TOK scanner emits a low-severity finding flagging it for tightening, since description text loads on every turn even when the body does not. +--- + +# Bloated skill body + +Minimal body. diff --git a/plugins/config-audit/tests/fixtures/skill-tight/skills/tight/SKILL.md b/plugins/config-audit/tests/fixtures/skill-tight/skills/tight/SKILL.md new file mode 100644 index 0000000..d4e43b1 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/skill-tight/skills/tight/SKILL.md @@ -0,0 +1,8 @@ +--- +name: tight +description: Tight skill description, well under 500 chars. +--- + +# Tight skill + +Minimal body. diff --git a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs index 77087b8..7249be6 100644 --- a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs +++ b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs @@ -189,6 +189,22 @@ describe('TOK scanner — hotspots contract', () => { }); }); +describe('TOK scanner — M2 skill description > 500 chars (v5)', () => { + it('flags skill with bloated description (low severity)', async () => { + const result = await runScanner('skill-bloated'); + const f = result.findings.find(x => /skill description/i.test(x.title || '')); + assert.ok(f, `expected skill-description finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'low', `expected low, got ${f.severity}`); + assert.match(f.evidence || '', /bloated/); + }); + + it('does NOT flag tight description (under 500 chars)', async () => { + const result = await runScanner('skill-tight'); + const f = result.findings.find(x => /skill description/i.test(x.title || '')); + assert.equal(f, undefined, `expected no skill-description finding; got: ${f?.title}`); + }); +}); + describe('TOK scanner — M4 cascade > 10k tokens (v5)', () => { it('flags CLAUDE.md cascade > 10k tokens with medium severity', async () => { const result = await runScanner('large-cascade');