diff --git a/scanners/lib/skill-listing-budget.mjs b/scanners/lib/skill-listing-budget.mjs index 4afed8d..d68682e 100644 --- a/scanners/lib/skill-listing-budget.mjs +++ b/scanners/lib/skill-listing-budget.mjs @@ -48,6 +48,19 @@ export const BUDGET_CALIBRATION_NOTE = `window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` + 'tok and you are likely within it. this is an estimate, not measured telemetry'; +// Skill-body size guidance (CA-SKL-003). A SKILL.md body over ~5,000 tokens +// (~500 lines / ~20k chars) should split reference content into supporting files +// (Claude Code skill-authoring guidance). Unlike the listing budget above, the +// body is an ON-DEMAND cost: it loads only when the skill is invoked, not every +// turn — so this is a LOW-severity efficiency signal, not an always-loaded bill. +export const BODY_TOKEN_THRESHOLD = 5000; + +// Honest framing for the body-size finding: distinguishes on-demand from +// always-loaded cost and flags the figure as an estimate. Appended to evidence. +export const BODY_CALIBRATION_NOTE = + 'this is the skill BODY (SKILL.md below the frontmatter), which loads ON DEMAND only when the ' + + 'skill is invoked - NOT every turn like the always-loaded listing. estimate (chars/4), not measured telemetry'; + /** * @typedef {object} BudgetAssessment * @property {number} scanned - number of descriptions assessed @@ -91,6 +104,9 @@ export function assessSkillListingBudget(descLengths) { * @property {string|null} pluginName * @property {string} path * @property {number} descLength + * @property {number} bodyChars - SKILL.md body length below the frontmatter (on-demand cost) + * @property {number} bodyLines - body line count + * @property {number} bodyTokens - estimateTokens(bodyChars, 'markdown') */ /** @@ -110,14 +126,20 @@ export async function measureActiveSkillListing() { if (!skill || typeof skill.path !== 'string') continue; const content = await readTextFile(skill.path); if (!content) continue; - const fm = parseFrontmatter(content)?.frontmatter || null; + const parsed = parseFrontmatter(content); + const fm = parsed?.frontmatter || null; const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + const body = (parsed && typeof parsed.body === 'string') ? parsed.body : ''; + const bodyChars = body.length; skills.push({ name: skill.name, source: skill.source, pluginName: skill.pluginName, path: skill.path, descLength: desc.length, + bodyChars, + bodyLines: bodyChars === 0 ? 0 : body.split('\n').length, + bodyTokens: estimateTokens(bodyChars, 'markdown'), }); } diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs index 838253a..cbbf16e 100644 --- a/scanners/skill-listing-scanner.mjs +++ b/scanners/skill-listing-scanner.mjs @@ -40,6 +40,8 @@ import { DESCRIPTION_CAP, AGGREGATE_BUDGET_TOKENS, BUDGET_CALIBRATION_NOTE, + BODY_TOKEN_THRESHOLD, + BODY_CALIBRATION_NOTE, measureActiveSkillListing, } from './lib/skill-listing-budget.mjs'; @@ -116,5 +118,38 @@ export async function scan(_targetPath, _discovery) { })); } + // CA-SKL-003 (oversized body). Emitted last so the common single-issue cases + // read cleanly. Unlike the listing budget, this is an ON-DEMAND cost — the body + // loads only when the skill is invoked, not every turn — hence low severity and + // an explicit on-demand calibration note. + for (const skill of skills) { + if (skill.bodyTokens <= BODY_TOKEN_THRESHOLD) continue; + + const sourceLabel = skill.source === 'plugin' + ? `plugin:${skill.pluginName}` + : 'user'; + + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Skill body is large (loads on demand when the skill runs)', + description: + `Skill "${skill.name}" (${sourceLabel}) has a body of about ${skill.bodyTokens} tokens ` + + `(${skill.bodyLines} lines), over the ~${BODY_TOKEN_THRESHOLD}-token guidance for a skill body. ` + + 'The body is not in the always-loaded listing — it loads only when the skill is invoked — but ' + + 'once loaded a large body consumes context for the rest of that session. Claude Code skill ' + + 'guidance is to keep the body lean and move heavy reference material into supporting files.', + file: skill.path, + evidence: + `body_tokens~${skill.bodyTokens}; body_lines=${skill.bodyLines}; body_chars=${skill.bodyChars}; ` + + `threshold=${BODY_TOKEN_THRESHOLD} tok; skill="${skill.name}"; source=${sourceLabel} - ${BODY_CALIBRATION_NOTE}`, + recommendation: + 'Move reference content into supporting files the skill loads only when needed, and consider ' + + '`context: fork` in the skill frontmatter for heavy skills so the body runs in a forked context ' + + 'instead of consuming the main thread.', + category: 'token-efficiency', + })); + } + return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start); } diff --git a/tests/scanners/skill-listing-scanner.test.mjs b/tests/scanners/skill-listing-scanner.test.mjs index aa5f31f..92d7b97 100644 --- a/tests/scanners/skill-listing-scanner.test.mjs +++ b/tests/scanners/skill-listing-scanner.test.mjs @@ -61,6 +61,22 @@ async function homeWithNUserSkills(count, descLen, prefix = 'agg') { return home; } +/** + * Write one user skill whose SKILL.md *body* (below the frontmatter) is about + * `bodyChars` chars, spread across multiple lines (81 chars incl. newline). + * The description stays tiny so body-size tests stay isolated from the cap/aggregate checks. + */ +async function addUserSkillWithBody(home, name, bodyChars) { + const dir = join(home, '.claude', 'skills', name); + await mkdir(dir, { recursive: true }); + const line = `${'x'.repeat(80)}\n`; + const body = line.repeat(Math.ceil(bodyChars / line.length)); + await writeFile( + join(dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: short.\n---\n${body}`, + ); +} + // Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor. const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000 // Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4). @@ -303,6 +319,81 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { }); }); +describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => { + const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || '')); + + it('flags a skill whose body exceeds ~5,000 tokens', async () => { + const home = uniqueDir('bigbody'); + await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body'); + assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity'); + assert.match(String(f.file), /heavy[\\/]SKILL\.md$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('does NOT flag a skill whose body is under the threshold', async () => { + const home = uniqueDir('smallbody'); + await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body + try { + const result = await runScannerWithHome(home); + assert.equal(findBody(result.findings), undefined, 'a small body must not fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => { + const home = uniqueDir('bodyev'); + await addUserSkillWithBody(home, 'heavy', 20_400); + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f); + const ev = String(f.evidence); + assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens'); + assert.match(ev, /body_lines=\d+/, 'evidence states the body line count'); + assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost'); + assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing'); + assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('recommends context:fork and moving reference content into supporting files', async () => { + const home = uniqueDir('bodyrec'); + await addUserSkillWithBody(home, 'heavy', 20_400); + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f); + const rec = String(f.recommendation); + assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork'); + assert.match(rec, /supporting file/i, 'recommendation should mention supporting files'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => { + const home = uniqueDir('bodyonly'); + await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description + try { + const result = await runScannerWithHome(home); + assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap'); + assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget'); + assert.ok(findBody(result.findings), 'only the body finding should fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + describe('SKL scanner — suppression compatibility', () => { it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => { const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');