feat(skl): flag oversized skill bodies on demand (v5.11 B7) [skip-docs]

New CA-SKL-003 (low): a SKILL.md body over ~5,000 tokens (~500 lines)
should split reference content into supporting files / use context: fork.

- measureActiveSkillListing() now returns body metrics (chars/lines/tokens);
  the body was already read in full, only the frontmatter was parsed before.
- Honest framing: BODY_CALIBRATION_NOTE marks this as ON-DEMAND cost (loads
  only when the skill is invoked, NOT every turn like the always-loaded
  listing) and an estimate — hence low severity.
- 5 new tests; full suite 1262 green; snapshots byte-stable (default branch
  untouched; new finding fires only on bodies >5k tok, none in fixtures).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 21:36:20 +02:00
commit 27988801be
3 changed files with 149 additions and 1 deletions

View file

@ -61,6 +61,22 @@ async function homeWithNUserSkills(count, descLen, prefix = 'agg') {
return home;
}
/**
* Write one user skill whose SKILL.md *body* (below the frontmatter) is about
* `bodyChars` chars, spread across multiple lines (81 chars incl. newline).
* The description stays tiny so body-size tests stay isolated from the cap/aggregate checks.
*/
async function addUserSkillWithBody(home, name, bodyChars) {
const dir = join(home, '.claude', 'skills', name);
await mkdir(dir, { recursive: true });
const line = `${'x'.repeat(80)}\n`;
const body = line.repeat(Math.ceil(bodyChars / line.length));
await writeFile(
join(dir, 'SKILL.md'),
`---\nname: ${name}\ndescription: short.\n---\n${body}`,
);
}
// Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor.
const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000
// Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4).
@ -303,6 +319,81 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
});
});
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));
it('flags a skill whose body exceeds ~5,000 tokens', async () => {
const home = uniqueDir('bigbody');
await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body
try {
const result = await runScannerWithHome(home);
const f = findBody(result.findings);
assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body');
assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity');
assert.match(String(f.file), /heavy[\\/]SKILL\.md$/);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('does NOT flag a skill whose body is under the threshold', async () => {
const home = uniqueDir('smallbody');
await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body
try {
const result = await runScannerWithHome(home);
assert.equal(findBody(result.findings), undefined, 'a small body must not fire');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => {
const home = uniqueDir('bodyev');
await addUserSkillWithBody(home, 'heavy', 20_400);
try {
const result = await runScannerWithHome(home);
const f = findBody(result.findings);
assert.ok(f);
const ev = String(f.evidence);
assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens');
assert.match(ev, /body_lines=\d+/, 'evidence states the body line count');
assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost');
assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing');
assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('recommends context:fork and moving reference content into supporting files', async () => {
const home = uniqueDir('bodyrec');
await addUserSkillWithBody(home, 'heavy', 20_400);
try {
const result = await runScannerWithHome(home);
const f = findBody(result.findings);
assert.ok(f);
const rec = String(f.recommendation);
assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork');
assert.match(rec, /supporting file/i, 'recommendation should mention supporting files');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => {
const home = uniqueDir('bodyonly');
await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description
try {
const result = await runScannerWithHome(home);
assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap');
assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget');
assert.ok(findBody(result.findings), 'only the body finding should fire');
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — suppression compatibility', () => {
it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');