feat(skl): flag oversized skill bodies on demand (v5.11 B7) [skip-docs]
New CA-SKL-003 (low): a SKILL.md body over ~5,000 tokens (~500 lines) should split reference content into supporting files / use context: fork. - measureActiveSkillListing() now returns body metrics (chars/lines/tokens); the body was already read in full, only the frontmatter was parsed before. - Honest framing: BODY_CALIBRATION_NOTE marks this as ON-DEMAND cost (loads only when the skill is invoked, NOT every turn like the always-loaded listing) and an estimate — hence low severity. - 5 new tests; full suite 1262 green; snapshots byte-stable (default branch untouched; new finding fires only on bodies >5k tok, none in fixtures). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c2e3a56a20
commit
27988801be
3 changed files with 149 additions and 1 deletions
|
|
@ -61,6 +61,22 @@ async function homeWithNUserSkills(count, descLen, prefix = 'agg') {
|
|||
return home;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write one user skill whose SKILL.md *body* (below the frontmatter) is about
|
||||
* `bodyChars` chars, spread across multiple lines (81 chars incl. newline).
|
||||
* The description stays tiny so body-size tests stay isolated from the cap/aggregate checks.
|
||||
*/
|
||||
async function addUserSkillWithBody(home, name, bodyChars) {
|
||||
const dir = join(home, '.claude', 'skills', name);
|
||||
await mkdir(dir, { recursive: true });
|
||||
const line = `${'x'.repeat(80)}\n`;
|
||||
const body = line.repeat(Math.ceil(bodyChars / line.length));
|
||||
await writeFile(
|
||||
join(dir, 'SKILL.md'),
|
||||
`---\nname: ${name}\ndescription: short.\n---\n${body}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor.
|
||||
const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000
|
||||
// Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4).
|
||||
|
|
@ -303,6 +319,81 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
|
||||
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));
|
||||
|
||||
it('flags a skill whose body exceeds ~5,000 tokens', async () => {
|
||||
const home = uniqueDir('bigbody');
|
||||
await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const f = findBody(result.findings);
|
||||
assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body');
|
||||
assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity');
|
||||
assert.match(String(f.file), /heavy[\\/]SKILL\.md$/);
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does NOT flag a skill whose body is under the threshold', async () => {
|
||||
const home = uniqueDir('smallbody');
|
||||
await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findBody(result.findings), undefined, 'a small body must not fire');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => {
|
||||
const home = uniqueDir('bodyev');
|
||||
await addUserSkillWithBody(home, 'heavy', 20_400);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const f = findBody(result.findings);
|
||||
assert.ok(f);
|
||||
const ev = String(f.evidence);
|
||||
assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens');
|
||||
assert.match(ev, /body_lines=\d+/, 'evidence states the body line count');
|
||||
assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost');
|
||||
assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing');
|
||||
assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('recommends context:fork and moving reference content into supporting files', async () => {
|
||||
const home = uniqueDir('bodyrec');
|
||||
await addUserSkillWithBody(home, 'heavy', 20_400);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const f = findBody(result.findings);
|
||||
assert.ok(f);
|
||||
const rec = String(f.recommendation);
|
||||
assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork');
|
||||
assert.match(rec, /supporting file/i, 'recommendation should mention supporting files');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => {
|
||||
const home = uniqueDir('bodyonly');
|
||||
await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap');
|
||||
assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget');
|
||||
assert.ok(findBody(result.findings), 'only the body finding should fire');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — suppression compatibility', () => {
|
||||
it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
|
||||
const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue