feat(skill-listing): add CA-SKL-002 aggregate listing-budget check
Syklus 2 of Fase 4 Items 2+3. Flags when the sum of active skill descriptions exceeds the listing budget (~2% of context, CC 2.1.32). Design (operator-confirmed "fact-first, 200k anchor"): - low severity (estimate) vs medium for the verified 1,536-char cap - each description counted up to the 1,536 cap (what actually loads in the listing) — avoids double-counting the tail CA-SKL-001 flags - fires when sum > 2% x 200k = 4000 tok; evidence leads with the measured sum + a calibration note that the budget scales 5x on 1M-context models - aggregate emitted after the per-skill loop so the common case reads 001=cap, 002=aggregate (finding IDs are a sequential counter, not stable semantic IDs — tests match on title, never NNN) Also: - tailored humanizer static entry for the aggregate title - fix latent HOME leak in posture-grade-stability.test.mjs: it spawned posture.mjs without hermeticEnv(), so a real ~/.claude leaked HOME-scoped SKL/COL findings into the baseline grade (Token Efficiency A->B). Now isolated like the 8 other CLI-spawning tests. - docs sync: test count 868->875, scanner-internals, gap-matrix, plan status Suite 875/875, no snapshot drift, self-audit clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
This commit is contained in:
parent
7bb254780a
commit
66433fee48
8 changed files with 215 additions and 11 deletions
|
|
@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
|
|
@ -11,9 +12,13 @@ const FIXTURES = resolve(__dirname, '../fixtures');
|
|||
const POSTURE_BIN = resolve(__dirname, '../../scanners/posture.mjs');
|
||||
|
||||
async function runPostureJson(fixturePath) {
|
||||
// Isolate HOME: SKL and COL are HOME-scoped (enumerateSkills/enumeratePlugins),
|
||||
// so a real ~/.claude would leak the developer's installed plugins/skills into
|
||||
// this baseline grade and make it machine-dependent. See tests/helpers/hermetic-home.mjs.
|
||||
const { stdout } = await exec('node', [POSTURE_BIN, fixturePath, '--json'], {
|
||||
timeout: 30000,
|
||||
cwd: resolve(__dirname, '../..'),
|
||||
env: hermeticEnv(),
|
||||
});
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,34 @@ async function homeWithUserSkill(name, descLen) {
|
|||
return home;
|
||||
}
|
||||
|
||||
/** Write one user skill (name + description of `descLen` chars) into an existing HOME. */
|
||||
async function addUserSkill(home, name, descLen) {
|
||||
const dir = join(home, '.claude', 'skills', name);
|
||||
await mkdir(dir, { recursive: true });
|
||||
const desc = 'a'.repeat(descLen);
|
||||
await writeFile(
|
||||
join(dir, 'SKILL.md'),
|
||||
`---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Build a fake HOME with `count` user skills, each description `descLen` chars. */
|
||||
async function homeWithNUserSkills(count, descLen, prefix = 'agg') {
|
||||
const home = uniqueDir(`${prefix}-${count}x${descLen}`);
|
||||
for (let i = 0; i < count; i++) {
|
||||
await addUserSkill(home, `${prefix}${i}`, descLen);
|
||||
}
|
||||
return home;
|
||||
}
|
||||
|
||||
// Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor.
|
||||
const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000
|
||||
// Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4).
|
||||
// 16 skills * 1000 chars = 16000 chars -> 4000 tok == budget (NOT over).
|
||||
// 17 skills * 1000 chars = 17000 chars -> 4250 tok > budget (over).
|
||||
const findAggregate = (findings) => findings.find(f => /aggregate/i.test(f.title));
|
||||
const findCaps = (findings) => findings.filter(f => /listing cap/i.test(f.title));
|
||||
|
||||
describe('SKL scanner — basic structure', () => {
|
||||
it('reports scanner prefix SKL', async () => {
|
||||
const home = uniqueDir('empty');
|
||||
|
|
@ -169,6 +197,112 @@ describe('SKL scanner — remediation levers (Item 2b folded in)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
|
||||
it('fires a low-severity aggregate finding when the description sum exceeds the 200k-anchored budget', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000); // 17000 chars -> ~4250 tok > 4000
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||
assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`);
|
||||
assert.match(agg.id, /^CA-SKL-\d{3}$/);
|
||||
// 17 skills of 1000 chars each are all under the 1536 cap, so no cap findings.
|
||||
assert.equal(findCaps(result.findings).length, 0,
|
||||
'no per-description cap findings expected when every description is under 1536');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does NOT fire when the aggregate is under budget', async () => {
|
||||
const home = await homeWithNUserSkills(3, 1000); // 3000 chars -> 750 tok < 4000
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findAggregate(result.findings), undefined,
|
||||
`expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('treats the budget as a boundary: exactly at budget yields no finding, one skill over yields one', async () => {
|
||||
const atBudget = await homeWithNUserSkills(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget
|
||||
try {
|
||||
const result = await runScannerWithHome(atBudget);
|
||||
assert.equal(findAggregate(result.findings), undefined,
|
||||
'aggregate exactly at the budget must not fire (strictly-greater rule)');
|
||||
} finally {
|
||||
await rm(atBudget, { recursive: true, force: true });
|
||||
}
|
||||
const overBudget = await homeWithNUserSkills(17, 1000, 'over'); // 17000 chars -> 4250 tok
|
||||
try {
|
||||
const result = await runScannerWithHome(overBudget);
|
||||
assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire');
|
||||
} finally {
|
||||
await rm(overBudget, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('counts each description capped at 1536 — a single huge skill does NOT inflate the aggregate', async () => {
|
||||
// One 5000-char skill: capped contribution = 1536 chars -> 384 tok, far under budget.
|
||||
// It must trigger the per-description cap finding but NOT the aggregate.
|
||||
const home = await homeWithUserSkill('huge', 5000);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findCaps(result.findings).length, 1, 'the oversized skill should trip the cap finding');
|
||||
assert.equal(findAggregate(result.findings), undefined,
|
||||
'aggregate must count the capped 1536 contribution, not the full 5000 chars');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('cap and aggregate findings are independent — both can fire in one run', async () => {
|
||||
// 1 oversized skill (cap) + 16 normal skills. Aggregate = min(5000,1536)+16000 = 17536 chars -> ~4384 tok.
|
||||
const home = await homeWithNUserSkills(16, 1000, 'norm');
|
||||
await addUserSkill(home, 'oversized', 5000);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findCaps(result.findings).length, 1, 'expected exactly one cap finding');
|
||||
assert.ok(findAggregate(result.findings), 'expected an aggregate finding alongside the cap finding');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('evidence carries the measured sum, the 200k budget, and an honest context-window calibration note', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg, 'expected an aggregate finding');
|
||||
const ev = String(agg.evidence);
|
||||
assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence should state the 4000-token budget');
|
||||
assert.match(ev, /200k/, 'evidence should anchor the budget on a 200k window');
|
||||
assert.match(ev, /CC 2\.1\.32/, 'evidence should cite the 2%-of-context changelog fact');
|
||||
assert.match(ev, /1,000,000|20,000/, 'evidence should note the 1M-context scaling');
|
||||
assert.match(ev, /estimate/i, 'evidence should flag this as an estimate, not measured telemetry');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('aggregate recommendation lists the same levers (disableBundledSkills, skillOverrides, trim)', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg, 'expected an aggregate finding to carry remediation');
|
||||
const rec = String(agg.recommendation);
|
||||
assert.match(rec, /disableBundledSkills/);
|
||||
assert.match(rec, /skillOverrides/);
|
||||
assert.match(rec, /trim/i);
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — suppression compatibility', () => {
|
||||
it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
|
||||
const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue