feat(skill-listing): add CA-SKL-002 aggregate listing-budget check

Syklus 2 of Fase 4 Items 2+3. Flags when the sum of active skill
descriptions exceeds the listing budget (~2% of context, CC 2.1.32).

Design (operator-confirmed "fact-first, 200k anchor"):
- low severity (estimate) vs medium for the verified 1,536-char cap
- each description counted up to the 1,536 cap (what actually loads in
  the listing) — avoids double-counting the tail CA-SKL-001 flags
- fires when sum > 2% x 200k = 4000 tok; evidence leads with the measured
  sum + a calibration note that the budget scales 5x on 1M-context models
- aggregate emitted after the per-skill loop so the common case reads
  001=cap, 002=aggregate (finding IDs are a sequential counter, not stable
  semantic IDs — tests match on title, never NNN)

Also:
- tailored humanizer static entry for the aggregate title
- fix latent HOME leak in posture-grade-stability.test.mjs: it spawned
  posture.mjs without hermeticEnv(), so a real ~/.claude leaked HOME-scoped
  SKL/COL findings into the baseline grade (Token Efficiency A->B). Now
  isolated like the 8 other CLI-spawning tests.
- docs sync: test count 868->875, scanner-internals, gap-matrix, plan status

Suite 875/875, no snapshot drift, self-audit clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
This commit is contained in:
Kjell Tore Guttormsen 2026-06-18 18:06:17 +02:00
commit 66433fee48
8 changed files with 215 additions and 11 deletions

View file

@ -4,6 +4,7 @@ import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
const exec = promisify(execFile);
const __dirname = fileURLToPath(new URL('.', import.meta.url));
@ -11,9 +12,13 @@ const FIXTURES = resolve(__dirname, '../fixtures');
const POSTURE_BIN = resolve(__dirname, '../../scanners/posture.mjs');
async function runPostureJson(fixturePath) {
// Isolate HOME: SKL and COL are HOME-scoped (enumerateSkills/enumeratePlugins),
// so a real ~/.claude would leak the developer's installed plugins/skills into
// this baseline grade and make it machine-dependent. See tests/helpers/hermetic-home.mjs.
const { stdout } = await exec('node', [POSTURE_BIN, fixturePath, '--json'], {
timeout: 30000,
cwd: resolve(__dirname, '../..'),
env: hermeticEnv(),
});
return JSON.parse(stdout);
}

View file

@ -41,6 +41,34 @@ async function homeWithUserSkill(name, descLen) {
return home;
}
/** Write one user skill (name + description of `descLen` chars) into an existing HOME. */
async function addUserSkill(home, name, descLen) {
const dir = join(home, '.claude', 'skills', name);
await mkdir(dir, { recursive: true });
const desc = 'a'.repeat(descLen);
await writeFile(
join(dir, 'SKILL.md'),
`---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`,
);
}
/** Build a fake HOME with `count` user skills, each description `descLen` chars. */
async function homeWithNUserSkills(count, descLen, prefix = 'agg') {
const home = uniqueDir(`${prefix}-${count}x${descLen}`);
for (let i = 0; i < count; i++) {
await addUserSkill(home, `${prefix}${i}`, descLen);
}
return home;
}
// Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor.
const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000
// Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4).
// 16 skills * 1000 chars = 16000 chars -> 4000 tok == budget (NOT over).
// 17 skills * 1000 chars = 17000 chars -> 4250 tok > budget (over).
const findAggregate = (findings) => findings.find(f => /aggregate/i.test(f.title));
const findCaps = (findings) => findings.filter(f => /listing cap/i.test(f.title));
describe('SKL scanner — basic structure', () => {
it('reports scanner prefix SKL', async () => {
const home = uniqueDir('empty');
@ -169,6 +197,112 @@ describe('SKL scanner — remediation levers (Item 2b folded in)', () => {
});
});
describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
it('fires a low-severity aggregate finding when the description sum exceeds the 200k-anchored budget', async () => {
const home = await homeWithNUserSkills(17, 1000); // 17000 chars -> ~4250 tok > 4000
try {
const result = await runScannerWithHome(home);
const agg = findAggregate(result.findings);
assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`);
assert.match(agg.id, /^CA-SKL-\d{3}$/);
// 17 skills of 1000 chars each are all under the 1536 cap, so no cap findings.
assert.equal(findCaps(result.findings).length, 0,
'no per-description cap findings expected when every description is under 1536');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('does NOT fire when the aggregate is under budget', async () => {
const home = await homeWithNUserSkills(3, 1000); // 3000 chars -> 750 tok < 4000
try {
const result = await runScannerWithHome(home);
assert.equal(findAggregate(result.findings), undefined,
`expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('treats the budget as a boundary: exactly at budget yields no finding, one skill over yields one', async () => {
const atBudget = await homeWithNUserSkills(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget
try {
const result = await runScannerWithHome(atBudget);
assert.equal(findAggregate(result.findings), undefined,
'aggregate exactly at the budget must not fire (strictly-greater rule)');
} finally {
await rm(atBudget, { recursive: true, force: true });
}
const overBudget = await homeWithNUserSkills(17, 1000, 'over'); // 17000 chars -> 4250 tok
try {
const result = await runScannerWithHome(overBudget);
assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire');
} finally {
await rm(overBudget, { recursive: true, force: true });
}
});
it('counts each description capped at 1536 — a single huge skill does NOT inflate the aggregate', async () => {
// One 5000-char skill: capped contribution = 1536 chars -> 384 tok, far under budget.
// It must trigger the per-description cap finding but NOT the aggregate.
const home = await homeWithUserSkill('huge', 5000);
try {
const result = await runScannerWithHome(home);
assert.equal(findCaps(result.findings).length, 1, 'the oversized skill should trip the cap finding');
assert.equal(findAggregate(result.findings), undefined,
'aggregate must count the capped 1536 contribution, not the full 5000 chars');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('cap and aggregate findings are independent — both can fire in one run', async () => {
// 1 oversized skill (cap) + 16 normal skills. Aggregate = min(5000,1536)+16000 = 17536 chars -> ~4384 tok.
const home = await homeWithNUserSkills(16, 1000, 'norm');
await addUserSkill(home, 'oversized', 5000);
try {
const result = await runScannerWithHome(home);
assert.equal(findCaps(result.findings).length, 1, 'expected exactly one cap finding');
assert.ok(findAggregate(result.findings), 'expected an aggregate finding alongside the cap finding');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('evidence carries the measured sum, the 200k budget, and an honest context-window calibration note', async () => {
const home = await homeWithNUserSkills(17, 1000);
try {
const result = await runScannerWithHome(home);
const agg = findAggregate(result.findings);
assert.ok(agg, 'expected an aggregate finding');
const ev = String(agg.evidence);
assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence should state the 4000-token budget');
assert.match(ev, /200k/, 'evidence should anchor the budget on a 200k window');
assert.match(ev, /CC 2\.1\.32/, 'evidence should cite the 2%-of-context changelog fact');
assert.match(ev, /1,000,000|20,000/, 'evidence should note the 1M-context scaling');
assert.match(ev, /estimate/i, 'evidence should flag this as an estimate, not measured telemetry');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('aggregate recommendation lists the same levers (disableBundledSkills, skillOverrides, trim)', async () => {
const home = await homeWithNUserSkills(17, 1000);
try {
const result = await runScannerWithHome(home);
const agg = findAggregate(result.findings);
assert.ok(agg, 'expected an aggregate finding to carry remediation');
const rec = String(agg.recommendation);
assert.match(rec, /disableBundledSkills/);
assert.match(rec, /skillOverrides/);
assert.match(rec, /trim/i);
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — suppression compatibility', () => {
it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');