import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { join } from 'node:path'; import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/skill-listing-scanner.mjs'; const CAP = 1536; // verified per-description listing cap (CC 2.1.105, changelog L1502) function uniqueDir(suffix) { return join(tmpdir(), `config-audit-skl-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); } /** * The SKL scanner enumerates active skills via process.env.HOME * (enumeratePlugins/enumerateSkills). Tests must override HOME, run, restore — * never rely on the developer's real ~/.claude. */ async function runScannerWithHome(home) { resetCounter(); const original = process.env.HOME; process.env.HOME = home; try { return await scan('/unused', { files: [] }); } finally { process.env.HOME = original; } } /** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */ async function runScannerWithCtx(home, contextWindow) { resetCounter(); const original = process.env.HOME; process.env.HOME = home; try { return await scan('/unused', { files: [] }, { contextWindow }); } finally { process.env.HOME = original; } } /** Build a fake HOME with one user skill whose description has `len` chars. */ async function homeWithUserSkill(name, descLen) { const home = uniqueDir(name); const dir = join(home, '.claude', 'skills', name); await mkdir(dir, { recursive: true }); const desc = 'a'.repeat(descLen); await writeFile( join(dir, 'SKILL.md'), `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`, ); return home; } /** Write one user skill (name + description of `descLen` chars) into an existing HOME. */ async function addUserSkill(home, name, descLen) { const dir = join(home, '.claude', 'skills', name); await mkdir(dir, { recursive: true }); const desc = 'a'.repeat(descLen); await writeFile( join(dir, 'SKILL.md'), `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`, ); } /** Build a fake HOME with `count` user skills, each description `descLen` chars. */ async function homeWithNUserSkills(count, descLen, prefix = 'agg') { const home = uniqueDir(`${prefix}-${count}x${descLen}`); for (let i = 0; i < count; i++) { await addUserSkill(home, `${prefix}${i}`, descLen); } return home; } /** * Write one user skill whose SKILL.md *body* (below the frontmatter) is about * `bodyChars` chars, spread across multiple lines (81 chars incl. newline). * The description stays tiny so body-size tests stay isolated from the cap/aggregate checks. */ async function addUserSkillWithBody(home, name, bodyChars) { const dir = join(home, '.claude', 'skills', name); await mkdir(dir, { recursive: true }); const line = `${'x'.repeat(80)}\n`; const body = line.repeat(Math.ceil(bodyChars / line.length)); await writeFile( join(dir, 'SKILL.md'), `---\nname: ${name}\ndescription: short.\n---\n${body}`, ); } // Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor. const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000 // Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4). // 16 skills * 1000 chars = 16000 chars -> 4000 tok == budget (NOT over). // 17 skills * 1000 chars = 17000 chars -> 4250 tok > budget (over). const findAggregate = (findings) => findings.find(f => /aggregate/i.test(f.title)); const findCaps = (findings) => findings.filter(f => /listing cap/i.test(f.title)); describe('SKL scanner — basic structure', () => { it('reports scanner prefix SKL', async () => { const home = uniqueDir('empty'); try { await mkdir(join(home, '.claude'), { recursive: true }); const result = await runScannerWithHome(home); assert.equal(result.scanner, 'SKL'); } finally { await rm(home, { recursive: true, force: true }); } }); it('finding IDs match CA-SKL-NNN pattern', async () => { const home = await homeWithUserSkill('longdesc', CAP + 64); try { const result = await runScannerWithHome(home); for (const f of result.findings) { assert.match(f.id, /^CA-SKL-\d{3}$/); } } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — per-description 1536-char cap (CA-SKL-001)', () => { it('flags a skill whose description exceeds the cap', async () => { const home = await homeWithUserSkill('toolong', CAP + 100); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /toolong/.test(x.evidence || x.description || '')); assert.ok(f, `expected a cap finding; got: ${result.findings.map(x => x.title).join(' | ')}`); assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`); assert.match(f.id, /^CA-SKL-001$/); } finally { await rm(home, { recursive: true, force: true }); } }); it('evidence carries the measured char count and the 1536 cap', async () => { const home = await homeWithUserSkill('measured', CAP + 200); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /measured/.test(x.evidence || '')); assert.ok(f, 'expected a finding for the oversized skill'); assert.match(String(f.evidence), new RegExp(String(CAP + 200))); assert.match(String(f.evidence), new RegExp(String(CAP))); } finally { await rm(home, { recursive: true, force: true }); } }); it('points file at the offending SKILL.md', async () => { const home = await homeWithUserSkill('filepath', CAP + 1); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /filepath/.test(x.evidence || '')); assert.ok(f); assert.match(String(f.file), /filepath[\\/]SKILL\.md$/); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — boundary and negative cases', () => { it('a description exactly at the cap (1536) yields no finding', async () => { const home = await homeWithUserSkill('exact', CAP); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0, `expected 0 findings at the cap; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('one char over the cap (1537) yields a finding', async () => { const home = await homeWithUserSkill('over', CAP + 1); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 1, `expected 1 finding at cap+1; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('a short description yields no finding', async () => { const home = await homeWithUserSkill('short', 80); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0, `expected 0 findings; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('an empty HOME (no skills) yields zero findings', async () => { const home = uniqueDir('noskills'); try { await mkdir(join(home, '.claude'), { recursive: true }); const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — remediation levers (Item 2b folded in)', () => { it('recommendation lists disableBundledSkills, skillOverrides, and trimming', async () => { const home = await homeWithUserSkill('levers', CAP + 300); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /levers/.test(x.evidence || '')); assert.ok(f, 'expected a finding to carry remediation'); const rec = String(f.recommendation); assert.match(rec, /disableBundledSkills/); assert.match(rec, /skillOverrides/); assert.match(rec, /trim/i); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { it('fires a low-severity aggregate finding when the description sum exceeds the 200k-anchored budget', async () => { const home = await homeWithNUserSkills(17, 1000); // 17000 chars -> ~4250 tok > 4000 try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`); assert.match(agg.id, /^CA-SKL-\d{3}$/); // 17 skills of 1000 chars each are all under the 1536 cap, so no cap findings. assert.equal(findCaps(result.findings).length, 0, 'no per-description cap findings expected when every description is under 1536'); } finally { await rm(home, { recursive: true, force: true }); } }); it('does NOT fire when the aggregate is under budget', async () => { const home = await homeWithNUserSkills(3, 1000); // 3000 chars -> 750 tok < 4000 try { const result = await runScannerWithHome(home); assert.equal(findAggregate(result.findings), undefined, `expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('treats the budget as a boundary: exactly at budget yields no finding, one skill over yields one', async () => { const atBudget = await homeWithNUserSkills(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget try { const result = await runScannerWithHome(atBudget); assert.equal(findAggregate(result.findings), undefined, 'aggregate exactly at the budget must not fire (strictly-greater rule)'); } finally { await rm(atBudget, { recursive: true, force: true }); } const overBudget = await homeWithNUserSkills(17, 1000, 'over'); // 17000 chars -> 4250 tok try { const result = await runScannerWithHome(overBudget); assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire'); } finally { await rm(overBudget, { recursive: true, force: true }); } }); it('counts each description capped at 1536 — a single huge skill does NOT inflate the aggregate', async () => { // One 5000-char skill: capped contribution = 1536 chars -> 384 tok, far under budget. // It must trigger the per-description cap finding but NOT the aggregate. const home = await homeWithUserSkill('huge', 5000); try { const result = await runScannerWithHome(home); assert.equal(findCaps(result.findings).length, 1, 'the oversized skill should trip the cap finding'); assert.equal(findAggregate(result.findings), undefined, 'aggregate must count the capped 1536 contribution, not the full 5000 chars'); } finally { await rm(home, { recursive: true, force: true }); } }); it('cap and aggregate findings are independent — both can fire in one run', async () => { // 1 oversized skill (cap) + 16 normal skills. Aggregate = min(5000,1536)+16000 = 17536 chars -> ~4384 tok. const home = await homeWithNUserSkills(16, 1000, 'norm'); await addUserSkill(home, 'oversized', 5000); try { const result = await runScannerWithHome(home); assert.equal(findCaps(result.findings).length, 1, 'expected exactly one cap finding'); assert.ok(findAggregate(result.findings), 'expected an aggregate finding alongside the cap finding'); } finally { await rm(home, { recursive: true, force: true }); } }); it('evidence carries the measured sum, the 200k budget, and an honest context-window calibration note', async () => { const home = await homeWithNUserSkills(17, 1000); try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, 'expected an aggregate finding'); const ev = String(agg.evidence); assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence should state the 4000-token budget'); assert.match(ev, /200k/, 'evidence should anchor the budget on a 200k window'); assert.match(ev, /CC 2\.1\.32/, 'evidence should cite the 2%-of-context changelog fact'); assert.match(ev, /1,000,000|20,000/, 'evidence should note the 1M-context scaling'); assert.match(ev, /estimate/i, 'evidence should flag this as an estimate, not measured telemetry'); } finally { await rm(home, { recursive: true, force: true }); } }); it('aggregate recommendation lists the same levers (disableBundledSkills, skillOverrides, trim)', async () => { const home = await homeWithNUserSkills(17, 1000); try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, 'expected an aggregate finding to carry remediation'); const rec = String(agg.recommendation); assert.match(rec, /disableBundledSkills/); assert.match(rec, /skillOverrides/); assert.match(rec, /trim/i); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — context-window calibration (B8)', () => { it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => { // 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k // budget, under the 20,000-tok 1M budget — the acceptance case. const home = await homeWithNUserSkills(50, 400, 'cw'); try { const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false }); assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor'); const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false }); assert.equal(findAggregate(at1m.findings), undefined, 'at a 1M context window the listing is within budget and must not fire'); } finally { await rm(home, { recursive: true, force: true }); } }); it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => { const home = await homeWithNUserSkills(17, 1000, 'cwadv'); try { const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true }); const agg = findAggregate(advisory.findings); assert.ok(agg, 'still surfaces the measurement (conservative anchor)'); assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info'); } finally { await rm(home, { recursive: true, force: true }); } }); it('no opts (default) is unchanged: fires low at the 200k anchor', async () => { const home = await homeWithNUserSkills(17, 1000, 'cwdef'); try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg); assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity'); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => { const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || '')); it('flags a skill whose body exceeds ~5,000 tokens', async () => { const home = uniqueDir('bigbody'); await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body try { const result = await runScannerWithHome(home); const f = findBody(result.findings); assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body'); assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity'); assert.match(String(f.file), /heavy[\\/]SKILL\.md$/); } finally { await rm(home, { recursive: true, force: true }); } }); it('does NOT flag a skill whose body is under the threshold', async () => { const home = uniqueDir('smallbody'); await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body try { const result = await runScannerWithHome(home); assert.equal(findBody(result.findings), undefined, 'a small body must not fire'); } finally { await rm(home, { recursive: true, force: true }); } }); it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => { const home = uniqueDir('bodyev'); await addUserSkillWithBody(home, 'heavy', 20_400); try { const result = await runScannerWithHome(home); const f = findBody(result.findings); assert.ok(f); const ev = String(f.evidence); assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens'); assert.match(ev, /body_lines=\d+/, 'evidence states the body line count'); assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost'); assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing'); assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry'); } finally { await rm(home, { recursive: true, force: true }); } }); it('recommends context:fork and moving reference content into supporting files', async () => { const home = uniqueDir('bodyrec'); await addUserSkillWithBody(home, 'heavy', 20_400); try { const result = await runScannerWithHome(home); const f = findBody(result.findings); assert.ok(f); const rec = String(f.recommendation); assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork'); assert.match(rec, /supporting file/i, 'recommendation should mention supporting files'); } finally { await rm(home, { recursive: true, force: true }); } }); it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => { const home = uniqueDir('bodyonly'); await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description try { const result = await runScannerWithHome(home); assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap'); assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget'); assert.ok(findBody(result.findings), 'only the body finding should fire'); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — suppression compatibility', () => { it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => { const { applySuppressions } = await import('../../scanners/lib/suppression.mjs'); const home = await homeWithUserSkill('suppress', CAP + 50); try { const result = await runScannerWithHome(home); assert.ok(result.findings.length > 0, 'precondition: at least one SKL finding'); const { active, suppressed } = applySuppressions(result.findings, [ { pattern: 'CA-TOK-*', source: 'test', sourceLine: 1 }, ]); assert.equal(active.length, result.findings.length, 'CA-TOK-* glob should not match CA-SKL-* findings'); assert.equal(suppressed.length, 0); } finally { await rm(home, { recursive: true, force: true }); } }); });