import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { join } from 'node:path'; import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/skill-listing-scanner.mjs'; const CAP = 1536; // verified per-description listing cap (CC 2.1.105, changelog L1502) function uniqueDir(suffix) { return join(tmpdir(), `config-audit-skl-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); } /** * The SKL scanner enumerates active skills via process.env.HOME * (enumeratePlugins/enumerateSkills). Tests must override HOME, run, restore — * never rely on the developer's real ~/.claude. */ async function runScannerWithHome(home) { resetCounter(); const original = process.env.HOME; process.env.HOME = home; try { return await scan('/unused', { files: [] }); } finally { process.env.HOME = original; } } /** Build a fake HOME with one user skill whose description has `len` chars. */ async function homeWithUserSkill(name, descLen) { const home = uniqueDir(name); const dir = join(home, '.claude', 'skills', name); await mkdir(dir, { recursive: true }); const desc = 'a'.repeat(descLen); await writeFile( join(dir, 'SKILL.md'), `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`, ); return home; } /** Write one user skill (name + description of `descLen` chars) into an existing HOME. */ async function addUserSkill(home, name, descLen) { const dir = join(home, '.claude', 'skills', name); await mkdir(dir, { recursive: true }); const desc = 'a'.repeat(descLen); await writeFile( join(dir, 'SKILL.md'), `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`, ); } /** Build a fake HOME with `count` user skills, each description `descLen` chars. */ async function homeWithNUserSkills(count, descLen, prefix = 'agg') { const home = uniqueDir(`${prefix}-${count}x${descLen}`); for (let i = 0; i < count; i++) { await addUserSkill(home, `${prefix}${i}`, descLen); } return home; } // Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor. const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000 // Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4). // 16 skills * 1000 chars = 16000 chars -> 4000 tok == budget (NOT over). // 17 skills * 1000 chars = 17000 chars -> 4250 tok > budget (over). const findAggregate = (findings) => findings.find(f => /aggregate/i.test(f.title)); const findCaps = (findings) => findings.filter(f => /listing cap/i.test(f.title)); describe('SKL scanner — basic structure', () => { it('reports scanner prefix SKL', async () => { const home = uniqueDir('empty'); try { await mkdir(join(home, '.claude'), { recursive: true }); const result = await runScannerWithHome(home); assert.equal(result.scanner, 'SKL'); } finally { await rm(home, { recursive: true, force: true }); } }); it('finding IDs match CA-SKL-NNN pattern', async () => { const home = await homeWithUserSkill('longdesc', CAP + 64); try { const result = await runScannerWithHome(home); for (const f of result.findings) { assert.match(f.id, /^CA-SKL-\d{3}$/); } } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — per-description 1536-char cap (CA-SKL-001)', () => { it('flags a skill whose description exceeds the cap', async () => { const home = await homeWithUserSkill('toolong', CAP + 100); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /toolong/.test(x.evidence || x.description || '')); assert.ok(f, `expected a cap finding; got: ${result.findings.map(x => x.title).join(' | ')}`); assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`); assert.match(f.id, /^CA-SKL-001$/); } finally { await rm(home, { recursive: true, force: true }); } }); it('evidence carries the measured char count and the 1536 cap', async () => { const home = await homeWithUserSkill('measured', CAP + 200); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /measured/.test(x.evidence || '')); assert.ok(f, 'expected a finding for the oversized skill'); assert.match(String(f.evidence), new RegExp(String(CAP + 200))); assert.match(String(f.evidence), new RegExp(String(CAP))); } finally { await rm(home, { recursive: true, force: true }); } }); it('points file at the offending SKILL.md', async () => { const home = await homeWithUserSkill('filepath', CAP + 1); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /filepath/.test(x.evidence || '')); assert.ok(f); assert.match(String(f.file), /filepath[\\/]SKILL\.md$/); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — boundary and negative cases', () => { it('a description exactly at the cap (1536) yields no finding', async () => { const home = await homeWithUserSkill('exact', CAP); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0, `expected 0 findings at the cap; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('one char over the cap (1537) yields a finding', async () => { const home = await homeWithUserSkill('over', CAP + 1); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 1, `expected 1 finding at cap+1; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('a short description yields no finding', async () => { const home = await homeWithUserSkill('short', 80); try { const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0, `expected 0 findings; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('an empty HOME (no skills) yields zero findings', async () => { const home = uniqueDir('noskills'); try { await mkdir(join(home, '.claude'), { recursive: true }); const result = await runScannerWithHome(home); assert.equal(result.findings.length, 0); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — remediation levers (Item 2b folded in)', () => { it('recommendation lists disableBundledSkills, skillOverrides, and trimming', async () => { const home = await homeWithUserSkill('levers', CAP + 300); try { const result = await runScannerWithHome(home); const f = result.findings.find(x => /levers/.test(x.evidence || '')); assert.ok(f, 'expected a finding to carry remediation'); const rec = String(f.recommendation); assert.match(rec, /disableBundledSkills/); assert.match(rec, /skillOverrides/); assert.match(rec, /trim/i); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { it('fires a low-severity aggregate finding when the description sum exceeds the 200k-anchored budget', async () => { const home = await homeWithNUserSkills(17, 1000); // 17000 chars -> ~4250 tok > 4000 try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`); assert.match(agg.id, /^CA-SKL-\d{3}$/); // 17 skills of 1000 chars each are all under the 1536 cap, so no cap findings. assert.equal(findCaps(result.findings).length, 0, 'no per-description cap findings expected when every description is under 1536'); } finally { await rm(home, { recursive: true, force: true }); } }); it('does NOT fire when the aggregate is under budget', async () => { const home = await homeWithNUserSkills(3, 1000); // 3000 chars -> 750 tok < 4000 try { const result = await runScannerWithHome(home); assert.equal(findAggregate(result.findings), undefined, `expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); } finally { await rm(home, { recursive: true, force: true }); } }); it('treats the budget as a boundary: exactly at budget yields no finding, one skill over yields one', async () => { const atBudget = await homeWithNUserSkills(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget try { const result = await runScannerWithHome(atBudget); assert.equal(findAggregate(result.findings), undefined, 'aggregate exactly at the budget must not fire (strictly-greater rule)'); } finally { await rm(atBudget, { recursive: true, force: true }); } const overBudget = await homeWithNUserSkills(17, 1000, 'over'); // 17000 chars -> 4250 tok try { const result = await runScannerWithHome(overBudget); assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire'); } finally { await rm(overBudget, { recursive: true, force: true }); } }); it('counts each description capped at 1536 — a single huge skill does NOT inflate the aggregate', async () => { // One 5000-char skill: capped contribution = 1536 chars -> 384 tok, far under budget. // It must trigger the per-description cap finding but NOT the aggregate. const home = await homeWithUserSkill('huge', 5000); try { const result = await runScannerWithHome(home); assert.equal(findCaps(result.findings).length, 1, 'the oversized skill should trip the cap finding'); assert.equal(findAggregate(result.findings), undefined, 'aggregate must count the capped 1536 contribution, not the full 5000 chars'); } finally { await rm(home, { recursive: true, force: true }); } }); it('cap and aggregate findings are independent — both can fire in one run', async () => { // 1 oversized skill (cap) + 16 normal skills. Aggregate = min(5000,1536)+16000 = 17536 chars -> ~4384 tok. const home = await homeWithNUserSkills(16, 1000, 'norm'); await addUserSkill(home, 'oversized', 5000); try { const result = await runScannerWithHome(home); assert.equal(findCaps(result.findings).length, 1, 'expected exactly one cap finding'); assert.ok(findAggregate(result.findings), 'expected an aggregate finding alongside the cap finding'); } finally { await rm(home, { recursive: true, force: true }); } }); it('evidence carries the measured sum, the 200k budget, and an honest context-window calibration note', async () => { const home = await homeWithNUserSkills(17, 1000); try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, 'expected an aggregate finding'); const ev = String(agg.evidence); assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence should state the 4000-token budget'); assert.match(ev, /200k/, 'evidence should anchor the budget on a 200k window'); assert.match(ev, /CC 2\.1\.32/, 'evidence should cite the 2%-of-context changelog fact'); assert.match(ev, /1,000,000|20,000/, 'evidence should note the 1M-context scaling'); assert.match(ev, /estimate/i, 'evidence should flag this as an estimate, not measured telemetry'); } finally { await rm(home, { recursive: true, force: true }); } }); it('aggregate recommendation lists the same levers (disableBundledSkills, skillOverrides, trim)', async () => { const home = await homeWithNUserSkills(17, 1000); try { const result = await runScannerWithHome(home); const agg = findAggregate(result.findings); assert.ok(agg, 'expected an aggregate finding to carry remediation'); const rec = String(agg.recommendation); assert.match(rec, /disableBundledSkills/); assert.match(rec, /skillOverrides/); assert.match(rec, /trim/i); } finally { await rm(home, { recursive: true, force: true }); } }); }); describe('SKL scanner — suppression compatibility', () => { it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => { const { applySuppressions } = await import('../../scanners/lib/suppression.mjs'); const home = await homeWithUserSkill('suppress', CAP + 50); try { const result = await runScannerWithHome(home); assert.ok(result.findings.length > 0, 'precondition: at least one SKL finding'); const { active, suppressed } = applySuppressions(result.findings, [ { pattern: 'CA-TOK-*', source: 'test', sourceLine: 1 }, ]); assert.equal(active.length, result.findings.length, 'CA-TOK-* glob should not match CA-SKL-* findings'); assert.equal(suppressed.length, 0); } finally { await rm(home, { recursive: true, force: true }); } }); });