diff --git a/CLAUDE.md b/CLAUDE.md index e951c58..bd5c733 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -109,7 +109,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -875 tests across 55 test files (16 lib + 29 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +887 tests across 56 test files (17 lib + 29 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ## Gotchas diff --git a/README.md b/README.md index e98869b..9ece13b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ![Commands](https://img.shields.io/badge/commands-18-green) ![Agents](https://img.shields.io/badge/agents-6-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-875+-brightgreen) +![Tests](https://img.shields.io/badge/tests-887+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 13 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, and cross-plugin collision detection. Zero external dependencies. diff --git a/scanners/lib/skill-listing-budget.mjs b/scanners/lib/skill-listing-budget.mjs new file mode 100644 index 0000000..af5f4ec --- /dev/null +++ b/scanners/lib/skill-listing-budget.mjs @@ -0,0 +1,125 @@ +/** + * Skill-listing budget — single source of truth. + * + * Claude Code shows the model a listing of every active skill's `description` + * so it can decide which skill to invoke. That listing is budgeted two ways: + * - per description: capped at 1,536 chars (CC 2.1.105, changelog L1502); + * anything past the cap is silently truncated. + * - in aggregate: the whole listing is allotted ~2% of the context window + * (CC 2.1.32, changelog L2860). We do NOT know the user's context window, + * so the aggregate budget anchors on a conservative 200k window (4,000 tok) + * and discloses the assumption. + * + * Two scanners consume this module so the budget is defined in exactly one place: + * - SKL (skill-listing-scanner) DIAGNOSES overflow (CA-SKL-001 per-description + * cap, CA-SKL-002 aggregate). + * - GAP (feature-gap-scanner) PRESCRIBES the remedy: when the listing is over + * budget and `disableBundledSkills` is un-pulled, it recommends that lever. + * + * Zero external dependencies. + */ + +import { estimateTokens, enumeratePlugins, enumerateSkills } from './active-config-reader.mjs'; +import { readTextFile } from './file-discovery.mjs'; +import { parseFrontmatter } from './yaml-parser.mjs'; + +// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502). +// Descriptions longer than this are truncated in the listing the model sees. +export const DESCRIPTION_CAP = 1536; + +// Aggregate listing budget (CC 2.1.32, changelog L2860): the skill listing the +// model reads is allotted ~2% of the context window. The context window is +// unknown, so we anchor on a conservative 200k window — the smallest common +// size, which fires earliest — and disclose the assumption in the evidence. +export const BUDGET_FRACTION = 0.02; +export const CONTEXT_WINDOW_ANCHOR = 200_000; +export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000 +export const LARGE_CONTEXT_WINDOW = 1_000_000; +export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000 + +// Dependency-free thousands separator (repo invariant: zero external deps). +export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); + +// The honest framing required because the budget depends on a context window we +// cannot observe (jf. TOK CALIBRATION_NOTE). Appended to budget-overflow evidence. +export const BUDGET_CALIBRATION_NOTE = + 'the budget scales with the context window - this anchors on a conservative 200k ' + + `window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` + + 'tok and you are likely within it. this is an estimate, not measured telemetry'; + +/** + * @typedef {object} BudgetAssessment + * @property {number} scanned - number of descriptions assessed + * @property {number} aggregateChars - sum of each length capped at DESCRIPTION_CAP + * @property {number} aggregateTokens - estimateTokens(aggregateChars, 'markdown') + * @property {number} budgetTokens - AGGREGATE_BUDGET_TOKENS (the 200k-anchored budget) + * @property {boolean} overBudget - aggregateTokens strictly greater than budgetTokens + * @property {number} overBy - tokens over budget (0 when not over) + */ + +/** + * Pure aggregate-budget assessment. Each description contributes only up to the + * cap (the tail past the cap is dropped from the listing, and CA-SKL-001 already + * flags it — so the aggregate does not double-count it). + * + * @param {number[]} descLengths - one entry per active skill (description char count) + * @returns {BudgetAssessment} + */ +export function assessSkillListingBudget(descLengths) { + let aggregateChars = 0; + for (const len of descLengths) { + const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0; + aggregateChars += Math.min(safe, DESCRIPTION_CAP); + } + const aggregateTokens = estimateTokens(aggregateChars, 'markdown'); + const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS; + return { + scanned: descLengths.length, + aggregateChars, + aggregateTokens, + budgetTokens: AGGREGATE_BUDGET_TOKENS, + overBudget, + overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0, + }; +} + +/** + * @typedef {object} ActiveSkillEntry + * @property {string} name + * @property {'user'|'plugin'} source + * @property {string|null} pluginName + * @property {string} path + * @property {number} descLength + */ + +/** + * Enumerate every active skill (user + plugin) and measure the listing budget. + * HOME-scoped: resolves ~/.claude via process.env.HOME (enumeratePlugins / + * enumerateSkills). Callers that run under test MUST override HOME (see the + * hermetic-home helper / runScannerWithHome pattern). + * + * @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>} + */ +export async function measureActiveSkillListing() { + const plugins = await enumeratePlugins(); + const allSkills = await enumerateSkills(plugins); + + const skills = []; + for (const skill of allSkills) { + if (!skill || typeof skill.path !== 'string') continue; + const content = await readTextFile(skill.path); + if (!content) continue; + const fm = parseFrontmatter(content)?.frontmatter || null; + const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + skills.push({ + name: skill.name, + source: skill.source, + pluginName: skill.pluginName, + path: skill.path, + descLength: desc.length, + }); + } + + const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength)); + return { skills, aggregate }; +} diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs index 563a693..838253a 100644 --- a/scanners/skill-listing-scanner.mjs +++ b/scanners/skill-listing-scanner.mjs @@ -18,10 +18,12 @@ * context window, so CA-SKL-002 anchors on a conservative 200k window and says * so loudly: it leads with the measured sum (a fact) and carries a calibration * note explaining the budget scales 5x on a 1M-context model. Severity is low - * (an estimate) versus medium for the verified per-description cap. Each - * description is counted only up to the 1,536-char cap, because that is all - * Claude Code loads into the listing — the tail past the cap is dropped and is - * already flagged by CA-SKL-001 (so the aggregate does not double-count it). + * (an estimate) versus medium for the verified per-description cap. + * + * The cap, the budget, and the enumerate-and-measure step all live in + * `lib/skill-listing-budget.mjs` — the single source of truth shared with the + * GAP scanner, which prescribes `disableBundledSkills` when this budget is + * exceeded. SKL only constructs the findings. * * Two-lens note vs TOK pattern F: TOK pattern F flags *project-local* skills * with descriptions > 500 chars as a structural per-turn "bloat" heuristic. @@ -34,36 +36,15 @@ import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; -import { enumeratePlugins, enumerateSkills, estimateTokens } from './lib/active-config-reader.mjs'; -import { readTextFile } from './lib/file-discovery.mjs'; -import { parseFrontmatter } from './lib/yaml-parser.mjs'; +import { + DESCRIPTION_CAP, + AGGREGATE_BUDGET_TOKENS, + BUDGET_CALIBRATION_NOTE, + measureActiveSkillListing, +} from './lib/skill-listing-budget.mjs'; const SCANNER = 'SKL'; -// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502). -// Descriptions longer than this are truncated in the listing the model sees. -const DESCRIPTION_CAP = 1536; - -// Aggregate listing budget (CC 2.1.32, changelog L2860): the skill listing the -// model reads is allotted ~2% of the context window. The context window is -// unknown, so we anchor on a conservative 200k window — the smallest common -// size, which fires earliest — and disclose the assumption in the evidence. -const BUDGET_FRACTION = 0.02; -const CONTEXT_WINDOW_ANCHOR = 200_000; -const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000 -const LARGE_CONTEXT_WINDOW = 1_000_000; -const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000 - -// Dependency-free thousands separator (repo invariant: zero external deps). -const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); - -// Appended to CA-SKL-002 evidence — the honest framing required because the -// budget depends on a context window we cannot observe (jf. TOK CALIBRATION_NOTE). -const BUDGET_CALIBRATION_NOTE = - 'the budget scales with the context window - this anchors on a conservative 200k ' + - `window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` + - 'tok and you are likely within it. this is an estimate, not measured telemetry'; - /** * Main scanner entry point. * @@ -74,22 +55,10 @@ export async function scan(_targetPath, _discovery) { const start = Date.now(); const findings = []; - const plugins = await enumeratePlugins(); - const allSkills = await enumerateSkills(plugins); + const { skills, aggregate } = await measureActiveSkillListing(); - let scanned = 0; - let aggregateChars = 0; - for (const skill of allSkills) { - if (!skill || typeof skill.path !== 'string') continue; - const content = await readTextFile(skill.path); - if (!content) continue; - scanned++; - const fm = parseFrontmatter(content)?.frontmatter || null; - const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; - // Aggregate budget counts only what loads in the listing: each description - // up to the cap (the tail past the cap is dropped, and CA-SKL-001 flags it). - aggregateChars += Math.min(desc.length, DESCRIPTION_CAP); - if (desc.length <= DESCRIPTION_CAP) continue; + for (const skill of skills) { + if (skill.descLength <= DESCRIPTION_CAP) continue; const sourceLabel = skill.source === 'plugin' ? `plugin:${skill.pluginName}` @@ -100,14 +69,14 @@ export async function scan(_targetPath, _discovery) { severity: SEVERITY.medium, title: 'Skill description exceeds the listing cap (Claude Code truncates it)', description: - `Skill "${skill.name}" (${sourceLabel}) has a description of ${desc.length} ` + + `Skill "${skill.name}" (${sourceLabel}) has a description of ${skill.descLength} ` + `characters (>${DESCRIPTION_CAP}). Claude Code caps each skill description in ` + 'the listing the model reads to choose a skill, so everything past ' + `${DESCRIPTION_CAP} characters is silently dropped — including any trigger ` + 'phrases at the tail meant to route invocation.', file: skill.path, evidence: - `description_chars=${desc.length}; cap=${DESCRIPTION_CAP}; ` + + `description_chars=${skill.descLength}; cap=${DESCRIPTION_CAP}; ` + `skill="${skill.name}"; source=${sourceLabel}`, recommendation: `Trim the description below ${DESCRIPTION_CAP} characters, leading with the ` + @@ -121,24 +90,23 @@ export async function scan(_targetPath, _discovery) { // CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common // "one oversized skill + aggregate" case reads 001=cap, 002=aggregate. - const aggregateTokens = estimateTokens(aggregateChars, 'markdown'); - if (aggregateTokens > AGGREGATE_BUDGET_TOKENS) { + if (aggregate.overBudget) { findings.push(finding({ scanner: SCANNER, severity: SEVERITY.low, title: 'Aggregate skill descriptions may exceed the listing budget', description: - `The ${scanned} active skills carry about ${aggregateTokens} tokens of description text ` + + `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` + 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + 'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' + 'is an estimate — the budget scales with your actual context window (see evidence).', evidence: - `active_skills_scanned=${scanned}; description_chars=${aggregateChars} (each capped at ` + - `${DESCRIPTION_CAP}); description_tokens~${aggregateTokens}; budget@200k=` + + `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + + `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` + `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` + - `${aggregateTokens - AGGREGATE_BUDGET_TOKENS} tok - ${BUDGET_CALIBRATION_NOTE}`, + `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, recommendation: 'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' + 'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' + @@ -148,5 +116,5 @@ export async function scan(_targetPath, _discovery) { })); } - return scannerResult(SCANNER, 'ok', findings, scanned, Date.now() - start); + return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start); } diff --git a/tests/lib/skill-listing-budget.test.mjs b/tests/lib/skill-listing-budget.test.mjs new file mode 100644 index 0000000..bcd1a23 --- /dev/null +++ b/tests/lib/skill-listing-budget.test.mjs @@ -0,0 +1,100 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + assessSkillListingBudget, + DESCRIPTION_CAP, + AGGREGATE_BUDGET_TOKENS, + CONTEXT_WINDOW_ANCHOR, + LARGE_CONTEXT_WINDOW, + LARGE_CONTEXT_BUDGET_TOKENS, + withCommas, + BUDGET_CALIBRATION_NOTE, +} from '../../scanners/lib/skill-listing-budget.mjs'; + +// This lib is the single source of truth for the skill-listing budget that both +// the SKL scanner (diagnoses overflow) and the GAP scanner (recommends the +// disableBundledSkills lever) consume. The constants and aggregate math here +// must match what SKL shipped in v5.2.0 — these tests pin that contract. + +describe('skill-listing-budget — constants', () => { + it('DESCRIPTION_CAP is the verified 1536-char listing cap (CC 2.1.105)', () => { + assert.equal(DESCRIPTION_CAP, 1536); + }); + + it('AGGREGATE_BUDGET_TOKENS is 2% of the conservative 200k anchor', () => { + assert.equal(CONTEXT_WINDOW_ANCHOR, 200_000); + assert.equal(AGGREGATE_BUDGET_TOKENS, 4000); + }); + + it('LARGE_CONTEXT_BUDGET_TOKENS is 2% of the 1M window', () => { + assert.equal(LARGE_CONTEXT_WINDOW, 1_000_000); + assert.equal(LARGE_CONTEXT_BUDGET_TOKENS, 20000); + }); +}); + +describe('withCommas', () => { + it('inserts thousands separators', () => { + assert.equal(withCommas(1_000_000), '1,000,000'); + assert.equal(withCommas(4000), '4,000'); + assert.equal(withCommas(999), '999'); + assert.equal(withCommas(0), '0'); + }); +}); + +describe('BUDGET_CALIBRATION_NOTE', () => { + it('discloses the 1M-context scaling and flags this as an estimate, not telemetry', () => { + assert.match(BUDGET_CALIBRATION_NOTE, /1,000,000/); + assert.match(BUDGET_CALIBRATION_NOTE, /20,000/); + assert.match(BUDGET_CALIBRATION_NOTE, /estimate/i); + }); +}); + +describe('assessSkillListingBudget — aggregate math', () => { + it('sums each description length capped at the 1536 cap', () => { + // 1000 + 1000 + min(5000,1536) = 3536 + const r = assessSkillListingBudget([1000, 1000, 5000]); + assert.equal(r.aggregateChars, 3536); + }); + + it('scanned counts every entry, including empty descriptions', () => { + const r = assessSkillListingBudget([0, 0, 100]); + assert.equal(r.scanned, 3); + assert.equal(r.aggregateChars, 100); + }); + + it('aggregateTokens mirrors estimateTokens markdown (ceil chars/4)', () => { + const r = assessSkillListingBudget([16]); // 16 chars -> 4 tokens + assert.equal(r.aggregateTokens, 4); + }); + + it('is NOT over budget exactly at the budget (16×1000 chars -> 4000 tok)', () => { + const r = assessSkillListingBudget(Array(16).fill(1000)); + assert.equal(r.aggregateChars, 16000); + assert.equal(r.aggregateTokens, 4000); + assert.equal(r.overBudget, false); + assert.equal(r.overBy, 0); + }); + + it('is over budget one skill past the budget (17×1000 chars -> 4250 tok)', () => { + const r = assessSkillListingBudget(Array(17).fill(1000)); + assert.equal(r.aggregateTokens, 4250); + assert.equal(r.overBudget, true); + assert.equal(r.overBy, 250); + }); + + it('exposes budgetTokens and handles the empty case', () => { + const r = assessSkillListingBudget([]); + assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS); + assert.equal(r.scanned, 0); + assert.equal(r.aggregateChars, 0); + assert.equal(r.aggregateTokens, 0); + assert.equal(r.overBudget, false); + assert.equal(r.overBy, 0); + }); + + it('treats negative or non-finite lengths as zero contribution', () => { + const r = assessSkillListingBudget([-5, NaN, 100]); + assert.equal(r.aggregateChars, 100); + assert.equal(r.scanned, 3); + }); +});