config-audit/scanners/lib/skill-listing-budget.mjs
Kjell Tore Guttormsen dfe9049b55 feat(feature-gap): recommend disableBundledSkills under skill-listing pressure
Chunk 2 of the disableBundledSkills GAP feature. Adds a conditional GAP check
that prescribes the `disableBundledSkills` lever — but only when the active
skill listing is measurably over budget (SKL's CA-SKL-002 overflow signal) and
the lever is un-pulled. It stays an opportunity, not noise.

Bundled skills (/code-review, /batch, /debug, /loop, /claude-api, …) live in the
CC binary, not on disk, so their exact cost is unmeasurable here — the finding
says so plainly, and frames the lever as zero-cost budget reclaim that leaves
the user's own skills untouched. CC 2.1.169+.

- Pure, exported bundledSkillsLeverFinding({leverPulled, aggregate}) → finding|null
  (severity low, category token-efficiency, CA-GAP-NNN), wired into scan() via the
  shared measureActiveSkillListing().
- Lever resolved via new isBundledSkillsDisabled(): env var + settings cascade
  read directly, because discovery does NOT tag ~/.claude/settings.json (its
  relPath lacks ".claude" when walked from the .claude root) — the dominant
  user-scope location for this global preference would otherwise be missed.
- GAP scan() now reads HOME → existing GAP tests retrofitted to withHermeticHome
  per the hermetic rule. Snapshots unchanged, contamination grep clean.
- 16 new tests (9 GAP, 7 lib). Suite 887 -> 903. README/CLAUDE.md document the
  cross-scanner remediation; test counts synced. self-audit: PASS, configGrade
  A 96, pluginGrade A 100, readme gate passed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
2026-06-18 21:38:19 +02:00

172 lines
7.2 KiB
JavaScript

/**
* Skill-listing budget — single source of truth.
*
* Claude Code shows the model a listing of every active skill's `description`
* so it can decide which skill to invoke. That listing is budgeted two ways:
* - per description: capped at 1,536 chars (CC 2.1.105, changelog L1502);
* anything past the cap is silently truncated.
* - in aggregate: the whole listing is allotted ~2% of the context window
* (CC 2.1.32, changelog L2860). We do NOT know the user's context window,
* so the aggregate budget anchors on a conservative 200k window (4,000 tok)
* and discloses the assumption.
*
* Two scanners consume this module so the budget is defined in exactly one place:
* - SKL (skill-listing-scanner) DIAGNOSES overflow (CA-SKL-001 per-description
* cap, CA-SKL-002 aggregate).
* - GAP (feature-gap-scanner) PRESCRIBES the remedy: when the listing is over
* budget and `disableBundledSkills` is un-pulled, it recommends that lever.
*
* Zero external dependencies.
*/
import { join } from 'node:path';
import { estimateTokens, enumeratePlugins, enumerateSkills } from './active-config-reader.mjs';
import { readTextFile } from './file-discovery.mjs';
import { parseFrontmatter, parseJson } from './yaml-parser.mjs';
// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502).
// Descriptions longer than this are truncated in the listing the model sees.
export const DESCRIPTION_CAP = 1536;
// Aggregate listing budget (CC 2.1.32, changelog L2860): the skill listing the
// model reads is allotted ~2% of the context window. The context window is
// unknown, so we anchor on a conservative 200k window — the smallest common
// size, which fires earliest — and disclose the assumption in the evidence.
export const BUDGET_FRACTION = 0.02;
export const CONTEXT_WINDOW_ANCHOR = 200_000;
export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000
export const LARGE_CONTEXT_WINDOW = 1_000_000;
export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000
// Dependency-free thousands separator (repo invariant: zero external deps).
export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
// The honest framing required because the budget depends on a context window we
// cannot observe (jf. TOK CALIBRATION_NOTE). Appended to budget-overflow evidence.
export const BUDGET_CALIBRATION_NOTE =
'the budget scales with the context window - this anchors on a conservative 200k ' +
`window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` +
'tok and you are likely within it. this is an estimate, not measured telemetry';
/**
* @typedef {object} BudgetAssessment
* @property {number} scanned - number of descriptions assessed
* @property {number} aggregateChars - sum of each length capped at DESCRIPTION_CAP
* @property {number} aggregateTokens - estimateTokens(aggregateChars, 'markdown')
* @property {number} budgetTokens - AGGREGATE_BUDGET_TOKENS (the 200k-anchored budget)
* @property {boolean} overBudget - aggregateTokens strictly greater than budgetTokens
* @property {number} overBy - tokens over budget (0 when not over)
*/
/**
* Pure aggregate-budget assessment. Each description contributes only up to the
* cap (the tail past the cap is dropped from the listing, and CA-SKL-001 already
* flags it — so the aggregate does not double-count it).
*
* @param {number[]} descLengths - one entry per active skill (description char count)
* @returns {BudgetAssessment}
*/
export function assessSkillListingBudget(descLengths) {
let aggregateChars = 0;
for (const len of descLengths) {
const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0;
aggregateChars += Math.min(safe, DESCRIPTION_CAP);
}
const aggregateTokens = estimateTokens(aggregateChars, 'markdown');
const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS;
return {
scanned: descLengths.length,
aggregateChars,
aggregateTokens,
budgetTokens: AGGREGATE_BUDGET_TOKENS,
overBudget,
overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0,
};
}
/**
* @typedef {object} ActiveSkillEntry
* @property {string} name
* @property {'user'|'plugin'} source
* @property {string|null} pluginName
* @property {string} path
* @property {number} descLength
*/
/**
* Enumerate every active skill (user + plugin) and measure the listing budget.
* HOME-scoped: resolves ~/.claude via process.env.HOME (enumeratePlugins /
* enumerateSkills). Callers that run under test MUST override HOME (see the
* hermetic-home helper / runScannerWithHome pattern).
*
* @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>}
*/
export async function measureActiveSkillListing() {
const plugins = await enumeratePlugins();
const allSkills = await enumerateSkills(plugins);
const skills = [];
for (const skill of allSkills) {
if (!skill || typeof skill.path !== 'string') continue;
const content = await readTextFile(skill.path);
if (!content) continue;
const fm = parseFrontmatter(content)?.frontmatter || null;
const desc = (fm && typeof fm.description === 'string') ? fm.description : '';
skills.push({
name: skill.name,
source: skill.source,
pluginName: skill.pluginName,
path: skill.path,
descLength: desc.length,
});
}
const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength));
return { skills, aggregate };
}
/**
* Read an env flag, treating null, "", "0", "false", "no", "off" as un-set.
* @param {string|undefined} v
* @returns {boolean}
*/
export function envFlag(v) {
if (v == null) return false;
const s = String(v).trim().toLowerCase();
return s !== '' && s !== '0' && s !== 'false' && s !== 'no' && s !== 'off';
}
/**
* Resolve whether the `disableBundledSkills` lever is effectively ON, reading the
* env var and the settings cascade directly (user ~/.claude, then project, then
* project-local).
*
* Reads the files directly rather than relying on config-discovery
* classification: when discovery walks ~/.claude from the .claude root, the
* user settings.json has a relPath of "settings.json" (no ".claude" segment)
* and is NOT tagged as settings-json — so the dominant user-scope location for
* this global preference would otherwise be missed. HOME-scoped via
* process.env.HOME.
*
* @param {string} [projectPath] - project root, to also read project + local settings
* @returns {Promise<boolean>}
*/
export async function isBundledSkillsDisabled(projectPath) {
if (envFlag(process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS)) return true;
const home = process.env.HOME || process.env.USERPROFILE || '';
const candidates = [];
if (home) candidates.push(join(home, '.claude', 'settings.json'));
if (projectPath) {
candidates.push(join(projectPath, '.claude', 'settings.json'));
candidates.push(join(projectPath, '.claude', 'settings.local.json'));
}
for (const p of candidates) {
const content = await readTextFile(p);
if (!content) continue;
const parsed = parseJson(content);
if (parsed && parsed.disableBundledSkills === true) return true;
}
return false;
}