config-audit/scanners/skill-listing-scanner.mjs

/**
 * SKL Scanner — Skill-listing token budget
 *
 * Claude Code shows the model a listing of every active skill's `description`
 * so it can decide which skill to invoke. That listing is budgeted: each
 * description is capped, and anything past the cap is silently truncated
 * (Claude Code raised the per-description cap 250 → 1,536 chars in v2.1.105,
 * changelog L1502, and added a startup/`/doctor` notice when truncation
 * happens). A description past the cap loses its tail from what the model
 * actually sees — the very trigger phrases meant to route invocation.
 *
 * Detection:
 *   CA-SKL-001  active skill description > 1,536 chars → truncated (medium)
 *   CA-SKL-002  sum of active descriptions exceeds the listing budget   (low)
 *
 * The aggregate listing budget (changelog L2860, CC 2.1.32) allots the skill
 * listing the model reads ~2% of the context window. We do NOT know the user's
 * context window, so CA-SKL-002 anchors on a conservative 200k window and says
 * so loudly: it leads with the measured sum (a fact) and carries a calibration
 * note explaining the budget scales 5x on a 1M-context model. Severity is low
 * (an estimate) versus medium for the verified per-description cap. Each
 * description is counted only up to the 1,536-char cap, because that is all
 * Claude Code loads into the listing — the tail past the cap is dropped and is
 * already flagged by CA-SKL-001 (so the aggregate does not double-count it).
 *
 * Two-lens note vs TOK pattern F: TOK pattern F flags *project-local* skills
 * with descriptions > 500 chars as a structural per-turn "bloat" heuristic.
 * SKL is a different lens — it scans ALL active skills (user + plugin) against
 * the *verified* 1,536-char hard truncation cap, not a bloat heuristic. The
 * two are intentionally distinct, not duplicates.
 *
 * Zero external dependencies.
 */

import { finding, scannerResult } from './lib/output.mjs';
import { SEVERITY } from './lib/severity.mjs';
import { enumeratePlugins, enumerateSkills, estimateTokens } from './lib/active-config-reader.mjs';
import { readTextFile } from './lib/file-discovery.mjs';
import { parseFrontmatter } from './lib/yaml-parser.mjs';

const SCANNER = 'SKL';

// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502).
// Descriptions longer than this are truncated in the listing the model sees.
const DESCRIPTION_CAP = 1536;

// Aggregate listing budget (CC 2.1.32, changelog L2860): the skill listing the
// model reads is allotted ~2% of the context window. The context window is
// unknown, so we anchor on a conservative 200k window — the smallest common
// size, which fires earliest — and disclose the assumption in the evidence.
const BUDGET_FRACTION = 0.02;
const CONTEXT_WINDOW_ANCHOR = 200_000;
const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000
const LARGE_CONTEXT_WINDOW = 1_000_000;
const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000

// Dependency-free thousands separator (repo invariant: zero external deps).
const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',');

// Appended to CA-SKL-002 evidence — the honest framing required because the
// budget depends on a context window we cannot observe (jf. TOK CALIBRATION_NOTE).
const BUDGET_CALIBRATION_NOTE =
  'the budget scales with the context window - this anchors on a conservative 200k ' +
  `window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` +
  'tok and you are likely within it. this is an estimate, not measured telemetry';

/**
 * Main scanner entry point.
 *
 * @param {string} _targetPath unused (skill listing is HOME-scoped)
 * @param {object} _discovery unused (ignores project discovery)
 */
export async function scan(_targetPath, _discovery) {
  const start = Date.now();
  const findings = [];

  const plugins = await enumeratePlugins();
  const allSkills = await enumerateSkills(plugins);

  let scanned = 0;
  let aggregateChars = 0;
  for (const skill of allSkills) {
    if (!skill || typeof skill.path !== 'string') continue;
    const content = await readTextFile(skill.path);
    if (!content) continue;
    scanned++;
    const fm = parseFrontmatter(content)?.frontmatter || null;
    const desc = (fm && typeof fm.description === 'string') ? fm.description : '';
    // Aggregate budget counts only what loads in the listing: each description
    // up to the cap (the tail past the cap is dropped, and CA-SKL-001 flags it).
    aggregateChars += Math.min(desc.length, DESCRIPTION_CAP);
    if (desc.length <= DESCRIPTION_CAP) continue;

    const sourceLabel = skill.source === 'plugin'
      ? `plugin:${skill.pluginName}`
      : 'user';

    findings.push(finding({
      scanner: SCANNER,
      severity: SEVERITY.medium,
      title: 'Skill description exceeds the listing cap (Claude Code truncates it)',
      description:
        `Skill "${skill.name}" (${sourceLabel}) has a description of ${desc.length} ` +
        `characters (>${DESCRIPTION_CAP}). Claude Code caps each skill description in ` +
        'the listing the model reads to choose a skill, so everything past ' +
        `${DESCRIPTION_CAP} characters is silently dropped — including any trigger ` +
        'phrases at the tail meant to route invocation.',
      file: skill.path,
      evidence:
        `description_chars=${desc.length}; cap=${DESCRIPTION_CAP}; ` +
        `skill="${skill.name}"; source=${sourceLabel}`,
      recommendation:
        `Trim the description below ${DESCRIPTION_CAP} characters, leading with the ` +
        'trigger phrases. To reclaim listing budget more broadly: set ' +
        '`disableBundledSkills: true` to drop bundled skills you do not use from the ' +
        'listing entirely, or use `skillOverrides` (`name-only` collapses a ' +
        'description, `off` removes a skill) on the heaviest entries.',
      category: 'token-efficiency',
    }));
  }

  // CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common
  // "one oversized skill + aggregate" case reads 001=cap, 002=aggregate.
  const aggregateTokens = estimateTokens(aggregateChars, 'markdown');
  if (aggregateTokens > AGGREGATE_BUDGET_TOKENS) {
    findings.push(finding({
      scanner: SCANNER,
      severity: SEVERITY.low,
      title: 'Aggregate skill descriptions may exceed the listing budget',
      description:
        `The ${scanned} active skills carry about ${aggregateTokens} tokens of description text ` +
        `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
        `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
        'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
        'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
        'is an estimate — the budget scales with your actual context window (see evidence).',
      evidence:
        `active_skills_scanned=${scanned}; description_chars=${aggregateChars} (each capped at ` +
        `${DESCRIPTION_CAP}); description_tokens~${aggregateTokens}; budget@200k=` +
        `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
        `${aggregateTokens - AGGREGATE_BUDGET_TOKENS} tok - ${BUDGET_CALIBRATION_NOTE}`,
      recommendation:
        'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
        'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
        '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
        'trigger phrases.',
      category: 'token-efficiency',
    }));
  }

  return scannerResult(SCANNER, 'ok', findings, scanned, Date.now() - start);
}