config-audit/scanners/skill-listing-scanner.mjs

/**
 * SKL Scanner — Skill-listing token budget
 *
 * Claude Code shows the model a listing of every active skill's `description`
 * so it can decide which skill to invoke. That listing is budgeted: each
 * description is capped, and anything past the cap is silently truncated
 * (Claude Code raised the per-description cap 250 → 1,536 chars in v2.1.105,
 * changelog L1502, and added a startup/`/doctor` notice when truncation
 * happens). A description past the cap loses its tail from what the model
 * actually sees — the very trigger phrases meant to route invocation.
 *
 * Detection:
 *   CA-SKL-001  active skill description > 1,536 chars → truncated (medium)
 *   CA-SKL-002  sum of active descriptions exceeds the listing budget   (low)
 *
 * The aggregate listing budget (changelog L2860, CC 2.1.32) allots the skill
 * listing the model reads ~2% of the context window. We do NOT know the user's
 * context window, so CA-SKL-002 anchors on a conservative 200k window and says
 * so loudly: it leads with the measured sum (a fact) and carries a calibration
 * note explaining the budget scales 5x on a 1M-context model. Severity is low
 * (an estimate) versus medium for the verified per-description cap.
 *
 * The cap, the budget, and the enumerate-and-measure step all live in
 * `lib/skill-listing-budget.mjs` — the single source of truth shared with the
 * GAP scanner, which prescribes `disableBundledSkills` when this budget is
 * exceeded. SKL only constructs the findings.
 *
 * Two-lens note vs TOK pattern F: TOK pattern F flags *project-local* skills
 * with descriptions > 500 chars as a structural per-turn "bloat" heuristic.
 * SKL is a different lens — it scans ALL active skills (user + plugin) against
 * the *verified* 1,536-char hard truncation cap, not a bloat heuristic. The
 * two are intentionally distinct, not duplicates.
 *
 * Zero external dependencies.
 */

import { finding, scannerResult } from './lib/output.mjs';
import { SEVERITY } from './lib/severity.mjs';
import {
  DESCRIPTION_CAP,
  AGGREGATE_BUDGET_TOKENS,
  BUDGET_CALIBRATION_NOTE,
  BODY_TOKEN_THRESHOLD,
  BODY_CALIBRATION_NOTE,
  measureActiveSkillListing,
} from './lib/skill-listing-budget.mjs';
import { CONTEXT_WINDOW_ANCHOR, scaleForWindow, withCommas } from './lib/context-window.mjs';

// Shared remediation for the aggregate-budget finding (byte-identical across the
// default and the B8 window-calibrated branches).
const AGGREGATE_RECOMMENDATION =
  'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
  'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
  '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
  'trigger phrases.';

const SCANNER = 'SKL';

/**
 * Main scanner entry point.
 *
 * @param {string} _targetPath unused (skill listing is HOME-scoped)
 * @param {object} _discovery unused (ignores project discovery)
 */
export async function scan(_targetPath, _discovery, opts = {}) {
  const start = Date.now();
  const findings = [];

  // B8 — calibrate the aggregate budget to the resolved context window. The
  // default (no opts) is the conservative 200k anchor at full severity, which is
  // byte-identical to the pre-B8 behavior. An unknown (advisory) window keeps the
  // anchor but downgrades the finding to info instead of firing it as a breach.
  const cw = opts.contextWindow;
  const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR;
  const advisory = !!(cw && cw.advisory);
  const isDefault = window === CONTEXT_WINDOW_ANCHOR && !advisory;
  const budgetTokens = scaleForWindow(AGGREGATE_BUDGET_TOKENS, window);

  const { skills, aggregate } = await measureActiveSkillListing(budgetTokens);

  for (const skill of skills) {
    if (skill.descLength <= DESCRIPTION_CAP) continue;

    const sourceLabel = skill.source === 'plugin'
      ? `plugin:${skill.pluginName}`
      : 'user';

    findings.push(finding({
      scanner: SCANNER,
      code: 'description-over-cap',
      severity: SEVERITY.medium,
      title: 'Skill description exceeds the listing cap (Claude Code truncates it)',
      description:
        `Skill "${skill.name}" (${sourceLabel}) has a description of ${skill.descLength} ` +
        `characters (>${DESCRIPTION_CAP}). Claude Code caps each skill description in ` +
        'the listing the model reads to choose a skill, so everything past ' +
        `${DESCRIPTION_CAP} characters is silently dropped — including any trigger ` +
        'phrases at the tail meant to route invocation.',
      file: skill.path,
      evidence:
        `description_chars=${skill.descLength}; cap=${DESCRIPTION_CAP}; ` +
        `skill="${skill.name}"; source=${sourceLabel}`,
      recommendation:
        `Trim the description below ${DESCRIPTION_CAP} characters, leading with the ` +
        'trigger phrases. To reclaim listing budget more broadly: set ' +
        '`disableBundledSkills: true` to drop bundled skills you do not use from the ' +
        'listing entirely, or use `skillOverrides` (`name-only` collapses a ' +
        'description, `off` removes a skill) on the heaviest entries.',
      category: 'token-efficiency',
    }));
  }

  // CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common
  // "one oversized skill + aggregate" case reads 001=cap, 002=aggregate.
  if (aggregate.overBudget) {
    if (isDefault) {
      // Conservative 200k anchor — byte-identical to the pre-B8 finding.
      findings.push(finding({
        scanner: SCANNER,
        code: 'aggregate-listing-budget',
        severity: SEVERITY.low,
        title: 'Aggregate skill descriptions may exceed the listing budget',
        description:
          `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
          `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
          `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
          'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
          'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
          'is an estimate — the budget scales with your actual context window (see evidence).',
        evidence:
          `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
          `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` +
          `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
          `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
        recommendation: AGGREGATE_RECOMMENDATION,
        category: 'token-efficiency',
      }));
    } else {
      // B8 — window-calibrated. Advisory (unknown window) downgrades to info.
      const winLabel = withCommas(window);
      findings.push(finding({
        scanner: SCANNER,
        code: 'aggregate-listing-budget',
        severity: advisory ? SEVERITY.info : SEVERITY.low,
        title: 'Aggregate skill descriptions may exceed the listing budget',
        description:
          `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
          `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
          `${budgetTokens}-token budget Claude Code allots the skill listing at a ${winLabel}-token ` +
          'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
          'Claude Code drops descriptions, so the model may stop seeing some skills entirely.' +
          (advisory
            ? ' Your context window is unknown, so this is advisory: it anchors on the conservative 200k window.'
            : ''),
        evidence:
          `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
          `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@${winLabel}=` +
          `${budgetTokens} tok (skill listing ~2% of context, CC 2.1.32); over_by~${aggregate.overBy} tok` +
          (advisory ? ` - ${BUDGET_CALIBRATION_NOTE}` : ' - this is an estimate, not measured telemetry'),
        recommendation: AGGREGATE_RECOMMENDATION,
        category: 'token-efficiency',
      }));
    }
  }

  // CA-SKL-003 (oversized body). Emitted last so the common single-issue cases
  // read cleanly. Unlike the listing budget, this is an ON-DEMAND cost — the body
  // loads only when the skill is invoked, not every turn — hence low severity and
  // an explicit on-demand calibration note.
  for (const skill of skills) {
    if (skill.bodyTokens <= BODY_TOKEN_THRESHOLD) continue;

    const sourceLabel = skill.source === 'plugin'
      ? `plugin:${skill.pluginName}`
      : 'user';

    findings.push(finding({
      scanner: SCANNER,
      code: 'oversized-body',
      severity: SEVERITY.low,
      title: 'Skill body is large (loads on demand when the skill runs)',
      description:
        `Skill "${skill.name}" (${sourceLabel}) has a body of about ${skill.bodyTokens} tokens ` +
        `(${skill.bodyLines} lines), over the ~${BODY_TOKEN_THRESHOLD}-token guidance for a skill body. ` +
        'The body is not in the always-loaded listing — it loads only when the skill is invoked — but ' +
        'once loaded a large body consumes context for the rest of that session. Claude Code skill ' +
        'guidance is to keep the body lean and move heavy reference material into supporting files.',
      file: skill.path,
      evidence:
        `body_tokens~${skill.bodyTokens}; body_lines=${skill.bodyLines}; body_chars=${skill.bodyChars}; ` +
        `threshold=${BODY_TOKEN_THRESHOLD} tok; skill="${skill.name}"; source=${sourceLabel} - ${BODY_CALIBRATION_NOTE}`,
      recommendation:
        'Move reference content into supporting files the skill loads only when needed, and consider ' +
        '`context: fork` in the skill frontmatter for heavy skills so the body runs in a forked context ' +
        'instead of consuming the main thread.',
      category: 'token-efficiency',
    }));
  }

  return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start);
}