feat(skill-listing): add SKL scanner for the skill-listing token budget
Fase 4 Items 2+3 (CC 2.1.114→181 gap-review). New orchestrated scanner `skill-listing-scanner.mjs` (prefix SKL) flags every active skill whose description exceeds the verified 1,536-char listing cap (CC 2.1.105, changelog L1502). Past the cap, Claude Code silently truncates the description the model reads to route skill invocation — dropping the trigger phrases at the tail. HOME-scoped over all user + plugin skills via enumerateSkills (COL is the model). - CA-SKL-001 (medium): description > 1,536 chars. Remediation folds in Item 2(b) — recommends disableBundledSkills + skillOverrides + trimming (designvalg A: no standalone GAP-check, which would fire for nearly everyone). - Designvalg B: v1 ships the verified cap ONLY. The aggregate 2%-of-context listing budget is deferred — it needs a context-window assumption that would turn a verified fact into a guess (would carry a CALIBRATION_NOTE if added). - Choice C: recognize the skillOverrides settings key (CC 2.1.129) in KNOWN_KEYS. Left OUT of TYPE_CHECKS — the value is a per-skill object (off/user-invocable-only/name-only), not a string; a 'string' check (as the plan sketched) would create a NEW false positive. Verify-first deviation. Registration: scan-orchestrator (13th scanner), humanizer (SKL → 'Wasted tokens' + static/_default translations), scoring SCANNER_AREA_MAP (→ Token Efficiency; no 11th area), README badge 12→13, CLAUDE.md (finding-id + test-count), docs/scanner-internals.md, gap-matrix + plan status notes. Snapshots reseeded hermetically (SEED_SNAPSHOT/UPDATE_SNAPSHOT): SKL entry with 0 findings in empty HOME, scanners_ok 11→12, claudeMdEstimatedTokens bump from the CLAUDE.md edits flowing through the cascade. Contamination grep clean. Suite 868/868 (856 baseline + 11 SKL + 1 skillOverrides). RED→GREEN logged per cycle. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
This commit is contained in:
parent
fcd8ad3048
commit
7bb254780a
22 changed files with 390 additions and 37 deletions
|
|
@ -730,4 +730,24 @@ export const TRANSLATIONS = {
|
|||
recommendation: 'See the details for what needs to change.',
|
||||
},
|
||||
},
|
||||
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
// SKL — Skill-Listing Budget
|
||||
// Category: Wasted tokens
|
||||
// ─────────────────────────────────────────────────────────────
|
||||
SKL: {
|
||||
static: {
|
||||
'Skill description exceeds the listing cap (Claude Code truncates it)': {
|
||||
title: 'A skill description is too long and gets cut off',
|
||||
description: 'Claude Code only shows the first part of each skill\'s description when it decides which skill to use. This one runs past the limit, so the end — often the trigger phrases — is silently dropped.',
|
||||
recommendation: 'Shorten the description so the trigger words come first and it fits under the limit. For skills you do not use, turning them off frees up the listing entirely.',
|
||||
},
|
||||
},
|
||||
patterns: [],
|
||||
_default: {
|
||||
title: 'A skill is using more of the listing budget than it should',
|
||||
description: 'A check on how much room your skills take in Claude Code\'s skill listing flagged something worth a look.',
|
||||
recommendation: 'See the details for which skill to trim or turn off.',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ const SCANNER_TO_CATEGORY = {
|
|||
COL: 'Conflict',
|
||||
TOK: 'Wasted tokens',
|
||||
CPS: 'Wasted tokens',
|
||||
SKL: 'Wasted tokens',
|
||||
DIS: 'Dead config',
|
||||
GAP: 'Missed opportunity',
|
||||
PLH: 'Configuration mistake',
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ const SCANNER_AREA_MAP = {
|
|||
GAP: 'Feature Coverage',
|
||||
TOK: 'Token Efficiency',
|
||||
CPS: 'Token Efficiency',
|
||||
SKL: 'Token Efficiency',
|
||||
DIS: 'Settings',
|
||||
COL: 'Plugin Hygiene',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import { scan as scanTokenHotspots } from './token-hotspots.mjs';
|
|||
import { scan as scanCachePrefix } from './cache-prefix-scanner.mjs';
|
||||
import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs';
|
||||
import { scan as scanCollision } from './collision-scanner.mjs';
|
||||
import { scan as scanSkillListing } from './skill-listing-scanner.mjs';
|
||||
|
||||
// Directory names that identify test fixture / example directories
|
||||
const FIXTURE_DIR_NAMES = ['tests', 'examples', '__tests__', 'test-fixtures'];
|
||||
|
|
@ -62,6 +63,7 @@ const SCANNERS = [
|
|||
{ name: 'CPS', fn: scanCachePrefix, label: 'Cache-Prefix Stability' },
|
||||
{ name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' },
|
||||
{ name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' },
|
||||
{ name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' },
|
||||
];
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ const PLUGIN_ROOT = resolve(__dirname, '..');
|
|||
// `plugin-health-scanner.mjs` is excluded from the main scanner count: it has
|
||||
// `export async function scan` but it runs standalone (not via scan-orchestrator)
|
||||
// and is documented under "Standalone Scanner" in README/CLAUDE.md. The badge
|
||||
// `scanners-12` reflects the orchestrated scanners that contribute to posture
|
||||
// `scanners-13` reflects the orchestrated scanners that contribute to posture
|
||||
// scoring.
|
||||
const SCANNER_EXCLUDES = new Set([
|
||||
'scan-orchestrator.mjs',
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ const KNOWN_KEYS = new Set([
|
|||
'outputStyle', 'parentSettingsBehavior', 'permissions', 'plansDirectory',
|
||||
'pluginSuggestionMarketplaces', 'pluginTrustMessage', 'prefersReducedMotion',
|
||||
'requiredMaximumVersion', 'requiredMinimumVersion', 'respectGitignore', 'sandbox',
|
||||
'showClearContextOnPlanAccept', 'showThinkingSummaries', 'spinnerTipsEnabled',
|
||||
'showClearContextOnPlanAccept', 'showThinkingSummaries', 'skillOverrides',
|
||||
'spinnerTipsEnabled',
|
||||
'spinnerTipsOverride', 'spinnerVerbs', 'statusLine', 'strictKnownMarketplaces',
|
||||
'useAutoModeDuringPlan', 'voiceEnabled', 'wheelScrollAccelerationEnabled',
|
||||
'worktree', '$schema',
|
||||
|
|
|
|||
94
scanners/skill-listing-scanner.mjs
Normal file
94
scanners/skill-listing-scanner.mjs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* SKL Scanner — Skill-listing token budget
|
||||
*
|
||||
* Claude Code shows the model a listing of every active skill's `description`
|
||||
* so it can decide which skill to invoke. That listing is budgeted: each
|
||||
* description is capped, and anything past the cap is silently truncated
|
||||
* (Claude Code raised the per-description cap 250 → 1,536 chars in v2.1.105,
|
||||
* changelog L1502, and added a startup/`/doctor` notice when truncation
|
||||
* happens). A description past the cap loses its tail from what the model
|
||||
* actually sees — the very trigger phrases meant to route invocation.
|
||||
*
|
||||
* Detection (v1, high confidence — no estimation):
|
||||
* CA-SKL-001 active skill description > 1,536 chars → truncated (medium)
|
||||
*
|
||||
* The aggregate listing budget (skill descriptions sum vs ~2% of the context
|
||||
* window, v2.1.32 L2860) is deliberately NOT scanned in v1: it requires
|
||||
* assuming the user's context-window size, which would turn a verified fact
|
||||
* into a guess. Lead with the cap; the aggregate check is a possible later
|
||||
* addition (would carry a CALIBRATION_NOTE).
|
||||
*
|
||||
* Two-lens note vs TOK pattern F: TOK pattern F flags *project-local* skills
|
||||
* with descriptions > 500 chars as a structural per-turn "bloat" heuristic.
|
||||
* SKL is a different lens — it scans ALL active skills (user + plugin) against
|
||||
* the *verified* 1,536-char hard truncation cap, not a bloat heuristic. The
|
||||
* two are intentionally distinct, not duplicates.
|
||||
*
|
||||
* Zero external dependencies.
|
||||
*/
|
||||
|
||||
import { finding, scannerResult } from './lib/output.mjs';
|
||||
import { SEVERITY } from './lib/severity.mjs';
|
||||
import { enumeratePlugins, enumerateSkills } from './lib/active-config-reader.mjs';
|
||||
import { readTextFile } from './lib/file-discovery.mjs';
|
||||
import { parseFrontmatter } from './lib/yaml-parser.mjs';
|
||||
|
||||
const SCANNER = 'SKL';
|
||||
|
||||
// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502).
|
||||
// Descriptions longer than this are truncated in the listing the model sees.
|
||||
const DESCRIPTION_CAP = 1536;
|
||||
|
||||
/**
|
||||
* Main scanner entry point.
|
||||
*
|
||||
* @param {string} _targetPath unused (skill listing is HOME-scoped)
|
||||
* @param {object} _discovery unused (ignores project discovery)
|
||||
*/
|
||||
export async function scan(_targetPath, _discovery) {
|
||||
const start = Date.now();
|
||||
const findings = [];
|
||||
|
||||
const plugins = await enumeratePlugins();
|
||||
const allSkills = await enumerateSkills(plugins);
|
||||
|
||||
let scanned = 0;
|
||||
for (const skill of allSkills) {
|
||||
if (!skill || typeof skill.path !== 'string') continue;
|
||||
const content = await readTextFile(skill.path);
|
||||
if (!content) continue;
|
||||
scanned++;
|
||||
const fm = parseFrontmatter(content)?.frontmatter || null;
|
||||
const desc = (fm && typeof fm.description === 'string') ? fm.description : '';
|
||||
if (desc.length <= DESCRIPTION_CAP) continue;
|
||||
|
||||
const sourceLabel = skill.source === 'plugin'
|
||||
? `plugin:${skill.pluginName}`
|
||||
: 'user';
|
||||
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.medium,
|
||||
title: 'Skill description exceeds the listing cap (Claude Code truncates it)',
|
||||
description:
|
||||
`Skill "${skill.name}" (${sourceLabel}) has a description of ${desc.length} ` +
|
||||
`characters (>${DESCRIPTION_CAP}). Claude Code caps each skill description in ` +
|
||||
'the listing the model reads to choose a skill, so everything past ' +
|
||||
`${DESCRIPTION_CAP} characters is silently dropped — including any trigger ` +
|
||||
'phrases at the tail meant to route invocation.',
|
||||
file: skill.path,
|
||||
evidence:
|
||||
`description_chars=${desc.length}; cap=${DESCRIPTION_CAP}; ` +
|
||||
`skill="${skill.name}"; source=${sourceLabel}`,
|
||||
recommendation:
|
||||
`Trim the description below ${DESCRIPTION_CAP} characters, leading with the ` +
|
||||
'trigger phrases. To reclaim listing budget more broadly: set ' +
|
||||
'`disableBundledSkills: true` to drop bundled skills you do not use from the ' +
|
||||
'listing entirely, or use `skillOverrides` (`name-only` collapses a ' +
|
||||
'description, `off` removes a skill) on the heaviest entries.',
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
}
|
||||
|
||||
return scannerResult(SCANNER, 'ok', findings, scanned, Date.now() - start);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue