From 7bb254780a1e8837c0345678a67105f79f8aab4b Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 18 Jun 2026 17:36:28 +0200 Subject: [PATCH] feat(skill-listing): add SKL scanner for the skill-listing token budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fase 4 Items 2+3 (CC 2.1.114→181 gap-review). New orchestrated scanner `skill-listing-scanner.mjs` (prefix SKL) flags every active skill whose description exceeds the verified 1,536-char listing cap (CC 2.1.105, changelog L1502). Past the cap, Claude Code silently truncates the description the model reads to route skill invocation — dropping the trigger phrases at the tail. HOME-scoped over all user + plugin skills via enumerateSkills (COL is the model). - CA-SKL-001 (medium): description > 1,536 chars. Remediation folds in Item 2(b) — recommends disableBundledSkills + skillOverrides + trimming (designvalg A: no standalone GAP-check, which would fire for nearly everyone). - Designvalg B: v1 ships the verified cap ONLY. The aggregate 2%-of-context listing budget is deferred — it needs a context-window assumption that would turn a verified fact into a guess (would carry a CALIBRATION_NOTE if added). - Choice C: recognize the skillOverrides settings key (CC 2.1.129) in KNOWN_KEYS. Left OUT of TYPE_CHECKS — the value is a per-skill object (off/user-invocable-only/name-only), not a string; a 'string' check (as the plan sketched) would create a NEW false positive. Verify-first deviation. Registration: scan-orchestrator (13th scanner), humanizer (SKL → 'Wasted tokens' + static/_default translations), scoring SCANNER_AREA_MAP (→ Token Efficiency; no 11th area), README badge 12→13, CLAUDE.md (finding-id + test-count), docs/scanner-internals.md, gap-matrix + plan status notes. Snapshots reseeded hermetically (SEED_SNAPSHOT/UPDATE_SNAPSHOT): SKL entry with 0 findings in empty HOME, scanners_ok 11→12, claudeMdEstimatedTokens bump from the CLAUDE.md edits flowing through the cascade. Contamination grep clean. Suite 868/868 (856 baseline + 11 SKL + 1 skillOverrides). RED→GREEN logged per cycle. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8 --- CLAUDE.md | 4 +- README.md | 2 +- docs/cc-2.1.x-fase4-items-2-3-plan.md | 2 +- docs/cc-2.1.x-gap-matrix.md | 4 +- docs/scanner-internals.md | 3 +- scanners/lib/humanizer-data.mjs | 20 ++ scanners/lib/humanizer.mjs | 1 + scanners/lib/scoring.mjs | 1 + scanners/scan-orchestrator.mjs | 2 + scanners/self-audit.mjs | 2 +- scanners/settings-validator.mjs | 3 +- scanners/skill-listing-scanner.mjs | 94 +++++++++ tests/lib/humanizer-data.test.mjs | 4 +- tests/lib/humanizer.test.mjs | 4 +- .../scan-orchestrator-humanizer.test.mjs | 2 +- tests/scanners/settings-validator.test.mjs | 3 +- tests/scanners/skill-listing-scanner.test.mjs | 189 ++++++++++++++++++ tests/snapshots/default-output/posture.json | 2 +- .../default-output/scan-orchestrator.json | 16 +- tests/snapshots/v5.0.0-stderr/posture.txt | 13 +- tests/snapshots/v5.0.0/posture.json | 30 ++- tests/snapshots/v5.0.0/scan-orchestrator.json | 26 ++- 22 files changed, 390 insertions(+), 37 deletions(-) create mode 100644 scanners/skill-listing-scanner.mjs create mode 100644 tests/scanners/skill-listing-scanner.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 920efbc..7777575 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,7 +101,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| ``` ### Finding ID Format -`CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001` +`CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001`, `CA-SKL-001` ## Testing @@ -109,7 +109,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -792 tests across 52 test files (15 lib + 28 scanner + 1 hook + 1 agent + 3 commands + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +868 tests across 55 test files (16 lib + 29 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ## Gotchas diff --git a/README.md b/README.md index 2b74256..b88934f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ ![Version](https://img.shields.io/badge/version-5.1.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) -![Scanners](https://img.shields.io/badge/scanners-12-cyan) +![Scanners](https://img.shields.io/badge/scanners-13-cyan) ![Commands](https://img.shields.io/badge/commands-18-green) ![Agents](https://img.shields.io/badge/agents-6-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) diff --git a/docs/cc-2.1.x-fase4-items-2-3-plan.md b/docs/cc-2.1.x-fase4-items-2-3-plan.md index 278a003..073d626 100644 --- a/docs/cc-2.1.x-fase4-items-2-3-plan.md +++ b/docs/cc-2.1.x-fase4-items-2-3-plan.md @@ -1,6 +1,6 @@ # Plan — Fase 4 Items 2 + 3 (skill-listing token-styring) -**Skrevet:** 2026-06-18 · **Scope:** kun `config-audit/` · **Status:** klar for utførelse (krever rask design-bekreftelse på 2 punkter FØR kode, så failing-test-først per syklus — Iron Law). +**Skrevet:** 2026-06-18 · **Scope:** kun `config-audit/` · **Status:** ✅ **LEVERT 2026-06-18.** Designvalg bekreftet: A=fold inn i SKL, B=v1 KUN 1 536-cap (aggregat utsatt), C=`skillOverrides` i KNOWN_KEYS (IKKE TYPE_CHECKS — verdien er objekt; avvik fra §C-skissen, verify-først). Syklus 1+3+4 ferdig; Syklus 2 (aggregat CA-SKL-002) gjenstår som valgfri stretch. Suite 868/868. Detaljer i `STATE.md`. **Forrige:** Fase 4 Item 1 (4.7→modell-nøytral) levert som `8376dab`. Baseline **856/856 grønn**. ## Mål diff --git a/docs/cc-2.1.x-gap-matrix.md b/docs/cc-2.1.x-gap-matrix.md index f4751be..a00860b 100644 --- a/docs/cc-2.1.x-gap-matrix.md +++ b/docs/cc-2.1.x-gap-matrix.md @@ -96,8 +96,8 @@ Netto-nytt scanner-arbeid (DIS param-bevissthet, PLH shadow-folder-check, permis | settings.json | env ANTHROPIC_WORKSPACE_ID | 2.1.141 | missing | update-knowledge | knowledge/claude-code-capabilities.md | L | S | | settings.json | env MAX_THINKING_TOKENS=0 disables thinking | 2.1.166 | missing | update-knowledge | knowledge/claude-code-capabilities.md | L | S | | settings.json | parentSettingsBehavior / policyHelper (UNVERIFIED) | 2.1.133 | missing | new-feature-gap-check | scanners/settings-validator.mjs | L | S | -| settings.json | skillOverrides (UNVERIFIED, below floor) | 2.1.129 | missing | new-feature-gap-check | scanners/settings-validator.mjs | L | S | -| settings.json | skillListingBudgetFraction / maxSkillDescriptionChars (UNVERIFIED, below floor) | 2.1.105 | missing | new-feature-gap-check | scanners/settings-validator.mjs | L | S | +| settings.json | skillOverrides | 2.1.129 | DONE | added to KNOWN_KEYS (NOT TYPE_CHECKS — value is per-skill object off/user-invocable-only/name-only) | scanners/settings-validator.mjs | L | S | +| settings.json | ~~skillListingBudgetFraction / maxSkillDescriptionChars~~ (keys DO NOT EXIST — verified) | 2.1.105 | DONE | budget=2% ctx + cap=1536 are internal CC mechanisms, not settings keys; the verified 1,536-char cap is enforced by the new SKL scanner (CA-SKL-001) | scanners/skill-listing-scanner.mjs | L | M | | CLAUDE.md / memory | 'too long' threshold now scales with context window | 2.1.169 | now-wrong | fix-false-positive | scanners/claude-md-linter.mjs | H | M | | CLAUDE.md / memory | --safe-mode / CLAUDE_CODE_SAFE_MODE disables customizations | 2.1.169 | missing | update-knowledge | knowledge/configuration-best-practices.md | L | S | | CLAUDE.md / memory | plugins in .claude/skills auto-load (cascade/token accounting) | 2.1.157 | partial | new-feature-gap-check | knowledge/configuration-best-practices.md | M | M | diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index d1fe7a9..404c57d 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -22,6 +22,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of CLAUDE.md cascade (beyond Pattern A's top-30 window) | | `disabled-in-schema-scanner.mjs` | DIS | Tools listed in BOTH `permissions.deny` AND `permissions.allow` — deny wins, allow entries are dead config | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions (low); user-vs-plugin overlaps (medium); `details.namespaces` payload | +| `skill-listing-scanner.mjs` | SKL | Active skill descriptions over the verified 1,536-char listing cap (CC 2.1.105) → silently truncated in the model's skill listing (medium). HOME-scoped (all user + plugin skills). Remediation surfaces `disableBundledSkills` / `skillOverrides`. Distinct lens from TOK pattern F (project-local 500-char bloat heuristic) | ## Scanner Lib (`scanners/lib/`) @@ -41,7 +42,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `active-config-reader.mjs` | Read-only inventory: readActiveConfig(), detectGitRoot(), walkClaudeMdCascade(), readClaudeJsonProjectSlice() (longest-prefix match), enumeratePlugins(), enumerateSkills(), readActiveHooks(), readActiveMcpServers() (with cache → package.json tool-count fallback), estimateTokens() (v5: `'mcp'` kind = 500 + toolCount × 200) | | `tokenizer-api.mjs` | Anthropic `count_tokens` wrapper for `--accurate-tokens` (v5 N5); 5s AbortController timeout, exponential 429 backoff, key masking | | `humanizer.mjs` | Plain-language output translator (v5.1.0): `humanizeFinding`, `humanizeFindings`, `humanizeEnvelope`, `computeRelevanceContext`. Pure functions; never mutate inputs. Adds `userImpactCategory`, `userActionLanguage`, `relevanceContext` fields and replaces title/description/recommendation when a translation exists. Bypassed by `--raw` and `--json` paths. | -| `humanizer-data.mjs` | TRANSLATIONS table for 13 scanner prefixes (CML/SET/HKV/RUL/MCP/IMP/CNF/COL/TOK/CPS/DIS/GAP/PLH). Three-step lookup: exact title → regex pattern → `_default` → fall through to original | +| `humanizer-data.mjs` | TRANSLATIONS table for 14 scanner prefixes (CML/SET/HKV/RUL/MCP/IMP/CNF/COL/TOK/CPS/DIS/GAP/PLH/SKL). Three-step lookup: exact title → regex pattern → `_default` → fall through to original | ## Action Engines (`scanners/`) diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index cc7feec..ce8e049 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -730,4 +730,24 @@ export const TRANSLATIONS = { recommendation: 'See the details for what needs to change.', }, }, + + // ───────────────────────────────────────────────────────────── + // SKL — Skill-Listing Budget + // Category: Wasted tokens + // ───────────────────────────────────────────────────────────── + SKL: { + static: { + 'Skill description exceeds the listing cap (Claude Code truncates it)': { + title: 'A skill description is too long and gets cut off', + description: 'Claude Code only shows the first part of each skill\'s description when it decides which skill to use. This one runs past the limit, so the end — often the trigger phrases — is silently dropped.', + recommendation: 'Shorten the description so the trigger words come first and it fits under the limit. For skills you do not use, turning them off frees up the listing entirely.', + }, + }, + patterns: [], + _default: { + title: 'A skill is using more of the listing budget than it should', + description: 'A check on how much room your skills take in Claude Code\'s skill listing flagged something worth a look.', + recommendation: 'See the details for which skill to trim or turn off.', + }, + }, }; diff --git a/scanners/lib/humanizer.mjs b/scanners/lib/humanizer.mjs index 62f1c46..2d8c321 100644 --- a/scanners/lib/humanizer.mjs +++ b/scanners/lib/humanizer.mjs @@ -37,6 +37,7 @@ const SCANNER_TO_CATEGORY = { COL: 'Conflict', TOK: 'Wasted tokens', CPS: 'Wasted tokens', + SKL: 'Wasted tokens', DIS: 'Dead config', GAP: 'Missed opportunity', PLH: 'Configuration mistake', diff --git a/scanners/lib/scoring.mjs b/scanners/lib/scoring.mjs index 1a99ccc..3347522 100644 --- a/scanners/lib/scoring.mjs +++ b/scanners/lib/scoring.mjs @@ -165,6 +165,7 @@ const SCANNER_AREA_MAP = { GAP: 'Feature Coverage', TOK: 'Token Efficiency', CPS: 'Token Efficiency', + SKL: 'Token Efficiency', DIS: 'Settings', COL: 'Plugin Hygiene', }; diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 08d180e..45c56b8 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -28,6 +28,7 @@ import { scan as scanTokenHotspots } from './token-hotspots.mjs'; import { scan as scanCachePrefix } from './cache-prefix-scanner.mjs'; import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs'; import { scan as scanCollision } from './collision-scanner.mjs'; +import { scan as scanSkillListing } from './skill-listing-scanner.mjs'; // Directory names that identify test fixture / example directories const FIXTURE_DIR_NAMES = ['tests', 'examples', '__tests__', 'test-fixtures']; @@ -62,6 +63,7 @@ const SCANNERS = [ { name: 'CPS', fn: scanCachePrefix, label: 'Cache-Prefix Stability' }, { name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' }, { name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' }, + { name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' }, ]; /** diff --git a/scanners/self-audit.mjs b/scanners/self-audit.mjs index 70be64b..cf6f88d 100644 --- a/scanners/self-audit.mjs +++ b/scanners/self-audit.mjs @@ -32,7 +32,7 @@ const PLUGIN_ROOT = resolve(__dirname, '..'); // `plugin-health-scanner.mjs` is excluded from the main scanner count: it has // `export async function scan` but it runs standalone (not via scan-orchestrator) // and is documented under "Standalone Scanner" in README/CLAUDE.md. The badge -// `scanners-12` reflects the orchestrated scanners that contribute to posture +// `scanners-13` reflects the orchestrated scanners that contribute to posture // scoring. const SCANNER_EXCLUDES = new Set([ 'scan-orchestrator.mjs', diff --git a/scanners/settings-validator.mjs b/scanners/settings-validator.mjs index e7225d4..f220f7a 100644 --- a/scanners/settings-validator.mjs +++ b/scanners/settings-validator.mjs @@ -32,7 +32,8 @@ const KNOWN_KEYS = new Set([ 'outputStyle', 'parentSettingsBehavior', 'permissions', 'plansDirectory', 'pluginSuggestionMarketplaces', 'pluginTrustMessage', 'prefersReducedMotion', 'requiredMaximumVersion', 'requiredMinimumVersion', 'respectGitignore', 'sandbox', - 'showClearContextOnPlanAccept', 'showThinkingSummaries', 'spinnerTipsEnabled', + 'showClearContextOnPlanAccept', 'showThinkingSummaries', 'skillOverrides', + 'spinnerTipsEnabled', 'spinnerTipsOverride', 'spinnerVerbs', 'statusLine', 'strictKnownMarketplaces', 'useAutoModeDuringPlan', 'voiceEnabled', 'wheelScrollAccelerationEnabled', 'worktree', '$schema', diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs new file mode 100644 index 0000000..ede78b6 --- /dev/null +++ b/scanners/skill-listing-scanner.mjs @@ -0,0 +1,94 @@ +/** + * SKL Scanner — Skill-listing token budget + * + * Claude Code shows the model a listing of every active skill's `description` + * so it can decide which skill to invoke. That listing is budgeted: each + * description is capped, and anything past the cap is silently truncated + * (Claude Code raised the per-description cap 250 → 1,536 chars in v2.1.105, + * changelog L1502, and added a startup/`/doctor` notice when truncation + * happens). A description past the cap loses its tail from what the model + * actually sees — the very trigger phrases meant to route invocation. + * + * Detection (v1, high confidence — no estimation): + * CA-SKL-001 active skill description > 1,536 chars → truncated (medium) + * + * The aggregate listing budget (skill descriptions sum vs ~2% of the context + * window, v2.1.32 L2860) is deliberately NOT scanned in v1: it requires + * assuming the user's context-window size, which would turn a verified fact + * into a guess. Lead with the cap; the aggregate check is a possible later + * addition (would carry a CALIBRATION_NOTE). + * + * Two-lens note vs TOK pattern F: TOK pattern F flags *project-local* skills + * with descriptions > 500 chars as a structural per-turn "bloat" heuristic. + * SKL is a different lens — it scans ALL active skills (user + plugin) against + * the *verified* 1,536-char hard truncation cap, not a bloat heuristic. The + * two are intentionally distinct, not duplicates. + * + * Zero external dependencies. + */ + +import { finding, scannerResult } from './lib/output.mjs'; +import { SEVERITY } from './lib/severity.mjs'; +import { enumeratePlugins, enumerateSkills } from './lib/active-config-reader.mjs'; +import { readTextFile } from './lib/file-discovery.mjs'; +import { parseFrontmatter } from './lib/yaml-parser.mjs'; + +const SCANNER = 'SKL'; + +// Verified per-description skill-listing cap (CC 2.1.105, changelog L1502). +// Descriptions longer than this are truncated in the listing the model sees. +const DESCRIPTION_CAP = 1536; + +/** + * Main scanner entry point. + * + * @param {string} _targetPath unused (skill listing is HOME-scoped) + * @param {object} _discovery unused (ignores project discovery) + */ +export async function scan(_targetPath, _discovery) { + const start = Date.now(); + const findings = []; + + const plugins = await enumeratePlugins(); + const allSkills = await enumerateSkills(plugins); + + let scanned = 0; + for (const skill of allSkills) { + if (!skill || typeof skill.path !== 'string') continue; + const content = await readTextFile(skill.path); + if (!content) continue; + scanned++; + const fm = parseFrontmatter(content)?.frontmatter || null; + const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + if (desc.length <= DESCRIPTION_CAP) continue; + + const sourceLabel = skill.source === 'plugin' + ? `plugin:${skill.pluginName}` + : 'user'; + + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.medium, + title: 'Skill description exceeds the listing cap (Claude Code truncates it)', + description: + `Skill "${skill.name}" (${sourceLabel}) has a description of ${desc.length} ` + + `characters (>${DESCRIPTION_CAP}). Claude Code caps each skill description in ` + + 'the listing the model reads to choose a skill, so everything past ' + + `${DESCRIPTION_CAP} characters is silently dropped — including any trigger ` + + 'phrases at the tail meant to route invocation.', + file: skill.path, + evidence: + `description_chars=${desc.length}; cap=${DESCRIPTION_CAP}; ` + + `skill="${skill.name}"; source=${sourceLabel}`, + recommendation: + `Trim the description below ${DESCRIPTION_CAP} characters, leading with the ` + + 'trigger phrases. To reclaim listing budget more broadly: set ' + + '`disableBundledSkills: true` to drop bundled skills you do not use from the ' + + 'listing entirely, or use `skillOverrides` (`name-only` collapses a ' + + 'description, `off` removes a skill) on the heaviest entries.', + category: 'token-efficiency', + })); + } + + return scannerResult(SCANNER, 'ok', findings, scanned, Date.now() - start); +} diff --git a/tests/lib/humanizer-data.test.mjs b/tests/lib/humanizer-data.test.mjs index ab35ba9..90321ee 100644 --- a/tests/lib/humanizer-data.test.mjs +++ b/tests/lib/humanizer-data.test.mjs @@ -8,7 +8,7 @@ import { TRANSLATIONS } from '../../scanners/lib/humanizer-data.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const FORBIDDEN_PATH = resolve(__dirname, '..', 'lint-forbidden-words.json'); -const EXPECTED_SCANNERS = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK', 'CPS', 'DIS', 'COL', 'PLH']; +const EXPECTED_SCANNERS = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK', 'CPS', 'DIS', 'COL', 'PLH', 'SKL']; function stripBacktickSpans(s) { return s.replace(/`[^`]*`/g, ''); @@ -24,7 +24,7 @@ test('TRANSLATIONS exports an object', () => { assert.ok(TRANSLATIONS !== null); }); -test('TRANSLATIONS covers all 13 expected scanner prefixes', () => { +test('TRANSLATIONS covers all 14 expected scanner prefixes', () => { for (const prefix of EXPECTED_SCANNERS) { assert.ok(TRANSLATIONS[prefix], `missing scanner prefix: ${prefix}`); } diff --git a/tests/lib/humanizer.test.mjs b/tests/lib/humanizer.test.mjs index a3f04e9..7fa58f6 100644 --- a/tests/lib/humanizer.test.mjs +++ b/tests/lib/humanizer.test.mjs @@ -171,8 +171,8 @@ test('humanizeFinding sets category Conflict for CNF/COL', () => { } }); -test('humanizeFinding sets category Wasted tokens for TOK/CPS', () => { - for (const s of ['TOK', 'CPS']) { +test('humanizeFinding sets category Wasted tokens for TOK/CPS/SKL', () => { + for (const s of ['TOK', 'CPS', 'SKL']) { const out = humanizeFinding(makeFinding({ scanner: s })); assert.equal(out.userImpactCategory, 'Wasted tokens'); } diff --git a/tests/scanners/scan-orchestrator-humanizer.test.mjs b/tests/scanners/scan-orchestrator-humanizer.test.mjs index 493c0ef..13c05a6 100644 --- a/tests/scanners/scan-orchestrator-humanizer.test.mjs +++ b/tests/scanners/scan-orchestrator-humanizer.test.mjs @@ -101,7 +101,7 @@ describe('scan-orchestrator humanizer wiring (Step 5)', () => { assert.ok(actual.meta, 'meta present'); assert.ok(Array.isArray(actual.scanners), 'scanners array present'); assert.ok(actual.aggregate, 'aggregate present'); - assert.equal(actual.scanners.length, 12, 'all 12 scanners present'); + assert.equal(actual.scanners.length, 13, 'all 13 scanners present'); }); it('preserves scanner shape (scanner/status/findings/counts)', async () => { diff --git a/tests/scanners/settings-validator.test.mjs b/tests/scanners/settings-validator.test.mjs index 85f90cf..1af2d61 100644 --- a/tests/scanners/settings-validator.test.mjs +++ b/tests/scanners/settings-validator.test.mjs @@ -131,7 +131,7 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi 'sandbox', 'fallbackModel', 'enforceAvailableModels', 'disableBundledSkills', 'pluginSuggestionMarketplaces', 'requiredMinimumVersion', 'requiredMaximumVersion', 'allowAllClaudeAiMcps', 'footerLinksRegexes', 'wheelScrollAccelerationEnabled', - 'parentSettingsBehavior', + 'parentSettingsBehavior', 'skillOverrides', ]; beforeEach(async () => { @@ -152,6 +152,7 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi footerLinksRegexes: ['^https://internal\\.'], // CC 2.1.181 wheelScrollAccelerationEnabled: true, // CC 2.1.174 parentSettingsBehavior: 'merge', // CC 2.1.133 + skillOverrides: { 'example-skill': 'name-only' }, // CC 2.1.129 (per-skill object) permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] }, }; await writeFile( diff --git a/tests/scanners/skill-listing-scanner.test.mjs b/tests/scanners/skill-listing-scanner.test.mjs new file mode 100644 index 0000000..ab98289 --- /dev/null +++ b/tests/scanners/skill-listing-scanner.test.mjs @@ -0,0 +1,189 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resetCounter } from '../../scanners/lib/output.mjs'; +import { scan } from '../../scanners/skill-listing-scanner.mjs'; + +const CAP = 1536; // verified per-description listing cap (CC 2.1.105, changelog L1502) + +function uniqueDir(suffix) { + return join(tmpdir(), `config-audit-skl-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); +} + +/** + * The SKL scanner enumerates active skills via process.env.HOME + * (enumeratePlugins/enumerateSkills). Tests must override HOME, run, restore — + * never rely on the developer's real ~/.claude. + */ +async function runScannerWithHome(home) { + resetCounter(); + const original = process.env.HOME; + process.env.HOME = home; + try { + return await scan('/unused', { files: [] }); + } finally { + process.env.HOME = original; + } +} + +/** Build a fake HOME with one user skill whose description has `len` chars. */ +async function homeWithUserSkill(name, descLen) { + const home = uniqueDir(name); + const dir = join(home, '.claude', 'skills', name); + await mkdir(dir, { recursive: true }); + const desc = 'a'.repeat(descLen); + await writeFile( + join(dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`, + ); + return home; +} + +describe('SKL scanner — basic structure', () => { + it('reports scanner prefix SKL', async () => { + const home = uniqueDir('empty'); + try { + await mkdir(join(home, '.claude'), { recursive: true }); + const result = await runScannerWithHome(home); + assert.equal(result.scanner, 'SKL'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('finding IDs match CA-SKL-NNN pattern', async () => { + const home = await homeWithUserSkill('longdesc', CAP + 64); + try { + const result = await runScannerWithHome(home); + for (const f of result.findings) { + assert.match(f.id, /^CA-SKL-\d{3}$/); + } + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('SKL scanner — per-description 1536-char cap (CA-SKL-001)', () => { + it('flags a skill whose description exceeds the cap', async () => { + const home = await homeWithUserSkill('toolong', CAP + 100); + try { + const result = await runScannerWithHome(home); + const f = result.findings.find(x => /toolong/.test(x.evidence || x.description || '')); + assert.ok(f, `expected a cap finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`); + assert.match(f.id, /^CA-SKL-001$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('evidence carries the measured char count and the 1536 cap', async () => { + const home = await homeWithUserSkill('measured', CAP + 200); + try { + const result = await runScannerWithHome(home); + const f = result.findings.find(x => /measured/.test(x.evidence || '')); + assert.ok(f, 'expected a finding for the oversized skill'); + assert.match(String(f.evidence), new RegExp(String(CAP + 200))); + assert.match(String(f.evidence), new RegExp(String(CAP))); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('points file at the offending SKILL.md', async () => { + const home = await homeWithUserSkill('filepath', CAP + 1); + try { + const result = await runScannerWithHome(home); + const f = result.findings.find(x => /filepath/.test(x.evidence || '')); + assert.ok(f); + assert.match(String(f.file), /filepath[\\/]SKILL\.md$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('SKL scanner — boundary and negative cases', () => { + it('a description exactly at the cap (1536) yields no finding', async () => { + const home = await homeWithUserSkill('exact', CAP); + try { + const result = await runScannerWithHome(home); + assert.equal(result.findings.length, 0, + `expected 0 findings at the cap; got: ${result.findings.map(f => f.title).join(' | ')}`); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('one char over the cap (1537) yields a finding', async () => { + const home = await homeWithUserSkill('over', CAP + 1); + try { + const result = await runScannerWithHome(home); + assert.equal(result.findings.length, 1, + `expected 1 finding at cap+1; got: ${result.findings.map(f => f.title).join(' | ')}`); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('a short description yields no finding', async () => { + const home = await homeWithUserSkill('short', 80); + try { + const result = await runScannerWithHome(home); + assert.equal(result.findings.length, 0, + `expected 0 findings; got: ${result.findings.map(f => f.title).join(' | ')}`); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('an empty HOME (no skills) yields zero findings', async () => { + const home = uniqueDir('noskills'); + try { + await mkdir(join(home, '.claude'), { recursive: true }); + const result = await runScannerWithHome(home); + assert.equal(result.findings.length, 0); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('SKL scanner — remediation levers (Item 2b folded in)', () => { + it('recommendation lists disableBundledSkills, skillOverrides, and trimming', async () => { + const home = await homeWithUserSkill('levers', CAP + 300); + try { + const result = await runScannerWithHome(home); + const f = result.findings.find(x => /levers/.test(x.evidence || '')); + assert.ok(f, 'expected a finding to carry remediation'); + const rec = String(f.recommendation); + assert.match(rec, /disableBundledSkills/); + assert.match(rec, /skillOverrides/); + assert.match(rec, /trim/i); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('SKL scanner — suppression compatibility', () => { + it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => { + const { applySuppressions } = await import('../../scanners/lib/suppression.mjs'); + const home = await homeWithUserSkill('suppress', CAP + 50); + try { + const result = await runScannerWithHome(home); + assert.ok(result.findings.length > 0, 'precondition: at least one SKL finding'); + const { active, suppressed } = applySuppressions(result.findings, [ + { pattern: 'CA-TOK-*', source: 'test', sourceLine: 1 }, + ]); + assert.equal(active.length, result.findings.length, + 'CA-TOK-* glob should not match CA-SKL-* findings'); + assert.equal(suppressed.length, 0); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/snapshots/default-output/posture.json b/tests/snapshots/default-output/posture.json index 8f1e118..935728d 100644 --- a/tests/snapshots/default-output/posture.json +++ b/tests/snapshots/default-output/posture.json @@ -1,4 +1,4 @@ { "kind": "text", - "payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + "payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" } diff --git a/tests/snapshots/default-output/scan-orchestrator.json b/tests/snapshots/default-output/scan-orchestrator.json index 534c529..2a017e3 100644 --- a/tests/snapshots/default-output/scan-orchestrator.json +++ b/tests/snapshots/default-output/scan-orchestrator.json @@ -539,6 +539,20 @@ "low": 0, "info": 0 } + }, + { + "scanner": "SKL", + "status": "ok", + "files_scanned": 0, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } } ], "aggregate": { @@ -553,7 +567,7 @@ "risk_score": 11, "risk_band": "Medium", "verdict": "PASS", - "scanners_ok": 11, + "scanners_ok": 12, "scanners_error": 0, "scanners_skipped": 1 } diff --git a/tests/snapshots/v5.0.0-stderr/posture.txt b/tests/snapshots/v5.0.0-stderr/posture.txt index 167d420..e1e5c09 100644 --- a/tests/snapshots/v5.0.0-stderr/posture.txt +++ b/tests/snapshots/v5.0.0-stderr/posture.txt @@ -1,15 +1,16 @@ [CML] CLAUDE.md Linter: 1 finding(s) (14ms) - [SET] Settings Validator: 0 finding(s) (2ms) - [HKV] Hook Validator: 0 finding(s) (1ms) + [SET] Settings Validator: 0 finding(s) (1ms) + [HKV] Hook Validator: 0 finding(s) (0ms) [RUL] Rules Validator: 0 finding(s) (0ms) [MCP] MCP Config Validator: 0 finding(s) (0ms) - [IMP] Import Resolver: 0 finding(s) (2ms) + [IMP] Import Resolver: 0 finding(s) (1ms) [CNF] Conflict Detector: 0 finding(s) (1ms) [GAP] Feature Gap Scanner: 17 finding(s) (2ms) [TOK] Token Hotspots: 1 finding(s) (8ms) - [CPS] Cache-Prefix Stability: 0 finding(s) (0ms) - [DIS] Disabled-In-Schema: 0 finding(s) (1ms) - [COL] Plugin Skill Collision: 0 finding(s) (0ms) + [CPS] Cache-Prefix Stability: 0 finding(s) (1ms) + [DIS] Disabled-In-Schema: 0 finding(s) (0ms) + [COL] Plugin Skill Collision: 0 finding(s) (1ms) + [SKL] Skill-Listing Budget: 0 finding(s) (0ms) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Config-Audit Health Score diff --git a/tests/snapshots/v5.0.0/posture.json b/tests/snapshots/v5.0.0/posture.json index 9d897b2..0d7d2ff 100644 --- a/tests/snapshots/v5.0.0/posture.json +++ b/tests/snapshots/v5.0.0/posture.json @@ -94,7 +94,7 @@ "scannerEnvelope": { "meta": { "target": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium", - "timestamp": "2026-06-18T12:19:30.125Z", + "timestamp": "2026-06-18T15:31:48.663Z", "version": "2.2.0", "tool": "config-audit" }, @@ -103,7 +103,7 @@ "scanner": "CML", "status": "ok", "files_scanned": 1, - "duration_ms": 2, + "duration_ms": 3, "findings": [ { "id": "CA-CML-001", @@ -145,7 +145,7 @@ "scanner": "HKV", "status": "ok", "files_scanned": 1, - "duration_ms": 2, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -173,7 +173,7 @@ "scanner": "MCP", "status": "ok", "files_scanned": 1, - "duration_ms": 0, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -187,7 +187,7 @@ "scanner": "IMP", "status": "ok", "files_scanned": 1, - "duration_ms": 2, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -526,7 +526,7 @@ ], "total_estimated_tokens": 801, "activeConfig": { - "claudeMdEstimatedTokens": 1639, + "claudeMdEstimatedTokens": 1647, "mcpServerCount": 1, "pluginCount": 0, "skillCount": 0 @@ -536,7 +536,7 @@ "scanner": "CPS", "status": "ok", "files_scanned": 1, - "duration_ms": 0, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -573,6 +573,20 @@ "low": 0, "info": 0 } + }, + { + "scanner": "SKL", + "status": "ok", + "files_scanned": 0, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } } ], "aggregate": { @@ -587,7 +601,7 @@ "risk_score": 11, "risk_band": "Medium", "verdict": "PASS", - "scanners_ok": 11, + "scanners_ok": 12, "scanners_error": 0, "scanners_skipped": 1 } diff --git a/tests/snapshots/v5.0.0/scan-orchestrator.json b/tests/snapshots/v5.0.0/scan-orchestrator.json index 4163784..e628935 100644 --- a/tests/snapshots/v5.0.0/scan-orchestrator.json +++ b/tests/snapshots/v5.0.0/scan-orchestrator.json @@ -1,7 +1,7 @@ { "meta": { "target": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium", - "timestamp": "2026-06-18T12:19:29.934Z", + "timestamp": "2026-06-18T15:31:48.479Z", "version": "2.2.0", "tool": "config-audit" }, @@ -80,7 +80,7 @@ "scanner": "MCP", "status": "ok", "files_scanned": 1, - "duration_ms": 0, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -94,7 +94,7 @@ "scanner": "IMP", "status": "ok", "files_scanned": 1, - "duration_ms": 2, + "duration_ms": 1, "findings": [], "counts": { "critical": 0, @@ -108,7 +108,7 @@ "scanner": "CNF", "status": "ok", "files_scanned": 2, - "duration_ms": 1, + "duration_ms": 0, "findings": [], "counts": { "critical": 0, @@ -433,7 +433,7 @@ ], "total_estimated_tokens": 801, "activeConfig": { - "claudeMdEstimatedTokens": 1639, + "claudeMdEstimatedTokens": 1647, "mcpServerCount": 1, "pluginCount": 0, "skillCount": 0 @@ -480,6 +480,20 @@ "low": 0, "info": 0 } + }, + { + "scanner": "SKL", + "status": "ok", + "files_scanned": 0, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } } ], "aggregate": { @@ -494,7 +508,7 @@ "risk_score": 11, "risk_band": "Medium", "verdict": "PASS", - "scanners_ok": 11, + "scanners_ok": 12, "scanners_error": 0, "scanners_skipped": 1 }