diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index 0a91879..1c0745d 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -18,7 +18,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `import-resolver.mjs` | IMP | Broken @imports, circular refs, deep chains, tilde paths | | `conflict-detector.mjs` | CNF | Settings conflicts, permission contradictions, hook duplicates | | `feature-gap-scanner.mjs` | GAP | 25 feature checks across 4 tiers — shown as opportunities, not grades | -| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascade, bloated SKILL.md descriptions, MCP tool-schema budget (prompt-cache patterns) | +| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascade, bloated SKILL.md descriptions, MCP tool-schema budget, MCP tool-schema deferral (CA-TOK-006), stale plugin-cache disk-cleanup (prompt-cache patterns) | | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of CLAUDE.md cascade (beyond Pattern A's top-30 window) | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries (low). (1) Tools in BOTH `permissions.deny` AND `permissions.allow` — deny wins; dominance is param-aware and treats the `Tool(*)` deny-all glob as equivalent to a bare deny (covers a bare allow). (2) Unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — CC accepts allow globs only after a literal glob-free `mcp____` prefix. Predicates shared with CNF live in `lib/permission-rules.mjs` | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions (low); user-vs-plugin overlaps (medium); `details.namespaces` payload | @@ -146,6 +146,45 @@ compare (proves the original schema is byte-identical), and the **SC-5 default-o track current output — diff reviewed as additive-only. **Lesson for any future hotspot/scanner-output field:** grep every frozen-v5.0.0 comparator (it is 5 files, not 2) before assuming the blast radius. +### token-hotspots — MCP tool-schema deferral (v5.10 B4, CA-TOK-006) + +By default Claude Code **defers** MCP tool schemas: only tool *names* enter the always-loaded prefix +(~120 tokens total) and full schemas load on demand via tool search. Several signals force the FULL +schemas into the prefix every turn instead. CA-TOK-006 detects them from **config files only**, so the +finding is deterministic and hermetic-safe (mirrors Pattern G's project-local scoping): + +| Signal | Source | Confidence | +|--------|--------|------------| +| `ENABLE_TOOL_SEARCH: "false"` | merged project+local settings.json `env` block | high | +| `"ToolSearch"` in `permissions.deny` | settings.json | high | +| configured `model` matches `/haiku/` | settings.json (Haiku lacks `tool_reference` support) | medium | +| per-server `alwaysLoad: true` | project `.mcp.json` (CC v2.1.121+) | high | +| `ENABLE_TOOL_SEARCH: "auto[:N]"` | settings `env` | threshold mode — **info, not a trigger** | + +The engine (`lib/mcp-deferral.mjs`) splits a **pure** `assessMcpDeferral({settings, mcpServers})` +(fully unit-tested, no IO) from a thin IO wrapper `assessMcpDeferralForRepo(repoPath, {mcpServers})` +shared by TOK and GAP. Severity scales with the aggregate forced-upfront token cost +(`severityForForcedSchemas`: ≥5000→high, ≥1500→medium; **medium-confidence reasons cap at medium**). + +**Honest scoping decision (Verifiseringsplikt).** The detector deliberately does **NOT** read +`process.env` shell vars. Tool search is also disabled on **Vertex AI**, with a custom +**`ANTHROPIC_BASE_URL`** (non-first-party host), or after a runtime **`/model`** switch to Haiku — but +those are launch/runtime state, not config files, so triggering on them would make the finding +machine-dependent (the marketplace-medium snapshot has MCP servers; an ambient `ANTHROPIC_BASE_URL` +would flap it). They are **disclosed** in every finding (`DEFERRAL_DISCLOSURE`), never triggered. +Tool-level `anthropic/alwaysLoad` (set server-side in the `tools/list` `_meta`) and claude.ai +connectors are likewise invisible to a static scan and disclosed. Mechanism verified 2026-06-23 +against `code.claude.com/docs`: `context-window.md` (MCP deferred, ~120 tok), +`mcp.md#configure-tool-search` + `#exempt-a-server-from-deferral`, `costs.md`. The prefix-cache +connect/disconnect-invalidation claim from the raw research was **`[NOT CONFIRMED]`** in docs and is +NOT asserted by this finding. + +**feature-gap companion.** `cliOverMcpLeverFinding` (GAP) fires **only** when CA-TOK-006's assessment +shows schemas forced upfront — recommends preferring CLI (`gh`/`aws`/`gcloud`) over MCP for common +operations (CLI adds zero context tokens until invoked). Deferred MCP is effectively free, so the +lever stays silent in the default case (opportunity, not noise — mirrors the bundledSkills lever). +`alwaysLoad` was added to CA-MCP's `VALID_SERVER_FIELDS` so it is never flagged as an unknown field. + ### CML scanner — context-window-scaled char budget Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors diff --git a/scanners/feature-gap-scanner.mjs b/scanners/feature-gap-scanner.mjs index 9e4c528..e6b4451 100644 --- a/scanners/feature-gap-scanner.mjs +++ b/scanners/feature-gap-scanner.mjs @@ -13,6 +13,7 @@ import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { measureActiveSkillListing, isBundledSkillsDisabled, BUDGET_CALIBRATION_NOTE } from './lib/skill-listing-budget.mjs'; +import { assessMcpDeferralForRepo } from './lib/mcp-deferral.mjs'; const SCANNER = 'GAP'; @@ -134,6 +135,46 @@ export function bundledSkillsLeverFinding({ leverPulled, aggregate }) { }); } +/** + * CLI-over-MCP lever — remediation companion to CA-TOK-006 (v5.10 B4). + * + * Fires ONLY when MCP tool schemas are forced into the always-loaded prefix + * (tool search disabled, or a per-server alwaysLoad), i.e. when MCP is actually + * costing always-loaded tokens. When schemas are deferred (the default), MCP is + * effectively free until used, so there is nothing to recommend and we stay + * silent — opportunity, not noise (mirrors the bundledSkills lever's "fire only + * under measured pressure" contract). CLI tools (gh / aws / gcloud) add zero + * context tokens until invoked, so they are the lever the deferral mechanism + * cannot reach for the forced-upfront servers. + * + * Pure and exported for unit testing. + * + * @param {{ assessment: (import('./lib/mcp-deferral.mjs').assessMcpDeferral)|null }} args + * @returns {object|null} a GAP finding, or null when nothing is forced upfront + */ +export function cliOverMcpLeverFinding({ assessment } = {}) { + if (!assessment || !assessment.forcedUpfront) return null; + const names = (assessment.affectedServers || []).map((m) => m.name).join(', '); + return finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Prefer CLI over MCP for common operations', + description: + `Your active project MCP tool schemas (~${assessment.aggregateTokens} tokens) are forced into the ` + + 'always-loaded prefix every turn rather than deferred (see CA-TOK-006). CLI tools (gh, aws, gcloud, …) ' + + 'add ZERO context tokens until you actually call them, so moving common operations off MCP and onto a ' + + 'CLI reclaims always-loaded budget the deferral mechanism cannot.', + evidence: + `forced_schema_tokens~${assessment.aggregateTokens}; servers=${names}; ` + + `reason=${assessment.reason || 'alwaysLoad'} (companion to CA-TOK-006)`, + recommendation: + 'For operations a CLI already covers (GitHub → gh, AWS → aws, GCP → gcloud), prefer the CLI over an ' + + 'MCP server — CLI output enters context only when invoked. Keep MCP for capabilities with no CLI ' + + 'equivalent, disable unused servers via /mcp, and re-enable tool-search deferral so the rest stay names-only.', + category: 'token-efficiency', + }); +} + /** @type {GapCheck[]} */ const GAP_CHECKS = [ // --- Tier 1: Foundation --- @@ -441,6 +482,13 @@ export async function scan(targetPath, sharedDiscovery) { const leverFinding = bundledSkillsLeverFinding({ leverPulled, aggregate }); if (leverFinding) findings.push(leverFinding); + // CLI-over-MCP lever — companion to CA-TOK-006: fires only when project-local + // MCP tool schemas are forced into the always-loaded prefix (tool search + // disabled or a per-server alwaysLoad). Reuses the same static assessment. + const mcpAssessment = await assessMcpDeferralForRepo(ctx.targetPath); + const cliLever = cliOverMcpLeverFinding({ assessment: mcpAssessment }); + if (cliLever) findings.push(cliLever); + const filesScanned = discovery.files.length; return scannerResult(SCANNER, 'ok', findings, filesScanned, Date.now() - start); } diff --git a/scanners/lib/active-config-reader.mjs b/scanners/lib/active-config-reader.mjs index a74f8cd..5fa017e 100644 --- a/scanners/lib/active-config-reader.mjs +++ b/scanners/lib/active-config-reader.mjs @@ -788,6 +788,7 @@ export async function readActiveMcpServers(repoPath, claudeJsonSlice = null, plu toolCount, toolCountUnknown: detected.toolCountUnknown, estimatedTokens: estimateTokens(0, 'mcp', { toolCount: toolCount ?? 0 }), + alwaysLoad: def?.alwaysLoad === true, }); } @@ -817,6 +818,7 @@ async function collectMcpFromFile(path, source, disabled, out, repoPath) { toolCount, toolCountUnknown: detected.toolCountUnknown, estimatedTokens: estimateTokens(0, 'mcp', { toolCount: toolCount ?? 0 }), + alwaysLoad: def?.alwaysLoad === true, }); } } diff --git a/scanners/lib/mcp-deferral.mjs b/scanners/lib/mcp-deferral.mjs new file mode 100644 index 0000000..528d140 --- /dev/null +++ b/scanners/lib/mcp-deferral.mjs @@ -0,0 +1,206 @@ +/** + * MCP tool-schema deferral assessment (v5.10 B4). + * + * By default Claude Code DEFERS MCP tool schemas: only tool *names* enter the + * always-loaded prefix (~120 tokens total) and full schemas load on demand via + * tool search. Certain conditions force ALL full schemas into the always-loaded + * prefix instead — paid on every turn whether or not a tool is used. + * + * This module is a STATIC, CONFIG-FILE assessment. It triggers only on signals + * that live in config files a scanner can read deterministically: + * - settings.json `env.ENABLE_TOOL_SEARCH` = "false" (HIGH confidence) + * - settings.json `permissions.deny` contains "ToolSearch" (HIGH confidence) + * - settings.json `model` is a Haiku model (MEDIUM confidence) + * - a per-server `.mcp.json` `alwaysLoad: true` (HIGH confidence) + * + * It deliberately does NOT read process.env shell variables. Tool search is also + * disabled on Vertex AI, with a custom ANTHROPIC_BASE_URL (non-first-party host), + * or after a runtime `/model` switch to Haiku — but those are launch/runtime + * state, not config files, so a static scan cannot see them without becoming + * machine-dependent. They are disclosed (DEFERRAL_DISCLOSURE), never triggered. + * + * Mechanism verified 2026-06-23 against code.claude.com/docs: + * context-window.md (MCP tools deferred, ~120 tok), mcp.md#configure-tool-search + * + #exempt-a-server-from-deferral, costs.md (reduce MCP server overhead). + * + * The pure `assessMcpDeferral` takes already-parsed inputs so it is fully + * unit-testable without file IO. `assessMcpDeferralForRepo` is the thin IO + * wrapper shared by token-hotspots and feature-gap. + */ + +import { resolve } from 'node:path'; +import { readTextFile } from './file-discovery.mjs'; +import { parseJson } from './yaml-parser.mjs'; +import { readActiveMcpServers } from './active-config-reader.mjs'; + +// Aggregate forced-upfront schema cost (tokens) → severity ladder. MCP token +// estimates are base 500 + ~200/tool (active-config-reader estimateTokens), so +// these anchor on a couple of small servers (medium) vs a large one / several +// (high). Heuristic, not measured — disclosed in every finding. +export const FORCED_SCHEMA_TOKENS_MEDIUM = 1500; +export const FORCED_SCHEMA_TOKENS_HIGH = 5000; + +// Appended to every CA-TOK-006 finding: the launch/runtime conditions a static +// config scan cannot see, so the user knows to check them manually. +export const DEFERRAL_DISCLOSURE = + 'static config-file check: tool search is ALSO disabled (all MCP schemas forced ' + + 'upfront) on Vertex AI, with a custom ANTHROPIC_BASE_URL (non-first-party host), ' + + 'or after a runtime /model switch to a Haiku model — none visible to a static ' + + 'scan, so verify at launch. Tool-level "anthropic/alwaysLoad" set server-side is ' + + 'likewise invisible. Per-server alwaysLoad requires Claude Code v2.1.121+.'; + +/** + * Does a permissions.deny list disable the ToolSearch tool? Matches a bare + * "ToolSearch" tool name (with or without an argument suffix), same shape Claude + * Code uses for built-in tool denies. + */ +function denyListDisablesToolSearch(deny) { + if (!Array.isArray(deny)) return false; + return deny.some((entry) => { + if (typeof entry !== 'string') return false; + const tool = entry.replace(/\(.*\)$/, '').trim(); + return tool === 'ToolSearch'; + }); +} + +function isHaikuModel(model) { + return typeof model === 'string' && /haiku/i.test(model); +} + +/** + * Map severity for the forced-upfront aggregate. High-confidence reasons scale + * with the token cost; medium-confidence reasons (inferred from a configured + * model) are capped at medium so the finding never overstates certainty. + * + * @param {number} aggregateTokens + * @param {'high'|'medium'} confidence + * @returns {'high'|'medium'|'low'} + */ +export function severityForForcedSchemas(aggregateTokens, confidence) { + const tok = typeof aggregateTokens === 'number' ? aggregateTokens : 0; + if (confidence === 'high') { + if (tok >= FORCED_SCHEMA_TOKENS_HIGH) return 'high'; + if (tok >= FORCED_SCHEMA_TOKENS_MEDIUM) return 'medium'; + return 'low'; + } + // medium confidence: cap at medium + if (tok >= FORCED_SCHEMA_TOKENS_HIGH) return 'medium'; + return 'low'; +} + +/** + * Assess whether MCP tool schemas are forced into the always-loaded prefix. + * + * @param {object} args + * @param {{ env?: object, permissions?: { deny?: string[] }, model?: string }} [args.settings] + * merged settings.json view (env block, permissions, model). + * @param {Array<{name:string, source?:string, enabled?:boolean, toolCount?:number, + * estimatedTokens?:number, alwaysLoad?:boolean}>} [args.mcpServers] + * @returns {{ + * toolSearchDisabled: boolean, reason: string|null, + * confidence: 'high'|'medium'|null, thresholdMode: boolean, + * alwaysLoadServers: object[], affectedServers: object[], + * aggregateTokens: number, forcedUpfront: boolean, + * }} + */ +export function assessMcpDeferral({ settings = {}, mcpServers = [] } = {}) { + const s = settings || {}; + const tsRaw = s.env && typeof s.env === 'object' ? s.env.ENABLE_TOOL_SEARCH : undefined; + const ts = String(tsRaw ?? '').trim().toLowerCase(); + const denyTS = denyListDisablesToolSearch(s.permissions?.deny); + const haiku = isHaikuModel(s.model); + + let toolSearchDisabled = false; + let reason = null; + let confidence = null; + let thresholdMode = false; + + if (ts === 'false') { + toolSearchDisabled = true; + reason = 'enable-tool-search-false'; + confidence = 'high'; + } else if (denyTS) { + toolSearchDisabled = true; + reason = 'deny-tool-search'; + confidence = 'high'; + } else if (haiku) { + // Haiku lacks tool_reference support, so tool search cannot run even when + // ENABLE_TOOL_SEARCH=true. Medium confidence: the configured model can be + // switched at runtime (/model), which a static scan cannot observe. + toolSearchDisabled = true; + reason = 'haiku-model'; + confidence = 'medium'; + } else if (ts === 'true') { + toolSearchDisabled = false; + } else if (ts.startsWith('auto')) { + // Threshold mode (auto / auto:N): schemas load upfront only when they fit a + // percentage of the context window — not a clear always-load. Info, not a + // forced-upfront trigger. + thresholdMode = true; + } + + const active = (Array.isArray(mcpServers) ? mcpServers : []).filter( + (m) => m && m.enabled !== false, + ); + const alwaysLoadServers = active.filter((m) => m.alwaysLoad === true); + const affectedServers = toolSearchDisabled ? active : alwaysLoadServers; + const aggregateTokens = affectedServers.reduce( + (sum, m) => sum + (typeof m.estimatedTokens === 'number' ? m.estimatedTokens : 0), + 0, + ); + + return { + toolSearchDisabled, + reason, + confidence, + thresholdMode, + alwaysLoadServers, + affectedServers, + aggregateTokens, + forcedUpfront: affectedServers.length > 0, + }; +} + +/** + * Merge project + local settings.json for the deferral check. Scoped to the + * audited path (NOT the user cascade) so the result stays deterministic and free + * of ambient HOME leakage — mirrors Pattern G's project-local scoping. Local + * overrides project; env/permissions shallow-merge, deny lists concatenate. + */ +async function readMergedProjectSettings(repoPath) { + const merged = { env: {}, permissions: {}, model: undefined }; + for (const rel of ['.claude/settings.json', '.claude/settings.local.json']) { + const content = await readTextFile(resolve(repoPath, rel)); + if (!content) continue; + const parsed = parseJson(content); + if (!parsed || typeof parsed !== 'object') continue; + if (parsed.env && typeof parsed.env === 'object') Object.assign(merged.env, parsed.env); + if (parsed.permissions && typeof parsed.permissions === 'object') { + const deny = [ + ...(Array.isArray(merged.permissions.deny) ? merged.permissions.deny : []), + ...(Array.isArray(parsed.permissions.deny) ? parsed.permissions.deny : []), + ]; + merged.permissions = { ...merged.permissions, ...parsed.permissions, deny }; + } + if (typeof parsed.model === 'string') merged.model = parsed.model; + } + return merged; +} + +/** + * IO wrapper: assess MCP deferral for a repo path. Scopes MCP servers to active + * project-local `.mcp.json` (plugin / ~/.claude.json servers are the manifest's + * concern). Pass `mcpServers` (e.g. an already-loaded activeConfig.mcpServers) to + * avoid a re-read; otherwise they are read via readActiveMcpServers. + * + * @param {string} repoPath + * @param {{ mcpServers?: object[] }} [opts] + */ +export async function assessMcpDeferralForRepo(repoPath, opts = {}) { + const all = Array.isArray(opts.mcpServers) + ? opts.mcpServers + : await readActiveMcpServers(repoPath); + const projectLocal = all.filter((m) => m && m.enabled && m.source === '.mcp.json'); + const settings = await readMergedProjectSettings(repoPath); + return assessMcpDeferral({ settings, mcpServers: projectLocal }); +} diff --git a/scanners/mcp-config-validator.mjs b/scanners/mcp-config-validator.mjs index 010dfa3..5e4b742 100644 --- a/scanners/mcp-config-validator.mjs +++ b/scanners/mcp-config-validator.mjs @@ -18,6 +18,9 @@ const VALID_SERVER_TYPES = new Set(['stdio', 'http', 'sse']); // not a per-server .mcp.json field. Verified against code.claude.com/docs 2026-06-18. const VALID_SERVER_FIELDS = new Set([ 'type', 'command', 'args', 'env', 'url', 'headers', 'timeout', + // alwaysLoad: exempt a server from MCP tool-schema deferral (CC v2.1.121+). + // Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral 2026-06-23. + 'alwaysLoad', ]); // Match only bare ${IDENTIFIER} references. POSIX expansions like ${VAR%pattern} diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index fc90e7f..1dc9eae 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -28,6 +28,11 @@ import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; +import { + assessMcpDeferralForRepo, + severityForForcedSchemas, + DEFERRAL_DISCLOSURE, +} from './lib/mcp-deferral.mjs'; const SCANNER = 'TOK'; @@ -580,6 +585,61 @@ export async function scan(targetPath, discovery) { })); } + // ── Pattern I: MCP tool-schema deferral (v5.10 B4, CA-TOK-006) ── + // By default MCP tool schemas are DEFERRED (names-only, ~120 tok); certain + // config signals force the FULL schemas into the always-loaded prefix every + // turn. Scope: project-local .mcp.json servers (mirrors Pattern G — plugin / + // global servers are the manifest's concern). Static config-file check only; + // runtime conditions (Vertex / ANTHROPIC_BASE_URL / runtime /model switch) are + // DISCLOSED (DEFERRAL_DISCLOSURE), not triggered, so the finding is deterministic. + const mcpForDeferral = (activeConfig && Array.isArray(activeConfig.mcpServers)) + ? activeConfig.mcpServers.filter(m => m && m.enabled && m.source === '.mcp.json') + : []; + if (mcpForDeferral.length > 0) { + const a = await assessMcpDeferralForRepo(targetPath, { mcpServers: activeConfig.mcpServers }); + if (a.forcedUpfront) { + const conf = a.confidence || 'high'; + const severity = severityForForcedSchemas(a.aggregateTokens, conf); + const names = a.affectedServers.map(m => m.name).join(', '); + const reasonText = { + 'enable-tool-search-false': 'ENABLE_TOOL_SEARCH is set to "false" in settings', + 'deny-tool-search': '"ToolSearch" is listed in permissions.deny', + 'haiku-model': 'the configured model is a Haiku model (Haiku has no tool-search support)', + }[a.reason]; + const description = a.toolSearchDisabled + ? `Tool search is disabled (${reasonText}), so the full tool schemas of ` + + `${a.affectedServers.length} active project MCP server${a.affectedServers.length === 1 ? '' : 's'} ` + + `(~${a.aggregateTokens} tokens) load into the always-loaded prefix on every turn instead of ` + + 'being deferred (tool names only, ~120 tokens total). Every schema token is re-sent each turn ' + + 'whether or not a tool is used.' + : `${a.alwaysLoadServers.length} project MCP server${a.alwaysLoadServers.length === 1 ? '' : 's'} ` + + `marked alwaysLoad (${names}) load their full tool schemas (~${a.aggregateTokens} tokens) into ` + + 'the always-loaded prefix on every turn regardless of tool search, instead of deferring them ' + + '(names only, ~120 tokens).'; + const evidence = + `reason=${a.reason || 'alwaysLoad'}; confidence=${conf}; servers=${names}; ` + + `forced_schema_tokens~${a.aggregateTokens} — ${CALIBRATION_NOTE}. ${DEFERRAL_DISCLOSURE}`; + const recommendation = a.toolSearchDisabled + ? 'Re-enable tool search so MCP schemas defer (names-only) by default: remove ' + + 'ENABLE_TOOL_SEARCH="false" / the "ToolSearch" deny, or stop defaulting to a Haiku model. ' + + 'Also disable unused servers via /mcp, and prefer CLI tools (gh / aws / gcloud) over MCP for ' + + 'common operations — CLI adds zero context tokens until used.' + : 'Drop alwaysLoad on large-schema servers so they defer (names-only) until a tool is needed; ' + + 'keep alwaysLoad only for small servers you call on most turns. Prefer CLI tools ' + + '(gh / aws / gcloud) over MCP for common operations.'; + findings.push(finding({ + scanner: SCANNER, + severity, + title: 'MCP tool schemas forced into the always-loaded prefix', + file: null, + evidence, + description, + recommendation, + category: 'token-efficiency', + })); + } + } + // ── Hotspots ranking ── const hotspots = await buildHotspots(discovery, targetPath, activeConfig); diff --git a/tests/fixtures/mcp-deferral/alwaysload/.mcp.json b/tests/fixtures/mcp-deferral/alwaysload/.mcp.json new file mode 100644 index 0000000..83df111 --- /dev/null +++ b/tests/fixtures/mcp-deferral/alwaysload/.mcp.json @@ -0,0 +1,6 @@ +{ + "mcpServers": { + "always-srv": { "command": "npx", "args": ["fake-pkg"], "alwaysLoad": true, "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }, + "deferred-srv": { "command": "npx", "args": ["other-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/fixtures/mcp-deferral/deferred/.mcp.json b/tests/fixtures/mcp-deferral/deferred/.mcp.json new file mode 100644 index 0000000..7be9be3 --- /dev/null +++ b/tests/fixtures/mcp-deferral/deferred/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "plain-srv": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json b/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json new file mode 100644 index 0000000..10a1000 --- /dev/null +++ b/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "srv-a": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/scanners/feature-gap-scanner.test.mjs b/tests/scanners/feature-gap-scanner.test.mjs index 91294bb..1e0066a 100644 --- a/tests/scanners/feature-gap-scanner.test.mjs +++ b/tests/scanners/feature-gap-scanner.test.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; -import { scan, opportunitySummary, bundledSkillsLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; +import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; import { withHermeticHome } from '../helpers/hermetic-home.mjs'; @@ -340,3 +340,27 @@ describe('GAP scanner — disableBundledSkills lever wiring (HOME-scoped)', () = } }); }); + +describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => { + it('returns null when nothing is forced upfront', () => { + assert.equal(cliOverMcpLeverFinding({ assessment: { forcedUpfront: false } }), null); + assert.equal(cliOverMcpLeverFinding({ assessment: null }), null); + assert.equal(cliOverMcpLeverFinding({}), null); + }); + + it('fires a low-severity opportunity when MCP schemas are forced upfront', () => { + resetCounter(); + const f = cliOverMcpLeverFinding({ + assessment: { + forcedUpfront: true, + aggregateTokens: 2500, + affectedServers: [{ name: 'srv-a' }], + reason: 'enable-tool-search-false', + }, + }); + assert.ok(f, 'expected a lever finding'); + assert.equal(f.severity, 'low'); + assert.equal(f.category, 'token-efficiency'); + assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/); + }); +}); diff --git a/tests/scanners/mcp-config-validator.test.mjs b/tests/scanners/mcp-config-validator.test.mjs index 95b971b..1f4440c 100644 --- a/tests/scanners/mcp-config-validator.test.mjs +++ b/tests/scanners/mcp-config-validator.test.mjs @@ -136,6 +136,36 @@ describe('MCP scanner — stray `trust` field is an unknown field (verify-first, }); }); +describe('MCP scanner — `alwaysLoad` is a valid field (v5.10 B4, verify-first 2026-06-23)', () => { + let result; + let tmpRoot; + + beforeEach(async () => { + resetCounter(); + tmpRoot = await mkdtemp(join(tmpdir(), 'ca-mcp-alwaysload-')); + // alwaysLoad exempts a server from MCP tool-schema deferral (CC v2.1.121+). + // Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral. + const mcp = { + mcpServers: { + core: { type: 'http', url: 'https://mcp.example.com/mcp', alwaysLoad: true }, + }, + }; + await writeFile(join(tmpRoot, '.mcp.json'), JSON.stringify(mcp, null, 2) + '\n', 'utf8'); + const discovery = await discoverConfigFiles(tmpRoot); + result = await scan(tmpRoot, discovery); + }); + + afterEach(async () => { + if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true }); + }); + + it('does NOT flag `alwaysLoad` as an unknown MCP server field', () => { + const f = result.findings.find(x => x.title.includes('Unknown MCP server field') + && /alwaysLoad/.test(x.description || '')); + assert.equal(f, undefined, 'alwaysLoad is a valid field and must not be flagged'); + }); +}); + describe('MCP scanner — env-var false positives (CC 2.1.139/2.1.142, Batch 1)', () => { let tmpRoot; let envFindings; diff --git a/tests/scanners/mcp-deferral.test.mjs b/tests/scanners/mcp-deferral.test.mjs new file mode 100644 index 0000000..d804526 --- /dev/null +++ b/tests/scanners/mcp-deferral.test.mjs @@ -0,0 +1,192 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + assessMcpDeferral, + severityForForcedSchemas, + FORCED_SCHEMA_TOKENS_HIGH, + FORCED_SCHEMA_TOKENS_MEDIUM, +} from '../../scanners/lib/mcp-deferral.mjs'; + +// A small helper to build server shapes the way active-config-reader exposes them. +const srv = (over = {}) => ({ + name: 'srv', + source: '.mcp.json', + enabled: true, + toolCount: 10, + estimatedTokens: 2500, + alwaysLoad: false, + ...over, +}); + +describe('assessMcpDeferral — default (deferred) case', () => { + it('no settings + a plain server → tool search NOT disabled, nothing forced upfront', () => { + const a = assessMcpDeferral({ settings: {}, mcpServers: [srv()] }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.reason, null); + assert.equal(a.confidence, null); + assert.equal(a.forcedUpfront, false); + assert.deepEqual(a.affectedServers, []); + assert.equal(a.aggregateTokens, 0); + }); + + it('empty inputs are tolerated', () => { + const a = assessMcpDeferral({}); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, false); + }); +}); + +describe('assessMcpDeferral — config-file disabling signals (HIGH confidence)', () => { + it('settings.env.ENABLE_TOOL_SEARCH="false" → disabled, high, all servers affected', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [srv({ name: 'a', estimatedTokens: 1000 }), srv({ name: 'b', estimatedTokens: 2000 })], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'enable-tool-search-false'); + assert.equal(a.confidence, 'high'); + assert.equal(a.forcedUpfront, true); + assert.equal(a.affectedServers.length, 2); + assert.equal(a.aggregateTokens, 3000); + }); + + it('permissions.deny including bare "ToolSearch" → disabled, high', () => { + const a = assessMcpDeferral({ + settings: { permissions: { deny: ['Read(./.env)', 'ToolSearch'] } }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'deny-tool-search'); + assert.equal(a.confidence, 'high'); + }); + + it('explicit ENABLE_TOOL_SEARCH="false" wins over a haiku model for the reason label', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' }, model: 'claude-haiku-4-5' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'enable-tool-search-false'); + assert.equal(a.confidence, 'high'); + }); +}); + +describe('assessMcpDeferral — configured Haiku model (MEDIUM confidence)', () => { + it('settings.model matching /haiku/ → disabled, medium, haiku-model', () => { + const a = assessMcpDeferral({ + settings: { model: 'claude-haiku-4-5-20251001' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'haiku-model'); + assert.equal(a.confidence, 'medium'); + assert.equal(a.forcedUpfront, true); + }); + + it('haiku disables even when ENABLE_TOOL_SEARCH="true" (Haiku lacks tool_reference support)', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'haiku' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'haiku-model'); + }); +}); + +describe('assessMcpDeferral — non-disabling values', () => { + it('ENABLE_TOOL_SEARCH="true" (non-haiku) → NOT disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'claude-sonnet-4-6' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, false); + }); + + it('ENABLE_TOOL_SEARCH="auto" → threshold mode, NOT disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'auto' } }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.thresholdMode, true); + assert.equal(a.forcedUpfront, false); + }); + + it('ENABLE_TOOL_SEARCH="auto:5" → threshold mode', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'auto:5' } }, + mcpServers: [srv()], + }); + assert.equal(a.thresholdMode, true); + assert.equal(a.toolSearchDisabled, false); + }); +}); + +describe('assessMcpDeferral — per-server alwaysLoad', () => { + it('alwaysLoad:true server with tool search enabled → that server forced upfront only', () => { + const a = assessMcpDeferral({ + settings: {}, + mcpServers: [ + srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }), + srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 9000 }), + ], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, true); + assert.equal(a.alwaysLoadServers.length, 1); + assert.equal(a.affectedServers.length, 1); + assert.equal(a.affectedServers[0].name, 'always'); + assert.equal(a.aggregateTokens, 1500); + }); + + it('when tool search is disabled, ALL active servers are affected (not just alwaysLoad)', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [ + srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }), + srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 2500 }), + ], + }); + assert.equal(a.affectedServers.length, 2); + assert.equal(a.aggregateTokens, 4000); + }); +}); + +describe('assessMcpDeferral — server filtering', () => { + it('disabled servers (enabled:false) are excluded from affected + aggregate', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [ + srv({ name: 'on', enabled: true, estimatedTokens: 1000 }), + srv({ name: 'off', enabled: false, estimatedTokens: 5000 }), + ], + }); + assert.equal(a.affectedServers.length, 1); + assert.equal(a.affectedServers[0].name, 'on'); + assert.equal(a.aggregateTokens, 1000); + }); + + it('no active servers → not forced upfront even when tool search disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.forcedUpfront, false); + assert.equal(a.aggregateTokens, 0); + }); +}); + +describe('severityForForcedSchemas', () => { + it('high confidence scales with aggregate tokens', () => { + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH, 'high'), 'high'); + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_MEDIUM, 'high'), 'medium'); + assert.equal(severityForForcedSchemas(100, 'high'), 'low'); + }); + + it('medium confidence is capped at medium', () => { + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH * 10, 'medium'), 'medium'); + assert.equal(severityForForcedSchemas(100, 'medium'), 'low'); + }); +}); diff --git a/tests/scanners/token-hotspots-deferral.test.mjs b/tests/scanners/token-hotspots-deferral.test.mjs new file mode 100644 index 0000000..1b19fb7 --- /dev/null +++ b/tests/scanners/token-hotspots-deferral.test.mjs @@ -0,0 +1,70 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resetCounter } from '../../scanners/lib/output.mjs'; +import { scan } from '../../scanners/token-hotspots.mjs'; +import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; +import { withHermeticHome } from '../helpers/hermetic-home.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const FIXTURES = resolve(__dirname, '../fixtures'); + +// Hermetic HOME so readActiveConfig does not leak the developer's real +// ~/.claude.json plugin MCP servers into the fixture result (mirrors the other +// TOK tests). The deferral check reads project+local settings + project .mcp.json. +async function runScanner(fixtureName) { + resetCounter(); + const path = resolve(FIXTURES, fixtureName); + const discovery = await discoverConfigFiles(path); + return withHermeticHome(() => scan(path, discovery)); +} + +const TITLE = 'MCP tool schemas forced into the always-loaded prefix'; +const findDeferral = (result) => result.findings.find(f => f.title === TITLE); + +describe('TOK scanner — MCP deferral (v5.10 B4, CA-TOK-006)', () => { + it('deferred (default) project → NO deferral finding', async () => { + const result = await runScanner('mcp-deferral/deferred'); + assert.equal(findDeferral(result), undefined, + 'a plain server with tool search on must not be flagged as forced-upfront'); + }); + + it('alwaysLoad:true server → deferral finding fires (medium severity)', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f, `expected a deferral finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'medium', `expected medium for ~2500 forced tokens, got ${f.severity}`); + assert.equal(f.category, 'token-efficiency'); + // Only the alwaysLoad server is named, not the deferred sibling. + assert.match(String(f.evidence || ''), /always-srv/); + assert.doesNotMatch(String(f.description || ''), /deferred-srv/); + assert.match(String(f.evidence || ''), /alwaysLoad/); + }); + + it('ENABLE_TOOL_SEARCH="false" in settings → deferral finding (all servers affected)', async () => { + const result = await runScanner('mcp-deferral/disabled-settings'); + const f = findDeferral(result); + assert.ok(f, `expected a deferral finding for tool-search-disabled fixture`); + assert.match(String(f.evidence || ''), /reason=enable-tool-search-false/); + assert.match(String(f.description || ''), /Tool search is disabled/); + }); + + it('every deferral finding discloses runtime conditions + calibration', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f); + // DEFERRAL_DISCLOSURE names the launch/runtime conditions a static scan can't see. + assert.match(String(f.evidence || ''), /Vertex AI/); + assert.match(String(f.evidence || ''), /ANTHROPIC_BASE_URL/); + // CALIBRATION_NOTE shared by all TOK pattern findings. + assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i); + }); + + it('finding ID matches CA-TOK-NNN format', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f); + assert.match(f.id, /^CA-TOK-\d{3}$/); + }); +});