From b2407a09b3f114d2e784e950f66435994676ef58 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 07:29:57 +0200 Subject: [PATCH] feat(config-audit): CA-TOK-005 MCP tool-schema budget (v5 N1) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds detectMcpToolBudget detection block in TOK scanner. Tiered severity per project-local .mcp.json server based on toolCount: - < 20: no finding - 20-49: low - 50-99: medium - 100+: high - null (manifest unparseable): low + "tool count unknown" message Scoped to source==='.mcp.json' to keep findings actionable for the audited path; plugin/user-level MCP servers are surfaced by the manifest scanner (Step 19 / N2). 5 fixtures (mcp-budget/{14,25,60,120,unknown}-tools) use inline `tools` arrays in .mcp.json — no node_modules needed for these tests. Tests assert title+severity (not exact ID) since TOK IDs are sequential per scan, not semantic per pattern. [skip-docs] reason: v5 plan fences off README/CLAUDE.md badge updates to Session 5; Forgejo pre-commit-docs-gate hook requires this tag on feat commits without doc changes. Tests: 586 → 593 (+7). Co-Authored-By: Claude Opus 4.7 --- .../config-audit/scanners/token-hotspots.mjs | 76 +++++++++++++++++++ .../fixtures/mcp-budget/120-tools/.mcp.json | 5 ++ .../fixtures/mcp-budget/14-tools/.mcp.json | 5 ++ .../fixtures/mcp-budget/25-tools/.mcp.json | 5 ++ .../fixtures/mcp-budget/60-tools/.mcp.json | 5 ++ .../mcp-budget/unknown-tools/.mcp.json | 5 ++ .../tests/scanners/token-hotspots.test.mjs | 63 +++++++++++++++ 7 files changed, 164 insertions(+) create mode 100644 plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json create mode 100644 plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json create mode 100644 plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json create mode 100644 plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json create mode 100644 plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json diff --git a/plugins/config-audit/scanners/token-hotspots.mjs b/plugins/config-audit/scanners/token-hotspots.mjs index 3d64f80..f5617e2 100644 --- a/plugins/config-audit/scanners/token-hotspots.mjs +++ b/plugins/config-audit/scanners/token-hotspots.mjs @@ -52,6 +52,18 @@ const CASCADE_TOKEN_THRESHOLD = 10_000; // not. Anything past this hints the description is doing the body's job. const SKILL_DESCRIPTION_THRESHOLD = 500; +// v5 N1: MCP tool-schema budget thresholds (CA-TOK-005). Tool descriptions +// load on every turn — high tool counts inflate the per-turn schema payload +// regardless of whether the tools are invoked. Tiered severity per server: +// < 20 → no finding +// 20–49 → low +// 50–99 → medium +// 100+ → high +// null → low ("tool count unknown" — manifest not parseable) +const MCP_BUDGET_LOW = 20; +const MCP_BUDGET_MEDIUM = 50; +const MCP_BUDGET_HIGH = 100; + const HOTSPOTS_MAX = 10; // v5 F7: shared evidence note appended to every TOK pattern finding. @@ -122,6 +134,25 @@ async function maxImportDepth(startFile, contentCache) { return maxDepth; } +/** + * Classify an MCP server's tool count into a budget tier (v5 N1). + * + * Returns null if no finding should be emitted (toolCount < 20). Otherwise + * returns { severity, tier, kind } where kind is 'unknown' (toolCount===null) + * or 'counted'. Threshold ladder: 20 → low, 50 → medium, 100 → high. Null + * toolCount maps to low + 'unknown' so users can see opaque servers without + * the scanner pretending they're free. + */ +function classifyMcpToolBudget(toolCount) { + if (toolCount === null) { + return { severity: SEVERITY.low, tier: 'unknown', kind: 'unknown' }; + } + if (typeof toolCount !== 'number' || toolCount < MCP_BUDGET_LOW) return null; + if (toolCount >= MCP_BUDGET_HIGH) return { severity: SEVERITY.high, tier: '100+', kind: 'counted' }; + if (toolCount >= MCP_BUDGET_MEDIUM) return { severity: SEVERITY.medium, tier: '50-99', kind: 'counted' }; + return { severity: SEVERITY.low, tier: '20-49', kind: 'counted' }; +} + /** * Detect cache-breaking volatile content in the first VOLATILE_TOP_LINES * lines of a CLAUDE.md file. @@ -372,6 +403,51 @@ export async function scan(targetPath, discovery) { })); } + // ── Pattern G: MCP tool-schema budget per server (v5 N1, CA-TOK-005) ── + // Scope: project-local .mcp.json only. Plugin- and ~/.claude.json-sourced + // servers are global concerns surfaced by the manifest scanner; scoping the + // finding here to .mcp.json keeps /config-audit actionable for the + // path the user is auditing. + if (activeConfig && Array.isArray(activeConfig.mcpServers)) { + for (const m of activeConfig.mcpServers) { + if (!m || !m.enabled) continue; + if (m.source !== '.mcp.json') continue; + const budget = classifyMcpToolBudget(m.toolCount); + if (!budget) continue; + const severity = budget.severity; + const sourceLabel = m.source ? `${m.name} (${m.source})` : m.name; + const isUnknown = budget.kind === 'unknown'; + const description = isUnknown + ? `MCP server "${sourceLabel}" has tool count unknown — could not parse manifest ` + + 'or cached tools/list. Tool schemas load on every turn; an unverified server ' + + 'may be inflating the per-turn payload silently.' + : `MCP server "${sourceLabel}" exposes ${m.toolCount} tools. Tool schemas load on ` + + 'every turn regardless of which tools the model actually invokes — high tool ' + + 'counts inflate the per-turn payload and crowd out usable context.'; + const evidence = isUnknown + ? `tool_count=unknown; server="${m.name}"; source="${m.source}" — ${CALIBRATION_NOTE}` + : `tool_count=${m.toolCount}; tier=${budget.tier}; server="${m.name}"; ` + + `source="${m.source}" — ${CALIBRATION_NOTE}`; + const recommendation = isUnknown + ? 'Install the package locally (so detect-mcp-tool-count can read its manifest), ' + + 'or run the server once and cache its tools/list response under ' + + '~/.claude/config-audit/mcp-cache/.json. See knowledge/cache-telemetry-recipe.md.' + : 'Use the server\'s `tools/filter` config (or equivalent) to expose only the tools ' + + 'this project actually needs. Consider splitting heavy MCP servers across project- ' + + 'and user-scopes so per-project budget stays tight.'; + findings.push(finding({ + scanner: SCANNER, + severity, + title: `High MCP tool-schema budget on server "${m.name}"`, + description, + file: m.source && m.source !== `mcp:${m.name}` ? m.source : null, + evidence, + recommendation, + category: 'token-efficiency', + })); + } + } + // ── Pattern E: CLAUDE.md cascade > CASCADE_TOKEN_THRESHOLD (v5 M4) ── if (activeConfig?.claudeMd?.estimatedTokens > CASCADE_TOKEN_THRESHOLD) { const cascadeTokens = activeConfig.claudeMd.estimatedTokens; diff --git a/plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json b/plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json new file mode 100644 index 0000000..905aa3a --- /dev/null +++ b/plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "budget-srv-120": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"},{"name":"t_25","description":"tool 25"},{"name":"t_26","description":"tool 26"},{"name":"t_27","description":"tool 27"},{"name":"t_28","description":"tool 28"},{"name":"t_29","description":"tool 29"},{"name":"t_30","description":"tool 30"},{"name":"t_31","description":"tool 31"},{"name":"t_32","description":"tool 32"},{"name":"t_33","description":"tool 33"},{"name":"t_34","description":"tool 34"},{"name":"t_35","description":"tool 35"},{"name":"t_36","description":"tool 36"},{"name":"t_37","description":"tool 37"},{"name":"t_38","description":"tool 38"},{"name":"t_39","description":"tool 39"},{"name":"t_40","description":"tool 40"},{"name":"t_41","description":"tool 41"},{"name":"t_42","description":"tool 42"},{"name":"t_43","description":"tool 43"},{"name":"t_44","description":"tool 44"},{"name":"t_45","description":"tool 45"},{"name":"t_46","description":"tool 46"},{"name":"t_47","description":"tool 47"},{"name":"t_48","description":"tool 48"},{"name":"t_49","description":"tool 49"},{"name":"t_50","description":"tool 50"},{"name":"t_51","description":"tool 51"},{"name":"t_52","description":"tool 52"},{"name":"t_53","description":"tool 53"},{"name":"t_54","description":"tool 54"},{"name":"t_55","description":"tool 55"},{"name":"t_56","description":"tool 56"},{"name":"t_57","description":"tool 57"},{"name":"t_58","description":"tool 58"},{"name":"t_59","description":"tool 59"},{"name":"t_60","description":"tool 60"},{"name":"t_61","description":"tool 61"},{"name":"t_62","description":"tool 62"},{"name":"t_63","description":"tool 63"},{"name":"t_64","description":"tool 64"},{"name":"t_65","description":"tool 65"},{"name":"t_66","description":"tool 66"},{"name":"t_67","description":"tool 67"},{"name":"t_68","description":"tool 68"},{"name":"t_69","description":"tool 69"},{"name":"t_70","description":"tool 70"},{"name":"t_71","description":"tool 71"},{"name":"t_72","description":"tool 72"},{"name":"t_73","description":"tool 73"},{"name":"t_74","description":"tool 74"},{"name":"t_75","description":"tool 75"},{"name":"t_76","description":"tool 76"},{"name":"t_77","description":"tool 77"},{"name":"t_78","description":"tool 78"},{"name":"t_79","description":"tool 79"},{"name":"t_80","description":"tool 80"},{"name":"t_81","description":"tool 81"},{"name":"t_82","description":"tool 82"},{"name":"t_83","description":"tool 83"},{"name":"t_84","description":"tool 84"},{"name":"t_85","description":"tool 85"},{"name":"t_86","description":"tool 86"},{"name":"t_87","description":"tool 87"},{"name":"t_88","description":"tool 88"},{"name":"t_89","description":"tool 89"},{"name":"t_90","description":"tool 90"},{"name":"t_91","description":"tool 91"},{"name":"t_92","description":"tool 92"},{"name":"t_93","description":"tool 93"},{"name":"t_94","description":"tool 94"},{"name":"t_95","description":"tool 95"},{"name":"t_96","description":"tool 96"},{"name":"t_97","description":"tool 97"},{"name":"t_98","description":"tool 98"},{"name":"t_99","description":"tool 99"},{"name":"t_100","description":"tool 100"},{"name":"t_101","description":"tool 101"},{"name":"t_102","description":"tool 102"},{"name":"t_103","description":"tool 103"},{"name":"t_104","description":"tool 104"},{"name":"t_105","description":"tool 105"},{"name":"t_106","description":"tool 106"},{"name":"t_107","description":"tool 107"},{"name":"t_108","description":"tool 108"},{"name":"t_109","description":"tool 109"},{"name":"t_110","description":"tool 110"},{"name":"t_111","description":"tool 111"},{"name":"t_112","description":"tool 112"},{"name":"t_113","description":"tool 113"},{"name":"t_114","description":"tool 114"},{"name":"t_115","description":"tool 115"},{"name":"t_116","description":"tool 116"},{"name":"t_117","description":"tool 117"},{"name":"t_118","description":"tool 118"},{"name":"t_119","description":"tool 119"}] } + } +} diff --git a/plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json b/plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json new file mode 100644 index 0000000..f9a2d34 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "budget-srv-14": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"}] } + } +} diff --git a/plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json b/plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json new file mode 100644 index 0000000..3221a85 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "budget-srv-25": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"}] } + } +} diff --git a/plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json b/plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json new file mode 100644 index 0000000..a5c0a71 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "budget-srv-60": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"},{"name":"t_25","description":"tool 25"},{"name":"t_26","description":"tool 26"},{"name":"t_27","description":"tool 27"},{"name":"t_28","description":"tool 28"},{"name":"t_29","description":"tool 29"},{"name":"t_30","description":"tool 30"},{"name":"t_31","description":"tool 31"},{"name":"t_32","description":"tool 32"},{"name":"t_33","description":"tool 33"},{"name":"t_34","description":"tool 34"},{"name":"t_35","description":"tool 35"},{"name":"t_36","description":"tool 36"},{"name":"t_37","description":"tool 37"},{"name":"t_38","description":"tool 38"},{"name":"t_39","description":"tool 39"},{"name":"t_40","description":"tool 40"},{"name":"t_41","description":"tool 41"},{"name":"t_42","description":"tool 42"},{"name":"t_43","description":"tool 43"},{"name":"t_44","description":"tool 44"},{"name":"t_45","description":"tool 45"},{"name":"t_46","description":"tool 46"},{"name":"t_47","description":"tool 47"},{"name":"t_48","description":"tool 48"},{"name":"t_49","description":"tool 49"},{"name":"t_50","description":"tool 50"},{"name":"t_51","description":"tool 51"},{"name":"t_52","description":"tool 52"},{"name":"t_53","description":"tool 53"},{"name":"t_54","description":"tool 54"},{"name":"t_55","description":"tool 55"},{"name":"t_56","description":"tool 56"},{"name":"t_57","description":"tool 57"},{"name":"t_58","description":"tool 58"},{"name":"t_59","description":"tool 59"}] } + } +} diff --git a/plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json b/plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json new file mode 100644 index 0000000..01add58 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "budget-srv-unknown": { "command": "npx", "args": ["nonexistent-pkg"] } + } +} diff --git a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs index 7249be6..04d2cc1 100644 --- a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs +++ b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs @@ -222,6 +222,69 @@ describe('TOK scanner — M4 cascade > 10k tokens (v5)', () => { }); }); +describe('TOK scanner — N1 MCP tool-schema budget (v5 CA-TOK-005)', () => { + // readActiveConfig pulls in ambient ~/.claude.json plugin MCP servers; tests + // filter to the fixture's own server name (budget-srv-) to avoid + // user-state leakage. Findings identified by title (not exact ID) — TOK IDs + // are sequential per scan. + const findFixtureBudget = (result, count) => + result.findings.find(f => + /MCP tool-schema budget/i.test(f.title || '') && + (f.title || '').includes(`budget-srv-${count}`) + ); + + it('14 tools → no budget finding (under 20-tool floor)', async () => { + const result = await runScanner('mcp-budget/14-tools'); + const f = findFixtureBudget(result, 14); + assert.equal(f, undefined, + `expected no budget finding for budget-srv-14 under 20 tools; got: ${f?.title}`); + }); + + it('25 tools → low severity', async () => { + const result = await runScanner('mcp-budget/25-tools'); + const f = findFixtureBudget(result, 25); + assert.ok(f, `expected budget finding for budget-srv-25; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'low', `expected low for 25 tools, got ${f.severity}`); + }); + + it('60 tools → medium severity', async () => { + const result = await runScanner('mcp-budget/60-tools'); + const f = findFixtureBudget(result, 60); + assert.ok(f, `expected budget finding for budget-srv-60`); + assert.equal(f.severity, 'medium', `expected medium for 60 tools, got ${f.severity}`); + }); + + it('120 tools → high severity', async () => { + const result = await runScanner('mcp-budget/120-tools'); + const f = findFixtureBudget(result, 120); + assert.ok(f, `expected budget finding for budget-srv-120`); + assert.equal(f.severity, 'high', `expected high for 120 tools, got ${f.severity}`); + }); + + it('unknown toolCount → low severity with "unknown" in evidence', async () => { + const result = await runScanner('mcp-budget/unknown-tools'); + const f = findFixtureBudget(result, 'unknown'); + assert.ok(f, `expected budget finding for budget-srv-unknown`); + assert.equal(f.severity, 'low', `expected low for unknown toolCount, got ${f.severity}`); + assert.match(String(f.evidence || ''), /unknown/i, + `expected "unknown" in evidence, got: ${f.evidence}`); + }); + + it('finding ID matches CA-TOK-NNN format', async () => { + const result = await runScanner('mcp-budget/120-tools'); + const f = findFixtureBudget(result, 120); + assert.ok(f); + assert.match(f.id, /^CA-TOK-\d{3}$/); + }); + + it('finding evidence carries calibration_note', async () => { + const result = await runScanner('mcp-budget/60-tools'); + const f = findFixtureBudget(result, 60); + assert.ok(f); + assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i); + }); +}); + describe('TOK scanner — F7 severity recalibration (v5)', () => { // Findings identified by title pattern, not finding ID — TOK IDs are // sequential per scan run, not semantic per pattern (output.mjs:31).