feat(config-audit): CA-TOK-005 MCP tool-schema budget (v5 N1) [skip-docs]
Adds detectMcpToolBudget detection block in TOK scanner. Tiered severity
per project-local .mcp.json server based on toolCount:
- < 20: no finding
- 20-49: low
- 50-99: medium
- 100+: high
- null (manifest unparseable): low + "tool count unknown" message
Scoped to source==='.mcp.json' to keep findings actionable for the
audited path; plugin/user-level MCP servers are surfaced by the
manifest scanner (Step 19 / N2).
5 fixtures (mcp-budget/{14,25,60,120,unknown}-tools) use inline `tools`
arrays in .mcp.json — no node_modules needed for these tests.
Tests assert title+severity (not exact ID) since TOK IDs are sequential
per scan, not semantic per pattern.
[skip-docs] reason: v5 plan fences off README/CLAUDE.md badge updates
to Session 5; Forgejo pre-commit-docs-gate hook requires this tag on
feat commits without doc changes.
Tests: 586 → 593 (+7).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen
parent
dd0d4bf738
commit
b2407a09b3
7 changed files with 164 additions and 0 deletions
|
|
@ -52,6 +52,18 @@ const CASCADE_TOKEN_THRESHOLD = 10_000;
|
|||
// not. Anything past this hints the description is doing the body's job.
|
||||
const SKILL_DESCRIPTION_THRESHOLD = 500;
|
||||
|
||||
// v5 N1: MCP tool-schema budget thresholds (CA-TOK-005). Tool descriptions
|
||||
// load on every turn — high tool counts inflate the per-turn schema payload
|
||||
// regardless of whether the tools are invoked. Tiered severity per server:
|
||||
// < 20 → no finding
|
||||
// 20–49 → low
|
||||
// 50–99 → medium
|
||||
// 100+ → high
|
||||
// null → low ("tool count unknown" — manifest not parseable)
|
||||
const MCP_BUDGET_LOW = 20;
|
||||
const MCP_BUDGET_MEDIUM = 50;
|
||||
const MCP_BUDGET_HIGH = 100;
|
||||
|
||||
const HOTSPOTS_MAX = 10;
|
||||
|
||||
// v5 F7: shared evidence note appended to every TOK pattern finding.
|
||||
|
|
@ -122,6 +134,25 @@ async function maxImportDepth(startFile, contentCache) {
|
|||
return maxDepth;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify an MCP server's tool count into a budget tier (v5 N1).
|
||||
*
|
||||
* Returns null if no finding should be emitted (toolCount < 20). Otherwise
|
||||
* returns { severity, tier, kind } where kind is 'unknown' (toolCount===null)
|
||||
* or 'counted'. Threshold ladder: 20 → low, 50 → medium, 100 → high. Null
|
||||
* toolCount maps to low + 'unknown' so users can see opaque servers without
|
||||
* the scanner pretending they're free.
|
||||
*/
|
||||
function classifyMcpToolBudget(toolCount) {
|
||||
if (toolCount === null) {
|
||||
return { severity: SEVERITY.low, tier: 'unknown', kind: 'unknown' };
|
||||
}
|
||||
if (typeof toolCount !== 'number' || toolCount < MCP_BUDGET_LOW) return null;
|
||||
if (toolCount >= MCP_BUDGET_HIGH) return { severity: SEVERITY.high, tier: '100+', kind: 'counted' };
|
||||
if (toolCount >= MCP_BUDGET_MEDIUM) return { severity: SEVERITY.medium, tier: '50-99', kind: 'counted' };
|
||||
return { severity: SEVERITY.low, tier: '20-49', kind: 'counted' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect cache-breaking volatile content in the first VOLATILE_TOP_LINES
|
||||
* lines of a CLAUDE.md file.
|
||||
|
|
@ -372,6 +403,51 @@ export async function scan(targetPath, discovery) {
|
|||
}));
|
||||
}
|
||||
|
||||
// ── Pattern G: MCP tool-schema budget per server (v5 N1, CA-TOK-005) ──
|
||||
// Scope: project-local .mcp.json only. Plugin- and ~/.claude.json-sourced
|
||||
// servers are global concerns surfaced by the manifest scanner; scoping the
|
||||
// finding here to .mcp.json keeps /config-audit <path> actionable for the
|
||||
// path the user is auditing.
|
||||
if (activeConfig && Array.isArray(activeConfig.mcpServers)) {
|
||||
for (const m of activeConfig.mcpServers) {
|
||||
if (!m || !m.enabled) continue;
|
||||
if (m.source !== '.mcp.json') continue;
|
||||
const budget = classifyMcpToolBudget(m.toolCount);
|
||||
if (!budget) continue;
|
||||
const severity = budget.severity;
|
||||
const sourceLabel = m.source ? `${m.name} (${m.source})` : m.name;
|
||||
const isUnknown = budget.kind === 'unknown';
|
||||
const description = isUnknown
|
||||
? `MCP server "${sourceLabel}" has tool count unknown — could not parse manifest ` +
|
||||
'or cached tools/list. Tool schemas load on every turn; an unverified server ' +
|
||||
'may be inflating the per-turn payload silently.'
|
||||
: `MCP server "${sourceLabel}" exposes ${m.toolCount} tools. Tool schemas load on ` +
|
||||
'every turn regardless of which tools the model actually invokes — high tool ' +
|
||||
'counts inflate the per-turn payload and crowd out usable context.';
|
||||
const evidence = isUnknown
|
||||
? `tool_count=unknown; server="${m.name}"; source="${m.source}" — ${CALIBRATION_NOTE}`
|
||||
: `tool_count=${m.toolCount}; tier=${budget.tier}; server="${m.name}"; ` +
|
||||
`source="${m.source}" — ${CALIBRATION_NOTE}`;
|
||||
const recommendation = isUnknown
|
||||
? 'Install the package locally (so detect-mcp-tool-count can read its manifest), ' +
|
||||
'or run the server once and cache its tools/list response under ' +
|
||||
'~/.claude/config-audit/mcp-cache/<name>.json. See knowledge/cache-telemetry-recipe.md.'
|
||||
: 'Use the server\'s `tools/filter` config (or equivalent) to expose only the tools ' +
|
||||
'this project actually needs. Consider splitting heavy MCP servers across project- ' +
|
||||
'and user-scopes so per-project budget stays tight.';
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity,
|
||||
title: `High MCP tool-schema budget on server "${m.name}"`,
|
||||
description,
|
||||
file: m.source && m.source !== `mcp:${m.name}` ? m.source : null,
|
||||
evidence,
|
||||
recommendation,
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pattern E: CLAUDE.md cascade > CASCADE_TOKEN_THRESHOLD (v5 M4) ──
|
||||
if (activeConfig?.claudeMd?.estimatedTokens > CASCADE_TOKEN_THRESHOLD) {
|
||||
const cascadeTokens = activeConfig.claudeMd.estimatedTokens;
|
||||
|
|
|
|||
5
plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json
vendored
Normal file
5
plugins/config-audit/tests/fixtures/mcp-budget/120-tools/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"budget-srv-120": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"},{"name":"t_25","description":"tool 25"},{"name":"t_26","description":"tool 26"},{"name":"t_27","description":"tool 27"},{"name":"t_28","description":"tool 28"},{"name":"t_29","description":"tool 29"},{"name":"t_30","description":"tool 30"},{"name":"t_31","description":"tool 31"},{"name":"t_32","description":"tool 32"},{"name":"t_33","description":"tool 33"},{"name":"t_34","description":"tool 34"},{"name":"t_35","description":"tool 35"},{"name":"t_36","description":"tool 36"},{"name":"t_37","description":"tool 37"},{"name":"t_38","description":"tool 38"},{"name":"t_39","description":"tool 39"},{"name":"t_40","description":"tool 40"},{"name":"t_41","description":"tool 41"},{"name":"t_42","description":"tool 42"},{"name":"t_43","description":"tool 43"},{"name":"t_44","description":"tool 44"},{"name":"t_45","description":"tool 45"},{"name":"t_46","description":"tool 46"},{"name":"t_47","description":"tool 47"},{"name":"t_48","description":"tool 48"},{"name":"t_49","description":"tool 49"},{"name":"t_50","description":"tool 50"},{"name":"t_51","description":"tool 51"},{"name":"t_52","description":"tool 52"},{"name":"t_53","description":"tool 53"},{"name":"t_54","description":"tool 54"},{"name":"t_55","description":"tool 55"},{"name":"t_56","description":"tool 56"},{"name":"t_57","description":"tool 57"},{"name":"t_58","description":"tool 58"},{"name":"t_59","description":"tool 59"},{"name":"t_60","description":"tool 60"},{"name":"t_61","description":"tool 61"},{"name":"t_62","description":"tool 62"},{"name":"t_63","description":"tool 63"},{"name":"t_64","description":"tool 64"},{"name":"t_65","description":"tool 65"},{"name":"t_66","description":"tool 66"},{"name":"t_67","description":"tool 67"},{"name":"t_68","description":"tool 68"},{"name":"t_69","description":"tool 69"},{"name":"t_70","description":"tool 70"},{"name":"t_71","description":"tool 71"},{"name":"t_72","description":"tool 72"},{"name":"t_73","description":"tool 73"},{"name":"t_74","description":"tool 74"},{"name":"t_75","description":"tool 75"},{"name":"t_76","description":"tool 76"},{"name":"t_77","description":"tool 77"},{"name":"t_78","description":"tool 78"},{"name":"t_79","description":"tool 79"},{"name":"t_80","description":"tool 80"},{"name":"t_81","description":"tool 81"},{"name":"t_82","description":"tool 82"},{"name":"t_83","description":"tool 83"},{"name":"t_84","description":"tool 84"},{"name":"t_85","description":"tool 85"},{"name":"t_86","description":"tool 86"},{"name":"t_87","description":"tool 87"},{"name":"t_88","description":"tool 88"},{"name":"t_89","description":"tool 89"},{"name":"t_90","description":"tool 90"},{"name":"t_91","description":"tool 91"},{"name":"t_92","description":"tool 92"},{"name":"t_93","description":"tool 93"},{"name":"t_94","description":"tool 94"},{"name":"t_95","description":"tool 95"},{"name":"t_96","description":"tool 96"},{"name":"t_97","description":"tool 97"},{"name":"t_98","description":"tool 98"},{"name":"t_99","description":"tool 99"},{"name":"t_100","description":"tool 100"},{"name":"t_101","description":"tool 101"},{"name":"t_102","description":"tool 102"},{"name":"t_103","description":"tool 103"},{"name":"t_104","description":"tool 104"},{"name":"t_105","description":"tool 105"},{"name":"t_106","description":"tool 106"},{"name":"t_107","description":"tool 107"},{"name":"t_108","description":"tool 108"},{"name":"t_109","description":"tool 109"},{"name":"t_110","description":"tool 110"},{"name":"t_111","description":"tool 111"},{"name":"t_112","description":"tool 112"},{"name":"t_113","description":"tool 113"},{"name":"t_114","description":"tool 114"},{"name":"t_115","description":"tool 115"},{"name":"t_116","description":"tool 116"},{"name":"t_117","description":"tool 117"},{"name":"t_118","description":"tool 118"},{"name":"t_119","description":"tool 119"}] }
|
||||
}
|
||||
}
|
||||
5
plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json
vendored
Normal file
5
plugins/config-audit/tests/fixtures/mcp-budget/14-tools/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"budget-srv-14": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"}] }
|
||||
}
|
||||
}
|
||||
5
plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json
vendored
Normal file
5
plugins/config-audit/tests/fixtures/mcp-budget/25-tools/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"budget-srv-25": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"}] }
|
||||
}
|
||||
}
|
||||
5
plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json
vendored
Normal file
5
plugins/config-audit/tests/fixtures/mcp-budget/60-tools/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"budget-srv-60": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name":"t_0","description":"tool 0"},{"name":"t_1","description":"tool 1"},{"name":"t_2","description":"tool 2"},{"name":"t_3","description":"tool 3"},{"name":"t_4","description":"tool 4"},{"name":"t_5","description":"tool 5"},{"name":"t_6","description":"tool 6"},{"name":"t_7","description":"tool 7"},{"name":"t_8","description":"tool 8"},{"name":"t_9","description":"tool 9"},{"name":"t_10","description":"tool 10"},{"name":"t_11","description":"tool 11"},{"name":"t_12","description":"tool 12"},{"name":"t_13","description":"tool 13"},{"name":"t_14","description":"tool 14"},{"name":"t_15","description":"tool 15"},{"name":"t_16","description":"tool 16"},{"name":"t_17","description":"tool 17"},{"name":"t_18","description":"tool 18"},{"name":"t_19","description":"tool 19"},{"name":"t_20","description":"tool 20"},{"name":"t_21","description":"tool 21"},{"name":"t_22","description":"tool 22"},{"name":"t_23","description":"tool 23"},{"name":"t_24","description":"tool 24"},{"name":"t_25","description":"tool 25"},{"name":"t_26","description":"tool 26"},{"name":"t_27","description":"tool 27"},{"name":"t_28","description":"tool 28"},{"name":"t_29","description":"tool 29"},{"name":"t_30","description":"tool 30"},{"name":"t_31","description":"tool 31"},{"name":"t_32","description":"tool 32"},{"name":"t_33","description":"tool 33"},{"name":"t_34","description":"tool 34"},{"name":"t_35","description":"tool 35"},{"name":"t_36","description":"tool 36"},{"name":"t_37","description":"tool 37"},{"name":"t_38","description":"tool 38"},{"name":"t_39","description":"tool 39"},{"name":"t_40","description":"tool 40"},{"name":"t_41","description":"tool 41"},{"name":"t_42","description":"tool 42"},{"name":"t_43","description":"tool 43"},{"name":"t_44","description":"tool 44"},{"name":"t_45","description":"tool 45"},{"name":"t_46","description":"tool 46"},{"name":"t_47","description":"tool 47"},{"name":"t_48","description":"tool 48"},{"name":"t_49","description":"tool 49"},{"name":"t_50","description":"tool 50"},{"name":"t_51","description":"tool 51"},{"name":"t_52","description":"tool 52"},{"name":"t_53","description":"tool 53"},{"name":"t_54","description":"tool 54"},{"name":"t_55","description":"tool 55"},{"name":"t_56","description":"tool 56"},{"name":"t_57","description":"tool 57"},{"name":"t_58","description":"tool 58"},{"name":"t_59","description":"tool 59"}] }
|
||||
}
|
||||
}
|
||||
5
plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json
vendored
Normal file
5
plugins/config-audit/tests/fixtures/mcp-budget/unknown-tools/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"budget-srv-unknown": { "command": "npx", "args": ["nonexistent-pkg"] }
|
||||
}
|
||||
}
|
||||
|
|
@ -222,6 +222,69 @@ describe('TOK scanner — M4 cascade > 10k tokens (v5)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('TOK scanner — N1 MCP tool-schema budget (v5 CA-TOK-005)', () => {
|
||||
// readActiveConfig pulls in ambient ~/.claude.json plugin MCP servers; tests
|
||||
// filter to the fixture's own server name (budget-srv-<count>) to avoid
|
||||
// user-state leakage. Findings identified by title (not exact ID) — TOK IDs
|
||||
// are sequential per scan.
|
||||
const findFixtureBudget = (result, count) =>
|
||||
result.findings.find(f =>
|
||||
/MCP tool-schema budget/i.test(f.title || '') &&
|
||||
(f.title || '').includes(`budget-srv-${count}`)
|
||||
);
|
||||
|
||||
it('14 tools → no budget finding (under 20-tool floor)', async () => {
|
||||
const result = await runScanner('mcp-budget/14-tools');
|
||||
const f = findFixtureBudget(result, 14);
|
||||
assert.equal(f, undefined,
|
||||
`expected no budget finding for budget-srv-14 under 20 tools; got: ${f?.title}`);
|
||||
});
|
||||
|
||||
it('25 tools → low severity', async () => {
|
||||
const result = await runScanner('mcp-budget/25-tools');
|
||||
const f = findFixtureBudget(result, 25);
|
||||
assert.ok(f, `expected budget finding for budget-srv-25; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
assert.equal(f.severity, 'low', `expected low for 25 tools, got ${f.severity}`);
|
||||
});
|
||||
|
||||
it('60 tools → medium severity', async () => {
|
||||
const result = await runScanner('mcp-budget/60-tools');
|
||||
const f = findFixtureBudget(result, 60);
|
||||
assert.ok(f, `expected budget finding for budget-srv-60`);
|
||||
assert.equal(f.severity, 'medium', `expected medium for 60 tools, got ${f.severity}`);
|
||||
});
|
||||
|
||||
it('120 tools → high severity', async () => {
|
||||
const result = await runScanner('mcp-budget/120-tools');
|
||||
const f = findFixtureBudget(result, 120);
|
||||
assert.ok(f, `expected budget finding for budget-srv-120`);
|
||||
assert.equal(f.severity, 'high', `expected high for 120 tools, got ${f.severity}`);
|
||||
});
|
||||
|
||||
it('unknown toolCount → low severity with "unknown" in evidence', async () => {
|
||||
const result = await runScanner('mcp-budget/unknown-tools');
|
||||
const f = findFixtureBudget(result, 'unknown');
|
||||
assert.ok(f, `expected budget finding for budget-srv-unknown`);
|
||||
assert.equal(f.severity, 'low', `expected low for unknown toolCount, got ${f.severity}`);
|
||||
assert.match(String(f.evidence || ''), /unknown/i,
|
||||
`expected "unknown" in evidence, got: ${f.evidence}`);
|
||||
});
|
||||
|
||||
it('finding ID matches CA-TOK-NNN format', async () => {
|
||||
const result = await runScanner('mcp-budget/120-tools');
|
||||
const f = findFixtureBudget(result, 120);
|
||||
assert.ok(f);
|
||||
assert.match(f.id, /^CA-TOK-\d{3}$/);
|
||||
});
|
||||
|
||||
it('finding evidence carries calibration_note', async () => {
|
||||
const result = await runScanner('mcp-budget/60-tools');
|
||||
const f = findFixtureBudget(result, 60);
|
||||
assert.ok(f);
|
||||
assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('TOK scanner — F7 severity recalibration (v5)', () => {
|
||||
// Findings identified by title pattern, not finding ID — TOK IDs are
|
||||
// sequential per scan run, not semantic per pattern (output.mjs:31).
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue