feat(config-audit): CA-TOK-005 MCP tool-schema budget (v5 N1) [skip-docs]
Adds detectMcpToolBudget detection block in TOK scanner. Tiered severity
per project-local .mcp.json server based on toolCount:
- < 20: no finding
- 20-49: low
- 50-99: medium
- 100+: high
- null (manifest unparseable): low + "tool count unknown" message
Scoped to source==='.mcp.json' to keep findings actionable for the
audited path; plugin/user-level MCP servers are surfaced by the
manifest scanner (Step 19 / N2).
5 fixtures (mcp-budget/{14,25,60,120,unknown}-tools) use inline `tools`
arrays in .mcp.json — no node_modules needed for these tests.
Tests assert title+severity (not exact ID) since TOK IDs are sequential
per scan, not semantic per pattern.
[skip-docs] reason: v5 plan fences off README/CLAUDE.md badge updates
to Session 5; Forgejo pre-commit-docs-gate hook requires this tag on
feat commits without doc changes.
Tests: 586 → 593 (+7).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
dd0d4bf738
commit
b2407a09b3
7 changed files with 164 additions and 0 deletions
|
|
@ -52,6 +52,18 @@ const CASCADE_TOKEN_THRESHOLD = 10_000;
|
|||
// not. Anything past this hints the description is doing the body's job.
|
||||
const SKILL_DESCRIPTION_THRESHOLD = 500;
|
||||
|
||||
// v5 N1: MCP tool-schema budget thresholds (CA-TOK-005). Tool descriptions
|
||||
// load on every turn — high tool counts inflate the per-turn schema payload
|
||||
// regardless of whether the tools are invoked. Tiered severity per server:
|
||||
// < 20 → no finding
|
||||
// 20–49 → low
|
||||
// 50–99 → medium
|
||||
// 100+ → high
|
||||
// null → low ("tool count unknown" — manifest not parseable)
|
||||
const MCP_BUDGET_LOW = 20;
|
||||
const MCP_BUDGET_MEDIUM = 50;
|
||||
const MCP_BUDGET_HIGH = 100;
|
||||
|
||||
const HOTSPOTS_MAX = 10;
|
||||
|
||||
// v5 F7: shared evidence note appended to every TOK pattern finding.
|
||||
|
|
@ -122,6 +134,25 @@ async function maxImportDepth(startFile, contentCache) {
|
|||
return maxDepth;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify an MCP server's tool count into a budget tier (v5 N1).
|
||||
*
|
||||
* Returns null if no finding should be emitted (toolCount < 20). Otherwise
|
||||
* returns { severity, tier, kind } where kind is 'unknown' (toolCount===null)
|
||||
* or 'counted'. Threshold ladder: 20 → low, 50 → medium, 100 → high. Null
|
||||
* toolCount maps to low + 'unknown' so users can see opaque servers without
|
||||
* the scanner pretending they're free.
|
||||
*/
|
||||
function classifyMcpToolBudget(toolCount) {
|
||||
if (toolCount === null) {
|
||||
return { severity: SEVERITY.low, tier: 'unknown', kind: 'unknown' };
|
||||
}
|
||||
if (typeof toolCount !== 'number' || toolCount < MCP_BUDGET_LOW) return null;
|
||||
if (toolCount >= MCP_BUDGET_HIGH) return { severity: SEVERITY.high, tier: '100+', kind: 'counted' };
|
||||
if (toolCount >= MCP_BUDGET_MEDIUM) return { severity: SEVERITY.medium, tier: '50-99', kind: 'counted' };
|
||||
return { severity: SEVERITY.low, tier: '20-49', kind: 'counted' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect cache-breaking volatile content in the first VOLATILE_TOP_LINES
|
||||
* lines of a CLAUDE.md file.
|
||||
|
|
@ -372,6 +403,51 @@ export async function scan(targetPath, discovery) {
|
|||
}));
|
||||
}
|
||||
|
||||
// ── Pattern G: MCP tool-schema budget per server (v5 N1, CA-TOK-005) ──
|
||||
// Scope: project-local .mcp.json only. Plugin- and ~/.claude.json-sourced
|
||||
// servers are global concerns surfaced by the manifest scanner; scoping the
|
||||
// finding here to .mcp.json keeps /config-audit <path> actionable for the
|
||||
// path the user is auditing.
|
||||
if (activeConfig && Array.isArray(activeConfig.mcpServers)) {
|
||||
for (const m of activeConfig.mcpServers) {
|
||||
if (!m || !m.enabled) continue;
|
||||
if (m.source !== '.mcp.json') continue;
|
||||
const budget = classifyMcpToolBudget(m.toolCount);
|
||||
if (!budget) continue;
|
||||
const severity = budget.severity;
|
||||
const sourceLabel = m.source ? `${m.name} (${m.source})` : m.name;
|
||||
const isUnknown = budget.kind === 'unknown';
|
||||
const description = isUnknown
|
||||
? `MCP server "${sourceLabel}" has tool count unknown — could not parse manifest ` +
|
||||
'or cached tools/list. Tool schemas load on every turn; an unverified server ' +
|
||||
'may be inflating the per-turn payload silently.'
|
||||
: `MCP server "${sourceLabel}" exposes ${m.toolCount} tools. Tool schemas load on ` +
|
||||
'every turn regardless of which tools the model actually invokes — high tool ' +
|
||||
'counts inflate the per-turn payload and crowd out usable context.';
|
||||
const evidence = isUnknown
|
||||
? `tool_count=unknown; server="${m.name}"; source="${m.source}" — ${CALIBRATION_NOTE}`
|
||||
: `tool_count=${m.toolCount}; tier=${budget.tier}; server="${m.name}"; ` +
|
||||
`source="${m.source}" — ${CALIBRATION_NOTE}`;
|
||||
const recommendation = isUnknown
|
||||
? 'Install the package locally (so detect-mcp-tool-count can read its manifest), ' +
|
||||
'or run the server once and cache its tools/list response under ' +
|
||||
'~/.claude/config-audit/mcp-cache/<name>.json. See knowledge/cache-telemetry-recipe.md.'
|
||||
: 'Use the server\'s `tools/filter` config (or equivalent) to expose only the tools ' +
|
||||
'this project actually needs. Consider splitting heavy MCP servers across project- ' +
|
||||
'and user-scopes so per-project budget stays tight.';
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity,
|
||||
title: `High MCP tool-schema budget on server "${m.name}"`,
|
||||
description,
|
||||
file: m.source && m.source !== `mcp:${m.name}` ? m.source : null,
|
||||
evidence,
|
||||
recommendation,
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pattern E: CLAUDE.md cascade > CASCADE_TOKEN_THRESHOLD (v5 M4) ──
|
||||
if (activeConfig?.claudeMd?.estimatedTokens > CASCADE_TOKEN_THRESHOLD) {
|
||||
const cascadeTokens = activeConfig.claudeMd.estimatedTokens;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue