feat(tokens): MCP tool-schema deferral check + CLI-over-MCP lever (v5.10 B4)
By default Claude Code defers MCP tool schemas (names-only, ~120 tok; full
schemas on demand via tool search). CA-TOK-006 detects config-file signals that
force the FULL schemas into the always-loaded prefix every turn:
- settings.json env.ENABLE_TOOL_SEARCH="false" (high)
- "ToolSearch" in permissions.deny (high)
- configured model is a Haiku model (medium)
- per-server .mcp.json alwaysLoad:true (CC v2.1.121+) (high)
auto[:N] is threshold mode (info, not a trigger).
New engine lib/mcp-deferral.mjs: pure assessMcpDeferral({settings,mcpServers})
(unit-tested, no IO) + thin IO wrapper assessMcpDeferralForRepo shared by TOK and
GAP. Severity scales with aggregate forced-upfront tokens (medium-confidence
reasons cap at medium). feature-gap cliOverMcpLeverFinding fires only as a
companion to CA-TOK-006 (prefer gh/aws/gcloud over MCP for common ops).
Honest scoping (Verifiseringsplikt): triggers on config files ONLY — never
process.env shell vars. Vertex / custom ANTHROPIC_BASE_URL / runtime /model
switch are launch state (would flap snapshots machine-dependently), so they are
DISCLOSED in every finding, not triggered. Tool-level anthropic/alwaysLoad and
claude.ai connectors likewise disclosed. Mechanism verified 2026-06-23 against
code.claude.com/docs (context-window.md, mcp.md#configure-tool-search +
#exempt-a-server-from-deferral, costs.md); the prefix-cache invalidation claim
was NOT-CONFIRMED in docs and is not asserted.
alwaysLoad added to CA-MCP VALID_SERVER_FIELDS (no longer flagged as unknown).
active-config-reader surfaces per-server alwaysLoad. Byte-stable: CA-TOK-006
fires only on new conditions; frozen v5.0.0 + SC-5 snapshots untouched.
Tests 1215 -> 1239 (engine 16, integration 5, lever 2, mcp-field guard 1).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
42e48514e4
commit
8f7e196046
13 changed files with 692 additions and 2 deletions
6
tests/fixtures/mcp-deferral/alwaysload/.mcp.json
vendored
Normal file
6
tests/fixtures/mcp-deferral/alwaysload/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"always-srv": { "command": "npx", "args": ["fake-pkg"], "alwaysLoad": true, "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] },
|
||||
"deferred-srv": { "command": "npx", "args": ["other-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
|
||||
}
|
||||
}
|
||||
5
tests/fixtures/mcp-deferral/deferred/.mcp.json
vendored
Normal file
5
tests/fixtures/mcp-deferral/deferred/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"plain-srv": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
|
||||
}
|
||||
}
|
||||
5
tests/fixtures/mcp-deferral/disabled-settings/.mcp.json
vendored
Normal file
5
tests/fixtures/mcp-deferral/disabled-settings/.mcp.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"mcpServers": {
|
||||
"srv-a": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { scan, opportunitySummary, bundledSkillsLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
|
||||
import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
|
||||
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
import { withHermeticHome } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
|
|
@ -340,3 +340,27 @@ describe('GAP scanner — disableBundledSkills lever wiring (HOME-scoped)', () =
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => {
|
||||
it('returns null when nothing is forced upfront', () => {
|
||||
assert.equal(cliOverMcpLeverFinding({ assessment: { forcedUpfront: false } }), null);
|
||||
assert.equal(cliOverMcpLeverFinding({ assessment: null }), null);
|
||||
assert.equal(cliOverMcpLeverFinding({}), null);
|
||||
});
|
||||
|
||||
it('fires a low-severity opportunity when MCP schemas are forced upfront', () => {
|
||||
resetCounter();
|
||||
const f = cliOverMcpLeverFinding({
|
||||
assessment: {
|
||||
forcedUpfront: true,
|
||||
aggregateTokens: 2500,
|
||||
affectedServers: [{ name: 'srv-a' }],
|
||||
reason: 'enable-tool-search-false',
|
||||
},
|
||||
});
|
||||
assert.ok(f, 'expected a lever finding');
|
||||
assert.equal(f.severity, 'low');
|
||||
assert.equal(f.category, 'token-efficiency');
|
||||
assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -136,6 +136,36 @@ describe('MCP scanner — stray `trust` field is an unknown field (verify-first,
|
|||
});
|
||||
});
|
||||
|
||||
describe('MCP scanner — `alwaysLoad` is a valid field (v5.10 B4, verify-first 2026-06-23)', () => {
|
||||
let result;
|
||||
let tmpRoot;
|
||||
|
||||
beforeEach(async () => {
|
||||
resetCounter();
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-mcp-alwaysload-'));
|
||||
// alwaysLoad exempts a server from MCP tool-schema deferral (CC v2.1.121+).
|
||||
// Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral.
|
||||
const mcp = {
|
||||
mcpServers: {
|
||||
core: { type: 'http', url: 'https://mcp.example.com/mcp', alwaysLoad: true },
|
||||
},
|
||||
};
|
||||
await writeFile(join(tmpRoot, '.mcp.json'), JSON.stringify(mcp, null, 2) + '\n', 'utf8');
|
||||
const discovery = await discoverConfigFiles(tmpRoot);
|
||||
result = await scan(tmpRoot, discovery);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('does NOT flag `alwaysLoad` as an unknown MCP server field', () => {
|
||||
const f = result.findings.find(x => x.title.includes('Unknown MCP server field')
|
||||
&& /alwaysLoad/.test(x.description || ''));
|
||||
assert.equal(f, undefined, 'alwaysLoad is a valid field and must not be flagged');
|
||||
});
|
||||
});
|
||||
|
||||
describe('MCP scanner — env-var false positives (CC 2.1.139/2.1.142, Batch 1)', () => {
|
||||
let tmpRoot;
|
||||
let envFindings;
|
||||
|
|
|
|||
192
tests/scanners/mcp-deferral.test.mjs
Normal file
192
tests/scanners/mcp-deferral.test.mjs
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
assessMcpDeferral,
|
||||
severityForForcedSchemas,
|
||||
FORCED_SCHEMA_TOKENS_HIGH,
|
||||
FORCED_SCHEMA_TOKENS_MEDIUM,
|
||||
} from '../../scanners/lib/mcp-deferral.mjs';
|
||||
|
||||
// A small helper to build server shapes the way active-config-reader exposes them.
|
||||
const srv = (over = {}) => ({
|
||||
name: 'srv',
|
||||
source: '.mcp.json',
|
||||
enabled: true,
|
||||
toolCount: 10,
|
||||
estimatedTokens: 2500,
|
||||
alwaysLoad: false,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — default (deferred) case', () => {
|
||||
it('no settings + a plain server → tool search NOT disabled, nothing forced upfront', () => {
|
||||
const a = assessMcpDeferral({ settings: {}, mcpServers: [srv()] });
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
assert.equal(a.reason, null);
|
||||
assert.equal(a.confidence, null);
|
||||
assert.equal(a.forcedUpfront, false);
|
||||
assert.deepEqual(a.affectedServers, []);
|
||||
assert.equal(a.aggregateTokens, 0);
|
||||
});
|
||||
|
||||
it('empty inputs are tolerated', () => {
|
||||
const a = assessMcpDeferral({});
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
assert.equal(a.forcedUpfront, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — config-file disabling signals (HIGH confidence)', () => {
|
||||
it('settings.env.ENABLE_TOOL_SEARCH="false" → disabled, high, all servers affected', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
|
||||
mcpServers: [srv({ name: 'a', estimatedTokens: 1000 }), srv({ name: 'b', estimatedTokens: 2000 })],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.reason, 'enable-tool-search-false');
|
||||
assert.equal(a.confidence, 'high');
|
||||
assert.equal(a.forcedUpfront, true);
|
||||
assert.equal(a.affectedServers.length, 2);
|
||||
assert.equal(a.aggregateTokens, 3000);
|
||||
});
|
||||
|
||||
it('permissions.deny including bare "ToolSearch" → disabled, high', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { permissions: { deny: ['Read(./.env)', 'ToolSearch'] } },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.reason, 'deny-tool-search');
|
||||
assert.equal(a.confidence, 'high');
|
||||
});
|
||||
|
||||
it('explicit ENABLE_TOOL_SEARCH="false" wins over a haiku model for the reason label', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'false' }, model: 'claude-haiku-4-5' },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.reason, 'enable-tool-search-false');
|
||||
assert.equal(a.confidence, 'high');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — configured Haiku model (MEDIUM confidence)', () => {
|
||||
it('settings.model matching /haiku/ → disabled, medium, haiku-model', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { model: 'claude-haiku-4-5-20251001' },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.reason, 'haiku-model');
|
||||
assert.equal(a.confidence, 'medium');
|
||||
assert.equal(a.forcedUpfront, true);
|
||||
});
|
||||
|
||||
it('haiku disables even when ENABLE_TOOL_SEARCH="true" (Haiku lacks tool_reference support)', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'haiku' },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.reason, 'haiku-model');
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — non-disabling values', () => {
|
||||
it('ENABLE_TOOL_SEARCH="true" (non-haiku) → NOT disabled', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'claude-sonnet-4-6' },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
assert.equal(a.forcedUpfront, false);
|
||||
});
|
||||
|
||||
it('ENABLE_TOOL_SEARCH="auto" → threshold mode, NOT disabled', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'auto' } },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
assert.equal(a.thresholdMode, true);
|
||||
assert.equal(a.forcedUpfront, false);
|
||||
});
|
||||
|
||||
it('ENABLE_TOOL_SEARCH="auto:5" → threshold mode', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'auto:5' } },
|
||||
mcpServers: [srv()],
|
||||
});
|
||||
assert.equal(a.thresholdMode, true);
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — per-server alwaysLoad', () => {
|
||||
it('alwaysLoad:true server with tool search enabled → that server forced upfront only', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: {},
|
||||
mcpServers: [
|
||||
srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }),
|
||||
srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 9000 }),
|
||||
],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, false);
|
||||
assert.equal(a.forcedUpfront, true);
|
||||
assert.equal(a.alwaysLoadServers.length, 1);
|
||||
assert.equal(a.affectedServers.length, 1);
|
||||
assert.equal(a.affectedServers[0].name, 'always');
|
||||
assert.equal(a.aggregateTokens, 1500);
|
||||
});
|
||||
|
||||
it('when tool search is disabled, ALL active servers are affected (not just alwaysLoad)', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
|
||||
mcpServers: [
|
||||
srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }),
|
||||
srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 2500 }),
|
||||
],
|
||||
});
|
||||
assert.equal(a.affectedServers.length, 2);
|
||||
assert.equal(a.aggregateTokens, 4000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessMcpDeferral — server filtering', () => {
|
||||
it('disabled servers (enabled:false) are excluded from affected + aggregate', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
|
||||
mcpServers: [
|
||||
srv({ name: 'on', enabled: true, estimatedTokens: 1000 }),
|
||||
srv({ name: 'off', enabled: false, estimatedTokens: 5000 }),
|
||||
],
|
||||
});
|
||||
assert.equal(a.affectedServers.length, 1);
|
||||
assert.equal(a.affectedServers[0].name, 'on');
|
||||
assert.equal(a.aggregateTokens, 1000);
|
||||
});
|
||||
|
||||
it('no active servers → not forced upfront even when tool search disabled', () => {
|
||||
const a = assessMcpDeferral({
|
||||
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
|
||||
mcpServers: [],
|
||||
});
|
||||
assert.equal(a.toolSearchDisabled, true);
|
||||
assert.equal(a.forcedUpfront, false);
|
||||
assert.equal(a.aggregateTokens, 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('severityForForcedSchemas', () => {
|
||||
it('high confidence scales with aggregate tokens', () => {
|
||||
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH, 'high'), 'high');
|
||||
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_MEDIUM, 'high'), 'medium');
|
||||
assert.equal(severityForForcedSchemas(100, 'high'), 'low');
|
||||
});
|
||||
|
||||
it('medium confidence is capped at medium', () => {
|
||||
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH * 10, 'medium'), 'medium');
|
||||
assert.equal(severityForForcedSchemas(100, 'medium'), 'low');
|
||||
});
|
||||
});
|
||||
70
tests/scanners/token-hotspots-deferral.test.mjs
Normal file
70
tests/scanners/token-hotspots-deferral.test.mjs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { scan } from '../../scanners/token-hotspots.mjs';
|
||||
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
import { withHermeticHome } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const FIXTURES = resolve(__dirname, '../fixtures');
|
||||
|
||||
// Hermetic HOME so readActiveConfig does not leak the developer's real
|
||||
// ~/.claude.json plugin MCP servers into the fixture result (mirrors the other
|
||||
// TOK tests). The deferral check reads project+local settings + project .mcp.json.
|
||||
async function runScanner(fixtureName) {
|
||||
resetCounter();
|
||||
const path = resolve(FIXTURES, fixtureName);
|
||||
const discovery = await discoverConfigFiles(path);
|
||||
return withHermeticHome(() => scan(path, discovery));
|
||||
}
|
||||
|
||||
const TITLE = 'MCP tool schemas forced into the always-loaded prefix';
|
||||
const findDeferral = (result) => result.findings.find(f => f.title === TITLE);
|
||||
|
||||
describe('TOK scanner — MCP deferral (v5.10 B4, CA-TOK-006)', () => {
|
||||
it('deferred (default) project → NO deferral finding', async () => {
|
||||
const result = await runScanner('mcp-deferral/deferred');
|
||||
assert.equal(findDeferral(result), undefined,
|
||||
'a plain server with tool search on must not be flagged as forced-upfront');
|
||||
});
|
||||
|
||||
it('alwaysLoad:true server → deferral finding fires (medium severity)', async () => {
|
||||
const result = await runScanner('mcp-deferral/alwaysload');
|
||||
const f = findDeferral(result);
|
||||
assert.ok(f, `expected a deferral finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
||||
assert.equal(f.severity, 'medium', `expected medium for ~2500 forced tokens, got ${f.severity}`);
|
||||
assert.equal(f.category, 'token-efficiency');
|
||||
// Only the alwaysLoad server is named, not the deferred sibling.
|
||||
assert.match(String(f.evidence || ''), /always-srv/);
|
||||
assert.doesNotMatch(String(f.description || ''), /deferred-srv/);
|
||||
assert.match(String(f.evidence || ''), /alwaysLoad/);
|
||||
});
|
||||
|
||||
it('ENABLE_TOOL_SEARCH="false" in settings → deferral finding (all servers affected)', async () => {
|
||||
const result = await runScanner('mcp-deferral/disabled-settings');
|
||||
const f = findDeferral(result);
|
||||
assert.ok(f, `expected a deferral finding for tool-search-disabled fixture`);
|
||||
assert.match(String(f.evidence || ''), /reason=enable-tool-search-false/);
|
||||
assert.match(String(f.description || ''), /Tool search is disabled/);
|
||||
});
|
||||
|
||||
it('every deferral finding discloses runtime conditions + calibration', async () => {
|
||||
const result = await runScanner('mcp-deferral/alwaysload');
|
||||
const f = findDeferral(result);
|
||||
assert.ok(f);
|
||||
// DEFERRAL_DISCLOSURE names the launch/runtime conditions a static scan can't see.
|
||||
assert.match(String(f.evidence || ''), /Vertex AI/);
|
||||
assert.match(String(f.evidence || ''), /ANTHROPIC_BASE_URL/);
|
||||
// CALIBRATION_NOTE shared by all TOK pattern findings.
|
||||
assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i);
|
||||
});
|
||||
|
||||
it('finding ID matches CA-TOK-NNN format', async () => {
|
||||
const result = await runScanner('mcp-deferral/alwaysload');
|
||||
const f = findDeferral(result);
|
||||
assert.ok(f);
|
||||
assert.match(f.id, /^CA-TOK-\d{3}$/);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue