feat(tokens): MCP tool-schema deferral check + CLI-over-MCP lever (v5.10 B4)

By default Claude Code defers MCP tool schemas (names-only, ~120 tok; full
schemas on demand via tool search). CA-TOK-006 detects config-file signals that
force the FULL schemas into the always-loaded prefix every turn:
  - settings.json env.ENABLE_TOOL_SEARCH="false"          (high)
  - "ToolSearch" in permissions.deny                       (high)
  - configured model is a Haiku model                      (medium)
  - per-server .mcp.json alwaysLoad:true (CC v2.1.121+)    (high)
auto[:N] is threshold mode (info, not a trigger).

New engine lib/mcp-deferral.mjs: pure assessMcpDeferral({settings,mcpServers})
(unit-tested, no IO) + thin IO wrapper assessMcpDeferralForRepo shared by TOK and
GAP. Severity scales with aggregate forced-upfront tokens (medium-confidence
reasons cap at medium). feature-gap cliOverMcpLeverFinding fires only as a
companion to CA-TOK-006 (prefer gh/aws/gcloud over MCP for common ops).

Honest scoping (Verifiseringsplikt): triggers on config files ONLY — never
process.env shell vars. Vertex / custom ANTHROPIC_BASE_URL / runtime /model
switch are launch state (would flap snapshots machine-dependently), so they are
DISCLOSED in every finding, not triggered. Tool-level anthropic/alwaysLoad and
claude.ai connectors likewise disclosed. Mechanism verified 2026-06-23 against
code.claude.com/docs (context-window.md, mcp.md#configure-tool-search +
#exempt-a-server-from-deferral, costs.md); the prefix-cache invalidation claim
was NOT-CONFIRMED in docs and is not asserted.

alwaysLoad added to CA-MCP VALID_SERVER_FIELDS (no longer flagged as unknown).
active-config-reader surfaces per-server alwaysLoad. Byte-stable: CA-TOK-006
fires only on new conditions; frozen v5.0.0 + SC-5 snapshots untouched.
Tests 1215 -> 1239 (engine 16, integration 5, lever 2, mcp-field guard 1).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 19:59:14 +02:00
commit 8f7e196046
13 changed files with 692 additions and 2 deletions

View file

@ -0,0 +1,6 @@
{
"mcpServers": {
"always-srv": { "command": "npx", "args": ["fake-pkg"], "alwaysLoad": true, "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] },
"deferred-srv": { "command": "npx", "args": ["other-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
}
}

View file

@ -0,0 +1,5 @@
{
"mcpServers": {
"plain-srv": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
}
}

View file

@ -0,0 +1,5 @@
{
"mcpServers": {
"srv-a": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }
}
}

View file

@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url';
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan, opportunitySummary, bundledSkillsLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
import { withHermeticHome } from '../helpers/hermetic-home.mjs';
@ -340,3 +340,27 @@ describe('GAP scanner — disableBundledSkills lever wiring (HOME-scoped)', () =
}
});
});
describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => {
it('returns null when nothing is forced upfront', () => {
assert.equal(cliOverMcpLeverFinding({ assessment: { forcedUpfront: false } }), null);
assert.equal(cliOverMcpLeverFinding({ assessment: null }), null);
assert.equal(cliOverMcpLeverFinding({}), null);
});
it('fires a low-severity opportunity when MCP schemas are forced upfront', () => {
resetCounter();
const f = cliOverMcpLeverFinding({
assessment: {
forcedUpfront: true,
aggregateTokens: 2500,
affectedServers: [{ name: 'srv-a' }],
reason: 'enable-tool-search-false',
},
});
assert.ok(f, 'expected a lever finding');
assert.equal(f.severity, 'low');
assert.equal(f.category, 'token-efficiency');
assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/);
});
});

View file

@ -136,6 +136,36 @@ describe('MCP scanner — stray `trust` field is an unknown field (verify-first,
});
});
describe('MCP scanner — `alwaysLoad` is a valid field (v5.10 B4, verify-first 2026-06-23)', () => {
let result;
let tmpRoot;
beforeEach(async () => {
resetCounter();
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-mcp-alwaysload-'));
// alwaysLoad exempts a server from MCP tool-schema deferral (CC v2.1.121+).
// Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral.
const mcp = {
mcpServers: {
core: { type: 'http', url: 'https://mcp.example.com/mcp', alwaysLoad: true },
},
};
await writeFile(join(tmpRoot, '.mcp.json'), JSON.stringify(mcp, null, 2) + '\n', 'utf8');
const discovery = await discoverConfigFiles(tmpRoot);
result = await scan(tmpRoot, discovery);
});
afterEach(async () => {
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
});
it('does NOT flag `alwaysLoad` as an unknown MCP server field', () => {
const f = result.findings.find(x => x.title.includes('Unknown MCP server field')
&& /alwaysLoad/.test(x.description || ''));
assert.equal(f, undefined, 'alwaysLoad is a valid field and must not be flagged');
});
});
describe('MCP scanner — env-var false positives (CC 2.1.139/2.1.142, Batch 1)', () => {
let tmpRoot;
let envFindings;

View file

@ -0,0 +1,192 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
assessMcpDeferral,
severityForForcedSchemas,
FORCED_SCHEMA_TOKENS_HIGH,
FORCED_SCHEMA_TOKENS_MEDIUM,
} from '../../scanners/lib/mcp-deferral.mjs';
// A small helper to build server shapes the way active-config-reader exposes them.
const srv = (over = {}) => ({
name: 'srv',
source: '.mcp.json',
enabled: true,
toolCount: 10,
estimatedTokens: 2500,
alwaysLoad: false,
...over,
});
describe('assessMcpDeferral — default (deferred) case', () => {
it('no settings + a plain server → tool search NOT disabled, nothing forced upfront', () => {
const a = assessMcpDeferral({ settings: {}, mcpServers: [srv()] });
assert.equal(a.toolSearchDisabled, false);
assert.equal(a.reason, null);
assert.equal(a.confidence, null);
assert.equal(a.forcedUpfront, false);
assert.deepEqual(a.affectedServers, []);
assert.equal(a.aggregateTokens, 0);
});
it('empty inputs are tolerated', () => {
const a = assessMcpDeferral({});
assert.equal(a.toolSearchDisabled, false);
assert.equal(a.forcedUpfront, false);
});
});
describe('assessMcpDeferral — config-file disabling signals (HIGH confidence)', () => {
it('settings.env.ENABLE_TOOL_SEARCH="false" → disabled, high, all servers affected', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
mcpServers: [srv({ name: 'a', estimatedTokens: 1000 }), srv({ name: 'b', estimatedTokens: 2000 })],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.reason, 'enable-tool-search-false');
assert.equal(a.confidence, 'high');
assert.equal(a.forcedUpfront, true);
assert.equal(a.affectedServers.length, 2);
assert.equal(a.aggregateTokens, 3000);
});
it('permissions.deny including bare "ToolSearch" → disabled, high', () => {
const a = assessMcpDeferral({
settings: { permissions: { deny: ['Read(./.env)', 'ToolSearch'] } },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.reason, 'deny-tool-search');
assert.equal(a.confidence, 'high');
});
it('explicit ENABLE_TOOL_SEARCH="false" wins over a haiku model for the reason label', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'false' }, model: 'claude-haiku-4-5' },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.reason, 'enable-tool-search-false');
assert.equal(a.confidence, 'high');
});
});
describe('assessMcpDeferral — configured Haiku model (MEDIUM confidence)', () => {
it('settings.model matching /haiku/ → disabled, medium, haiku-model', () => {
const a = assessMcpDeferral({
settings: { model: 'claude-haiku-4-5-20251001' },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.reason, 'haiku-model');
assert.equal(a.confidence, 'medium');
assert.equal(a.forcedUpfront, true);
});
it('haiku disables even when ENABLE_TOOL_SEARCH="true" (Haiku lacks tool_reference support)', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'haiku' },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.reason, 'haiku-model');
});
});
describe('assessMcpDeferral — non-disabling values', () => {
it('ENABLE_TOOL_SEARCH="true" (non-haiku) → NOT disabled', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'claude-sonnet-4-6' },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, false);
assert.equal(a.forcedUpfront, false);
});
it('ENABLE_TOOL_SEARCH="auto" → threshold mode, NOT disabled', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'auto' } },
mcpServers: [srv()],
});
assert.equal(a.toolSearchDisabled, false);
assert.equal(a.thresholdMode, true);
assert.equal(a.forcedUpfront, false);
});
it('ENABLE_TOOL_SEARCH="auto:5" → threshold mode', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'auto:5' } },
mcpServers: [srv()],
});
assert.equal(a.thresholdMode, true);
assert.equal(a.toolSearchDisabled, false);
});
});
describe('assessMcpDeferral — per-server alwaysLoad', () => {
it('alwaysLoad:true server with tool search enabled → that server forced upfront only', () => {
const a = assessMcpDeferral({
settings: {},
mcpServers: [
srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }),
srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 9000 }),
],
});
assert.equal(a.toolSearchDisabled, false);
assert.equal(a.forcedUpfront, true);
assert.equal(a.alwaysLoadServers.length, 1);
assert.equal(a.affectedServers.length, 1);
assert.equal(a.affectedServers[0].name, 'always');
assert.equal(a.aggregateTokens, 1500);
});
it('when tool search is disabled, ALL active servers are affected (not just alwaysLoad)', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
mcpServers: [
srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }),
srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 2500 }),
],
});
assert.equal(a.affectedServers.length, 2);
assert.equal(a.aggregateTokens, 4000);
});
});
describe('assessMcpDeferral — server filtering', () => {
it('disabled servers (enabled:false) are excluded from affected + aggregate', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
mcpServers: [
srv({ name: 'on', enabled: true, estimatedTokens: 1000 }),
srv({ name: 'off', enabled: false, estimatedTokens: 5000 }),
],
});
assert.equal(a.affectedServers.length, 1);
assert.equal(a.affectedServers[0].name, 'on');
assert.equal(a.aggregateTokens, 1000);
});
it('no active servers → not forced upfront even when tool search disabled', () => {
const a = assessMcpDeferral({
settings: { env: { ENABLE_TOOL_SEARCH: 'false' } },
mcpServers: [],
});
assert.equal(a.toolSearchDisabled, true);
assert.equal(a.forcedUpfront, false);
assert.equal(a.aggregateTokens, 0);
});
});
describe('severityForForcedSchemas', () => {
it('high confidence scales with aggregate tokens', () => {
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH, 'high'), 'high');
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_MEDIUM, 'high'), 'medium');
assert.equal(severityForForcedSchemas(100, 'high'), 'low');
});
it('medium confidence is capped at medium', () => {
assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH * 10, 'medium'), 'medium');
assert.equal(severityForForcedSchemas(100, 'medium'), 'low');
});
});

View file

@ -0,0 +1,70 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/token-hotspots.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
import { withHermeticHome } from '../helpers/hermetic-home.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURES = resolve(__dirname, '../fixtures');
// Hermetic HOME so readActiveConfig does not leak the developer's real
// ~/.claude.json plugin MCP servers into the fixture result (mirrors the other
// TOK tests). The deferral check reads project+local settings + project .mcp.json.
async function runScanner(fixtureName) {
resetCounter();
const path = resolve(FIXTURES, fixtureName);
const discovery = await discoverConfigFiles(path);
return withHermeticHome(() => scan(path, discovery));
}
const TITLE = 'MCP tool schemas forced into the always-loaded prefix';
const findDeferral = (result) => result.findings.find(f => f.title === TITLE);
describe('TOK scanner — MCP deferral (v5.10 B4, CA-TOK-006)', () => {
it('deferred (default) project → NO deferral finding', async () => {
const result = await runScanner('mcp-deferral/deferred');
assert.equal(findDeferral(result), undefined,
'a plain server with tool search on must not be flagged as forced-upfront');
});
it('alwaysLoad:true server → deferral finding fires (medium severity)', async () => {
const result = await runScanner('mcp-deferral/alwaysload');
const f = findDeferral(result);
assert.ok(f, `expected a deferral finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
assert.equal(f.severity, 'medium', `expected medium for ~2500 forced tokens, got ${f.severity}`);
assert.equal(f.category, 'token-efficiency');
// Only the alwaysLoad server is named, not the deferred sibling.
assert.match(String(f.evidence || ''), /always-srv/);
assert.doesNotMatch(String(f.description || ''), /deferred-srv/);
assert.match(String(f.evidence || ''), /alwaysLoad/);
});
it('ENABLE_TOOL_SEARCH="false" in settings → deferral finding (all servers affected)', async () => {
const result = await runScanner('mcp-deferral/disabled-settings');
const f = findDeferral(result);
assert.ok(f, `expected a deferral finding for tool-search-disabled fixture`);
assert.match(String(f.evidence || ''), /reason=enable-tool-search-false/);
assert.match(String(f.description || ''), /Tool search is disabled/);
});
it('every deferral finding discloses runtime conditions + calibration', async () => {
const result = await runScanner('mcp-deferral/alwaysload');
const f = findDeferral(result);
assert.ok(f);
// DEFERRAL_DISCLOSURE names the launch/runtime conditions a static scan can't see.
assert.match(String(f.evidence || ''), /Vertex AI/);
assert.match(String(f.evidence || ''), /ANTHROPIC_BASE_URL/);
// CALIBRATION_NOTE shared by all TOK pattern findings.
assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i);
});
it('finding ID matches CA-TOK-NNN format', async () => {
const result = await runScanner('mcp-deferral/alwaysload');
const f = findDeferral(result);
assert.ok(f);
assert.match(f.id, /^CA-TOK-\d{3}$/);
});
});