feat(post-mcp-verify): E14 part 2 — cumulative-drift MEDIUM advisory [skip-docs]
Wave C step C2: surface the cumulative-drift signal from checkDescriptionDrift() (added in C1) as a separate MEDIUM advisory with finding category mcp-cumulative-drift. Independent of the existing per-update drift advisory — a slow-burn rug-pull that keeps each update below the 10% per-update threshold but cumulatively drifts >=25% from the sticky baseline now triggers the new advisory without ever crossing the per-update bar. The advisory references /security mcp-baseline-reset (added in C3) so the user knows how to acknowledge a legitimate MCP server upgrade. CLAUDE.md updates: - post-mcp-verify hooks-table row mentions per-update + cumulative drift - mcp-description-cache lib bullet documents baseline schema, history, cumulative threshold policy key, and LLM_SECURITY_MCP_CACHE_FILE override. Tests: 2 new hook tests using LLM_SECURITY_MCP_CACHE_FILE for cache isolation. Existing 68 still pass; total 70. Plugin README and root marketplace README updates land in C3 alongside the new /security mcp-baseline-reset slash command (combined Wave-C doc update per plan §"Wave C — Touch" list). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
eaac830300
commit
427b68eca9
3 changed files with 109 additions and 4 deletions
|
|
@ -11,8 +11,10 @@
|
|||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve } from 'node:path';
|
||||
import { runHook } from './hook-helper.mjs';
|
||||
import { resolve, join } from 'node:path';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { runHook, runHookWithEnv } from './hook-helper.mjs';
|
||||
|
||||
const SCRIPT = resolve(import.meta.dirname, '../../hooks/scripts/post-mcp-verify.mjs');
|
||||
|
||||
|
|
@ -399,6 +401,91 @@ describe('post-mcp-verify — MCP description drift detection', () => {
|
|||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MCP cumulative description drift (E14 / v7.3.0)
|
||||
// Five sub-10% updates that cumulatively diverge >25% from baseline.
|
||||
// LLM_SECURITY_MCP_CACHE_FILE isolates the cache file so the test does not
|
||||
// pollute the user's real ~/.cache/llm-security/mcp-descriptions.json.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('post-mcp-verify — MCP cumulative drift advisory (E14)', () => {
|
||||
it('emits MEDIUM mcp-cumulative-drift advisory after slow-burn drift', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'mcp-cumdrift-test-'));
|
||||
const cacheFile = join(dir, 'mcp-descriptions.json');
|
||||
const env = { LLM_SECURITY_MCP_CACHE_FILE: cacheFile };
|
||||
const tool = 'mcp__creep__search';
|
||||
|
||||
// Seed the baseline with a long description
|
||||
const v0 = 'Search the web for current information about technology and science topics from reliable sources.';
|
||||
let result = await runHookWithEnv(SCRIPT, {
|
||||
tool_name: tool,
|
||||
tool_input: { description: v0 },
|
||||
tool_output: 'A clean output line padded with extra characters so the injection scan threshold is met.',
|
||||
}, env);
|
||||
assert.equal(result.code, 0);
|
||||
assert.equal(parseAdvisory(result.stdout), null, 'first call seeds baseline, no advisory');
|
||||
|
||||
// Five small mutations that each stay below 10% per-update drift
|
||||
const mutations = [
|
||||
'Search the web for current information about technology and science topics from trusted sources.',
|
||||
'Search the web for recent information about technology and science topics from trusted sources.',
|
||||
'Search the web for recent information about technology and science topics including trusted sources.',
|
||||
'Search the web for recent information about technology, science, and engineering topics including trusted sources.',
|
||||
'Search the web for recent information about technology, science, engineering, and medicine topics including trusted sources.',
|
||||
];
|
||||
|
||||
let lastResult = null;
|
||||
for (const m of mutations) {
|
||||
lastResult = await runHookWithEnv(SCRIPT, {
|
||||
tool_name: tool,
|
||||
tool_input: { description: m },
|
||||
tool_output: 'A clean output line padded with extra characters so the injection scan threshold is met.',
|
||||
}, env);
|
||||
assert.equal(lastResult.code, 0);
|
||||
}
|
||||
|
||||
const adv = parseAdvisory(lastResult.stdout);
|
||||
assert.ok(adv, 'cumulative drift advisory emitted on the final mutation');
|
||||
assert.ok(
|
||||
adv.systemMessage.includes('mcp-cumulative-drift'),
|
||||
'advisory includes finding category mcp-cumulative-drift',
|
||||
);
|
||||
assert.ok(adv.systemMessage.includes('MEDIUM'), 'advisory severity is MEDIUM');
|
||||
assert.ok(adv.systemMessage.includes('MCP05'), 'advisory references OWASP MCP05');
|
||||
assert.ok(
|
||||
adv.systemMessage.includes('/security mcp-baseline-reset'),
|
||||
'advisory mentions reset command for legitimate upgrades',
|
||||
);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('no cumulative-drift advisory for stable descriptions across many calls', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'mcp-cumdrift-stable-'));
|
||||
const cacheFile = join(dir, 'mcp-descriptions.json');
|
||||
const env = { LLM_SECURITY_MCP_CACHE_FILE: cacheFile };
|
||||
const tool = 'mcp__stable__t';
|
||||
const desc = 'A stable, descriptive tool for searching the public web.';
|
||||
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const result = await runHookWithEnv(SCRIPT, {
|
||||
tool_name: tool,
|
||||
tool_input: { description: desc },
|
||||
tool_output: 'Clean output line padded with extra characters so the injection scan threshold is met.',
|
||||
}, env);
|
||||
assert.equal(result.code, 0);
|
||||
const adv = parseAdvisory(result.stdout);
|
||||
// Either null (no advisory) or no cumulative-drift mention
|
||||
if (adv) {
|
||||
assert.ok(
|
||||
!adv.systemMessage.includes('mcp-cumulative-drift'),
|
||||
'no cumulative-drift advisory for stable description',
|
||||
);
|
||||
}
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MCP per-tool volume tracking (NEW in v4.3.0)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue