feat(commands): E14 part 3 — /security mcp-baseline-reset slash command
Wave C step C3: closes E14 with the user-facing reset command. After a legitimate MCP server upgrade the sticky baseline (added in C1) becomes a stale "what the tool used to say" anchor and every subsequent post-mcp-verify advisory will re-flag the change. /security mcp-baseline-reset lets the user acknowledge the upgrade so the next call seeds a fresh baseline. New files: - scanners/mcp-baseline-reset.mjs — small CLI wrapper around clearBaseline / listBaselines. Modes: --list (read-only), --target <name>, no-args (all). Outputs JSON summary on stdout. Exit 0 always (idempotent). - commands/mcp-baseline-reset.md — dispatcher following mcp-inspect.md shape. Frontmatter: name=security:mcp-baseline-reset, sonnet model, Read/Bash/AskUserQuestion tools. 4-step body (list -> confirm scope -> execute -> confirm result). - tests/scanners/mcp-baseline-reset.test.mjs — 10 CLI tests across --list, --target, clear-all, idempotency, history preservation, and bare-positional sugar. Updated: - commands/security.md — new row in commands table after mcp-inspect. - CLAUDE.md — new commands-table row + new v7.3.0 narrative section describing the baseline schema, cumulative-drift detection, reset semantics, and the LLM_SECURITY_MCP_CACHE_FILE override. - Plugin README.md — new MCP-baseline-reset row in commands table, scanner count 12 standalone -> 13 standalone, new "MCP Description Drift (E14, v7.3.0)" subsection explaining the sticky baseline, cumulative threshold, reset semantics, and env-var override. - Root marketplace README.md — scanner count 22 -> 23 (10 orchestrated + 13 standalone), command count 19 -> 20, test count 1511 -> 1768. Wave C complete: 1738 -> 1768 tests (+30 across C1/C2/C3). Per plan, Wave C does NOT bump the plugin version — that lands at the wave-bundle release. The advisory text in post-mcp-verify already references the new command path so the user has a ready remediation step. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen
parent
427b68eca9
commit
001df2ebe8
7 changed files with 454 additions and 5 deletions
244
plugins/llm-security/tests/scanners/mcp-baseline-reset.test.mjs
Normal file
244
plugins/llm-security/tests/scanners/mcp-baseline-reset.test.mjs
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
// mcp-baseline-reset.test.mjs — CLI tests for scanners/mcp-baseline-reset.mjs
|
||||
// Zero external dependencies: node:test + node:assert + child_process.execFile.
|
||||
//
|
||||
// LLM_SECURITY_MCP_CACHE_FILE controls the cache path so the test does not
|
||||
// pollute the user's real ~/.cache/llm-security/mcp-descriptions.json.
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync, rmSync, existsSync, readFileSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const SCRIPT = resolve(import.meta.dirname, '../../scanners/mcp-baseline-reset.mjs');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function runCli(args, env) {
|
||||
return new Promise((res) => {
|
||||
execFile(
|
||||
'node',
|
||||
[SCRIPT, ...args],
|
||||
{ env: { ...process.env, ...env }, timeout: 5000 },
|
||||
(err, stdout, stderr) => {
|
||||
res({
|
||||
code: err && typeof err.code === 'number' ? err.code : 0,
|
||||
stdout: stdout || '',
|
||||
stderr: stderr || '',
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function makeTmpCache() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'baseline-reset-test-'));
|
||||
const cacheFile = join(dir, 'mcp-descriptions.json');
|
||||
return { dir, cacheFile };
|
||||
}
|
||||
|
||||
function cleanup(dir) {
|
||||
try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
function seedCache(cacheFile, entries) {
|
||||
writeFileSync(cacheFile, JSON.stringify(entries, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
function parseJson(stdout) {
|
||||
return JSON.parse(stdout.trim());
|
||||
}
|
||||
|
||||
const NOW = Date.now();
|
||||
|
||||
function makeEntry(desc, opts = {}) {
|
||||
return {
|
||||
description: desc,
|
||||
firstSeen: NOW - 10000,
|
||||
lastSeen: NOW,
|
||||
baseline: opts.noBaseline ? undefined : { description: desc, seenAt: NOW - 10000 },
|
||||
history: opts.history || [],
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// --list mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('mcp-baseline-reset CLI — --list mode', () => {
|
||||
it('returns mode=list with empty baselines on empty cache', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
const result = await runCli(['--list'], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.mode, 'list');
|
||||
assert.equal(json.count, 0);
|
||||
assert.deepEqual(json.baselines, []);
|
||||
cleanup(dir);
|
||||
});
|
||||
|
||||
it('lists all entries with baseline metadata', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
'mcp__beta__t': makeEntry('Beta description text long enough'),
|
||||
});
|
||||
const result = await runCli(['--list'], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.mode, 'list');
|
||||
assert.equal(json.count, 2);
|
||||
const tools = json.baselines.map((b) => b.tool).sort();
|
||||
assert.deepEqual(tools, ['mcp__alpha__t', 'mcp__beta__t']);
|
||||
for (const b of json.baselines) {
|
||||
assert.ok(typeof b.baseline_excerpt === 'string');
|
||||
assert.ok(typeof b.seen_at === 'number');
|
||||
assert.ok(typeof b.last_seen === 'number');
|
||||
assert.ok(typeof b.history_events === 'number');
|
||||
}
|
||||
cleanup(dir);
|
||||
});
|
||||
|
||||
it('--list does not mutate the cache', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
const before = {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
};
|
||||
seedCache(cacheFile, before);
|
||||
await runCli(['--list'], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
const after = JSON.parse(readFileSync(cacheFile, 'utf-8'));
|
||||
assert.ok(after['mcp__alpha__t'].baseline, 'baseline preserved by --list');
|
||||
cleanup(dir);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// --target mode (single tool)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('mcp-baseline-reset CLI — --target mode', () => {
|
||||
it('clears one named baseline and reports it', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
'mcp__beta__t': makeEntry('Beta description text long enough'),
|
||||
});
|
||||
const result = await runCli(
|
||||
['--target', 'mcp__alpha__t'],
|
||||
{ LLM_SECURITY_MCP_CACHE_FILE: cacheFile },
|
||||
);
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.mode, 'reset');
|
||||
assert.equal(json.cleared, 1);
|
||||
assert.deepEqual(json.tools, ['mcp__alpha__t']);
|
||||
assert.equal(json.remaining, 1, 'beta baseline still present');
|
||||
|
||||
// Verify on disk
|
||||
const after = JSON.parse(readFileSync(cacheFile, 'utf-8'));
|
||||
assert.equal(after['mcp__alpha__t'].baseline, undefined, 'alpha baseline cleared');
|
||||
assert.ok(after['mcp__beta__t'].baseline, 'beta baseline preserved');
|
||||
cleanup(dir);
|
||||
});
|
||||
|
||||
it('idempotent — clearing nonexistent target reports 0 cleared', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
});
|
||||
const result = await runCli(
|
||||
['--target', 'mcp__no_such__tool'],
|
||||
{ LLM_SECURITY_MCP_CACHE_FILE: cacheFile },
|
||||
);
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.cleared, 0);
|
||||
assert.deepEqual(json.tools, []);
|
||||
assert.equal(json.remaining, 1, 'unrelated baseline untouched');
|
||||
cleanup(dir);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clear-all mode (no args)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('mcp-baseline-reset CLI — clear-all mode', () => {
|
||||
it('with no args, clears all baselines', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
'mcp__beta__t': makeEntry('Beta description text long enough'),
|
||||
'mcp__gamma__t': makeEntry('Gamma description text long enough'),
|
||||
});
|
||||
const result = await runCli([], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.mode, 'reset');
|
||||
assert.equal(json.cleared, 3);
|
||||
assert.equal(json.remaining, 0);
|
||||
assert.equal(json.tools.length, 3);
|
||||
|
||||
const after = JSON.parse(readFileSync(cacheFile, 'utf-8'));
|
||||
for (const key of ['mcp__alpha__t', 'mcp__beta__t', 'mcp__gamma__t']) {
|
||||
assert.equal(after[key].baseline, undefined);
|
||||
}
|
||||
cleanup(dir);
|
||||
});
|
||||
|
||||
it('idempotent — clear-all on empty cache returns 0', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
const result = await runCli([], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.cleared, 0);
|
||||
assert.equal(json.remaining, 0);
|
||||
cleanup(dir);
|
||||
});
|
||||
|
||||
it('preserves description and history after clear', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough', {
|
||||
history: [{ description: 'older', seenAt: NOW - 5000, distance: 4 }],
|
||||
}),
|
||||
});
|
||||
await runCli([], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
const after = JSON.parse(readFileSync(cacheFile, 'utf-8'));
|
||||
const entry = after['mcp__alpha__t'];
|
||||
assert.equal(entry.baseline, undefined);
|
||||
assert.equal(entry.description, 'Alpha description text long enough', 'description preserved');
|
||||
assert.ok(typeof entry.firstSeen === 'number');
|
||||
assert.equal(entry.history.length, 1, 'history preserved');
|
||||
cleanup(dir);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Help / unknown args
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('mcp-baseline-reset CLI — misc', () => {
|
||||
it('--help prints usage and exits 0', async () => {
|
||||
const result = await runCli(['--help'], {});
|
||||
assert.equal(result.code, 0);
|
||||
assert.ok(/Usage:/i.test(result.stdout));
|
||||
});
|
||||
|
||||
it('bare positional argument is treated as --target', async () => {
|
||||
const { dir, cacheFile } = makeTmpCache();
|
||||
seedCache(cacheFile, {
|
||||
'mcp__alpha__t': makeEntry('Alpha description text long enough'),
|
||||
'mcp__beta__t': makeEntry('Beta description text long enough'),
|
||||
});
|
||||
const result = await runCli(['mcp__alpha__t'], { LLM_SECURITY_MCP_CACHE_FILE: cacheFile });
|
||||
assert.equal(result.code, 0);
|
||||
const json = parseJson(result.stdout);
|
||||
assert.equal(json.cleared, 1);
|
||||
assert.deepEqual(json.tools, ['mcp__alpha__t']);
|
||||
cleanup(dir);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue