From 07629e9dae469fa1985f0a6ffa24f7123f95e5eb Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 18:21:31 +0200 Subject: [PATCH] test(humanizer): default-output snapshot test (SC-5) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 12 of v5.1.0 humanizer Wave 4. Adds tests/snapshot-default-output .test.mjs and seeds three snapshots in tests/snapshots/default-output/ that capture humanized default-mode output for representative CLIs. Coverage: - scan-orchestrator: stdout JSON envelope (humanized findings); time fields normalized. - token-hotspots-cli: stdout JSON envelope (humanized payload.findings); duration_ms normalized. - posture: stderr humanized scorecard; (Xms) durations normalized. Snapshot envelope is uniform on disk: { kind: 'json', payload: ... } for JSON streams and { kind: 'text', payload: '...' } for stderr text. This keeps the snapshot files self-describing and easy to read. Re-seeding requires UPDATE_SNAPSHOT=1 — drift fails the test by design, so any humanizer prose change is intentional and re-approved. Tests: 764 to 767 (+3 SC-5 cases). Full suite passes. Co-Authored-By: Claude Opus 4.7 --- .../tests/snapshot-default-output.test.mjs | 164 +++++ .../snapshots/default-output/posture.json | 4 + .../default-output/scan-orchestrator.json | 608 ++++++++++++++++++ .../default-output/token-hotspots.json | 101 +++ 4 files changed, 877 insertions(+) create mode 100644 plugins/config-audit/tests/snapshot-default-output.test.mjs create mode 100644 plugins/config-audit/tests/snapshots/default-output/posture.json create mode 100644 plugins/config-audit/tests/snapshots/default-output/scan-orchestrator.json create mode 100644 plugins/config-audit/tests/snapshots/default-output/token-hotspots.json diff --git a/plugins/config-audit/tests/snapshot-default-output.test.mjs b/plugins/config-audit/tests/snapshot-default-output.test.mjs new file mode 100644 index 0000000..993eb8b --- /dev/null +++ b/plugins/config-audit/tests/snapshot-default-output.test.mjs @@ -0,0 +1,164 @@ +/** + * SC-5 — default-output snapshot test (Wave 4 Step 12). + * + * Captures the humanized stdout of three representative CLIs running in + * default mode against tests/fixtures/marketplace-medium and asserts + * byte-equal output against tests/snapshots/default-output/.json. + * + * Set UPDATE_SNAPSHOT=1 to seed or refresh a snapshot. Subsequent runs + * assert byte-equal — any drift fails the test, so humanizer prose + * changes must be intentional and re-approved by re-running with + * UPDATE_SNAPSHOT=1. + * + * Time-varying fields are normalized before comparison (timestamp, + * target path, duration_ms). Humanizer-added prose fields + * (titleHumanized / descriptionHumanized / recommendationHumanized, + * userImpactCategory, userActionLanguage, relevanceContext) are kept — + * they are the contract being snapshotted. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { readFile, writeFile } from 'node:fs/promises'; + +const exec = promisify(execFile); +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(__dirname, '..'); +const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium'); +const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/default-output'); + +const UPDATE = process.env.UPDATE_SNAPSHOT === '1'; + +async function runCli(scriptPath, args) { + try { + const { stdout, stderr } = await exec('node', [scriptPath, ...args], { + timeout: 60000, + cwd: REPO, + maxBuffer: 10 * 1024 * 1024, + }); + return { stdout: stdout || '', stderr: stderr || '' }; + } catch (err) { + return { stdout: err.stdout || '', stderr: err.stderr || '' }; + } +} + +// --------------------------------------------------------------------------- +// Normalizers — same shape per CLI as json-backcompat / cli-humanizer tests. +// --------------------------------------------------------------------------- + +function normalizeScanOrchestrator(env) { + const out = JSON.parse(JSON.stringify(env)); + if (out.meta) { + out.meta.target = ''; + out.meta.timestamp = ''; + } + if (Array.isArray(out.scanners)) { + for (const s of out.scanners) { + s.duration_ms = 0; + } + } + return out; +} + +function normalizeTokenHotspots(p) { + const out = JSON.parse(JSON.stringify(p)); + out.duration_ms = 0; + return out; +} + +const CLIS = [ + { + name: 'scan-orchestrator', + script: 'scanners/scan-orchestrator.mjs', + snapshotName: 'scan-orchestrator.json', + normalize: normalizeScanOrchestrator, + captureStream: 'stdout', + }, + { + name: 'token-hotspots', + script: 'scanners/token-hotspots-cli.mjs', + snapshotName: 'token-hotspots.json', + normalize: normalizeTokenHotspots, + captureStream: 'stdout', + }, + { + name: 'posture', + script: 'scanners/posture.mjs', + snapshotName: 'posture.json', + // Posture default mode emits the humanized scorecard to stderr; stdout is + // empty unless --json/--raw. Snapshot the scorecard text. + normalize: (s) => s.replace(/\(\d+ms\)/g, '(0ms)'), + captureStream: 'stderr-text', + }, +]; + +async function captureForCli(cli) { + const script = resolve(REPO, cli.script); + const { stdout, stderr } = await runCli(script, [FIXTURE]); + + if (cli.captureStream === 'stdout') { + const parsed = JSON.parse(stdout); + return { + kind: 'json', + payload: cli.normalize(parsed), + }; + } + + if (cli.captureStream === 'stderr-text') { + return { + kind: 'text', + payload: cli.normalize(stderr.trim()), + }; + } + + throw new Error(`unknown captureStream: ${cli.captureStream}`); +} + +async function loadSnapshot(snapshotPath) { + const raw = await readFile(snapshotPath, 'utf8'); + // Snapshot files are stored as JSON envelopes — text snapshots are wrapped + // as { kind: 'text', payload: '...' } so all snapshots look uniform on disk. + return JSON.parse(raw); +} + +async function writeSnapshot(snapshotPath, captured) { + const serialized = JSON.stringify(captured, null, 2) + '\n'; + await writeFile(snapshotPath, serialized, 'utf8'); +} + +describe('SC-5 default-output snapshot test', () => { + for (const cli of CLIS) { + it(`${cli.name} default mode matches tests/snapshots/default-output/${cli.snapshotName}`, async () => { + const captured = await captureForCli(cli); + const snapshotPath = resolve(SNAPSHOT_DIR, cli.snapshotName); + + if (UPDATE) { + await writeSnapshot(snapshotPath, captured); + return; + } + + let expected; + try { + expected = await loadSnapshot(snapshotPath); + } catch (err) { + if (err.code === 'ENOENT') { + assert.fail( + `Snapshot missing: ${snapshotPath}. ` + + `Re-run with UPDATE_SNAPSHOT=1 to seed it.`, + ); + } + throw err; + } + + assert.deepStrictEqual( + captured, + expected, + `${cli.name}: default-output drift detected. ` + + `If intentional, re-run with UPDATE_SNAPSHOT=1.`, + ); + }); + } +}); diff --git a/plugins/config-audit/tests/snapshots/default-output/posture.json b/plugins/config-audit/tests/snapshots/default-output/posture.json new file mode 100644 index 0000000..127003d --- /dev/null +++ b/plugins/config-audit/tests/snapshots/default-output/posture.json @@ -0,0 +1,4 @@ +{ + "kind": "text", + "payload": "`[CML] CLAUDE.md Linter`: 0 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 1 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 1 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (97/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (100) `Settings` ............ A (90)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (90)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +} diff --git a/plugins/config-audit/tests/snapshots/default-output/scan-orchestrator.json b/plugins/config-audit/tests/snapshots/default-output/scan-orchestrator.json new file mode 100644 index 0000000..18c4bd7 --- /dev/null +++ b/plugins/config-audit/tests/snapshots/default-output/scan-orchestrator.json @@ -0,0 +1,608 @@ +{ + "kind": "json", + "payload": { + "meta": { + "target": "", + "timestamp": "", + "version": "2.2.0", + "tool": "config-audit" + }, + "scanners": [ + { + "scanner": "CML", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "SET", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "HKV", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "RUL", + "status": "skipped", + "files_scanned": 0, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "MCP", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "IMP", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "CNF", + "status": "ok", + "files_scanned": 2, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "GAP", + "status": "ok", + "files_scanned": 4, + "duration_ms": 0, + "findings": [ + { + "id": "CA-GAP-001", + "scanner": "GAP", + "severity": "medium", + "title": "You haven't added any custom shortcuts yet", + "description": "Custom skills give you `/your-shortcut` invocations for tasks you do often.", + "file": null, + "line": null, + "evidence": null, + "category": "t1", + "recommendation": "Create a skill in `.claude/skills/` for a workflow you find yourself repeating.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Fix when convenient", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-002", + "scanner": "GAP", + "severity": "low", + "title": "You only have settings at one level", + "description": "Settings can live at user, project, or local-only scope. Using more than one lets you keep personal preferences separate from team-shared ones.", + "file": null, + "line": null, + "evidence": null, + "category": "t2", + "recommendation": "Consider moving team-wide settings to project scope and keeping personal ones at user or local scope.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-003", + "scanner": "GAP", + "severity": "low", + "title": "Your rules all load on every conversation", + "description": "Path-scoped rules only load when you're working with files that match — keeps each conversation focused.", + "file": null, + "line": null, + "evidence": null, + "category": "t2", + "recommendation": "Add scoping to your rules so they only load for the files they apply to.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-004", + "scanner": "GAP", + "severity": "low", + "title": "Your automations all listen to similar events", + "description": "Listening to a wider range of events (before-tool, after-tool, session-start, etc.) lets you catch more workflow opportunities.", + "file": null, + "line": null, + "evidence": null, + "category": "t2", + "recommendation": "Look at the events your current automations skip and consider adding one or two.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-005", + "scanner": "GAP", + "severity": "low", + "title": "You haven't set up any specialized helper agents yet", + "description": "Subagents handle parallel work in separate contexts (research, code review, testing) without crowding your main conversation.", + "file": null, + "line": null, + "evidence": null, + "category": "t2", + "recommendation": "Create a subagent in `.claude/agents/` for a task you delegate often.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-006", + "scanner": "GAP", + "severity": "low", + "title": "You haven't pinned a model preference", + "description": "Setting a default model lets you choose between speed and depth of reasoning for your work.", + "file": null, + "line": null, + "evidence": null, + "category": "t2", + "recommendation": "Add a `model` setting in your settings file.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-007", + "scanner": "GAP", + "severity": "info", + "title": "You haven't set up a status line yet", + "description": "A status line shows live context (token usage, current branch, time) at the bottom of your terminal.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Add a `statusLine` setting if you want this information at a glance.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-008", + "scanner": "GAP", + "severity": "info", + "title": "You haven't set up any custom keybindings", + "description": "Custom keybindings let you trigger your most-used skills with a keystroke.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Add bindings in your settings for skills you run often.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-009", + "scanner": "GAP", + "severity": "info", + "title": "You're using the default output style", + "description": "Output styles let you change how Claude formats responses (concise, verbose, bullet-heavy, etc.).", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Try a different `outputStyle` setting if you have a strong preference.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-010", + "scanner": "GAP", + "severity": "info", + "title": "You haven't set up parallel worktree support", + "description": "Worktrees let Claude work on a branch in an isolated copy of the repo without disturbing your main checkout.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Enable worktrees if you regularly work on multiple branches at once.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-011", + "scanner": "GAP", + "severity": "info", + "title": "Your skills don't use the richer settings block", + "description": "Adding richer settings at the top of a skill lets you control when it loads, what tools it uses, and more.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Add fields like `model`, `tools`, or `description` to your skill files where useful.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-012", + "scanner": "GAP", + "severity": "info", + "title": "Your subagents share Claude's main work folder", + "description": "Isolated subagents run in their own copy of the repo so they can't accidentally disturb your main work.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Add `isolation: worktree` to subagents that do destructive or experimental work.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-013", + "scanner": "GAP", + "severity": "info", + "title": "Your skills don't include live context", + "description": "Dynamic context lets a skill see fresh information (file contents, command output) at the moment it runs, not at the time it was written.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Use the dynamic-context block in skills that need up-to-date information.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-014", + "scanner": "GAP", + "severity": "info", + "title": "You haven't set up auto-mode classification", + "description": "Auto-mode classification helps Claude decide when to act on its own vs. ask you, based on the kind of task.", + "file": null, + "line": null, + "evidence": null, + "category": "t3", + "recommendation": "Add an auto-mode classifier in your settings if you want this nuance.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-015", + "scanner": "GAP", + "severity": "info", + "title": "You haven't built a custom plugin yet", + "description": "Plugins let you bundle skills, automations, and connected services that you want available across many projects.", + "file": null, + "line": null, + "evidence": null, + "category": "t4", + "recommendation": "If you have workflows you repeat across projects, consider packaging them as a plugin.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-016", + "scanner": "GAP", + "severity": "info", + "title": "Your project has no settings managed by your organization", + "description": "Managed settings let your organization apply rules everyone has to follow.", + "file": null, + "line": null, + "evidence": null, + "category": "t4", + "recommendation": "If you work in a team setting, consider whether managed settings would help.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + }, + { + "id": "CA-GAP-017", + "scanner": "GAP", + "severity": "info", + "title": "You haven't connected Claude to your editor's language servers", + "description": "Language-server connections let Claude see types, error messages, and definitions the same way your editor does.", + "file": null, + "line": null, + "evidence": null, + "category": "t4", + "recommendation": "Set up LSP integration if you work in a typed language.", + "autoFixable": false, + "userImpactCategory": "Missed opportunity", + "userActionLanguage": "FYI", + "relevanceContext": "affects-everyone" + } + ], + "counts": { + "critical": 0, + "high": 0, + "medium": 1, + "low": 5, + "info": 11 + } + }, + { + "scanner": "TOK", + "status": "ok", + "files_scanned": 2, + "duration_ms": 0, + "findings": [ + { + "id": "CA-TOK-001", + "scanner": "TOK", + "severity": "low", + "title": "A connected service exposes many tools, all loading on every turn", + "description": "Each tool a connected service exposes adds its description to every turn. Services with many tools eat space fast.", + "file": ".mcp.json", + "line": null, + "evidence": "tool_count=unknown; server=\"memory\"; source=\".mcp.json\" — severity reflects estimated tokens/turn based on structural heuristic; not measured against runtime telemetry", + "category": "token-efficiency", + "recommendation": "Limit which tools the service exposes (often via a `tools` allow-list), or disconnect services you rarely use.", + "autoFixable": false, + "userImpactCategory": "Wasted tokens", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + } + ], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 1, + "info": 0 + }, + "hotspots": [ + { + "source": "mcp:memory (.mcp.json)", + "estimated_tokens": 500, + "rank": 1, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "mcp:sadhguru-wisdom (plugin:sadhguru-wisdom)", + "estimated_tokens": 500, + "rank": 2, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "mcp:vegnorm-rag (plugin:vegnormalene)", + "estimated_tokens": 500, + "rank": 3, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "CLAUDE.md", + "estimated_tokens": 116, + "rank": 4, + "recommendations": [ + "Move volatile top-of-file content to the bottom or extract to an @import-ed file.", + "Split overlong CLAUDE.md into focused @imports (≤200 lines each)." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/CLAUDE.md" + }, + { + "source": "hooks/hooks.json", + "estimated_tokens": 81, + "rank": 5, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/hooks/hooks.json" + }, + { + "source": ".claude/settings.json", + "estimated_tokens": 59, + "rank": 6, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json" + }, + { + "source": ".mcp.json", + "estimated_tokens": 53, + "rank": 7, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.mcp.json" + } + ], + "total_estimated_tokens": 1809, + "activeConfig": { + "claudeMdEstimatedTokens": 5716, + "mcpServerCount": 3, + "pluginCount": 41, + "skillCount": 65 + } + }, + { + "scanner": "CPS", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, + { + "scanner": "DIS", + "status": "ok", + "files_scanned": 1, + "duration_ms": 0, + "findings": [ + { + "id": "CA-DIS-001", + "scanner": "DIS", + "severity": "low", + "title": "A tool is in both the let-in list and the shut-out list", + "description": "When a tool is in both lists, the shut-out always wins, so the let-in entry does nothing. It looks like the tool is approved, but it isn't.", + "file": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json", + "line": null, + "evidence": "Read: allow=\"Read(src/**)\" + deny=\"Read(./.env)\"", + "category": "permissions-hygiene", + "recommendation": "Decide whether the tool should be allowed or denied, and remove it from the other list.", + "autoFixable": false, + "userImpactCategory": "Dead config", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "test-fixture-no-impact" + } + ], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 1, + "info": 0 + } + }, + { + "scanner": "COL", + "status": "ok", + "files_scanned": 65, + "duration_ms": 0, + "findings": [ + { + "id": "CA-COL-001", + "scanner": "COL", + "severity": "low", + "title": "Two plugins both define a skill with the same name", + "description": "When two plugins offer the same skill name, only one wins, and which one is hard to predict.", + "file": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/okr/skills/okr-offentlig-sektor/SKILL.md", + "line": null, + "evidence": "name=\"okr-offentlig-sektor\"; plugins=okr,okr", + "category": "plugin-hygiene", + "recommendation": "Rename the skill in one of the plugins, or disable the one you don't use.", + "autoFixable": false, + "userImpactCategory": "Conflict", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone", + "details": { + "namespaces": [ + { + "source": "plugin:okr", + "name": "okr-offentlig-sektor", + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/okr/skills/okr-offentlig-sektor/SKILL.md" + }, + { + "source": "plugin:okr", + "name": "okr-offentlig-sektor", + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-privat/plugins/okr/skills/okr-offentlig-sektor/SKILL.md" + } + ] + } + } + ], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 1, + "info": 0 + } + } + ], + "aggregate": { + "total_findings": 20, + "counts": { + "critical": 0, + "high": 0, + "medium": 1, + "low": 8, + "info": 11 + }, + "risk_score": 12, + "risk_band": "Medium", + "verdict": "PASS", + "scanners_ok": 11, + "scanners_error": 0, + "scanners_skipped": 1 + } + } +} diff --git a/plugins/config-audit/tests/snapshots/default-output/token-hotspots.json b/plugins/config-audit/tests/snapshots/default-output/token-hotspots.json new file mode 100644 index 0000000..17d032a --- /dev/null +++ b/plugins/config-audit/tests/snapshots/default-output/token-hotspots.json @@ -0,0 +1,101 @@ +{ + "kind": "json", + "payload": { + "scanner": "TOK", + "status": "ok", + "files_scanned": 2, + "duration_ms": 0, + "total_estimated_tokens": 1809, + "hotspots": [ + { + "source": "mcp:memory (.mcp.json)", + "estimated_tokens": 500, + "rank": 1, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "mcp:sadhguru-wisdom (plugin:sadhguru-wisdom)", + "estimated_tokens": 500, + "rank": 2, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "mcp:vegnorm-rag (plugin:vegnormalene)", + "estimated_tokens": 500, + "rank": 3, + "recommendations": [ + "Review whether this source needs to load on every turn." + ] + }, + { + "source": "CLAUDE.md", + "estimated_tokens": 116, + "rank": 4, + "recommendations": [ + "Move volatile top-of-file content to the bottom or extract to an @import-ed file.", + "Split overlong CLAUDE.md into focused @imports (≤200 lines each)." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/CLAUDE.md" + }, + { + "source": "hooks/hooks.json", + "estimated_tokens": 81, + "rank": 5, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/hooks/hooks.json" + }, + { + "source": ".claude/settings.json", + "estimated_tokens": 59, + "rank": 6, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json" + }, + { + "source": ".mcp.json", + "estimated_tokens": 53, + "rank": 7, + "recommendations": [ + "Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.", + "Move rarely-used permissions to a project-local override." + ], + "path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.mcp.json" + } + ], + "findings": [ + { + "id": "CA-TOK-001", + "scanner": "TOK", + "severity": "low", + "title": "A connected service exposes many tools, all loading on every turn", + "description": "Each tool a connected service exposes adds its description to every turn. Services with many tools eat space fast.", + "file": ".mcp.json", + "line": null, + "evidence": "tool_count=unknown; server=\"memory\"; source=\".mcp.json\" — severity reflects estimated tokens/turn based on structural heuristic; not measured against runtime telemetry", + "category": "token-efficiency", + "recommendation": "Limit which tools the service exposes (often via a `tools` allow-list), or disconnect services you rarely use.", + "autoFixable": false, + "userImpactCategory": "Wasted tokens", + "userActionLanguage": "Optional cleanup", + "relevanceContext": "affects-everyone" + } + ], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 1, + "info": 0 + } + } +}