/** * Session #50 — command-template output-discipline tests. * * Dogfooding the four read commands (`posture`, `tokens`, `manifest`, * `whats-active`) surfaced three defect classes that all live in the same * seam: what a command template PROMISES versus what the scanner behind it * actually does. * * 1. M-BUG-43 — a scanner invoked with `--raw`/`--json` writes the payload * to stdout *even when `--output-file` is set* * (`token-hotspots-cli.mjs:137`, `manifest.mjs:293`, * `whats-active.mjs:60`, `posture.mjs:101`, `drift-cli.mjs`, * `plugin-health-scanner.mjs`). The command templates redirect only * stderr, so the payload lands in the transcript. Measured on this repo: * posture 255 182 B, whats-active 35 922 B, drift 28 316 B, * manifest 23 825 B, tokens 8 768 B. `.claude/rules/ux-rules.md` rule 1 * says NEVER show raw JSON — and a plugin that exists to cut token cost * must not dump a quarter-megabyte to report one grade. * * 2. Flags documented in a command's prose that never reach any shell * (`tokens.md`: `--json`, `--with-telemetry-recipe`). The scanner * supports them; the template silently swallows them. Same "green is * worse than an error" shape as the arg-sink class. * * 3. Render contracts that name a field the scanner never emits * (`manifest.md` asked for `{load}`; the payload carries `loadPattern`), * so the column the command's own prose calls the whole point renders * blank for every row. * * Test 3 runs the real scanners against a fixture rather than asserting * against a hardcoded key list — a hardcoded list drifts, a live payload * cannot. */ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFile, readdir, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; const execFileAsync = promisify(execFile); const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(__dirname, '..', '..'); const COMMANDS_DIR = join(ROOT, 'commands'); const FIXTURE = join(ROOT, 'tests', 'fixtures', 'marketplace-medium'); async function commandFiles() { const entries = await readdir(COMMANDS_DIR); return entries.filter((e) => e.endsWith('.md')).sort(); } /** Strip `#` comments so the tests never match their own explanatory prose. */ function stripComment(line) { const h = line.indexOf('#'); return h === -1 ? line : line.slice(0, h); } /** Yield [lineNumber, line] for lines inside ```bash fences only. */ function bashLines(content) { const out = []; let open = null; content.split('\n').forEach((line, i) => { const m = line.match(/^\s*```(\w*)/); if (m) { open = open === null ? m[1] : null; return; } if (open === 'bash') out.push([i + 1, line]); }); return out; } /** Yield lines inside ```markdown fences — the render contracts. */ function markdownFenceLines(content) { const out = []; let open = null; content.split('\n').forEach((line) => { const m = line.match(/^\s*```(\w*)/); if (m) { open = open === null ? m[1] : null; return; } if (open === 'markdown') out.push(line); }); return out; } test('Output discipline: a scanner asked for --output-file never also prints to stdout', async () => { // The rule is blanket and needs no per-scanner allowlist: if the command // asked for the payload in a FILE, the same invocation must not let it reach // the transcript. Invocations with no --output-file are out of scope — there // the payload has nowhere else to go (e.g. `drift --save`, 112 B). const violations = []; for (const name of await commandFiles()) { const content = await readFile(join(COMMANDS_DIR, name), 'utf-8'); for (const [lineNo, raw] of bashLines(content)) { const line = stripComment(raw); if (!/node \$\{CLAUDE_PLUGIN_ROOT\}\/scanners\//.test(line)) continue; if (!line.includes('--output-file')) continue; const rawish = /--json\b|--raw\b|\$RAW_FLAG/.test(line); if (!rawish) continue; // Mask `2>` so a stderr redirect is never mistaken for a stdout one. const masked = line.replace(/2>/g, '2@'); if (/(^|\s)>\s*\S+/.test(masked)) continue; violations.push( `${name}:${lineNo} runs a scanner in raw/json mode with --output-file but never redirects stdout — the payload lands in the transcript`, ); } } assert.deepEqual(violations, [], `Unredirected scanner payloads:\n${violations.join('\n')}`); }); test('Flag threading: every flag documented in prose reaches a shell', async () => { // A flag the user is told to pass must arrive somewhere. Two legitimate // destinations exist: a bash fence (threaded to the scanner) or a documented // control-flow branch handled by the model. The allowlist below carries only // the second kind, each verified by reading the code: // posture --drift / --plugin-health : step 5 runs *different* scanners // fix --dry-run : dry-run is fix-cli's DEFAULT and the // flag is an explicit no-op alias // (`scanners/fix-cli.mjs:18-21`) // knowledge-refresh --no-candidates : skips a web-poll step; never a CLI flag // manifest/whats-active --json : served by `cat` of the payload, whose // content is byte-identical to --raw // (both scanners document --raw as a // no-op; verified by diffing payloads) const PROSE_HANDLED = new Set([ 'posture.md:--drift', 'posture.md:--plugin-health', 'fix.md:--dry-run', 'knowledge-refresh.md:--no-candidates', 'manifest.md:--json', 'whats-active.md:--json', ]); const violations = []; for (const name of await commandFiles()) { const content = await readFile(join(COMMANDS_DIR, name), 'utf-8'); const bash = bashLines(content).map(([, l]) => l).join('\n'); const documented = new Set(); for (const line of content.split('\n')) { const m = line.match(/^\s*[-*]\s+`(--[a-z][a-z0-9-]*)`/); if (m) documented.add(m[1]); } for (const flag of documented) { if (bash.includes(flag)) continue; if (PROSE_HANDLED.has(`${name}:${flag}`)) continue; violations.push(`${name} documents \`${flag}\` but no bash fence ever passes it`); } } assert.deepEqual(violations, [], `Flags dropped between docs and shell:\n${violations.join('\n')}`); }); test('Render contract: every {field} in a render fence exists in the real payload', async () => { // Derived values the model computes rather than reads. Kept deliberately // short — every entry here is a field the render fence does NOT get from the // scanner, so a long list would hollow out the test. const DERIVED = new Set(['rank']); const CASES = [ { command: 'manifest.md', scanner: 'manifest.mjs', args: [] }, { command: 'tokens.md', scanner: 'token-hotspots-cli.mjs', args: [] }, { command: 'whats-active.md', scanner: 'whats-active.mjs', args: ['--verbose', '--suggest-disables'], }, ]; const dir = await mkdtemp(join(tmpdir(), 'ca-render-')); try { const violations = []; for (const { command, scanner, args } of CASES) { const outFile = join(dir, `${scanner}.json`); await execFileAsync('node', [ join(ROOT, 'scanners', scanner), FIXTURE, '--output-file', outFile, ...args, ]); const payload = JSON.parse(await readFile(outFile, 'utf-8')); const keys = new Set(); const walk = (node) => { if (Array.isArray(node)) node.slice(0, 5).forEach(walk); else if (node && typeof node === 'object') { for (const k of Object.keys(node)) { keys.add(k); walk(node[k]); } } }; walk(payload); const content = await readFile(join(COMMANDS_DIR, command), 'utf-8'); const refs = new Set(); for (const line of markdownFenceLines(content)) { const re = /\{([a-zA-Z][a-zA-Z0-9_.]*)\}/g; let m; while ((m = re.exec(line)) !== null) refs.add(m[1]); } for (const ref of refs) { if (DERIVED.has(ref)) continue; if (ref.endsWith('.length')) { // `{foo.length}` is a count of a collection — assert the collection // itself exists, since that is the part the scanner owns. const base = ref.slice(0, -'.length'.length).split('.').pop(); if (!keys.has(base)) { violations.push(`${command}: {${ref}} — no \`${base}\` collection in the payload`); } continue; } const leaf = ref.split('.').pop(); if (!keys.has(leaf) && !keys.has(ref)) { violations.push(`${command}: {${ref}} is never emitted by ${scanner}`); } } } assert.deepEqual(violations, [], `Render fields the scanner never emits:\n${violations.join('\n')}`); } finally { await rm(dir, { recursive: true, force: true }); } });