/** * Q2 — the command layer's contract with the CLI layer. * * The templates tell an agent to run a CLI with a particular argv. Until now * nothing checked that the CLI on the other end still *accepts* that argv. The * measured cost of that gap is M-BUG-45: `--stale-after` reached * `knowledge-refresh-cli` malformed, the CLI ignored it, and the command * reported "✓ all 14 entries re-verified within the last 90 days" about a * threshold the user had just overridden. A flag that quietly stops existing * produces the same sentence. * * Three properties are load-bearing here, each learned from a guard that was * green on its own defect: * * 1. **The argv is built from the template's own text**, never hand-typed * ([[dogfood-the-command-not-the-cli]], #63 — a hand-written call is a path * no user takes). `tests/helpers/command-invocations.mjs` reads all three * forms a flag appears in, including the comment-only form * (`GLOBAL_FLAG="" # --global`), which is the one that can die unobserved * because the default path leaves the variable empty. * * 2. **The probe proves itself per CLI before it is trusted.** A CLI that * exits on a required-arg check before reaching flag parsing would report * nothing about any flag, and every pair for it would pass vacuously. So each * CLI must first be seen rejecting a flag that certainly does not exist. An * empty sweep certifies nothing (#63, #64). Measured when this was written: * 15 of 15 report the unknown flag first, so no prefix argv is needed — a * fact worth re-deriving rather than assuming, which is why it is asserted. * * 3. **"Unknown" is distinguished from "needs a value" by the CLI's own * words**, so a value-taking flag passed last is not mistaken for a dead one. * That distinction is only sound because every CLI classifies the two * correctly — measured 14/15 when this was written; the fifteenth * (`campaign-export-cli`, the last hand-rolled parser) called its own * required `--repo` an unknown flag and was moved onto the shared * `requireValidArgs` gate in the same chunk. A guard that special-cased it * instead would have rebuilt, in test code, the prose exception Q1 deleted. * * Not asserted here: that a template calling a gated writer also calls * `write-scope-cli`. That arm belongs to write-scope-gate-shape.test.mjs, and * deriving it from the writer set would be false-red — `discover` and * `config-audit` invoke `scan-orchestrator` without ever reaching its * `--save-baseline` write. */ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { spawn } from 'node:child_process'; import { access, mkdtemp, readdir, rm } from 'node:fs/promises'; import { resolve, dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { allInvocations, commandTemplates, extractInvocations } from '../helpers/command-invocations.mjs'; import { hermeticEnv } from '../helpers/hermetic-home.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const SCANNERS_DIR = resolve(__dirname, '..', '..', 'scanners'); /** A flag no CLI can plausibly define — the probe's own control. */ const IMPOSSIBLE_FLAG = '--zzz-not-a-real-flag'; /** * Probing a flag means RUNNING the CLI, and some of these flags are writers. * Measured the first time this file ran, against the real environment: * `drift-cli --save` takes its target from the working directory and its name * from a default, so the probe scanned the temp dir and overwrote the * operator's `~/.config-audit/baselines/default.json` — an ungated write * outside the repo, produced by the guard whose whole subject is ungated writes * outside the repo. `fix-cli --apply` is the same shape one step worse. * * So every probe runs with HOME redirected into an empty temp dir AND its own * empty working directory, and the working directory is asserted to have stayed * empty. Isolation that is only a convention is not isolation. */ async function run(cli, argv) { const sandbox = await mkdtemp(join(tmpdir(), 'ca-contract-')); try { const { code, stderr } = await new Promise((res) => { const child = spawn(process.execPath, [resolve(SCANNERS_DIR, cli), ...argv], { cwd: sandbox, env: hermeticEnv(), }); let err = ''; child.stderr.on('data', (d) => { err += d; }); child.stdout.on('data', () => {}); child.on('close', (c) => res({ code: c, stderr: err })); }); return { code, stderr, wrote: await readdir(sandbox) }; } finally { await rm(sandbox, { recursive: true, force: true }); } } /** * Did the CLI call THIS flag unknown? * * Both wordings in the tree name the flag on the same line — `unknown flag * "--x"` (shared `cli-args`) and `Unknown option: --x` (the BOOL_FLAGS * parsers). "needs a value" / "requires a value" are deliberately NOT matched: * a value-taking flag passed last is well-formed as far as this contract goes. */ function reportedUnknown(stderr, flag) { const escaped = flag.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&'); return new RegExp(`unknown (flag|option)[^\\n]*${escaped}`, 'i').test(stderr); } const invocations = await allInvocations(); const clis = [...new Set(invocations.map((i) => i.cli))].sort(); /** `Map>` — every flag a template hands each CLI. */ const surface = new Map(); for (const inv of invocations) { if (!surface.has(inv.cli)) surface.set(inv.cli, new Map()); const flags = surface.get(inv.cli); for (const flag of inv.flags) { if (!flags.has(flag)) flags.set(flag, []); flags.get(flag).push(`${inv.file}:${inv.line} (${inv.source.get(flag)})`); } } /** * The extractor is the floor everything else stands on. A regex that stops * matching would make every assertion below vacuously green (#63, #64), and the * failure would look exactly like success. Coverage is derived from the tree * rather than pinned to a literal count, which would only be a drift point * (#60, #61). */ test('the extractor finds the invocations that are actually in the templates', async () => { assert.ok(invocations.length > 0, 'No CLI invocation was extracted from any command template.'); assert.ok(clis.length > 0, 'Invocations were found but named no CLI.'); const missed = []; for (const { file, content } of await commandTemplates()) { const namesACli = /node\s+\$\{CLAUDE_PLUGIN_ROOT\}\/scanners\//.test(content); if (namesACli && extractInvocations(content).length === 0) missed.push(file); } assert.deepEqual( missed, [], 'A template writes an anchored `node ${CLAUDE_PLUGIN_ROOT}/scanners/…` call that the\n' + 'extractor did not see. Every assertion in this file is silent about whatever it cannot\n' + 'parse, so a shrinking sweep reads as a passing one.', ); }); test('every scanner a template names exists on disk', async () => { const dead = []; for (const cli of clis) { try { await access(resolve(SCANNERS_DIR, cli)); } catch { const sites = invocations.filter((i) => i.cli === cli).map((i) => `${i.file}:${i.line}`); dead.push(`${cli} <- ${sites.join(', ')}`); } } assert.deepEqual( dead, [], 'A command template tells the agent to run a scanner that is not there. The user gets a\n' + "node stack trace the ux-rules promise they never see, because the command's own\n" + '`2>/dev/null` hides the one line that would explain it.', ); }); for (const cli of clis) { const flags = surface.get(cli); test(`${cli} accepts every flag the command templates hand it`, async () => { // Property 2 — the probe proves itself against this CLI before its silence // is allowed to mean anything. const control = await run(cli, [IMPOSSIBLE_FLAG]); assert.ok( reportedUnknown(control.stderr, IMPOSSIBLE_FLAG), `${cli} did not report ${IMPOSSIBLE_FLAG} as unknown (exit ${control.code}), so this test\n` + 'cannot tell an accepted flag from an unreachable parser — every flag below would pass\n' + 'for the wrong reason. Give this CLI the argv it needs to reach its flag parsing, or\n' + 'fix the CLI. stderr was: ' + JSON.stringify(control.stderr), ); const dead = []; const spilled = []; for (const [flag, sites] of [...flags].sort()) { const { stderr, wrote } = await run(cli, [flag]); if (reportedUnknown(stderr, flag)) dead.push(`${flag} <- ${sites.join(', ')}`); if (wrote.length) spilled.push(`${flag} left ${JSON.stringify(wrote)}`); } assert.deepEqual( spilled, [], `${cli} wrote into the working directory while being probed with a single flag and no\n` + '`--output-file`. Either the CLI writes somewhere it should not, or this probe is no\n' + 'longer contained — and an uncontained probe of a writer is how this test overwrote a\n' + 'real baseline the first time it ran.', ); assert.deepEqual( dead, [], `${cli} rejects a flag a command template still passes it. The template keeps working\n` + 'right up to the moment a user turns that knob on, and then the CLI exits 3 behind the\n' + "command's `2>/dev/null` — or, before the unknown-flag guard existed, answered a\n" + 'question nobody asked (M-BUG-45). Flags:\n ' + dead.join('\n '), ); }); }