config-audit/tests/commands/command-placeholder-shell-safety.test.mjs

/**
 * Session #56 — placeholders inside runnable bash fences must not be shell-active.
 *
 * Dogfooding the ROUTER (`commands/config-audit.md`) surfaced a defect one layer
 * below the CLIs: the fence on the router's step 3 carries the placeholder
 * `<target-path>` **bare** — unquoted, preceded by a space. `<` and `>` are
 * redirection operators. Measured under the shell the Bash tool actually runs
 * (zsh), in a scratch directory:
 *
 *   $ node …/scan-orchestrator.mjs <target-path> --output-file scan.json \
 *       >/dev/null 2>/dev/null; node …/posture.mjs <target-path> \
 *       --output-file posture.json 2>/dev/null; echo $?
 *   zsh:1: no such file or directory: target-path
 *   zsh:1: no such file or directory: target-path
 *   1
 *   → neither file written, neither CLI ever started
 *
 * Three properties make this worse than a plain typo:
 *
 *  1. **The CLI never runs.** Redirection is resolved by the shell before the
 *     command is executed, so the CLI's own argument validation — the layer that
 *     `cli-unknown-flag-rejection.test.mjs` hardened — never sees it.
 *  2. **The failure is quiet where it counts.** The echoed status is `1`, and
 *     `1` is inside the band the router's own step 3 classifies as
 *     "continue normally" (0/1/2 = PASS/WARNING/FAIL; only 3 is a real error).
 *     A total non-execution is indistinguishable from a healthy WARNING run.
 *  3. **The file already knew about the neighbouring hazard.** Two lines above
 *     the offending call sits a comment warning that a *square-bracket*
 *     placeholder "does not start with a dash, so both CLIs' arg loops would
 *     take it as the TARGET PATH instead of a flag" — awareness of the
 *     placeholder class, while carrying a strictly worse member of it.
 *
 * The invariant is not "substitute your placeholders" (a template cannot enforce
 * that). It is that an UNSUBSTITUTED placeholder must fail **loudly, in the
 * CLI**, not silently in the shell. Quoting achieves exactly that: `"<path>"`
 * reaches argv as a literal, the CLI reports an unreadable target, and the exit
 * code means what the router thinks it means.
 *
 * Measured breadth at the time of writing: 13 of 21 command files, 33 sites.
 */

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFile, readdir } from 'node:fs/promises';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const COMMANDS_DIR = resolve(__dirname, '..', '..', 'commands');

async function commandFiles() {
  const entries = await readdir(COMMANDS_DIR);
  return entries.filter((e) => e.endsWith('.md')).sort();
}

/**
 * Yield the lines that live inside ```bash fences, with their 1-based file line
 * numbers. Only bash-tagged fences count: an untagged or json fence is prose.
 */
function bashFenceLines(content) {
  const out = [];
  let inFence = false;
  content.split('\n').forEach((line, i) => {
    if (/^\s*```/.test(line)) {
      inFence = /^\s*```bash\s*$/.test(line);
      return;
    }
    if (inFence) out.push({ line, n: i + 1 });
  });
  return out;
}

/**
 * A placeholder that the shell would read as a redirection: `<` at the start of
 * a word (start of line, or after whitespace / `;` / `|` / `&`), a lowercase
 * placeholder name, then `>`. A quoted placeholder (`"<path>"`) is excluded by
 * construction — the `<` is preceded by a quote, not a word boundary.
 */
const BARE_PLACEHOLDER = /(?:^|[\s;|&(])(<[a-z][a-z0-9._-]*>)/;

test('no runnable bash fence carries a bare (shell-active) angle-bracket placeholder', async () => {
  const offenders = [];

  for (const file of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, file), 'utf-8');
    for (const { line, n } of bashFenceLines(content)) {
      const m = line.match(BARE_PLACEHOLDER);
      if (m) offenders.push(`${file}:${n}  ${m[1]}  in: ${line.trim().slice(0, 90)}`);
    }
  }

  assert.deepEqual(
    offenders,
    [],
    'A bare `<name>` inside a bash fence is a REDIRECTION, not an argument. Left\n' +
      'unsubstituted it fails in the shell before the CLI starts — no output file,\n' +
      'no CLI diagnostics, and an exit code (1) that the router reads as a normal\n' +
      'WARNING run. Quote the placeholder (`"<name>"`) so an unsubstituted template\n' +
      'fails loudly in the CLI instead.\n' +
      'Offending sites:\n  ' + offenders.join('\n  '),
  );
});