config-audit/tests/commands/command-shell-state-shape.test.mjs

/**
 * Session #49 — command-template shell-state shape tests.
 *
 * Dogfooding the `plan` + `implement` chunk surfaced one root defect with
 * several arms: **command templates assume shell state survives between
 * fenced blocks.** It does not. Every ```bash fence is executed as its own
 * Bash tool call, in its own process:
 *
 *   - A variable assigned in block N is empty in block N+1.
 *   - `$$` (the PID) differs between calls, so a `/tmp/foo-$$.json` path
 *     created in one block can never be reconstructed in a later one.
 *   - The Read tool expands neither shell variables nor `$$` nor globs; it
 *     takes one literal path.
 *
 * Measured arms at the time of writing (all fixed by the accompanying commit):
 *   - `$RAW_FLAG` referenced inside a (non-bash) agent-prompt fence in
 *     analyze.md, plan.md, implement.md — the agent received the literal
 *     string `$RAW_FLAG`, confirmed at runtime by the planner-agent itself.
 *   - `$TMPFILE` referenced across blocks in manifest.md, tokens.md,
 *     whats-active.md, plugin-health.md.
 *   - `$GLOBAL_FLAG` across blocks in fix.md.
 *   - `$TODAY` across blocks in campaign.md (6 sites).
 *   - `$$` temp paths referenced outside their creating fence in fix.md.
 *   - plan.md asked the Read tool to expand
 *     `~/.claude/config-audit/sessions/*_/state.yaml`.
 *
 * The hardened pattern already present in drift.md is the target shape: a
 * fixed literal temp path, repeated literally in every block that needs it.
 */

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFile, readdir } from 'node:fs/promises';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const COMMANDS_DIR = resolve(__dirname, '..', '..', 'commands');

/** Shell variables supplied by the environment, not by a prior block. */
const AMBIENT = new Set([
  'CLAUDE_PLUGIN_ROOT', 'ARGUMENTS', 'HOME', 'PATH', 'PWD', 'USER', 'TMPDIR',
]);

async function commandFiles() {
  const entries = await readdir(COMMANDS_DIR);
  return entries.filter((e) => e.endsWith('.md')).sort();
}

/**
 * Parse fenced blocks. Returns { lines, blockIndexOf(lineIdx) } where
 * blockIndexOf returns -1 for prose outside any fence.
 */
function parseFences(content) {
  const lines = content.split('\n');
  const blocks = [];
  let open = null;
  lines.forEach((line, i) => {
    const m = line.match(/^\s*```(\w*)/);
    if (!m) return;
    if (open === null) open = { lang: m[1], start: i };
    else {
      blocks.push({ lang: open.lang, start: open.start, end: i });
      open = null;
    }
  });
  const blockIndexOf = (i) => blocks.findIndex((b) => i > b.start && i < b.end);
  return { lines, blocks, blockIndexOf };
}

/** Strip `#` comments from a bash line so the test never matches its own prose. */
function stripComment(line) {
  const h = line.indexOf('#');
  return h === -1 ? line : line.slice(0, h);
}

test('Shell state: no variable is referenced outside the block that assigned it', async () => {
  const violations = [];
  for (const name of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
    const { lines, blocks, blockIndexOf } = parseFences(content);

    const assignedIn = new Map();
    lines.forEach((raw, i) => {
      const bi = blockIndexOf(i);
      if (bi < 0) return;
      // An assignment starts a command, and a command starts at the beginning of
      // the line OR after a separator. Capturing only the line-initial form would
      // miss the idiomatic status capture `node …; STATUS=$?` — the one form that
      // MUST trail a command — and report the variable as never assigned.
      const re = /(?:^|[;&|]|&&|\|\|)\s*([A-Z_][A-Z0-9_]*)=/g;
      let m;
      while ((m = re.exec(stripComment(raw))) !== null) {
        if (!assignedIn.has(m[1])) assignedIn.set(m[1], new Set());
        assignedIn.get(m[1]).add(bi);
      }
    });

    lines.forEach((raw, i) => {
      const line = stripComment(raw);
      const bi = blockIndexOf(i);
      const re = /\$\{?([A-Z_][A-Z0-9_]*)\}?/g;
      let m;
      while ((m = re.exec(line)) !== null) {
        const v = m[1];
        if (AMBIENT.has(v)) continue;
        // Skip the assignment site itself (`FOO=$FOO...` right-hand side is fine).
        const eq = line.indexOf('=');
        if (/^\s*[A-Z_][A-Z0-9_]*=/.test(line) && line.indexOf(m[0]) < eq) continue;
        const where = assignedIn.get(v);
        if (!where) {
          violations.push(`${name}:${i + 1} $${v} is never assigned in any block`);
        } else if (bi < 0) {
          violations.push(
            `${name}:${i + 1} $${v} referenced in prose/agent-prompt — no shell expands it there`,
          );
        } else if (!where.has(bi)) {
          const lang = blocks[bi].lang || 'none';
          violations.push(
            `${name}:${i + 1} $${v} referenced in block ${bi} (lang=${lang}) but assigned only in block(s) ${[...where].join(', ')} — separate Bash calls, separate processes`,
          );
        }
      }
    });
  }
  assert.deepEqual(violations, [], `Cross-block shell-variable references:\n${violations.join('\n')}`);
});

test('Shell state: no $$ temp path is referenced outside the block that created it', async () => {
  const violations = [];
  for (const name of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
    const { lines, blockIndexOf } = parseFences(content);

    const firstSeen = new Map();
    const scan = (raw, i, cb) => {
      const re = /(\/tmp\/[A-Za-z0-9._-]*\$\$[A-Za-z0-9._-]*)/g;
      let m;
      while ((m = re.exec(stripComment(raw))) !== null) cb(m[1], i);
    };
    lines.forEach((raw, i) => scan(raw, i, (p) => {
      if (!firstSeen.has(p)) firstSeen.set(p, { blk: blockIndexOf(i), line: i + 1 });
    }));
    lines.forEach((raw, i) => scan(raw, i, (p) => {
      const origin = firstSeen.get(p);
      if (origin.line === i + 1) return;
      const bi = blockIndexOf(i);
      if (bi < 0) {
        violations.push(`${name}:${i + 1} ${p} referenced in prose — the Read tool cannot expand $$`);
      } else if (bi !== origin.blk) {
        violations.push(
          `${name}:${i + 1} ${p} referenced in block ${bi} but created in block ${origin.blk} — $$ is a different PID there`,
        );
      }
    }));
  }
  assert.deepEqual(violations, [], `Unresolvable $$ temp paths:\n${violations.join('\n')}`);
});

test('Shell state: no $$ appears in any temp path at all', async () => {
  // Session #50 closed a blind spot in the test above: it only flags a `$$`
  // path that is *referenced twice*, because it compares each occurrence to
  // the block that created it. A path written once and then read via prose
  // ("Read the JSON output file using the Read tool") has no second
  // occurrence — so posture.md sat green through #49 while being unreadable
  // by construction: the PID is never printed, so no later step can name the
  // file. Measured live: written by PID 21614, read attempted from PID 23772.
  //
  // The invariant is therefore blanket, not relational: a command template
  // must not put `$$` in a temp path at all. The hardened pattern from
  // drift.md — one fixed literal path, repeated literally — is the only
  // shape that survives the fence boundary.
  const violations = [];
  for (const name of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
    content.split('\n').forEach((raw, i) => {
      const re = /(\/tmp\/[A-Za-z0-9._-]*\$\$[A-Za-z0-9._-]*)/g;
      let m;
      while ((m = re.exec(stripComment(raw))) !== null) {
        violations.push(
          `${name}:${i + 1} ${m[1]} — $$ differs per Bash call; no later step can name this file`,
        );
      }
    });
  }
  assert.deepEqual(violations, [], `Unresolvable $$ temp paths:\n${violations.join('\n')}`);
});

test('Read tool: never asked to expand a glob', async () => {
  const violations = [];
  for (const name of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
    content.split('\n').forEach((raw, i) => {
      // Only flag when the glob is the *object* of a Read instruction, i.e.
      // "Read tool on `<glob>`" / "Read `<glob>`". Prose that merely explains
      // that Read cannot expand a glob is the fix, not the defect — matching
      // any co-occurrence on the line would flag this repo's own warning text.
      const m = raw.match(/\bRead(?:\s+the)?(?:\s+tool)?\s+(?:tool\s+)?on\s+`([~/][^`]*)`|\bRead\s+`([~/][^`]*)`/);
      if (!m) return;
      const path = m[1] ?? m[2];
      if (!path.includes('*')) return;
      violations.push(`${name}:${i + 1} Read tool pointed at a glob \`${path}\` — use Glob`);
    });
  }
  assert.deepEqual(violations, [], `Read-tool glob misuse:\n${violations.join('\n')}`);
});

test('state.yaml: phase commands name all four fields the rule requires', async () => {
  // .claude/rules/state-management.md mandates current_phase, completed_phases,
  // next_phase, updated_at after EVERY phase. A command that writes the file
  // while naming only two fields silently drops the other two.
  const REQUIRED = ['current_phase', 'completed_phases', 'next_phase', 'updated_at'];
  const violations = [];
  for (const name of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
    if (!/Update\s+`?state\.yaml`?|Update state/i.test(content)) continue;
    const missing = REQUIRED.filter((f) => !content.includes(f));
    if (missing.length) {
      violations.push(`${name} updates state.yaml but never names: ${missing.join(', ')}`);
    }
  }
  assert.deepEqual(violations, [], `Incomplete state.yaml contracts:\n${violations.join('\n')}`);
});

/**
 * Session #51 — the instruction that CAUSED the $TODAY defect must not outlive its fix.
 *
 * #49 fixed campaign.md by re-deriving `TODAY=$(date +%F)` inside each of the six write
 * blocks. But step 1 still carried the original prose — "Set a shared date stamp for any
 * write: `TODAY=$(date +%F)`" — which is not in a fence, cannot set anything, and directly
 * contradicts the six comments added below it. A template that argues with itself is not a
 * contract, and the next edit is the one that believes the wrong half.
 *
 * The guard above asserts fences; this one asserts that no PROSE line instructs the reader
 * to establish shell state for later blocks.
 */
test('no command template instructs shell state to be set outside a fence', async () => {
  const offenders = [];

  for (const file of await commandFiles()) {
    const content = await readFile(resolve(COMMANDS_DIR, file), 'utf-8');
    const { lines, blockIndexOf } = parseFences(content);
    lines.forEach((line, i) => {
      if (blockIndexOf(i) !== -1) return; // inside a fence: that is where state belongs
      if (/`[A-Z_][A-Z0-9_]*=\$\(/.test(line) || /^\s*[A-Z_][A-Z0-9_]*=\$\(/.test(line)) {
        offenders.push(`${file}:${i + 1}  ${line.trim()}`);
      }
    });
  }

  assert.deepEqual(
    offenders,
    [],
    'Prose that tells the reader to set a shell variable implies it survives to a later\n' +
      'block. It does not — every fence is its own process. Delete the instruction; the\n' +
      'blocks that need the value derive it themselves:\n  ' + offenders.join('\n  '),
  );
});