config-audit/tests/scanners/campaign-write-cli.test.mjs
Kjell Tore Guttormsen caea8aca23 fix(commands): stop answering questions the caller did not ask
Dogfooding `campaign` + `knowledge-refresh` against a throwaway ledger. Seven
defects, all found by running the commands as written and measuring, not by
reading them.

The headline pair only existed together. `knowledge-refresh` built
`STALE_AFTER="--stale-after 30"` and expanded it unquoted, trusting the shell to
split it in two. bash does; zsh — the macOS default, and what the Bash tool runs
here — does not. The CLI got one argv entry, matched no flag, and because it had
no unknown-flag branch, silently kept the 90-day default and reported "✓ All 14
register entries were re-verified within the last 90 days": a true-sounding
sentence about a threshold the user had just overridden. Fixing either half alone
leaves a silent wrong answer or a loud one; both are fixed, and a guard now
rejects any template that packs a flag and its value into one variable.

`knowledge-refresh` also read one register and wrote another: step 6 named an
unanchored `knowledge/best-practices.json` while the CLI reads
`${CLAUDE_PLUGIN_ROOT}/…`, which for an installed plugin is the cache. The
validation gate then ran the cached test against the cached register — green no
matter what was written. The two copies were byte-identical that day, which is
exactly why it was invisible.

`campaign` vouched for repos it could not read. `add /finnes/ikke` returned
`added` + exit 0; `refresh-tokens` then put the phantom in `swept[]` with a
0-token delta and left `skipped[]` empty, so the machine-wide bill claimed
coverage of three repos on a machine with two. Paths stay tracked — an unmounted
volume is a legitimate absence — but are reported as `addedUnverified`, and the
command names them.

Two class sweeps, both measured rather than assumed. `posture` was the single
scanner (1 of 14) whose fatal catch exited 1, which ux-rules defines as a normal
WARNING grade — a crash indistinguishable from a result. And all 13 payload
writers failed on a `--output-file` whose parent did not exist, which on a fresh
machine turned `campaign`'s first run into "the ledger may be corrupt"; they now
share `scanners/lib/write-output.mjs`.

Predicted breadth was too wide for the first time in five sessions: 6 of 8 CLIs
predicted to lack unknown-flag rejection, 4 measured. `drift` and `fix` already
reject them, via a construct the grep did not recognise — a grep matches an
implementation, the invariant is a behaviour. The sweep was rewritten to run each
CLI with a bogus flag and read the exit code.

Suite 1453 → 1469/0. Frozen snapshots untouched. `optimize-lens-cli` and
`token-hotspots-cli` share the unknown-flag defect and are deferred to the v5.14
argument-handling chunk with their positional-swallow arm; the count is recorded
in the guard rather than rounded down to zero.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012NHWjN8EnoxSqRvMTLK2NE
2026-08-01 21:26:39 +02:00

393 lines
16 KiB
JavaScript

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { readFileSync, mkdtempSync, existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import {
loadLedger,
validateLedger,
rollUp,
} from '../../scanners/lib/campaign-ledger.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLI = resolve(__dirname, '../../scanners/campaign-write-cli.mjs');
const NOW = '2026-06-22';
// Every test passes an explicit --ledger-file in a temp dir + an injected --reference-date,
// so the write-CLI never touches the real default path under HOME and never reads the clock.
// The suite is hermetic + deterministic by construction.
function runWrite(args, env = {}) {
try {
const stdout = execFileSync('node', [CLI, ...args], {
encoding: 'utf-8',
timeout: 30000,
env: { ...process.env, ...env },
});
return { status: 0, stdout };
} catch (err) {
return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' };
}
}
function newDir() {
return mkdtempSync(join(tmpdir(), 'camp-write-'));
}
// Read the persisted ledger back through the real lib so a test proves the on-disk file
// is exactly what the command layer would later load.
async function readLedger(file) {
const ledger = await loadLedger(file);
return { ledger, validation: validateLedger(ledger) };
}
describe('campaign-write-cli — init', () => {
it('creates a valid, empty, versioned ledger (exit 0)', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
assert.equal(status, 0);
const out = JSON.parse(stdout);
assert.equal(out.action, 'init');
assert.equal(out.written, true);
assert.equal(out.alreadyInitialized, false);
assert.equal(out.ledgerPath, resolve(file));
const { ledger, validation } = await readLedger(file);
assert.ok(validation.valid, validation.errors.join('; '));
assert.equal(ledger.schemaVersion, 1);
assert.equal(ledger.createdDate, NOW);
assert.equal(ledger.updatedDate, NOW);
assert.deepEqual(ledger.repos, []);
});
it('refuses to clobber an existing ledger (exit 1, file untouched)', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
// First init, then add a repo so we can prove a second init does NOT wipe it.
runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
runWrite(['add', '/r/a', '--ledger-file', file, '--reference-date', NOW]);
const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
assert.equal(status, 1, 'advisory: already initialized');
const out = JSON.parse(stdout);
assert.equal(out.written, false);
assert.equal(out.alreadyInitialized, true);
const { ledger } = await readLedger(file);
assert.equal(ledger.repos.length, 1, 'existing repo preserved — not clobbered');
});
});
describe('campaign-write-cli — add', () => {
it('auto-initializes when no ledger exists, then adds repos (exit 0)', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
// Real directories: since #51 `add` reports a path it cannot read under
// `addedUnverified` instead of vouching for it. This test is about auto-init and
// tracking, so its fixtures must be repos that actually exist.
const a = newDir();
const b = newDir();
assert.ok(!existsSync(file));
const { status, stdout } = runWrite([
'add', a, b, '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 0);
const out = JSON.parse(stdout);
assert.equal(out.action, 'add');
assert.equal(out.autoInitialized, true);
assert.deepEqual(out.added.sort(), [resolve(a), resolve(b)].sort());
assert.deepEqual(out.addedUnverified, []);
const { ledger, validation } = await readLedger(file);
assert.ok(validation.valid, validation.errors.join('; '));
assert.equal(ledger.repos.length, 2);
assert.ok(ledger.repos.every((r) => r.status === 'pending'));
});
it('appends to an existing ledger and is idempotent on re-add (added vs skipped)', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
const a = newDir();
const b = newDir();
const c = newDir();
runWrite(['add', a, b, '--ledger-file', file, '--reference-date', NOW]);
const { status, stdout } = runWrite([
'add', a, c, '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 0);
const out = JSON.parse(stdout);
assert.equal(out.autoInitialized, false);
assert.deepEqual(out.added, [resolve(c)]);
assert.deepEqual(out.skipped, [resolve(a)]);
const { ledger } = await readLedger(file);
assert.equal(ledger.repos.length, 3);
});
it('uses --name for a single path; derives basenames for multiple', async () => {
const dir = newDir();
const solo = join(dir, 'solo.json');
runWrite(['add', '/r/x', '--name', 'custom', '--ledger-file', solo, '--reference-date', NOW]);
const { ledger: l1 } = await readLedger(solo);
assert.equal(l1.repos[0].name, 'custom');
const multi = join(dir, 'multi.json');
runWrite(['add', '/r/alpha', '/r/beta', '--ledger-file', multi, '--reference-date', NOW]);
const { ledger: l2 } = await readLedger(multi);
assert.deepEqual(l2.repos.map((r) => r.name).sort(), ['alpha', 'beta']);
});
});
describe('campaign-write-cli — set-status', () => {
async function seeded() {
const dir = newDir();
const file = join(dir, 'ledger.json');
runWrite(['add', '/r/a', '/r/b', '--ledger-file', file, '--reference-date', NOW]);
return file;
}
it('transitions a tracked repo and attaches findings + session (exit 0)', async () => {
const file = await seeded();
const findings = { critical: 1, high: 2, medium: 0, low: 4 };
const { status, stdout } = runWrite([
'set-status', '/r/a', 'audited',
'--findings', JSON.stringify(findings),
'--session', '20260622_120000',
'--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 0);
const out = JSON.parse(stdout);
assert.equal(out.action, 'set-status');
assert.equal(out.repo.status, 'audited');
assert.deepEqual(out.repo.findingsBySeverity, findings);
assert.equal(out.repo.sessionId, '20260622_120000');
const { ledger } = await readLedger(file);
assert.deepEqual(rollUp(ledger).bySeverity, findings);
assert.deepEqual(out.rollUp, rollUp(ledger));
});
it('exits 3 on an invalid status', async () => {
const file = await seeded();
const { status } = runWrite([
'set-status', '/r/a', 'nonsense', '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 3);
});
it('exits 3 when the repo is not tracked', async () => {
const file = await seeded();
const { status } = runWrite([
'set-status', '/r/ghost', 'audited', '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 3);
});
it('exits 3 when no ledger exists yet', () => {
const dir = newDir();
const { status } = runWrite([
'set-status', '/r/a', 'audited',
'--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW,
]);
assert.equal(status, 3);
});
});
describe('campaign-write-cli — refresh-tokens (live cross-repo sweep, v5.9 B2b)', () => {
// Fixture: a fake HOME with a LARGE global CLAUDE.md (the shared layer, made to
// dominate so the counted-once guard is meaningful) + two repos OUTSIDE that HOME,
// each with a small project CLAUDE.md (the per-repo delta). repos sit beside the
// fake home so the cascade walk never picks up the global file as a project file.
function buildSweepFixture() {
const root = mkdtempSync(join(tmpdir(), 'camp-sweep-'));
const fakeHome = join(root, 'home');
mkdirSync(join(fakeHome, '.claude'), { recursive: true });
writeFileSync(
join(fakeHome, '.claude', 'CLAUDE.md'),
`# Global user config\n\n${'Shared instruction line that every repo pays for. '.repeat(120)}\n`,
);
const repoA = join(root, 'repo-a');
const repoB = join(root, 'repo-b');
const bodies = [
[repoA, '# Repo A\n\nProject A note.\n'],
[repoB, '# Repo B\n\nProject B has a little more local text here.\n'],
];
for (const [repo, body] of bodies) {
mkdirSync(join(repo, '.git'), { recursive: true });
writeFileSync(join(repo, '.git', 'HEAD'), 'ref: refs/heads/main\n');
writeFileSync(join(repo, 'CLAUDE.md'), body);
}
return { root, fakeHome, repoA, repoB };
}
it('sweeps every tracked repo, counts the shared layer ONCE, attributes deltas per repo', async () => {
const { root, fakeHome, repoA, repoB } = buildSweepFixture();
try {
const file = join(root, 'ledger.json');
runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome });
const { status, stdout } = runWrite(
['refresh-tokens', '--ledger-file', file, '--reference-date', NOW],
{ HOME: fakeHome },
);
assert.equal(status, 0, stdout);
const out = JSON.parse(stdout);
assert.equal(out.action, 'refresh-tokens');
assert.equal(out.written, true);
assert.deepEqual(out.swept.slice().sort(), [resolve(repoA), resolve(repoB)].sort());
assert.deepEqual(out.skipped, []);
// NOTE on shapes (B2a contract): the STORED summaries — out.sharedGlobal and each
// repo.tokens — carry {always:{tokens,count}}, but rollUp().tokens flattens its
// sharedGlobal/perRepoDelta/machineWide buckets to {always:<number>} (bucketTokens).
const t = out.rollUp.tokens;
assert.ok(t.sharedGlobal.always > 0, 'shared global layer should be non-empty');
assert.equal(t.reposWithTokens, 2);
assert.equal(t.byRepo.length, 2);
// Persisted ledger carries the new token fields and stays schema-valid.
const ledgerOut = await loadLedger(file);
assert.ok(validateLedger(ledgerOut).valid, validateLedger(ledgerOut).errors.join('; '));
assert.deepEqual(ledgerOut.sharedGlobal, out.sharedGlobal);
// Counted-ONCE guard: each repo's stored delta must NOT contain the dominant
// shared global CLAUDE.md. Had the sweep folded shared into the per-repo tokens,
// each delta would EXCEED the shared layer instead of being a small fraction.
for (const repo of ledgerOut.repos) {
assert.ok(repo.tokens, `${repo.name} should have tokens`);
assert.ok(
repo.tokens.always.tokens < t.sharedGlobal.always,
`${repo.name} delta (${repo.tokens.always.tokens}) must be < shared (${t.sharedGlobal.always})`,
);
}
// rollUp invariant: machineWide = shared (once) + sum of per-repo deltas.
const deltaSum = ledgerOut.repos.reduce((s, r) => s + r.tokens.always.tokens, 0);
assert.equal(t.perRepoDelta.always, deltaSum);
assert.equal(t.machineWide.always, t.sharedGlobal.always + deltaSum);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it('is idempotent — a second sweep replaces, never accumulates', async () => {
const { root, fakeHome, repoA, repoB } = buildSweepFixture();
try {
const file = join(root, 'ledger.json');
runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome });
const first = JSON.parse(
runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout,
);
const second = JSON.parse(
runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout,
);
assert.deepEqual(second.rollUp.tokens, first.rollUp.tokens);
} finally {
rmSync(root, { recursive: true, force: true });
}
});
it('exits 3 when no ledger exists yet', () => {
const dir = newDir();
const { status } = runWrite([
'refresh-tokens', '--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW,
]);
assert.equal(status, 3);
});
});
describe('campaign-write-cli — determinism + --output-file', () => {
it('stamps updatedDate from --reference-date and writes the payload to --output-file', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
runWrite(['init', '--ledger-file', file, '--reference-date', '2026-01-01']);
const report = join(dir, 'report.json');
const { stdout } = runWrite([
'add', '/r/a', '--ledger-file', file, '--reference-date', '2026-03-15',
'--output-file', report,
]);
assert.equal(stdout.trim(), '', 'stdout is silent when --output-file is given');
const written = JSON.parse(readFileSync(report, 'utf-8'));
assert.equal(written.action, 'add');
const { ledger } = await readLedger(file);
assert.equal(ledger.createdDate, '2026-01-01', 'created stamp preserved');
assert.equal(ledger.updatedDate, '2026-03-15', 'updated stamp from the later --reference-date');
});
it('exits 3 on an unknown subcommand', () => {
const dir = newDir();
const { status } = runWrite(['frobnicate', '--ledger-file', join(dir, 'x.json')]);
assert.equal(status, 3);
});
});
// ── Session #51: honest coverage — the ledger must not vouch for repos it cannot read ──
//
// Dogfooding the campaign lifecycle put a path that does not exist into the ledger and
// swept it. Both halves lied, quietly:
//
// add → added: ["/finnes/absolutt/ikke/noe-repo"], exit 0, no warning.
// sweep → swept: [… , "/finnes/absolutt/ikke/noe-repo"], skipped: [], byRepo entry with 0
// tokens, reposWithTokens: 3 for a machine with 2 real repos.
//
// The root cause is that `readActiveConfig` resolves a path and every sub-reader tolerates
// ENOENT, so a missing repo yields an EMPTY config rather than an error — and the sweep's
// try/catch only routes THROWN errors to `skipped`. The command's own honesty clause ("If
// anything was skipped, name those repos plainly so the user knows the bill omits them")
// could therefore never fire. A token bill that silently counts phantom repos as 0 is worse
// than one that refuses to answer: it looks complete.
describe('campaign-write-cli — honest coverage for unreadable repos', () => {
it('add flags a path that does not exist instead of vouching for it', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
const real = newDir();
const phantom = join(dir, 'no-such-repo');
const { status, stdout } = runWrite([
'add', real, phantom, '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 0, 'a missing path is reported, not rejected — it may be unmounted');
const out = JSON.parse(stdout);
assert.deepEqual(out.added, [resolve(real)], 'only the readable repo is vouched for');
assert.deepEqual(
out.addedUnverified,
[resolve(phantom)],
'a path that does not exist must be reported separately so the command can say so',
);
// Still tracked — the campaign is a work register, and an unmounted volume is a
// legitimate reason for a path to be absent today and present tomorrow.
const { ledger } = await readLedger(file);
assert.equal(ledger.repos.length, 2);
});
it('refresh-tokens skips an unreadable repo instead of counting it as 0 tokens', async () => {
const dir = newDir();
const file = join(dir, 'ledger.json');
const phantom = join(dir, 'no-such-repo');
runWrite(['add', phantom, '--ledger-file', file, '--reference-date', NOW]);
const { status, stdout } = runWrite([
'refresh-tokens', '--ledger-file', file, '--reference-date', NOW,
]);
assert.equal(status, 0);
const out = JSON.parse(stdout);
assert.deepEqual(out.swept, [], 'a repo that cannot be read was never actually swept');
assert.equal(out.skipped.length, 1, 'it belongs in skipped, with a reason the user can act on');
assert.equal(out.skipped[0].path, resolve(phantom));
assert.match(out.skipped[0].reason, /not readable|does not exist|ENOENT/i);
assert.equal(
out.rollUp.tokens.reposWithTokens,
0,
'the machine-wide bill must not claim coverage of a repo it could not read',
);
});
});