Dogfooding `campaign` + `knowledge-refresh` against a throwaway ledger. Seven
defects, all found by running the commands as written and measuring, not by
reading them.
The headline pair only existed together. `knowledge-refresh` built
`STALE_AFTER="--stale-after 30"` and expanded it unquoted, trusting the shell to
split it in two. bash does; zsh — the macOS default, and what the Bash tool runs
here — does not. The CLI got one argv entry, matched no flag, and because it had
no unknown-flag branch, silently kept the 90-day default and reported "✓ All 14
register entries were re-verified within the last 90 days": a true-sounding
sentence about a threshold the user had just overridden. Fixing either half alone
leaves a silent wrong answer or a loud one; both are fixed, and a guard now
rejects any template that packs a flag and its value into one variable.
`knowledge-refresh` also read one register and wrote another: step 6 named an
unanchored `knowledge/best-practices.json` while the CLI reads
`${CLAUDE_PLUGIN_ROOT}/…`, which for an installed plugin is the cache. The
validation gate then ran the cached test against the cached register — green no
matter what was written. The two copies were byte-identical that day, which is
exactly why it was invisible.
`campaign` vouched for repos it could not read. `add /finnes/ikke` returned
`added` + exit 0; `refresh-tokens` then put the phantom in `swept[]` with a
0-token delta and left `skipped[]` empty, so the machine-wide bill claimed
coverage of three repos on a machine with two. Paths stay tracked — an unmounted
volume is a legitimate absence — but are reported as `addedUnverified`, and the
command names them.
Two class sweeps, both measured rather than assumed. `posture` was the single
scanner (1 of 14) whose fatal catch exited 1, which ux-rules defines as a normal
WARNING grade — a crash indistinguishable from a result. And all 13 payload
writers failed on a `--output-file` whose parent did not exist, which on a fresh
machine turned `campaign`'s first run into "the ledger may be corrupt"; they now
share `scanners/lib/write-output.mjs`.
Predicted breadth was too wide for the first time in five sessions: 6 of 8 CLIs
predicted to lack unknown-flag rejection, 4 measured. `drift` and `fix` already
reject them, via a construct the grep did not recognise — a grep matches an
implementation, the invariant is a behaviour. The sweep was rewritten to run each
CLI with a bogus flag and read the exit code.
Suite 1453 → 1469/0. Frozen snapshots untouched. `optimize-lens-cli` and
`token-hotspots-cli` share the unknown-flag defect and are deferred to the v5.14
argument-handling chunk with their positional-swallow arm; the count is recorded
in the guard rather than rounded down to zero.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012NHWjN8EnoxSqRvMTLK2NE
393 lines
16 KiB
JavaScript
393 lines
16 KiB
JavaScript
import { describe, it } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { readFileSync, mkdtempSync, existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import {
|
|
loadLedger,
|
|
validateLedger,
|
|
rollUp,
|
|
} from '../../scanners/lib/campaign-ledger.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const CLI = resolve(__dirname, '../../scanners/campaign-write-cli.mjs');
|
|
|
|
const NOW = '2026-06-22';
|
|
|
|
// Every test passes an explicit --ledger-file in a temp dir + an injected --reference-date,
|
|
// so the write-CLI never touches the real default path under HOME and never reads the clock.
|
|
// The suite is hermetic + deterministic by construction.
|
|
function runWrite(args, env = {}) {
|
|
try {
|
|
const stdout = execFileSync('node', [CLI, ...args], {
|
|
encoding: 'utf-8',
|
|
timeout: 30000,
|
|
env: { ...process.env, ...env },
|
|
});
|
|
return { status: 0, stdout };
|
|
} catch (err) {
|
|
return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' };
|
|
}
|
|
}
|
|
|
|
function newDir() {
|
|
return mkdtempSync(join(tmpdir(), 'camp-write-'));
|
|
}
|
|
|
|
// Read the persisted ledger back through the real lib so a test proves the on-disk file
|
|
// is exactly what the command layer would later load.
|
|
async function readLedger(file) {
|
|
const ledger = await loadLedger(file);
|
|
return { ledger, validation: validateLedger(ledger) };
|
|
}
|
|
|
|
describe('campaign-write-cli — init', () => {
|
|
it('creates a valid, empty, versioned ledger (exit 0)', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
|
|
assert.equal(status, 0);
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.action, 'init');
|
|
assert.equal(out.written, true);
|
|
assert.equal(out.alreadyInitialized, false);
|
|
assert.equal(out.ledgerPath, resolve(file));
|
|
|
|
const { ledger, validation } = await readLedger(file);
|
|
assert.ok(validation.valid, validation.errors.join('; '));
|
|
assert.equal(ledger.schemaVersion, 1);
|
|
assert.equal(ledger.createdDate, NOW);
|
|
assert.equal(ledger.updatedDate, NOW);
|
|
assert.deepEqual(ledger.repos, []);
|
|
});
|
|
|
|
it('refuses to clobber an existing ledger (exit 1, file untouched)', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
// First init, then add a repo so we can prove a second init does NOT wipe it.
|
|
runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
|
|
runWrite(['add', '/r/a', '--ledger-file', file, '--reference-date', NOW]);
|
|
|
|
const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]);
|
|
assert.equal(status, 1, 'advisory: already initialized');
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.written, false);
|
|
assert.equal(out.alreadyInitialized, true);
|
|
|
|
const { ledger } = await readLedger(file);
|
|
assert.equal(ledger.repos.length, 1, 'existing repo preserved — not clobbered');
|
|
});
|
|
});
|
|
|
|
describe('campaign-write-cli — add', () => {
|
|
it('auto-initializes when no ledger exists, then adds repos (exit 0)', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
// Real directories: since #51 `add` reports a path it cannot read under
|
|
// `addedUnverified` instead of vouching for it. This test is about auto-init and
|
|
// tracking, so its fixtures must be repos that actually exist.
|
|
const a = newDir();
|
|
const b = newDir();
|
|
assert.ok(!existsSync(file));
|
|
const { status, stdout } = runWrite([
|
|
'add', a, b, '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 0);
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.action, 'add');
|
|
assert.equal(out.autoInitialized, true);
|
|
assert.deepEqual(out.added.sort(), [resolve(a), resolve(b)].sort());
|
|
assert.deepEqual(out.addedUnverified, []);
|
|
|
|
const { ledger, validation } = await readLedger(file);
|
|
assert.ok(validation.valid, validation.errors.join('; '));
|
|
assert.equal(ledger.repos.length, 2);
|
|
assert.ok(ledger.repos.every((r) => r.status === 'pending'));
|
|
});
|
|
|
|
it('appends to an existing ledger and is idempotent on re-add (added vs skipped)', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
const a = newDir();
|
|
const b = newDir();
|
|
const c = newDir();
|
|
runWrite(['add', a, b, '--ledger-file', file, '--reference-date', NOW]);
|
|
|
|
const { status, stdout } = runWrite([
|
|
'add', a, c, '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 0);
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.autoInitialized, false);
|
|
assert.deepEqual(out.added, [resolve(c)]);
|
|
assert.deepEqual(out.skipped, [resolve(a)]);
|
|
|
|
const { ledger } = await readLedger(file);
|
|
assert.equal(ledger.repos.length, 3);
|
|
});
|
|
|
|
it('uses --name for a single path; derives basenames for multiple', async () => {
|
|
const dir = newDir();
|
|
const solo = join(dir, 'solo.json');
|
|
runWrite(['add', '/r/x', '--name', 'custom', '--ledger-file', solo, '--reference-date', NOW]);
|
|
const { ledger: l1 } = await readLedger(solo);
|
|
assert.equal(l1.repos[0].name, 'custom');
|
|
|
|
const multi = join(dir, 'multi.json');
|
|
runWrite(['add', '/r/alpha', '/r/beta', '--ledger-file', multi, '--reference-date', NOW]);
|
|
const { ledger: l2 } = await readLedger(multi);
|
|
assert.deepEqual(l2.repos.map((r) => r.name).sort(), ['alpha', 'beta']);
|
|
});
|
|
});
|
|
|
|
describe('campaign-write-cli — set-status', () => {
|
|
async function seeded() {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
runWrite(['add', '/r/a', '/r/b', '--ledger-file', file, '--reference-date', NOW]);
|
|
return file;
|
|
}
|
|
|
|
it('transitions a tracked repo and attaches findings + session (exit 0)', async () => {
|
|
const file = await seeded();
|
|
const findings = { critical: 1, high: 2, medium: 0, low: 4 };
|
|
const { status, stdout } = runWrite([
|
|
'set-status', '/r/a', 'audited',
|
|
'--findings', JSON.stringify(findings),
|
|
'--session', '20260622_120000',
|
|
'--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 0);
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.action, 'set-status');
|
|
assert.equal(out.repo.status, 'audited');
|
|
assert.deepEqual(out.repo.findingsBySeverity, findings);
|
|
assert.equal(out.repo.sessionId, '20260622_120000');
|
|
|
|
const { ledger } = await readLedger(file);
|
|
assert.deepEqual(rollUp(ledger).bySeverity, findings);
|
|
assert.deepEqual(out.rollUp, rollUp(ledger));
|
|
});
|
|
|
|
it('exits 3 on an invalid status', async () => {
|
|
const file = await seeded();
|
|
const { status } = runWrite([
|
|
'set-status', '/r/a', 'nonsense', '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 3);
|
|
});
|
|
|
|
it('exits 3 when the repo is not tracked', async () => {
|
|
const file = await seeded();
|
|
const { status } = runWrite([
|
|
'set-status', '/r/ghost', 'audited', '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 3);
|
|
});
|
|
|
|
it('exits 3 when no ledger exists yet', () => {
|
|
const dir = newDir();
|
|
const { status } = runWrite([
|
|
'set-status', '/r/a', 'audited',
|
|
'--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 3);
|
|
});
|
|
});
|
|
|
|
describe('campaign-write-cli — refresh-tokens (live cross-repo sweep, v5.9 B2b)', () => {
|
|
// Fixture: a fake HOME with a LARGE global CLAUDE.md (the shared layer, made to
|
|
// dominate so the counted-once guard is meaningful) + two repos OUTSIDE that HOME,
|
|
// each with a small project CLAUDE.md (the per-repo delta). repos sit beside the
|
|
// fake home so the cascade walk never picks up the global file as a project file.
|
|
function buildSweepFixture() {
|
|
const root = mkdtempSync(join(tmpdir(), 'camp-sweep-'));
|
|
const fakeHome = join(root, 'home');
|
|
mkdirSync(join(fakeHome, '.claude'), { recursive: true });
|
|
writeFileSync(
|
|
join(fakeHome, '.claude', 'CLAUDE.md'),
|
|
`# Global user config\n\n${'Shared instruction line that every repo pays for. '.repeat(120)}\n`,
|
|
);
|
|
const repoA = join(root, 'repo-a');
|
|
const repoB = join(root, 'repo-b');
|
|
const bodies = [
|
|
[repoA, '# Repo A\n\nProject A note.\n'],
|
|
[repoB, '# Repo B\n\nProject B has a little more local text here.\n'],
|
|
];
|
|
for (const [repo, body] of bodies) {
|
|
mkdirSync(join(repo, '.git'), { recursive: true });
|
|
writeFileSync(join(repo, '.git', 'HEAD'), 'ref: refs/heads/main\n');
|
|
writeFileSync(join(repo, 'CLAUDE.md'), body);
|
|
}
|
|
return { root, fakeHome, repoA, repoB };
|
|
}
|
|
|
|
it('sweeps every tracked repo, counts the shared layer ONCE, attributes deltas per repo', async () => {
|
|
const { root, fakeHome, repoA, repoB } = buildSweepFixture();
|
|
try {
|
|
const file = join(root, 'ledger.json');
|
|
runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome });
|
|
|
|
const { status, stdout } = runWrite(
|
|
['refresh-tokens', '--ledger-file', file, '--reference-date', NOW],
|
|
{ HOME: fakeHome },
|
|
);
|
|
assert.equal(status, 0, stdout);
|
|
const out = JSON.parse(stdout);
|
|
assert.equal(out.action, 'refresh-tokens');
|
|
assert.equal(out.written, true);
|
|
assert.deepEqual(out.swept.slice().sort(), [resolve(repoA), resolve(repoB)].sort());
|
|
assert.deepEqual(out.skipped, []);
|
|
|
|
// NOTE on shapes (B2a contract): the STORED summaries — out.sharedGlobal and each
|
|
// repo.tokens — carry {always:{tokens,count}}, but rollUp().tokens flattens its
|
|
// sharedGlobal/perRepoDelta/machineWide buckets to {always:<number>} (bucketTokens).
|
|
const t = out.rollUp.tokens;
|
|
assert.ok(t.sharedGlobal.always > 0, 'shared global layer should be non-empty');
|
|
assert.equal(t.reposWithTokens, 2);
|
|
assert.equal(t.byRepo.length, 2);
|
|
|
|
// Persisted ledger carries the new token fields and stays schema-valid.
|
|
const ledgerOut = await loadLedger(file);
|
|
assert.ok(validateLedger(ledgerOut).valid, validateLedger(ledgerOut).errors.join('; '));
|
|
assert.deepEqual(ledgerOut.sharedGlobal, out.sharedGlobal);
|
|
|
|
// Counted-ONCE guard: each repo's stored delta must NOT contain the dominant
|
|
// shared global CLAUDE.md. Had the sweep folded shared into the per-repo tokens,
|
|
// each delta would EXCEED the shared layer instead of being a small fraction.
|
|
for (const repo of ledgerOut.repos) {
|
|
assert.ok(repo.tokens, `${repo.name} should have tokens`);
|
|
assert.ok(
|
|
repo.tokens.always.tokens < t.sharedGlobal.always,
|
|
`${repo.name} delta (${repo.tokens.always.tokens}) must be < shared (${t.sharedGlobal.always})`,
|
|
);
|
|
}
|
|
|
|
// rollUp invariant: machineWide = shared (once) + sum of per-repo deltas.
|
|
const deltaSum = ledgerOut.repos.reduce((s, r) => s + r.tokens.always.tokens, 0);
|
|
assert.equal(t.perRepoDelta.always, deltaSum);
|
|
assert.equal(t.machineWide.always, t.sharedGlobal.always + deltaSum);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it('is idempotent — a second sweep replaces, never accumulates', async () => {
|
|
const { root, fakeHome, repoA, repoB } = buildSweepFixture();
|
|
try {
|
|
const file = join(root, 'ledger.json');
|
|
runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome });
|
|
const first = JSON.parse(
|
|
runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout,
|
|
);
|
|
const second = JSON.parse(
|
|
runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout,
|
|
);
|
|
assert.deepEqual(second.rollUp.tokens, first.rollUp.tokens);
|
|
} finally {
|
|
rmSync(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
it('exits 3 when no ledger exists yet', () => {
|
|
const dir = newDir();
|
|
const { status } = runWrite([
|
|
'refresh-tokens', '--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 3);
|
|
});
|
|
});
|
|
|
|
describe('campaign-write-cli — determinism + --output-file', () => {
|
|
it('stamps updatedDate from --reference-date and writes the payload to --output-file', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
runWrite(['init', '--ledger-file', file, '--reference-date', '2026-01-01']);
|
|
const report = join(dir, 'report.json');
|
|
const { stdout } = runWrite([
|
|
'add', '/r/a', '--ledger-file', file, '--reference-date', '2026-03-15',
|
|
'--output-file', report,
|
|
]);
|
|
assert.equal(stdout.trim(), '', 'stdout is silent when --output-file is given');
|
|
const written = JSON.parse(readFileSync(report, 'utf-8'));
|
|
assert.equal(written.action, 'add');
|
|
|
|
const { ledger } = await readLedger(file);
|
|
assert.equal(ledger.createdDate, '2026-01-01', 'created stamp preserved');
|
|
assert.equal(ledger.updatedDate, '2026-03-15', 'updated stamp from the later --reference-date');
|
|
});
|
|
|
|
it('exits 3 on an unknown subcommand', () => {
|
|
const dir = newDir();
|
|
const { status } = runWrite(['frobnicate', '--ledger-file', join(dir, 'x.json')]);
|
|
assert.equal(status, 3);
|
|
});
|
|
});
|
|
|
|
// ── Session #51: honest coverage — the ledger must not vouch for repos it cannot read ──
|
|
//
|
|
// Dogfooding the campaign lifecycle put a path that does not exist into the ledger and
|
|
// swept it. Both halves lied, quietly:
|
|
//
|
|
// add → added: ["/finnes/absolutt/ikke/noe-repo"], exit 0, no warning.
|
|
// sweep → swept: [… , "/finnes/absolutt/ikke/noe-repo"], skipped: [], byRepo entry with 0
|
|
// tokens, reposWithTokens: 3 for a machine with 2 real repos.
|
|
//
|
|
// The root cause is that `readActiveConfig` resolves a path and every sub-reader tolerates
|
|
// ENOENT, so a missing repo yields an EMPTY config rather than an error — and the sweep's
|
|
// try/catch only routes THROWN errors to `skipped`. The command's own honesty clause ("If
|
|
// anything was skipped, name those repos plainly so the user knows the bill omits them")
|
|
// could therefore never fire. A token bill that silently counts phantom repos as 0 is worse
|
|
// than one that refuses to answer: it looks complete.
|
|
|
|
describe('campaign-write-cli — honest coverage for unreadable repos', () => {
|
|
it('add flags a path that does not exist instead of vouching for it', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
const real = newDir();
|
|
const phantom = join(dir, 'no-such-repo');
|
|
|
|
const { status, stdout } = runWrite([
|
|
'add', real, phantom, '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 0, 'a missing path is reported, not rejected — it may be unmounted');
|
|
|
|
const out = JSON.parse(stdout);
|
|
assert.deepEqual(out.added, [resolve(real)], 'only the readable repo is vouched for');
|
|
assert.deepEqual(
|
|
out.addedUnverified,
|
|
[resolve(phantom)],
|
|
'a path that does not exist must be reported separately so the command can say so',
|
|
);
|
|
|
|
// Still tracked — the campaign is a work register, and an unmounted volume is a
|
|
// legitimate reason for a path to be absent today and present tomorrow.
|
|
const { ledger } = await readLedger(file);
|
|
assert.equal(ledger.repos.length, 2);
|
|
});
|
|
|
|
it('refresh-tokens skips an unreadable repo instead of counting it as 0 tokens', async () => {
|
|
const dir = newDir();
|
|
const file = join(dir, 'ledger.json');
|
|
const phantom = join(dir, 'no-such-repo');
|
|
|
|
runWrite(['add', phantom, '--ledger-file', file, '--reference-date', NOW]);
|
|
const { status, stdout } = runWrite([
|
|
'refresh-tokens', '--ledger-file', file, '--reference-date', NOW,
|
|
]);
|
|
assert.equal(status, 0);
|
|
|
|
const out = JSON.parse(stdout);
|
|
assert.deepEqual(out.swept, [], 'a repo that cannot be read was never actually swept');
|
|
assert.equal(out.skipped.length, 1, 'it belongs in skipped, with a reason the user can act on');
|
|
assert.equal(out.skipped[0].path, resolve(phantom));
|
|
assert.match(out.skipped[0].reason, /not readable|does not exist|ENOENT/i);
|
|
assert.equal(
|
|
out.rollUp.tokens.reposWithTokens,
|
|
0,
|
|
'the machine-wide bill must not claim coverage of a repo it could not read',
|
|
);
|
|
});
|
|
});
|