fix(verification): red proof prints the list it ran and refuses a plan it cannot read

Operator decision 30 (14.10); review findings 793fc1aa, 09ae46d8, d01df508, 11ee78c5 and 289e59cd.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-14 11:50:37 +02:00
commit 73e53f3134
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 141 additions and 28 deletions

View file

@ -400,3 +400,98 @@ test('red proof --plan/--step: an empty plan list (1.7 without Test first, 1.8 F
}
} finally { rmSync(dir, { recursive: true, force: true }); }
});
// Operator decision 30 (14.10; reviews 793fc1aa, 09ae46d8, d01df508, 11ee78c5):
// with --plan and --step, red proof reads the step's test files from the plan,
// prints the list it ran (`tests`, and a ` tests:` line in text mode), and a
// plan or step it cannot read is not-run (`plan-unreadable`), --tests or not.
test('red proof --plan: a plan that cannot be read is not-run, exit 2 (plan-unreadable), with and without --tests', () => {
const { dir, base } = repo('a - b');
try {
addTest(dir); // red at the start point: a missing plan must not read as proven
const missing = join(dir, 'nope.md');
for (const given of [[], ['--tests', 'tests/sum.test.mjs']]) {
const c = cli(['--red-only', '--cwd', dir, '--base', base, '--cmd', TEST_CMD, ...given, '--plan', missing, '--step', '1', '--json']);
assert.equal(c.status, 2, `${given.join(' ')}: ${c.stdout}${c.stderr}`);
const out = JSON.parse(c.stdout);
assert.equal(out.status, 'not-run', JSON.stringify(out));
assert.equal(typeof out.reason, 'string', JSON.stringify(out));
assert.match(out.reason, /^plan-unreadable: cannot read the plan: /, JSON.stringify(out));
}
// A readable plan without the step named.
const plan = planFile(dir);
for (const given of [[], ['--tests', 'tests/sum.test.mjs']]) {
const c = cli(['--red-only', '--cwd', dir, '--base', base, '--cmd', TEST_CMD, ...given, '--plan', plan, '--step', '9', '--json']);
assert.equal(c.status, 2, `${given.join(' ')}: ${c.stdout}${c.stderr}`);
const out = JSON.parse(c.stdout);
assert.equal(out.status, 'not-run', JSON.stringify(out));
assert.equal(out.reason, 'plan-unreadable: no "### Step 9:" in the plan', JSON.stringify(out));
}
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof --plan: a green start point with a missing plan and --tests is exit 2 plan-unreadable, not exit 1', () => {
const { dir, base } = repo('a + b');
try {
addTest(dir);
const missing = join(dir, 'nope.md');
for (const mode of [['--red-only'], []]) {
const c = cli([...mode, '--cwd', dir, '--base', base, '--cmd', TEST_CMD, '--tests', 'tests/sum.test.mjs', '--plan', missing, '--step', '1', '--json']);
assert.equal(c.status, 2, `${mode.join(' ')}: ${c.stdout}${c.stderr}`);
const out = JSON.parse(c.stdout);
assert.equal(out.status, 'not-run', JSON.stringify(out));
assert.equal(typeof out.reason, 'string', JSON.stringify(out));
assert.match(out.reason, /^plan-unreadable/, JSON.stringify(out));
}
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof --plan/--step, the documented call (no --tests): the result names the list it ran in `tests`, and text mode prints it', () => {
const { dir, base } = repo('a - b');
try {
addTest(dir);
const plan = planFile(dir);
const args = ['--red-only', '--cwd', dir, '--base', base, '--cmd', TEST_CMD, '--plan', plan, '--step', '1'];
const json = cli([...args, '--json']);
assert.equal(json.status, 0, json.stdout + json.stderr);
const out = JSON.parse(json.stdout);
assert.equal(out.status, 'proven', JSON.stringify(out));
assert.deepEqual(out.tests, ['tests/sum.test.mjs'], JSON.stringify(out));
const text = cli(args);
assert.equal(text.status, 0, text.stdout + text.stderr);
assert.equal(typeof text.stdout, 'string');
assert.match(text.stdout, /^ {2}tests: tests\/sum\.test\.mjs$/m, text.stdout);
// Built: the full run on a red start point and a green result names the same list.
writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n');
const full = cli(['--cwd', dir, '--base', base, '--cmd', TEST_CMD, '--plan', plan, '--step', '1', '--json']);
assert.equal(full.status, 0, full.stdout + full.stderr);
const f = JSON.parse(full.stdout);
assert.equal(f.status, 'proven', JSON.stringify(f));
assert.deepEqual(f.tests, ['tests/sum.test.mjs'], JSON.stringify(f));
} finally { rmSync(dir, { recursive: true, force: true }); }
// The full run on a green start point: rejected, with the same list.
const green = repo('a + b');
try {
addTest(green.dir);
const plan = planFile(green.dir);
const c = cli(['--cwd', green.dir, '--base', green.base, '--cmd', TEST_CMD, '--plan', plan, '--step', '1', '--json']);
assert.equal(c.status, 1, c.stdout + c.stderr);
const out = JSON.parse(c.stdout);
assert.equal(out.reason, 'not-red-at-base', JSON.stringify(out));
assert.deepEqual(out.tests, ['tests/sum.test.mjs'], JSON.stringify(out));
} finally { rmSync(green.dir, { recursive: true, force: true }); }
});
test('red proof --plan/--step: a --tests naming another test is exit 2, and text mode prints the plan\'s list on a plan_tests line', () => {
const { dir, base } = repo('a - b');
try {
addTest(dir);
writeFileSync(join(dir, 'tests', 'other.test.mjs'), OTHER_TEST);
const plan = planFile(dir);
const c = cli(['--red-only', '--cwd', dir, '--base', base, '--cmd', TEST_CMD, '--tests', 'tests/other.test.mjs', '--plan', plan, '--step', '1']);
assert.equal(c.status, 2, c.stdout + c.stderr);
assert.equal(typeof c.stdout, 'string');
assert.match(c.stdout, /^ {2}plan_tests: tests\/sum\.test\.mjs$/m, c.stdout);
} finally { rmSync(dir, { recursive: true, force: true }); }
});