open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

fix(verification): red proof checks the test list against the plan

Browse source 
Review finding cf241526; the list is read as operator decision 20 sets it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-13 18:43:44 +02:00
commit 734b4a23ae
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 180 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

49
lib/verification/red-proof.mjs
View file

@ -19,6 +19,14 @@
// red and goes up to the plan. The plan's `Files:` alone decides (see
// `testsOnly` below).
//
// With `--plan <plan.md> --step <N>` the test list is also checked against the
// step's `Test first:` File line (plan-schema `testFirstFiles`: the leading
// backticked paths, never a path in the description - operator decision 20).
// No `--tests`: the plan's list is used. A `--tests` whose set differs from the
// plan's is `not-run` with reason `tests-mismatch` and the plan's list in
// `plan_tests` (review cf241526). A step whose plan list is empty (a 1.7 step
// without `Test first:`) or cannot be read changes nothing.
//
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
// 2 = could not run (`not-run`: no test files, an unknown start point, a command
// outside the test-runner allowlist, a command that cannot start, a control run
@ -61,7 +69,7 @@ import { tmpdir } from 'node:os';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { DEFAULT_TIMEOUT_MS, allowedCommand, runCommand, screenCommand, stoppedBy } from './criteria-runner.mjs';
import { commandEnv } from '../util/command-env.mjs';
import { extractPlanVersion, parseStepFields, sliceSteps, stepTestsOnlyFromFields } from '../parsers/plan-schema.mjs';
import { extractPlanVersion, parseStepFields, sliceSteps, stepTestsOnlyFromFields, testFirstFiles } from '../parsers/plan-schema.mjs';
import { isMain } from '../util/is-main.mjs';
// The one timeout, defined in criteria-runner.mjs (release review of 6.3.0, round 7).
@ -131,6 +139,22 @@ function testsOnly(o) {
return { tests_only: stepTestsOnlyFromFields(parseStepFields(step.body), extractPlanVersion(text)).testsOnly };
}
// With `--plan` and `--step`: the step's test files as its `Test first:` File
// line names them (testFirstFiles, operator decision 20), read as `testsOnly`
// reads the plan. null = the plan or the step could not be read.
function planTests(o) {
if (!o.plan) return null;
let text;
try { text = readFileSync(o.plan, 'utf8'); }
catch { return null; }
const step = sliceSteps(text).find((s) => Number(s.n) === Number(o.step));
if (!step) return null;
return testFirstFiles(parseStepFields(step.body), extractPlanVersion(text));
}
/** A test path as the plan's list holds it: `/` separators, no leading `./`. */
const normTest = (p) => String(p).replace(/\\/g, '/').replace(/^\.\//, '');
// A program that could not start (a spawn error: not installed), or one this
// run stopped (the timeout, output past the buffer, a signal), is not a test
// verdict at all (criteria-runner `stoppedBy`). With no shell, an exit 126/127
@ -185,16 +209,30 @@ function applyResult(root, tree, base) {
}
/**
* @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], redOnly?: boolean, timeoutMs?: number, plan?: string, step?: number}} o
* @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object, control?: object}}
* @param {{cwd: string, base: string, cmd: string, tests?: string[], link?: string[], redOnly?: boolean, timeoutMs?: number, plan?: string, step?: number}} o
* @returns {{status: 'proven'|'rejected'|'not-run', reason: string, plan_tests?: string[], base?: object, result?: object, control?: object}}
*/
export function proveRed(o) {
const given = resolve(o.cwd ?? process.cwd());
const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const tests = (o.tests ?? []).filter(Boolean);
let tests = (o.tests ?? []).filter(Boolean);
if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`);
const cwd = repoRoot(given);
if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`);
// The step's own test list (review cf241526): read from the plan when none is
// given, and a given list that is another set is refused. An empty or
// unreadable plan list changes nothing.
const fromPlan = planTests(o);
if (fromPlan && fromPlan.length > 0) {
if (tests.length === 0) tests = [...fromPlan];
else {
const a = new Set(tests.map(normTest));
const b = new Set(fromPlan.map(normTest));
if (a.size !== b.size || [...a].some((t) => !b.has(t))) {
return notRun(`tests-mismatch: --tests names ${tests.join(',')}; the plan's Test first File line names ${fromPlan.join(',')}`, { plan_tests: fromPlan });
}
}
}
if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red');
for (const t of tests) {
if (!safeRel(t)) return notRun(`test path must be repo-relative: ${t}`);
@ -254,7 +292,8 @@ export function proveRed(o) {
// --- CLI ------------------------------------------------------------------------
const USAGE =
'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" --tests <a,b> [--plan <plan.md> --step <N>] [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]';
'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" [--tests <a,b>] [--plan <plan.md> --step <N>] [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]\n' +
' --tests is required unless --plan/--step name a step whose Test first File line lists its tests; given with them, it must name the same set';
const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean);