open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

fix: round 7 of the release review of 6.3.0, for 6.3.1

Browse source 
Round 7 (516dffa..45aa9e1, WARN: 12 MAJOR, 0 BLOCKER) and the fence defect
found while writing its review.md. Each fix began with a test that failed on
0f400d3 (v6.3.0); suite 2208 (2206/0/2), 14 tests more.

- Tests-only (operator 2026-10-11): the plan's Files: alone decides, and the
  text says so (red-proof, trekexecute Sub-step B, step-implementer,
  HANDOVER-CONTRACTS). red-proof no longer reads the step's diff, so a change
  already in the tree when the step began no longer counts against it, and a
  plan placed outside .claude/ reads the same. The round-6 test that had the
  diff overrule the plan is rewritten to the new rule.
- The Files: reader keeps a plain file beside a backticked path
  (", Dockerfile", "the CLI entry src/cli.mjs", "and Makefile"). Measured on
  the 520 Files: fields in 182 plans: no field reads differently.
- Red proof reports a command its timeout killed as killed after the timeout,
  not "could not start"; the release check tells a timeout from output past
  its buffer and from another signal. One reading of a stop
  (criteria-runner stoppedBy) for red proof, the criteria runner and the
  release check.
- A trailing `# comment` is dropped in a backticked check too, and in a
  command passed as written, so the words screened are the words run.
- The release check reads a relative --review in --repo.
- The 15-minute timeout is defined once (criteria-runner.mjs).
- The findings check reads each cited file once per validation.
- One isMain helper (lib/util/is-main.mjs) for every CLI: the 31 copies of
  the realpath guard and the 20 modules still on the file:// guard, which
  exited 0 having done nothing under a symlink or a path with a space.
- The Bash guard writes its no-one-to-ask refusal once, on a direct condition.
- The findings check ends a ```json block only at a fence on its own line, so
  three backticks inside a JSON string no longer end it (also in 6.2.0).
- Brief-conformance notes: a test for test-lock green on exit 127 (it
  passed at once; it fails when 127 is read as could-not-run); the prose now
  names the git revision `@~1` as the one leading-@ exception.

Not changed: the example plan's steps 1-2 keep their tests-only warning
(operator decision 2026-10-09; the r5 test records it). Brief note 4 (a
Regression: whose runner is not installed goes to the fix rounds) is a
routing question for the operator.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-11 09:11:57 +02:00
commit 29b4ac91b3
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
65 changed files with 582 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

70
lib/verification/red-proof.mjs
View file

@ -15,9 +15,9 @@
// run, and only the full run is authoritative.
//
// `--plan <plan.md> --step <N>` adds `tests_only` to a `not-red-at-base`: true
// when every file in the step's `Files:` is a test and so is every file the
// step changed since its start point, so the step can never be red and goes
// up to the plan (see `testsOnly` below).
// when every file in the step's `Files:` is a test, so the step can never be
// red and goes up to the plan. The plan's `Files:` alone decides (see
// `testsOnly` below).
//
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
// 2 = could not run (`not-run`: no test files, an unknown start point, a command
@ -55,16 +55,17 @@
// Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first
// rule is an instruction there; here a command proves it.
import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync } from 'node:fs';
import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { allowedCommand, runCommand, screenCommand } from './criteria-runner.mjs';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { DEFAULT_TIMEOUT_MS, allowedCommand, runCommand, screenCommand, stoppedBy } from './criteria-runner.mjs';
import { commandEnv } from '../util/command-env.mjs';
import { parseStepFields, sliceSteps, stepTestFile, stepTestsOnlyFromFields } from '../parsers/plan-schema.mjs';
import { parseStepFields, sliceSteps, stepTestsOnlyFromFields } from '../parsers/plan-schema.mjs';
import { isMain } from '../util/is-main.mjs';
export const DEFAULT_TIMEOUT_MS = 900_000;
// The one timeout, defined in criteria-runner.mjs (release review of 6.3.0, round 7).
export { DEFAULT_TIMEOUT_MS };
const TAIL = 2000;
function tap(output) {
@ -96,7 +97,8 @@ const LOAD_ERROR = /^\s*SyntaxError: |ERR_MODULE_NOT_FOUND|does not provide an e
function run(cmd, cwd, timeoutMs) {
const r = runCommand(cmd, { cwd, timeoutMs });
const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null, loadError: LOAD_ERROR.test(output) };
const stopped = stoppedBy(r, { timeoutMs });
return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null, stopped: stopped?.why ?? null, loadError: LOAD_ERROR.test(output) };
}
/** What the red at the start point was: the test file failing to load, or a test failing. */
@ -112,42 +114,31 @@ const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra })
// (voyage-lekerepo trial 3, 2026-10-09). `null` = the plan or the step could
// not be read; without `--plan` the field is absent and nothing new happens.
//
// The plan's `Files:` is prose, read by a reader that can misread it; the
// step's own diff is a fact. So a step the plan calls tests-only is so only
// when every file it changed since the start point is a test too (release
// review of 6.3.0, round 6). Before building (`--red-only`) the diff holds the
// step's test alone, so there the plan decides. The plan file itself and
// Voyage's own files under .claude/ (progress, approvals) are not the step's change.
function testsOnly(o, cwd, baseSha) {
// The plan's `Files:` alone decides (operator 2026-10-11): the step's diff is
// not read. In the documented flow the helper writes only the test before this
// gate, so the diff could never overrule the plan, and a change already in the
// tree when the step began (a dirty tree is allowed, trekexecute Sub-step 0)
// made a true tests-only step look like it built something (release review of
// 6.3.0, round 7). A path the step changed outside `Files:` is the
// controller's scope finding (trekexecute Sub-step C, Hard rule 2), not this check's.
function testsOnly(o) {
if (!o.plan) return {};
let text;
try { text = readFileSync(o.plan, 'utf8'); }
catch (e) { return { tests_only: null, tests_only_note: `cannot read the plan: ${e.message}` }; }
const step = sliceSteps(text).find((s) => Number(s.n) === Number(o.step));
if (!step) return { tests_only: null, tests_only_note: `no "### Step ${o.step}:" in the plan` };
const fields = parseStepFields(step.body);
if (!stepTestsOnlyFromFields(fields).testsOnly) return { tests_only: false };
const isTest = stepTestFile(fields);
const given = new Set(o.tests ?? []);
const changed = [
...lines(git(cwd, ['diff', '--name-only', '--no-renames', baseSha]).stdout),
...lines(git(cwd, ['ls-files', '--others', '--exclude-standard']).stdout),
];
const planRel = relative(cwd, realpathSync(o.plan)).split(sep).join('/');
const other = [...new Set(changed)].filter((p) => p !== planRel && !p.startsWith('.claude/') && !given.has(p) && !isTest(p));
if (other.length === 0) return { tests_only: true };
return { tests_only: false, tests_only_note: `the plan's Files: names only tests, but the step's diff also changes ${other.slice(0, 5).join(', ')}` };
return { tests_only: stepTestsOnlyFromFields(parseStepFields(step.body)).testsOnly };
}
// A program that could not start (a spawn error: not installed) or a signal
// (the timeout) is not a test verdict at all. With no shell, an exit 126/127
// A program that could not start (a spawn error: not installed), or one this
// run stopped (the timeout, output past the buffer, a signal), is not a test
// verdict at all (criteria-runner `stoppedBy`). With no shell, an exit 126/127
// is the test program's own (a script whose tool the step has not built yet):
// a red like any other, and the control run tells an environment fault apart
// (release review of 6.3.0, round 6).
function unrunnable(r) {
if (r.spawnError) return `the command could not start: ${r.spawnError}`;
if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`;
return null;
return r.stopped ? `the command ${r.stopped}` : null;
}
const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
@ -241,7 +232,7 @@ export function proveRed(o) {
const baseBroken = unrunnable(base);
if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result, linked });
const out = { base_sha: baseSha, linked, base, result };
if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...testsOnly(o, cwd, baseSha), ...out };
if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...testsOnly(o), ...out };
if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', red_kind: redKind(base), base_sha: baseSha, linked, base };
if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out };
@ -249,6 +240,8 @@ export function proveRed(o) {
// copy itself cannot run the test green, so its red proved nothing.
applyResult(cwd, tree, baseSha);
const control = run(o.cmd, tree, timeoutMs);
const controlBroken = unrunnable(control);
if (controlBroken) return notRun(`in the control run: ${controlBroken}`, { ...out, control });
if (control.exit !== 0) {
return notRun('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control });
}
@ -302,9 +295,4 @@ export function main(argv) {
return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2;
}
function isMain() {
try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
catch { return false; }
}
if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));
if (isMain(import.meta.url)) process.exitCode = main(process.argv.slice(2));