fix(execute): the step loop survives an adversarial review

An independent review of abc3083 found 15 defects, the two worst verified
in scratch repos. Each fix started with a test that failed.

- Proof of red could be "proven" by the environment: a start point
  unpacked with git archive had no node_modules and no .git, so any test
  failed there. The start point is now a git clone --shared checked out at
  start_sha, gitignored top-level entries are linked in, and a control run
  (start point + the change) must pass, or the result is not-run.
- The Test first file was never committed (the template lists it apart
  from Files:) and leaked into the next step's diff. The step's scope is
  now Files: + test files, staged at the checkpoint.
- New files outside Files: were invisible to every check. Every untracked
  file is registered with --intent-to-add; a revert unregisters them.
- The floor guard paired renames (foo.test.js -> .bak read as clean) and
  missed skipIf and node's t.skip/t.todo; prose files no longer count as
  stubs.
- Phase 7.5 audited from the last step's start_sha; it now uses the run's
  session_start_sha.
- The done line is written after the checkpoint and checked with --commit.
- fix_rounds never passes 5; attempts are never reset; re-control keeps
  the spec/quality verdicts and earlier floor-guard rulings; the closed
  stop list maps every existing stop; failed vs stopped made consistent;
  the controller's effort is said, not assumed.

Tests: 1470 (1468 pass, 0 fail, 2 skipped).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-06 20:17:39 +02:00
commit 44dcc779a9
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
11 changed files with 466 additions and 148 deletions

View file

@ -81,8 +81,13 @@ test('done line: --check finds the line for that step only', () => {
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 1, 'no report file = not done');
writeFileSync(report, '# Step 3 report\nDONE step 3 (typed by hand)\n');
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 1, 'a hand-typed line is not the script line');
// A well-formed line typed by hand, at a sha that is not the step's commit.
writeFileSync(report, `# Step 3 report\nDONE step 3: \`node --test tests/ok.test.mjs\` exit 0 at ${'0'.repeat(40)} 2026-10-06T12:00:00.000Z\n`);
const head = sh(dir, 'git', ['rev-parse', 'HEAD']);
assert.equal(run(['--check', '--step', '3', '--report', report, '--commit', head]).status, 1, 'the line must be at the step commit');
run(['--cwd', dir, '--step', '3', '--cmd', 'node --test tests/ok.test.mjs', '--report', report]);
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 0);
assert.equal(run(['--check', '--step', '3', '--report', report, '--commit', head]).status, 0);
assert.equal(run(['--check', '--step', '4', '--report', report]).status, 1);
} finally { rmSync(dir, { recursive: true, force: true }); }
});

View file

@ -141,16 +141,61 @@ test('K8: the test lock is used through its CLI, and a missing CLI is said out l
});
test('P9 (2): the floor guard runs on the step diff and exit 2 is never green', () => {
const text = read(CMD);
assert.match(text, /lib\/verification\/test-integrity\.mjs --base \{start_sha\}/);
assert.match(text, /exit 2[^\n]*never (green|a pass)/i);
const d3 = section(read(CMD), '#### Sub-step D3 — Floor guard');
assert.match(d3, /lib\/verification\/test-integrity\.mjs --base \{start_sha\}/);
assert.match(d3, /exit 2 is never green/i);
assert.match(d3, /justified/);
});
test('P9 (4): the done line is written by a script, only when the test command exits 0', () => {
test('P9 (4): the done line is written by a script, only when the test command exits 0, at the step commit', () => {
const g = section(read(CMD), '#### Sub-step G — Checkpoint, done line, unlock');
assert.match(g, /lib\/verification\/done-line\.mjs --step \{N\}/);
assert.match(g, /done-line\.mjs --check --step \{N\} --report \{report_path\} --commit \{hash\}/);
assert.match(g, /writes its line only when the test command\s+exits 0/i);
// The checkpoint comes first, so the line names the step's own commit.
assert.ok(g.indexOf('**Checkpoint.**') < g.indexOf('**The done line**'), 'checkpoint before the done line');
});
// --- the independent review of 2026-10-06 (15 findings): what the fixes must keep
test('review: the step test files are part of the scope and committed at the checkpoint', () => {
const text = read(CMD);
assert.match(text, /lib\/verification\/done-line\.mjs/);
assert.match(text, /done-line\.mjs --check/);
assert.match(text, /only (when|if) the test command exits 0/i);
assert.match(section(text, '#### Sub-step B — Test first and proof of red'), /scope = its `Files:` \+ its test files/);
assert.match(section(text, '#### Sub-step G — Checkpoint, done line, unlock'), /its `Files:` and its test files/);
});
test('review: every new file is registered (not only listed ones), so no check is blind to it', () => {
const c = section(read(CMD), '#### Sub-step C — Implement through one fresh helper');
assert.match(c, /git ls-files --others --exclude-standard/);
assert.match(c, /git add --intent-to-add -- \{path\}/);
});
test('review: proof of red has a control run and a real git checkout as its start point', () => {
const b = section(read(CMD), '#### Sub-step B — Test first and proof of red');
assert.match(b, /control run/);
assert.match(b, /git clone --shared/);
});
test('review: Phase 7.5 audits the run from session_start_sha, never a step start_sha', () => {
const text = read(CMD);
const audit = text.slice(text.indexOf('## Phase 7.5'), text.indexOf('### Machine-verifiable completion gate'));
assert.match(audit, /\{session_start_sha\}\.\.HEAD/);
assert.doesNotMatch(audit, /\{start_sha\}/);
});
test('review: no sixth round, attempts never reset, a step at 5 is never built again', () => {
const text = read(CMD);
assert.match(section(text, '#### Sub-step F — Fix rounds'), /no sixth round/);
const c = section(text, '#### Sub-step C — Implement through one fresh helper');
assert.match(c, /never\s+reset/);
assert.match(c, /A step at `fix_rounds` 5 is never built again/);
});
test('review: the helper appends its report, and passes tests as one comma-separated list', () => {
const agent = read(AGENT);
assert.match(agent, /APPEND it to `report_path`/);
assert.match(agent, /ONE comma-separated argument/);
assert.match(agent, /A step with no `Test first:`/);
});
test('P9 (6): without a run-the-thing check the result is "not verified end to end", never green', () => {

View file

@ -143,6 +143,76 @@ test('red proof: --cwd in a subdirectory still runs at the repo root (plan paths
} finally { rmSync(dir, { recursive: true, force: true }); }
});
// Review finding 2 (2026-10-06): a start-point copy that cannot run the test at
// all failed "red" for the wrong reason and was called proven. Three guards:
// the copy is a real git checkout, gitignored dependency dirs are linked in, and
// the same copy WITH the change must go green (the control run).
test('red proof: a gitignored dependency the test needs is linked into the start point - no false red', () => {
const { dir, base } = repo('a + b');
try {
writeFileSync(join(dir, '.gitignore'), 'node_modules/\n');
sh(dir, 'git', ['add', '.gitignore']);
sh(dir, 'git', ['commit', '-q', '-m', 'ignore']);
const base2 = sh(dir, 'git', ['rev-parse', 'HEAD']);
mkdirSync(join(dir, 'node_modules', 'dep'), { recursive: true });
writeFileSync(join(dir, 'node_modules', 'dep', 'index.mjs'), 'export const one = 1;\n');
mkdirSync(join(dir, 'tests'));
writeFileSync(join(dir, 'tests', 'dep.test.mjs'), [
"import { test } from 'node:test';",
"import { strict as assert } from 'node:assert';",
"import { one } from '../node_modules/dep/index.mjs';",
"import { sum } from '../lib/sum.mjs';",
"test('dep', () => { assert.equal(sum(one, one), 2); });",
'',
].join('\n'));
// The test already passes without any change: it proves nothing.
const r = proveRed({ cwd: dir, base: base2, cmd: 'node --test tests/dep.test.mjs', tests: ['tests/dep.test.mjs'] });
assert.equal(r.status, 'rejected', JSON.stringify({ status: r.status, reason: r.reason }));
assert.equal(r.reason, 'not-red-at-base');
void base;
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof: the start point is a git checkout - a git-reading command does not fail there for lack of .git', () => {
const { dir, base } = repo('a - b');
try {
addTest(dir);
const r = proveRed({ cwd: dir, base, cmd: 'git status', tests: ['tests/sum.test.mjs'] });
assert.notEqual(r.status, 'proven', JSON.stringify({ status: r.status, reason: r.reason }));
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof: a copy that cannot go green even with the change is "not-run" (environment), never proven', () => {
const { dir, base } = repo('a - b');
try {
mkdirSync(join(dir, 'tests'));
// Red anywhere outside the real working tree, whatever the code says.
writeFileSync(join(dir, 'tests', 'env.test.mjs'), [
"import { test } from 'node:test';",
"import { strict as assert } from 'node:assert';",
"test('env', () => { assert.ok(!process.cwd().includes('voyage-red-proof-')); });",
'',
].join('\n'));
writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n');
const r = proveRed({ cwd: dir, base, cmd: 'node --test tests/env.test.mjs', tests: ['tests/env.test.mjs'] });
assert.equal(r.status, 'not-run', JSON.stringify({ status: r.status, reason: r.reason }));
assert.match(r.reason, /control/);
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof: an untracked file the change adds (not a listed test) is part of the result, so the control run goes green', () => {
const { dir, base } = repo('a - b');
try {
addTest(dir);
// The fix lives in a NEW untracked module the step adds.
writeFileSync(join(dir, 'lib', 'add.mjs'), 'export const add = (a, b) => a + b;\n');
writeFileSync(join(dir, 'lib', 'sum.mjs'), "export { add as sum } from './add.mjs';\n");
const r = proveRed({ cwd: dir, base, cmd: TEST_CMD, tests: ['tests/sum.test.mjs'] });
assert.equal(r.status, 'proven', JSON.stringify({ status: r.status, reason: r.reason }));
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('red proof: the start-point run leaves the working tree untouched', () => {
const { dir, base } = repo('a - b');
try {

View file

@ -82,6 +82,54 @@ test('floor guard: hunk line counts decide what is content - a removed "-- x" li
assert.deepEqual(r.findings.map((f) => [f.kind, f.file]), [['removed-assert', 'tests/a.test.mjs']]);
});
test('floor guard: skipIf and node\'s lowercase t.skip() / t.todo() are skips too', () => {
const diff = (line) => [
'diff --git a/tests/a.test.mjs b/tests/a.test.mjs',
'--- a/tests/a.test.mjs',
'+++ b/tests/a.test.mjs',
'@@ -1,1 +1,2 @@',
" test('a', (t) => {",
`+${line}`,
].join('\n');
for (const l of [" it.skipIf(process.env.CI)('x', () => {});", ' t.skip();', " t.todo('later');", "test('b', { todo: true }, () => {});"]) {
assert.deepEqual(analyzeDiff(diff(l)).findings.map((f) => f.kind), ['skipped-test'], l);
}
});
test('floor guard: renaming a test out of the runner\'s pattern is a deleted test (--base never pairs renames)', () => {
const dir = mkdtempSync(join(tmpdir(), 'floor-guard-rename-'));
const git = (...a) => {
const r = spawnSync('git', a, { cwd: dir, encoding: 'utf8' });
assert.equal(r.status, 0, `git ${a.join(' ')}: ${r.stderr}`);
return r.stdout.trim();
};
try {
git('init', '-q');
git('config', 'user.email', 't@example.invalid');
git('config', 'user.name', 'test');
writeFileSync(join(dir, 'foo.test.js'), "test('a', () => {\n assert.equal(1, 1);\n});\n");
git('add', 'foo.test.js');
git('commit', '-q', '-m', 'base');
const base = git('rev-parse', 'HEAD');
git('mv', 'foo.test.js', 'foo.test.js.bak');
const r = cli(['--base', base, '--cwd', dir, '--json']);
assert.equal(r.status, 1, r.stdout + r.stderr);
assert.ok(JSON.parse(r.stdout).findings.some((f) => f.kind === 'deleted-test'), r.stdout);
} finally { rmSync(dir, { recursive: true, force: true }); }
});
test('floor guard: prose files are not code - "not implemented" in a .md is not a stub', () => {
const diff = [
'diff --git a/docs/guide.md b/docs/guide.md',
'--- a/docs/guide.md',
'+++ b/docs/guide.md',
'@@ -1,1 +1,2 @@',
' # Guide',
"+Never write `throw new Error('not implemented')` or an eslint-disable as a fix.",
].join('\n');
assert.deepEqual(analyzeDiff(diff).findings, []);
});
test('floor guard: an empty diff is clean (nothing changed weakens nothing)', () => {
assert.equal(analyzeDiff('').status, 'clean');
});