fix(execute): the step loop survives an adversarial review
An independent review of abc3083 found 15 defects, the two worst verified
in scratch repos. Each fix started with a test that failed.
- Proof of red could be "proven" by the environment: a start point
unpacked with git archive had no node_modules and no .git, so any test
failed there. The start point is now a git clone --shared checked out at
start_sha, gitignored top-level entries are linked in, and a control run
(start point + the change) must pass, or the result is not-run.
- The Test first file was never committed (the template lists it apart
from Files:) and leaked into the next step's diff. The step's scope is
now Files: + test files, staged at the checkpoint.
- New files outside Files: were invisible to every check. Every untracked
file is registered with --intent-to-add; a revert unregisters them.
- The floor guard paired renames (foo.test.js -> .bak read as clean) and
missed skipIf and node's t.skip/t.todo; prose files no longer count as
stubs.
- Phase 7.5 audited from the last step's start_sha; it now uses the run's
session_start_sha.
- The done line is written after the checkpoint and checked with --commit.
- fix_rounds never passes 5; attempts are never reset; re-control keeps
the spec/quality verdicts and earlier floor-guard rulings; the closed
stop list maps every existing stop; failed vs stopped made consistent;
the controller's effort is said, not assumed.
Tests: 1470 (1468 pass, 0 fail, 2 skipped).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
382d5ccba2
commit
44dcc779a9
11 changed files with 466 additions and 148 deletions
|
|
@ -81,8 +81,13 @@ test('done line: --check finds the line for that step only', () => {
|
|||
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 1, 'no report file = not done');
|
||||
writeFileSync(report, '# Step 3 report\nDONE step 3 (typed by hand)\n');
|
||||
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 1, 'a hand-typed line is not the script line');
|
||||
// A well-formed line typed by hand, at a sha that is not the step's commit.
|
||||
writeFileSync(report, `# Step 3 report\nDONE step 3: \`node --test tests/ok.test.mjs\` exit 0 at ${'0'.repeat(40)} 2026-10-06T12:00:00.000Z\n`);
|
||||
const head = sh(dir, 'git', ['rev-parse', 'HEAD']);
|
||||
assert.equal(run(['--check', '--step', '3', '--report', report, '--commit', head]).status, 1, 'the line must be at the step commit');
|
||||
run(['--cwd', dir, '--step', '3', '--cmd', 'node --test tests/ok.test.mjs', '--report', report]);
|
||||
assert.equal(run(['--check', '--step', '3', '--report', report]).status, 0);
|
||||
assert.equal(run(['--check', '--step', '3', '--report', report, '--commit', head]).status, 0);
|
||||
assert.equal(run(['--check', '--step', '4', '--report', report]).status, 1);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
|
|
|||
|
|
@ -141,16 +141,61 @@ test('K8: the test lock is used through its CLI, and a missing CLI is said out l
|
|||
});
|
||||
|
||||
test('P9 (2): the floor guard runs on the step diff and exit 2 is never green', () => {
|
||||
const text = read(CMD);
|
||||
assert.match(text, /lib\/verification\/test-integrity\.mjs --base \{start_sha\}/);
|
||||
assert.match(text, /exit 2[^\n]*never (green|a pass)/i);
|
||||
const d3 = section(read(CMD), '#### Sub-step D3 — Floor guard');
|
||||
assert.match(d3, /lib\/verification\/test-integrity\.mjs --base \{start_sha\}/);
|
||||
assert.match(d3, /exit 2 is never green/i);
|
||||
assert.match(d3, /justified/);
|
||||
});
|
||||
|
||||
test('P9 (4): the done line is written by a script, only when the test command exits 0', () => {
|
||||
test('P9 (4): the done line is written by a script, only when the test command exits 0, at the step commit', () => {
|
||||
const g = section(read(CMD), '#### Sub-step G — Checkpoint, done line, unlock');
|
||||
assert.match(g, /lib\/verification\/done-line\.mjs --step \{N\}/);
|
||||
assert.match(g, /done-line\.mjs --check --step \{N\} --report \{report_path\} --commit \{hash\}/);
|
||||
assert.match(g, /writes its line only when the test command\s+exits 0/i);
|
||||
// The checkpoint comes first, so the line names the step's own commit.
|
||||
assert.ok(g.indexOf('**Checkpoint.**') < g.indexOf('**The done line**'), 'checkpoint before the done line');
|
||||
});
|
||||
|
||||
// --- the independent review of 2026-10-06 (15 findings): what the fixes must keep
|
||||
|
||||
test('review: the step test files are part of the scope and committed at the checkpoint', () => {
|
||||
const text = read(CMD);
|
||||
assert.match(text, /lib\/verification\/done-line\.mjs/);
|
||||
assert.match(text, /done-line\.mjs --check/);
|
||||
assert.match(text, /only (when|if) the test command exits 0/i);
|
||||
assert.match(section(text, '#### Sub-step B — Test first and proof of red'), /scope = its `Files:` \+ its test files/);
|
||||
assert.match(section(text, '#### Sub-step G — Checkpoint, done line, unlock'), /its `Files:` and its test files/);
|
||||
});
|
||||
|
||||
test('review: every new file is registered (not only listed ones), so no check is blind to it', () => {
|
||||
const c = section(read(CMD), '#### Sub-step C — Implement through one fresh helper');
|
||||
assert.match(c, /git ls-files --others --exclude-standard/);
|
||||
assert.match(c, /git add --intent-to-add -- \{path\}/);
|
||||
});
|
||||
|
||||
test('review: proof of red has a control run and a real git checkout as its start point', () => {
|
||||
const b = section(read(CMD), '#### Sub-step B — Test first and proof of red');
|
||||
assert.match(b, /control run/);
|
||||
assert.match(b, /git clone --shared/);
|
||||
});
|
||||
|
||||
test('review: Phase 7.5 audits the run from session_start_sha, never a step start_sha', () => {
|
||||
const text = read(CMD);
|
||||
const audit = text.slice(text.indexOf('## Phase 7.5'), text.indexOf('### Machine-verifiable completion gate'));
|
||||
assert.match(audit, /\{session_start_sha\}\.\.HEAD/);
|
||||
assert.doesNotMatch(audit, /\{start_sha\}/);
|
||||
});
|
||||
|
||||
test('review: no sixth round, attempts never reset, a step at 5 is never built again', () => {
|
||||
const text = read(CMD);
|
||||
assert.match(section(text, '#### Sub-step F — Fix rounds'), /no sixth round/);
|
||||
const c = section(text, '#### Sub-step C — Implement through one fresh helper');
|
||||
assert.match(c, /never\s+reset/);
|
||||
assert.match(c, /A step at `fix_rounds` 5 is never built again/);
|
||||
});
|
||||
|
||||
test('review: the helper appends its report, and passes tests as one comma-separated list', () => {
|
||||
const agent = read(AGENT);
|
||||
assert.match(agent, /APPEND it to `report_path`/);
|
||||
assert.match(agent, /ONE comma-separated argument/);
|
||||
assert.match(agent, /A step with no `Test first:`/);
|
||||
});
|
||||
|
||||
test('P9 (6): without a run-the-thing check the result is "not verified end to end", never green', () => {
|
||||
|
|
|
|||
|
|
@ -143,6 +143,76 @@ test('red proof: --cwd in a subdirectory still runs at the repo root (plan paths
|
|||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// Review finding 2 (2026-10-06): a start-point copy that cannot run the test at
|
||||
// all failed "red" for the wrong reason and was called proven. Three guards:
|
||||
// the copy is a real git checkout, gitignored dependency dirs are linked in, and
|
||||
// the same copy WITH the change must go green (the control run).
|
||||
|
||||
test('red proof: a gitignored dependency the test needs is linked into the start point - no false red', () => {
|
||||
const { dir, base } = repo('a + b');
|
||||
try {
|
||||
writeFileSync(join(dir, '.gitignore'), 'node_modules/\n');
|
||||
sh(dir, 'git', ['add', '.gitignore']);
|
||||
sh(dir, 'git', ['commit', '-q', '-m', 'ignore']);
|
||||
const base2 = sh(dir, 'git', ['rev-parse', 'HEAD']);
|
||||
mkdirSync(join(dir, 'node_modules', 'dep'), { recursive: true });
|
||||
writeFileSync(join(dir, 'node_modules', 'dep', 'index.mjs'), 'export const one = 1;\n');
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
writeFileSync(join(dir, 'tests', 'dep.test.mjs'), [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"import { one } from '../node_modules/dep/index.mjs';",
|
||||
"import { sum } from '../lib/sum.mjs';",
|
||||
"test('dep', () => { assert.equal(sum(one, one), 2); });",
|
||||
'',
|
||||
].join('\n'));
|
||||
// The test already passes without any change: it proves nothing.
|
||||
const r = proveRed({ cwd: dir, base: base2, cmd: 'node --test tests/dep.test.mjs', tests: ['tests/dep.test.mjs'] });
|
||||
assert.equal(r.status, 'rejected', JSON.stringify({ status: r.status, reason: r.reason }));
|
||||
assert.equal(r.reason, 'not-red-at-base');
|
||||
void base;
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('red proof: the start point is a git checkout - a git-reading command does not fail there for lack of .git', () => {
|
||||
const { dir, base } = repo('a - b');
|
||||
try {
|
||||
addTest(dir);
|
||||
const r = proveRed({ cwd: dir, base, cmd: 'git status', tests: ['tests/sum.test.mjs'] });
|
||||
assert.notEqual(r.status, 'proven', JSON.stringify({ status: r.status, reason: r.reason }));
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('red proof: a copy that cannot go green even with the change is "not-run" (environment), never proven', () => {
|
||||
const { dir, base } = repo('a - b');
|
||||
try {
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
// Red anywhere outside the real working tree, whatever the code says.
|
||||
writeFileSync(join(dir, 'tests', 'env.test.mjs'), [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"test('env', () => { assert.ok(!process.cwd().includes('voyage-red-proof-')); });",
|
||||
'',
|
||||
].join('\n'));
|
||||
writeFileSync(join(dir, 'lib', 'sum.mjs'), 'export function sum(a, b) { return a + b; }\n');
|
||||
const r = proveRed({ cwd: dir, base, cmd: 'node --test tests/env.test.mjs', tests: ['tests/env.test.mjs'] });
|
||||
assert.equal(r.status, 'not-run', JSON.stringify({ status: r.status, reason: r.reason }));
|
||||
assert.match(r.reason, /control/);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('red proof: an untracked file the change adds (not a listed test) is part of the result, so the control run goes green', () => {
|
||||
const { dir, base } = repo('a - b');
|
||||
try {
|
||||
addTest(dir);
|
||||
// The fix lives in a NEW untracked module the step adds.
|
||||
writeFileSync(join(dir, 'lib', 'add.mjs'), 'export const add = (a, b) => a + b;\n');
|
||||
writeFileSync(join(dir, 'lib', 'sum.mjs'), "export { add as sum } from './add.mjs';\n");
|
||||
const r = proveRed({ cwd: dir, base, cmd: TEST_CMD, tests: ['tests/sum.test.mjs'] });
|
||||
assert.equal(r.status, 'proven', JSON.stringify({ status: r.status, reason: r.reason }));
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('red proof: the start-point run leaves the working tree untouched', () => {
|
||||
const { dir, base } = repo('a - b');
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -82,6 +82,54 @@ test('floor guard: hunk line counts decide what is content - a removed "-- x" li
|
|||
assert.deepEqual(r.findings.map((f) => [f.kind, f.file]), [['removed-assert', 'tests/a.test.mjs']]);
|
||||
});
|
||||
|
||||
test('floor guard: skipIf and node\'s lowercase t.skip() / t.todo() are skips too', () => {
|
||||
const diff = (line) => [
|
||||
'diff --git a/tests/a.test.mjs b/tests/a.test.mjs',
|
||||
'--- a/tests/a.test.mjs',
|
||||
'+++ b/tests/a.test.mjs',
|
||||
'@@ -1,1 +1,2 @@',
|
||||
" test('a', (t) => {",
|
||||
`+${line}`,
|
||||
].join('\n');
|
||||
for (const l of [" it.skipIf(process.env.CI)('x', () => {});", ' t.skip();', " t.todo('later');", "test('b', { todo: true }, () => {});"]) {
|
||||
assert.deepEqual(analyzeDiff(diff(l)).findings.map((f) => f.kind), ['skipped-test'], l);
|
||||
}
|
||||
});
|
||||
|
||||
test('floor guard: renaming a test out of the runner\'s pattern is a deleted test (--base never pairs renames)', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'floor-guard-rename-'));
|
||||
const git = (...a) => {
|
||||
const r = spawnSync('git', a, { cwd: dir, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, `git ${a.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
};
|
||||
try {
|
||||
git('init', '-q');
|
||||
git('config', 'user.email', 't@example.invalid');
|
||||
git('config', 'user.name', 'test');
|
||||
writeFileSync(join(dir, 'foo.test.js'), "test('a', () => {\n assert.equal(1, 1);\n});\n");
|
||||
git('add', 'foo.test.js');
|
||||
git('commit', '-q', '-m', 'base');
|
||||
const base = git('rev-parse', 'HEAD');
|
||||
git('mv', 'foo.test.js', 'foo.test.js.bak');
|
||||
const r = cli(['--base', base, '--cwd', dir, '--json']);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.ok(JSON.parse(r.stdout).findings.some((f) => f.kind === 'deleted-test'), r.stdout);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('floor guard: prose files are not code - "not implemented" in a .md is not a stub', () => {
|
||||
const diff = [
|
||||
'diff --git a/docs/guide.md b/docs/guide.md',
|
||||
'--- a/docs/guide.md',
|
||||
'+++ b/docs/guide.md',
|
||||
'@@ -1,1 +1,2 @@',
|
||||
' # Guide',
|
||||
"+Never write `throw new Error('not implemented')` or an eslint-disable as a fix.",
|
||||
].join('\n');
|
||||
assert.deepEqual(analyzeDiff(diff).findings, []);
|
||||
});
|
||||
|
||||
test('floor guard: an empty diff is clean (nothing changed weakens nothing)', () => {
|
||||
assert.equal(analyzeDiff('').status, 'clean');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue