test(r8): move the live tests out of the round files

R8 of cleanup R-a (plan .claude/projects/2026-11-07-cleanup-r-a, appendix A).
The live tests of the round files move into the test named after their
source; each moved test was seen red under a mutation of its source.
New: tests/lib/lib-imports.test.mjs, tests/lib/timing.test.mjs (from
timing-under-load). The [#1] --gates rule and shim-line pins move into
tests/lib/autonomy-gate.test.mjs.

Removed: the round test files (retterunde, retterunde-prose-pins,
trial-mechanical-fixes, trial2-fixes, trial2-minor, trial2-minor-prose-pins,
fix-round-4-5a-prose, fix-round-4-5b2-b-prose, timing-under-load), the 8 R8
e2e scripts, and seven round test files on the R-a removal list that pinned
them (fix-round-4-5b-prose, fix-round-4-5b-part-2/3/4-scripts,
fix-round-4-5b-part-4-prose, fix-round-4-5b2-b/c-scripts). The diff-mode e2e
pin (test 24) and the red-script round-wrapper cases go too.

tests/synthetic/plan-run-C.md becomes tests/fixtures/plan-run-c.md,
regenerated from buildPlan({ version: '1.9' }). The red judge, reporter and
judge test headers now name the shared red script.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-07 10:19:49 +01:00
commit 651be3468b
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
60 changed files with 1755 additions and 7965 deletions

View file

@ -451,3 +451,68 @@ test('CLI — --min-version still locates the file positional after the value to
rmSync(dir, { recursive: true, force: true });
}
});
const codes = (list) => list.map((e) => e.code);
function briefWith(sections) {
return [
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
...sections,
].join('\n');
}
// ---- BRIEF_SC_SUBPOINTS: sub-points belong to their criterion -----------------
test('BRIEF_SC_SUBPOINTS: a criterion with sub-points and no command of its own is a warning that names it', () => {
const r = validateBriefContent(briefWith([
'## Success Criteria', '',
'- `npm test` exits 0.',
'- The separator is honoured:',
' - `-` joins words',
' - `_` joins words',
' - `.` joins words',
'- `bash tests/e2e.sh` exits 0.', '',
]));
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_SUBPOINTS');
assert.equal(w.length, 1, JSON.stringify(r.warnings));
assert.match(w[0].message, /SC2/);
});
// ---- BRIEF_SC_BASH_NOT_PINNED: a bash 3.2 constraint is measured under bash 3.2 ----
const BASH32 = ['## Constraints', '', '- Scripts must run under bash 3.2 (macOS system bash).', ''];
test('BRIEF_SC_BASH_NOT_PINNED: a brief that requires bash 3.2 and runs a criterion with plain `bash` gets a warning pointing at /bin/bash', () => {
const r = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_BASH_NOT_PINNED');
assert.equal(w.length, 1, JSON.stringify(r.warnings));
assert.match(w[0].hint, /\/bin\/bash tests\//);
});
test('BRIEF_SC_BASH_NOT_PINNED: `/bin/bash`, or no bash 3.2 constraint, raises no such warning', () => {
const pinned = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `/bin/bash tests/contract.sh` exits 0.', '']));
assert.ok(!codes(pinned.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
const free = validateBriefContent(briefWith(['## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
assert.ok(!codes(free.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
});
test('BRIEF_SC_BASH_NOT_PINNED: a negated bash 3.2 constraint, an env prefix and `bash -c` are handled', () => {
const neg = validateBriefContent(briefWith(['## Constraints', '', '- bash 3.2 is not required here.', '',
'## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
assert.ok(!codes(neg.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'), 'negated constraint');
const env = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `LC_ALL=C bash tests/contract.sh` exits 0.', '']));
assert.ok(codes(env.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'), 'env-prefixed bash');
const dashC = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `bash -c true` exits 0.', '']));
for (const w of dashC.warnings.filter((x) => x.code === 'BRIEF_SC_BASH_NOT_PINNED')) {
assert.doesNotMatch(w.hint, /\/bin\/bash -c/, 'never suggests a form the runner refuses');
}
});
test('/trekbrief proposes /bin/bash when the brief requires bash 3.2, and names both warnings', () => {
const md = readFileSync(new URL('../../commands/trekbrief.md', import.meta.url), 'utf8');
assert.match(md, /requires\s+bash\s+3\.2[^`]*`\/bin\/bash tests\/<script>\.sh`/);
assert.match(md, /BRIEF_SC_BASH_NOT_PINNED/);
assert.match(md, /BRIEF_SC_SUBPOINTS/);
});

View file

@ -691,7 +691,6 @@ test('acceptOpen after round 3 lists only the real majors, never a lowered one',
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const GATE = join(ROOT, 'lib', 'validators', 'plan-gate.mjs');
const E2E = join(ROOT, 'tests', 'e2e-critic-diff-mode.sh');
const GIT_CLEAN = { GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_INDEX_FILE: undefined };
function gate(cwd, args, input) {
@ -935,13 +934,63 @@ test('23 (CLI): --record on a plan outside any git repo keeps no snapshot anywhe
}
});
test('24: the e2e script is ASCII only and bash -n clean', () => {
assert.ok(existsSync(E2E), 'tests/e2e-critic-diff-mode.sh exists');
const buf = readFileSync(E2E);
assert.equal(buf.findIndex(b => b > 127), -1, 'ASCII only');
assert.match(buf.toString('utf-8'), /e2e: ok/);
const r = spawnSync('/bin/bash', ['-n', E2E], { encoding: 'utf-8', env: childEnv() });
assert.equal(r.status, 0, r.stderr);
// The pipe /trekplan Phase 9 runs: the raw reviewer payload through the dedup
// CLI's --stdin, its stdout into the gate's --record (no in-process dedup).
const DEDUP = join(ROOT, 'lib', 'review', 'plan-review-dedup.mjs');
/** One round through the two CLIs: --hash, dedup --stdin on `payload`, then --record on its output. */
function pipeRound(dir, p, payload) {
const reviewed = hashOf(dir, p);
const d = spawnSync(process.execPath, [DEDUP, '--stdin'], { cwd: dir, input: JSON.stringify(payload), encoding: 'utf-8', env: childEnv(GIT_CLEAN) });
assert.equal(d.status, 0, `plan-review-dedup --stdin: ${d.stdout}${d.stderr}`);
const r = gate(dir, ['--record', p, '--reviewed', reviewed], d.stdout);
const lines = r.out.split('\n').filter(l => l !== '');
return { ...r, reviewed, j: lines.length ? JSON.parse(lines[lines.length - 1]) : null };
}
test('CLI pipe: dedup --stdin into --record counts round 2 by rule A, and a refused and a malformed lowering reach the gate as blockers', () => {
withRepo(buildPlan(), (p, dir) => {
const t1 = readPlan(p);
const raw = (findings, previous) => ({ findings, ...(previous ? { previous } : {}) });
const r1 = pipeRound(dir, p, {
plan_critic: raw([f('major', at(t1, S1_VERIFIES), 'Step 1 verification never states the expected return value')]),
scope_guardian: raw([]),
});
assert.equal(r1.code, 0, r1.out + r1.err);
assert.equal(r1.j.verdict, 'REVISE', r1.out);
const id = verdictIn(p).findings[0].id;
assert.match(id, /^[0-9a-f]{40}$/);
// Round 2: the round-1 major resolved by both, a new major on a line outside the edit.
const t2 = editA(readPlan(p));
writeFileSync(p, t2);
const prev = [{ id, resolved: true }];
const r2 = pipeRound(dir, p, { plan_critic: raw([guard(t2)], prev), scope_guardian: raw([], prev) });
assert.equal(r2.code, 0, r2.out + r2.err);
assert.equal(r2.j.verdict, 'PASS', r2.out);
assert.equal(r2.j.diff_base, r1.reviewed, 'round 2 diffs against round 1');
const g2 = gate(dir, [p]);
assert.equal(g2.code, 0, g2.out + g2.err);
// Round 3 (plan unchanged): a minor merged with its copy that claims a lowering
// from blocker, and a minor whose lowered_from is not a severity.
const t3 = readPlan(p);
const M9 = 'Step 1 never names the exit code its verify command expects';
const G9 = 'The revert for step one is never run against a dirty tree';
const r3 = pipeRound(dir, p, {
plan_critic: raw([f('minor', at(t3, S1_VERIFIES), M9)]),
scope_guardian: raw([
f('minor', at(t3, S1_VERIFIES), M9, { lowered_from: 'blocker', caught_by: 'npm test' }),
{ ...f('minor', at(t3, S1_ROLLBACK), G9, { lowered_from: 'high', caught_by: 'npm test' }), rule_key: 'gap' },
]),
});
assert.equal(r3.code, 0, r3.out + r3.err);
assert.equal(r3.j.verdict, 'BLOCK', r3.out);
assert.match(r3.out, /"blocker":2/, `round 3 counts two blockers: ${r3.out}`);
const g3 = gate(dir, [p]);
assert.equal(g3.code, 1, g3.out + g3.err);
assert.match(g3.out + g3.err, /PLAN_GATE_OPEN_BLOCKER/);
});
});
// ---- the matcher needs overlapping texts; a new id never repeats an old one ----

View file

@ -1,6 +1,8 @@
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
import { ON_FAILURE_ACTIONS } from '../../lib/parsers/plan-schema.mjs';
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
const VALID_PLAN = `---
plan_version: "1.7"
@ -111,3 +113,87 @@ test('validatePlan — prose plan_version line parses, no PLAN_NO_VERSION (defec
assert.equal(r.parsed.planVersion, '1.7');
assert.equal(r.warnings.find(w => w.code === 'PLAN_NO_VERSION'), undefined);
});
const codes = (list) => list.map((e) => e.code);
// ---- PLAN_STEP_TESTS_ONLY: a step whose Files are all tests can never be proven red ----
// Step 2 changes only tests for behaviour step 1 delivered.
const TESTS_ONLY_PLAN = buildPlan({
steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e-guard.test.mjs', endToEnd: 'bash tests/e2e.sh' }],
});
test('a step whose Files are all tests gives PLAN_STEP_TESTS_ONLY, a warning on that step', () => {
const r = validatePlanContent(TESTS_ONLY_PLAN);
const w = r.warnings.filter((x) => x.code === 'PLAN_STEP_TESTS_ONLY');
assert.equal(w.length, 1, codes(r.warnings).join(','));
assert.match(w[0].message, /Step 2/);
assert.ok(!codes(r.errors).includes('PLAN_STEP_TESTS_ONLY'), 'a warning, never an error');
assert.equal(r.valid, true, codes(r.errors).join(','));
});
test('a step with code and its test gives no PLAN_STEP_TESTS_ONLY', () => {
const r = validatePlanContent(buildPlan());
assert.ok(!codes(r.warnings).includes('PLAN_STEP_TESTS_ONLY'), codes(r.warnings).join(','));
});
// ---- K10 `Regression:` and K11 `On failure:` -----------------------------------
// Step 1's Verify line in the builder's plan; the new field goes right under it.
const STEP1_VERIFY = '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass';
const withRegression = (line, plan = buildPlan()) => plan.replace(STEP1_VERIFY, `${STEP1_VERIFY}\n${line}`);
const withOnFailure = (text, plan = buildPlan()) =>
plan.replace(/^- \*\*On failure:\*\* .*$/m, `- **On failure:** ${text}`);
test('K10: `Regression: npm test | tee x` is refused — one command, unpiped', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x` → expected: exit 0'));
assert.equal(r.valid, false);
const e = r.errors.find((x) => x.code === 'PLAN_STEP_REGRESSION_NOT_RUNNABLE');
assert.ok(e, codes(r.errors).join(','));
assert.match(e.message, /Step 1/);
assert.match(e.message, /shell operator/);
});
test('K10: `Regression: npm test` is accepted', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test` → expected: exit 0'));
assert.deepEqual(codes(r.errors), []);
assert.ok(!codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
test('K10: without the field the plan validates exactly as before', () => {
const r = validatePlanContent(buildPlan());
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.deepEqual(codes(r.warnings), []);
});
test('K10: `npm test && bash tests/e2e.sh`, a bare word, or no backticks is refused', () => {
for (const line of [
'- **Regression:** `npm test && bash tests/e2e.sh`',
'- **Regression:** `./run-all-tests`',
'- **Regression:** npm test',
'- **Regression:**',
]) {
const r = validatePlanContent(withRegression(line));
assert.ok(codes(r.errors).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), `${line}: ${codes(r.errors).join(',')}`);
}
});
test('K10: --soft reports an unrunnable Regression as a warning, not an error', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x`'), { strict: false });
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.ok(codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
test('K11: `On failure: try again` warns, and the plan stays valid', () => {
const r = validatePlanContent(withOnFailure('try again'));
assert.equal(r.valid, true, codes(r.errors).join(','));
const w = r.warnings.find((x) => x.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN');
assert.ok(w, codes(r.warnings).join(','));
assert.match(w.message, /Step 1/);
for (const word of ON_FAILURE_ACTIONS) assert.match(w.hint, new RegExp(word));
});
test('K11: `On failure: retry once, then escalate` gives no warning', () => {
const r = validatePlanContent(withOnFailure('retry once, then escalate'));
assert.ok(!codes(r.warnings).includes('PLAN_STEP_ON_FAILURE_UNKNOWN'), codes(r.warnings).join(','));
});