test(r8): move the live tests out of the round files
R8 of cleanup R-a (plan .claude/projects/2026-11-07-cleanup-r-a, appendix A). The live tests of the round files move into the test named after their source; each moved test was seen red under a mutation of its source. New: tests/lib/lib-imports.test.mjs, tests/lib/timing.test.mjs (from timing-under-load). The [#1] --gates rule and shim-line pins move into tests/lib/autonomy-gate.test.mjs. Removed: the round test files (retterunde, retterunde-prose-pins, trial-mechanical-fixes, trial2-fixes, trial2-minor, trial2-minor-prose-pins, fix-round-4-5a-prose, fix-round-4-5b2-b-prose, timing-under-load), the 8 R8 e2e scripts, and seven round test files on the R-a removal list that pinned them (fix-round-4-5b-prose, fix-round-4-5b-part-2/3/4-scripts, fix-round-4-5b-part-4-prose, fix-round-4-5b2-b/c-scripts). The diff-mode e2e pin (test 24) and the red-script round-wrapper cases go too. tests/synthetic/plan-run-C.md becomes tests/fixtures/plan-run-c.md, regenerated from buildPlan({ version: '1.9' }). The red judge, reporter and judge test headers now name the shared red script. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fa0d4f036c
commit
651be3468b
60 changed files with 1755 additions and 7965 deletions
|
|
@ -451,3 +451,68 @@ test('CLI — --min-version still locates the file positional after the value to
|
|||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
const codes = (list) => list.map((e) => e.code);
|
||||
|
||||
function briefWith(sections) {
|
||||
return [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
...sections,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
// ---- BRIEF_SC_SUBPOINTS: sub-points belong to their criterion -----------------
|
||||
|
||||
test('BRIEF_SC_SUBPOINTS: a criterion with sub-points and no command of its own is a warning that names it', () => {
|
||||
const r = validateBriefContent(briefWith([
|
||||
'## Success Criteria', '',
|
||||
'- `npm test` exits 0.',
|
||||
'- The separator is honoured:',
|
||||
' - `-` joins words',
|
||||
' - `_` joins words',
|
||||
' - `.` joins words',
|
||||
'- `bash tests/e2e.sh` exits 0.', '',
|
||||
]));
|
||||
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_SUBPOINTS');
|
||||
assert.equal(w.length, 1, JSON.stringify(r.warnings));
|
||||
assert.match(w[0].message, /SC2/);
|
||||
});
|
||||
|
||||
// ---- BRIEF_SC_BASH_NOT_PINNED: a bash 3.2 constraint is measured under bash 3.2 ----
|
||||
|
||||
const BASH32 = ['## Constraints', '', '- Scripts must run under bash 3.2 (macOS system bash).', ''];
|
||||
|
||||
test('BRIEF_SC_BASH_NOT_PINNED: a brief that requires bash 3.2 and runs a criterion with plain `bash` gets a warning pointing at /bin/bash', () => {
|
||||
const r = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
|
||||
const w = r.warnings.filter((x) => x.code === 'BRIEF_SC_BASH_NOT_PINNED');
|
||||
assert.equal(w.length, 1, JSON.stringify(r.warnings));
|
||||
assert.match(w[0].hint, /\/bin\/bash tests\//);
|
||||
});
|
||||
|
||||
test('BRIEF_SC_BASH_NOT_PINNED: `/bin/bash`, or no bash 3.2 constraint, raises no such warning', () => {
|
||||
const pinned = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `/bin/bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(!codes(pinned.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
|
||||
const free = validateBriefContent(briefWith(['## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(!codes(free.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'));
|
||||
});
|
||||
|
||||
test('BRIEF_SC_BASH_NOT_PINNED: a negated bash 3.2 constraint, an env prefix and `bash -c` are handled', () => {
|
||||
const neg = validateBriefContent(briefWith(['## Constraints', '', '- bash 3.2 is not required here.', '',
|
||||
'## Success Criteria', '', '- `bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(!codes(neg.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'), 'negated constraint');
|
||||
const env = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `LC_ALL=C bash tests/contract.sh` exits 0.', '']));
|
||||
assert.ok(codes(env.warnings).includes('BRIEF_SC_BASH_NOT_PINNED'), 'env-prefixed bash');
|
||||
const dashC = validateBriefContent(briefWith([...BASH32, '## Success Criteria', '', '- `bash -c true` exits 0.', '']));
|
||||
for (const w of dashC.warnings.filter((x) => x.code === 'BRIEF_SC_BASH_NOT_PINNED')) {
|
||||
assert.doesNotMatch(w.hint, /\/bin\/bash -c/, 'never suggests a form the runner refuses');
|
||||
}
|
||||
});
|
||||
|
||||
test('/trekbrief proposes /bin/bash when the brief requires bash 3.2, and names both warnings', () => {
|
||||
const md = readFileSync(new URL('../../commands/trekbrief.md', import.meta.url), 'utf8');
|
||||
assert.match(md, /requires\s+bash\s+3\.2[^`]*`\/bin\/bash tests\/<script>\.sh`/);
|
||||
assert.match(md, /BRIEF_SC_BASH_NOT_PINNED/);
|
||||
assert.match(md, /BRIEF_SC_SUBPOINTS/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -691,7 +691,6 @@ test('acceptOpen after round 3 lists only the real majors, never a lowered one',
|
|||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const GATE = join(ROOT, 'lib', 'validators', 'plan-gate.mjs');
|
||||
const E2E = join(ROOT, 'tests', 'e2e-critic-diff-mode.sh');
|
||||
const GIT_CLEAN = { GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_INDEX_FILE: undefined };
|
||||
|
||||
function gate(cwd, args, input) {
|
||||
|
|
@ -935,13 +934,63 @@ test('23 (CLI): --record on a plan outside any git repo keeps no snapshot anywhe
|
|||
}
|
||||
});
|
||||
|
||||
test('24: the e2e script is ASCII only and bash -n clean', () => {
|
||||
assert.ok(existsSync(E2E), 'tests/e2e-critic-diff-mode.sh exists');
|
||||
const buf = readFileSync(E2E);
|
||||
assert.equal(buf.findIndex(b => b > 127), -1, 'ASCII only');
|
||||
assert.match(buf.toString('utf-8'), /e2e: ok/);
|
||||
const r = spawnSync('/bin/bash', ['-n', E2E], { encoding: 'utf-8', env: childEnv() });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
// The pipe /trekplan Phase 9 runs: the raw reviewer payload through the dedup
|
||||
// CLI's --stdin, its stdout into the gate's --record (no in-process dedup).
|
||||
const DEDUP = join(ROOT, 'lib', 'review', 'plan-review-dedup.mjs');
|
||||
|
||||
/** One round through the two CLIs: --hash, dedup --stdin on `payload`, then --record on its output. */
|
||||
function pipeRound(dir, p, payload) {
|
||||
const reviewed = hashOf(dir, p);
|
||||
const d = spawnSync(process.execPath, [DEDUP, '--stdin'], { cwd: dir, input: JSON.stringify(payload), encoding: 'utf-8', env: childEnv(GIT_CLEAN) });
|
||||
assert.equal(d.status, 0, `plan-review-dedup --stdin: ${d.stdout}${d.stderr}`);
|
||||
const r = gate(dir, ['--record', p, '--reviewed', reviewed], d.stdout);
|
||||
const lines = r.out.split('\n').filter(l => l !== '');
|
||||
return { ...r, reviewed, j: lines.length ? JSON.parse(lines[lines.length - 1]) : null };
|
||||
}
|
||||
|
||||
test('CLI pipe: dedup --stdin into --record counts round 2 by rule A, and a refused and a malformed lowering reach the gate as blockers', () => {
|
||||
withRepo(buildPlan(), (p, dir) => {
|
||||
const t1 = readPlan(p);
|
||||
const raw = (findings, previous) => ({ findings, ...(previous ? { previous } : {}) });
|
||||
const r1 = pipeRound(dir, p, {
|
||||
plan_critic: raw([f('major', at(t1, S1_VERIFIES), 'Step 1 verification never states the expected return value')]),
|
||||
scope_guardian: raw([]),
|
||||
});
|
||||
assert.equal(r1.code, 0, r1.out + r1.err);
|
||||
assert.equal(r1.j.verdict, 'REVISE', r1.out);
|
||||
const id = verdictIn(p).findings[0].id;
|
||||
assert.match(id, /^[0-9a-f]{40}$/);
|
||||
|
||||
// Round 2: the round-1 major resolved by both, a new major on a line outside the edit.
|
||||
const t2 = editA(readPlan(p));
|
||||
writeFileSync(p, t2);
|
||||
const prev = [{ id, resolved: true }];
|
||||
const r2 = pipeRound(dir, p, { plan_critic: raw([guard(t2)], prev), scope_guardian: raw([], prev) });
|
||||
assert.equal(r2.code, 0, r2.out + r2.err);
|
||||
assert.equal(r2.j.verdict, 'PASS', r2.out);
|
||||
assert.equal(r2.j.diff_base, r1.reviewed, 'round 2 diffs against round 1');
|
||||
const g2 = gate(dir, [p]);
|
||||
assert.equal(g2.code, 0, g2.out + g2.err);
|
||||
|
||||
// Round 3 (plan unchanged): a minor merged with its copy that claims a lowering
|
||||
// from blocker, and a minor whose lowered_from is not a severity.
|
||||
const t3 = readPlan(p);
|
||||
const M9 = 'Step 1 never names the exit code its verify command expects';
|
||||
const G9 = 'The revert for step one is never run against a dirty tree';
|
||||
const r3 = pipeRound(dir, p, {
|
||||
plan_critic: raw([f('minor', at(t3, S1_VERIFIES), M9)]),
|
||||
scope_guardian: raw([
|
||||
f('minor', at(t3, S1_VERIFIES), M9, { lowered_from: 'blocker', caught_by: 'npm test' }),
|
||||
{ ...f('minor', at(t3, S1_ROLLBACK), G9, { lowered_from: 'high', caught_by: 'npm test' }), rule_key: 'gap' },
|
||||
]),
|
||||
});
|
||||
assert.equal(r3.code, 0, r3.out + r3.err);
|
||||
assert.equal(r3.j.verdict, 'BLOCK', r3.out);
|
||||
assert.match(r3.out, /"blocker":2/, `round 3 counts two blockers: ${r3.out}`);
|
||||
const g3 = gate(dir, [p]);
|
||||
assert.equal(g3.code, 1, g3.out + g3.err);
|
||||
assert.match(g3.out + g3.err, /PLAN_GATE_OPEN_BLOCKER/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---- the matcher needs overlapping texts; a new id never repeats an old one ----
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
||||
import { ON_FAILURE_ACTIONS } from '../../lib/parsers/plan-schema.mjs';
|
||||
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
|
||||
|
||||
const VALID_PLAN = `---
|
||||
plan_version: "1.7"
|
||||
|
|
@ -111,3 +113,87 @@ test('validatePlan — prose plan_version line parses, no PLAN_NO_VERSION (defec
|
|||
assert.equal(r.parsed.planVersion, '1.7');
|
||||
assert.equal(r.warnings.find(w => w.code === 'PLAN_NO_VERSION'), undefined);
|
||||
});
|
||||
|
||||
const codes = (list) => list.map((e) => e.code);
|
||||
|
||||
// ---- PLAN_STEP_TESTS_ONLY: a step whose Files are all tests can never be proven red ----
|
||||
|
||||
// Step 2 changes only tests for behaviour step 1 delivered.
|
||||
const TESTS_ONLY_PLAN = buildPlan({
|
||||
steps: [{}, { file: 'tests/e2e.sh', test: 'tests/e2e-guard.test.mjs', endToEnd: 'bash tests/e2e.sh' }],
|
||||
});
|
||||
|
||||
test('a step whose Files are all tests gives PLAN_STEP_TESTS_ONLY, a warning on that step', () => {
|
||||
const r = validatePlanContent(TESTS_ONLY_PLAN);
|
||||
const w = r.warnings.filter((x) => x.code === 'PLAN_STEP_TESTS_ONLY');
|
||||
assert.equal(w.length, 1, codes(r.warnings).join(','));
|
||||
assert.match(w[0].message, /Step 2/);
|
||||
assert.ok(!codes(r.errors).includes('PLAN_STEP_TESTS_ONLY'), 'a warning, never an error');
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
});
|
||||
|
||||
test('a step with code and its test gives no PLAN_STEP_TESTS_ONLY', () => {
|
||||
const r = validatePlanContent(buildPlan());
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_TESTS_ONLY'), codes(r.warnings).join(','));
|
||||
});
|
||||
|
||||
// ---- K10 `Regression:` and K11 `On failure:` -----------------------------------
|
||||
|
||||
// Step 1's Verify line in the builder's plan; the new field goes right under it.
|
||||
const STEP1_VERIFY = '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass';
|
||||
const withRegression = (line, plan = buildPlan()) => plan.replace(STEP1_VERIFY, `${STEP1_VERIFY}\n${line}`);
|
||||
const withOnFailure = (text, plan = buildPlan()) =>
|
||||
plan.replace(/^- \*\*On failure:\*\* .*$/m, `- **On failure:** ${text}`);
|
||||
|
||||
test('K10: `Regression: npm test | tee x` is refused — one command, unpiped', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x` → expected: exit 0'));
|
||||
assert.equal(r.valid, false);
|
||||
const e = r.errors.find((x) => x.code === 'PLAN_STEP_REGRESSION_NOT_RUNNABLE');
|
||||
assert.ok(e, codes(r.errors).join(','));
|
||||
assert.match(e.message, /Step 1/);
|
||||
assert.match(e.message, /shell operator/);
|
||||
});
|
||||
|
||||
test('K10: `Regression: npm test` is accepted', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test` → expected: exit 0'));
|
||||
assert.deepEqual(codes(r.errors), []);
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
|
||||
});
|
||||
|
||||
test('K10: without the field the plan validates exactly as before', () => {
|
||||
const r = validatePlanContent(buildPlan());
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
assert.deepEqual(codes(r.warnings), []);
|
||||
});
|
||||
|
||||
test('K10: `npm test && bash tests/e2e.sh`, a bare word, or no backticks is refused', () => {
|
||||
for (const line of [
|
||||
'- **Regression:** `npm test && bash tests/e2e.sh`',
|
||||
'- **Regression:** `./run-all-tests`',
|
||||
'- **Regression:** npm test',
|
||||
'- **Regression:**',
|
||||
]) {
|
||||
const r = validatePlanContent(withRegression(line));
|
||||
assert.ok(codes(r.errors).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), `${line}: ${codes(r.errors).join(',')}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('K10: --soft reports an unrunnable Regression as a warning, not an error', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x`'), { strict: false });
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
assert.ok(codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
|
||||
});
|
||||
|
||||
test('K11: `On failure: try again` warns, and the plan stays valid', () => {
|
||||
const r = validatePlanContent(withOnFailure('try again'));
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
const w = r.warnings.find((x) => x.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN');
|
||||
assert.ok(w, codes(r.warnings).join(','));
|
||||
assert.match(w.message, /Step 1/);
|
||||
for (const word of ON_FAILURE_ACTIONS) assert.match(w.hint, new RegExp(word));
|
||||
});
|
||||
|
||||
test('K11: `On failure: retry once, then escalate` gives no warning', () => {
|
||||
const r = validatePlanContent(withOnFailure('retry once, then escalate'));
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_ON_FAILURE_UNKNOWN'), codes(r.warnings).join(','));
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue