test(proevesett): red — case 4 can halt on the intent gate; says-fail rejects FAILED

Case 4 (plan-halts-without-phase-signals) measured on dc9b480, 6 runs:
5 of 6. The failing run called only intent-approval.mjs, halted on
BRIEF_INTENT_NOT_APPROVED, and so failed validator-ran and
halt-names-signals: the fixture has no intent marker, so the case can
measure the wrong gate. PM: 6 of 12 on dc9b480, 7 of 10 on 66e1fa1.

New tests: the case-4 fixture must pass intent-approval --check and fail
only BRIEF_V51_MISSING_SIGNALS; a not_contains grader must fail the case
on any BRIEF_INTENT_* code in the trace. says-fail / says-pass must take
FAILED / PASSED as the verdict word (v2's word boundary dropped them).

Red: 3 of 9 in tests/lib/proevesett.test.mjs fail, each for its own reason.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 10:45:23 +02:00
commit afb2745cd0
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q

View file

@ -96,6 +96,39 @@ test('proevesett: the gate case fixture passes the sequencing gate and stops at
} finally { rmSync(ws, { recursive: true, force: true }); }
});
// ---- case 4: only the sequencing gate may stop it ----
// Measured 2026-09-23: with no intent marker the child sometimes ran the
// intent gate first and halted there (PM: 6 of 12 on dc9b480, 7 of 10 on
// 66e1fa1). The halt was right, but the case then measured the wrong gate.
const SEQ_CASE = 'plan-halts-without-phase-signals';
test('plan-halts-without-phase-signals: the fixture carries a current intent approval and fails only the sequencing gate', () => {
const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
try {
const s = spawnSync('bash', [join(EVALS, SEQ_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
assert.equal(s.status, 0, s.stderr);
const brief = join(ws, 'proj', 'brief.md');
const g = spawnSync('node', [join(ROOT, 'lib/validators/intent-approval.mjs'), '--check', '--json', brief], { encoding: 'utf8' });
assert.equal(g.status, 0, `the intent gate must pass so only the sequencing gate can stop the run: ${g.stdout}`);
const v = spawnSync('node', [join(ROOT, 'lib/validators/brief-validator.mjs'), '--soft', '--json', brief], { encoding: 'utf8' });
assert.deepEqual(JSON.parse(v.stdout).errors.map((e) => e.code), ['BRIEF_V51_MISSING_SIGNALS']);
} finally { rmSync(ws, { recursive: true, force: true }); }
});
test('plan-halts-without-phase-signals: a halt at the intent gate fails the case', () => {
assert.ok(graders(SEQ_CASE).includes('no-intent-halt.md'), graders(SEQ_CASE).join(','));
const g = grader(`evals/${SEQ_CASE}/graders/no-intent-halt.md`);
assert.deepEqual([g.type, g.target, g.match], ['regex', 'trace', 'not_contains']);
const re = new RegExp(g.pattern);
for (const code of ['BRIEF_INTENT_NOT_APPROVED', 'BRIEF_INTENT_APPROVAL_STALE', 'BRIEF_INTENT_APPROVAL_INVALID']) {
assert.match(inTrace({ valid: false, errors: [{ code }] }, 2), re, `${code} must fail the case`);
}
assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
assert.doesNotMatch(COMMAND_PROSE, re, 'the command prose must not trip the grader');
});
// ---- case 5: the missing-flag error comes from code ----
test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
@ -153,6 +186,11 @@ test('says-pass / says-fail are not raw substrings: a negation does not pass the
for (const s of ['The review did not PASS.', 'NOT PASS', 'PASSED_WITH_WARNINGS']) assert.doesNotMatch(s, pass);
for (const s of ['review-validator: FAIL proj/review.md', 'Result: **FAIL**']) assert.match(s, fail);
for (const s of ['It did not FAIL.', 'NOT FAIL', 'FAILOVER']) assert.doesNotMatch(s, fail);
// The verdict word, not one spelling of it: v1's raw substring took these,
// v2's word boundary dropped them (a correct reply would fail the case).
for (const s of ['Validation FAILED', '**FAILED**: proj/review.md']) assert.match(s, fail);
for (const s of ['Validation PASSED', '**PASSED**: proj/review.md']) assert.match(s, pass);
for (const s of ['It did not FAILED.', 'FAILEDX']) assert.doesNotMatch(s, fail);
});
test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {