test(proevesett): red — case 4 can halt on the intent gate; says-fail rejects FAILED
Case 4 (plan-halts-without-phase-signals) measured ondc9b480, 6 runs: 5 of 6. The failing run called only intent-approval.mjs, halted on BRIEF_INTENT_NOT_APPROVED, and so failed validator-ran and halt-names-signals: the fixture has no intent marker, so the case can measure the wrong gate. PM: 6 of 12 ondc9b480, 7 of 10 on66e1fa1. New tests: the case-4 fixture must pass intent-approval --check and fail only BRIEF_V51_MISSING_SIGNALS; a not_contains grader must fail the case on any BRIEF_INTENT_* code in the trace. says-fail / says-pass must take FAILED / PASSED as the verdict word (v2's word boundary dropped them). Red: 3 of 9 in tests/lib/proevesett.test.mjs fail, each for its own reason. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
dc9b480780
commit
afb2745cd0
1 changed files with 38 additions and 0 deletions
|
|
@ -96,6 +96,39 @@ test('proevesett: the gate case fixture passes the sequencing gate and stops at
|
||||||
} finally { rmSync(ws, { recursive: true, force: true }); }
|
} finally { rmSync(ws, { recursive: true, force: true }); }
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ---- case 4: only the sequencing gate may stop it ----
|
||||||
|
// Measured 2026-09-23: with no intent marker the child sometimes ran the
|
||||||
|
// intent gate first and halted there (PM: 6 of 12 on dc9b480, 7 of 10 on
|
||||||
|
// 66e1fa1). The halt was right, but the case then measured the wrong gate.
|
||||||
|
|
||||||
|
const SEQ_CASE = 'plan-halts-without-phase-signals';
|
||||||
|
|
||||||
|
test('plan-halts-without-phase-signals: the fixture carries a current intent approval and fails only the sequencing gate', () => {
|
||||||
|
const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
|
||||||
|
try {
|
||||||
|
const s = spawnSync('bash', [join(EVALS, SEQ_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
|
||||||
|
assert.equal(s.status, 0, s.stderr);
|
||||||
|
const brief = join(ws, 'proj', 'brief.md');
|
||||||
|
const g = spawnSync('node', [join(ROOT, 'lib/validators/intent-approval.mjs'), '--check', '--json', brief], { encoding: 'utf8' });
|
||||||
|
assert.equal(g.status, 0, `the intent gate must pass so only the sequencing gate can stop the run: ${g.stdout}`);
|
||||||
|
const v = spawnSync('node', [join(ROOT, 'lib/validators/brief-validator.mjs'), '--soft', '--json', brief], { encoding: 'utf8' });
|
||||||
|
assert.deepEqual(JSON.parse(v.stdout).errors.map((e) => e.code), ['BRIEF_V51_MISSING_SIGNALS']);
|
||||||
|
} finally { rmSync(ws, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('plan-halts-without-phase-signals: a halt at the intent gate fails the case', () => {
|
||||||
|
assert.ok(graders(SEQ_CASE).includes('no-intent-halt.md'), graders(SEQ_CASE).join(','));
|
||||||
|
const g = grader(`evals/${SEQ_CASE}/graders/no-intent-halt.md`);
|
||||||
|
assert.deepEqual([g.type, g.target, g.match], ['regex', 'trace', 'not_contains']);
|
||||||
|
const re = new RegExp(g.pattern);
|
||||||
|
for (const code of ['BRIEF_INTENT_NOT_APPROVED', 'BRIEF_INTENT_APPROVAL_STALE', 'BRIEF_INTENT_APPROVAL_INVALID']) {
|
||||||
|
assert.match(inTrace({ valid: false, errors: [{ code }] }, 2), re, `${code} must fail the case`);
|
||||||
|
}
|
||||||
|
assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
|
||||||
|
assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
|
||||||
|
assert.doesNotMatch(COMMAND_PROSE, re, 'the command prose must not trip the grader');
|
||||||
|
});
|
||||||
|
|
||||||
// ---- case 5: the missing-flag error comes from code ----
|
// ---- case 5: the missing-flag error comes from code ----
|
||||||
|
|
||||||
test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
|
test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
|
||||||
|
|
@ -153,6 +186,11 @@ test('says-pass / says-fail are not raw substrings: a negation does not pass the
|
||||||
for (const s of ['The review did not PASS.', 'NOT PASS', 'PASSED_WITH_WARNINGS']) assert.doesNotMatch(s, pass);
|
for (const s of ['The review did not PASS.', 'NOT PASS', 'PASSED_WITH_WARNINGS']) assert.doesNotMatch(s, pass);
|
||||||
for (const s of ['review-validator: FAIL proj/review.md', 'Result: **FAIL**']) assert.match(s, fail);
|
for (const s of ['review-validator: FAIL proj/review.md', 'Result: **FAIL**']) assert.match(s, fail);
|
||||||
for (const s of ['It did not FAIL.', 'NOT FAIL', 'FAILOVER']) assert.doesNotMatch(s, fail);
|
for (const s of ['It did not FAIL.', 'NOT FAIL', 'FAILOVER']) assert.doesNotMatch(s, fail);
|
||||||
|
// The verdict word, not one spelling of it: v1's raw substring took these,
|
||||||
|
// v2's word boundary dropped them (a correct reply would fail the case).
|
||||||
|
for (const s of ['Validation FAILED', '**FAILED**: proj/review.md']) assert.match(s, fail);
|
||||||
|
for (const s of ['Validation PASSED', '**PASSED**: proj/review.md']) assert.match(s, pass);
|
||||||
|
for (const s of ['It did not FAILED.', 'FAILEDX']) assert.doesNotMatch(s, fail);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {
|
test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue