test(proevesett): a capitalised opposite verdict refuses says-pass/says-fail (red)
Measured on687710a: a correct case-6 reply said "**FAIL**: ... did not pass ... The other checks passed." With lower case accepted, says-pass matches that reply, so a wrong case-8 reply of the same shape would pass the case. The test pins that a reply whose capitalised verdict is the opposite one is refused, while the opposite word in lower case (a side clause) or negated is not. Red on687710a: 11 of 12 in tests/lib/proevesett.test.mjs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
687710a975
commit
f838e3c527
1 changed files with 20 additions and 0 deletions
|
|
@ -239,6 +239,26 @@ test('says-pass / says-fail take the verdict in any letter case, and still refus
|
|||
for (const s of lowerPass) assert.doesNotMatch(s, fail, s);
|
||||
});
|
||||
|
||||
test('says-pass / says-fail refuse a reply whose capitalised verdict is the opposite one', () => {
|
||||
const pass = new RegExp(grader('evals/review-validate-passes-clean-review/graders/says-pass.md').pattern);
|
||||
const fail = new RegExp(grader('evals/review-validate-flags-bad-finding-id/graders/says-fail.md').pattern);
|
||||
// A correct case-6 reply recorded on 687710a: the verdict is FAIL, and a
|
||||
// side clause says "passed". Lower case alone would let it satisfy says-pass.
|
||||
const mixedFail = '**FAIL**: `proj/review.md` did not pass the strict validator (exit 1).\n\n'
|
||||
+ 'The first entry is `F-001`. The other checks passed.';
|
||||
assert.match(mixedFail, fail);
|
||||
assert.doesNotMatch(mixedFail, pass);
|
||||
assert.doesNotMatch('Result: FAILED\nEverything else passed.', pass);
|
||||
assert.doesNotMatch('**PASS**: proj/review.md is valid.\nNothing failed.', fail);
|
||||
assert.doesNotMatch('**PASSED**\n`review.md` fails nothing.', fail);
|
||||
// The opposite word in lower case, or negated in capitals, does not refuse.
|
||||
assert.match('**PASS**: nothing failed.', pass);
|
||||
assert.match('It did not FAIL: `proj/review.md` passed.', pass);
|
||||
assert.match('It did not PASS: `proj/review.md` failed.', fail);
|
||||
// Multi-line: the refusal looks at the whole reply, not the line of the verdict.
|
||||
assert.doesNotMatch('`proj/review.md` passed one check.\n\n**FAIL**', pass);
|
||||
});
|
||||
|
||||
test('every case with a no-write grader also has no-bash-write, and it sees a write through Bash', () => {
|
||||
const withNoWrite = cases().filter((c) => graders(c).includes('no-write.md'));
|
||||
assert.ok(withNoWrite.length >= 5, withNoWrite.join(','));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue