test(eval): two-sided gate coverage for BRIEF_* BLOCKERs
This commit is contained in:
parent
971604d870
commit
844492fbcf
1 changed files with 244 additions and 0 deletions
244
tests/validators/brief-gate-coverage.test.mjs
Normal file
244
tests/validators/brief-gate-coverage.test.mjs
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
// tests/validators/brief-gate-coverage.test.mjs
|
||||
// SKAL-1·4a — two-sided (class-balanced) coverage for every error-severity
|
||||
// BRIEF_* BLOCKER the brief-validator can emit.
|
||||
//
|
||||
// brief-validator.test.mjs already tests many codes; several were FAILURE-only
|
||||
// (only the raising case asserted) and three were untested. This meta-test pins
|
||||
// a class-balanced pair { fail, pass } for EACH in-scope BLOCKER and a parity
|
||||
// guard so a code can't sit in IN_SCOPE_CODES without a paired case. It removes
|
||||
// the one-sided-optimization risk programmatically, not by inspection.
|
||||
//
|
||||
// In scope = ERROR-severity BRIEF_* codes the validator pushes to errors[]
|
||||
// (BLOCKERs). EXCLUDED: warning-severity codes (BRIEF_TLDR_TOO_LONG,
|
||||
// BRIEF_VERSION_FORMAT, BRIEF_VERSION_BELOW_MINIMUM, BRIEF_PARTIAL_SKIPPED) — a
|
||||
// "passing" counterpart to a warning is not a BLOCKER concern — and the
|
||||
// file-level BRIEF_NOT_FOUND / BRIEF_READ_ERROR which live in validateBrief()
|
||||
// (filesystem), not validateBriefContent().
|
||||
//
|
||||
// LIMITATION: brief-validator.mjs does not export its code set (codes are string
|
||||
// literals inside issue(...)), so IN_SCOPE_CODES is HAND-MAINTAINED. A NEW
|
||||
// validator BLOCKER must be added here manually; the parity guard catches
|
||||
// list/case drift, not validator drift. (Operator chose the hand-maintained list
|
||||
// over exporting BRIEF_ERROR_CODES from the forbidden_paths-guarded validator.)
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
|
||||
|
||||
// ---- valid bases (mirror tests/validators/brief-validator.test.mjs) ----------
|
||||
|
||||
const GOOD_BRIEF = `---
|
||||
type: trekbrief
|
||||
brief_version: "2.0"
|
||||
created: 2026-04-30
|
||||
task: "Add JWT auth to API"
|
||||
slug: jwt-auth
|
||||
project_dir: .claude/projects/2026-04-30-jwt-auth/
|
||||
research_topics: 2
|
||||
research_status: pending
|
||||
auto_research: false
|
||||
interview_turns: 5
|
||||
source: interview
|
||||
---
|
||||
|
||||
# Task: JWT auth
|
||||
|
||||
## Intent
|
||||
|
||||
Why this matters.
|
||||
|
||||
## Goal
|
||||
|
||||
What success looks like.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- All tests pass.
|
||||
`;
|
||||
|
||||
const GOOD_BRIEF_22 = `---
|
||||
type: trekbrief
|
||||
brief_version: "2.2"
|
||||
created: 2026-06-18
|
||||
task: "Add JWT auth to API"
|
||||
slug: jwt-auth
|
||||
project_dir: .claude/projects/2026-06-18-jwt-auth/
|
||||
research_topics: 0
|
||||
research_status: complete
|
||||
auto_research: false
|
||||
interview_turns: 5
|
||||
source: interview
|
||||
framing: new-direction
|
||||
phase_signals_partial: true
|
||||
---
|
||||
|
||||
# Task: JWT auth
|
||||
|
||||
## TL;DR
|
||||
|
||||
Net-new JWT auth; no prior brief to anchor against.
|
||||
|
||||
## Intent
|
||||
|
||||
Why this matters.
|
||||
|
||||
## Goal
|
||||
|
||||
What success looks like.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- All tests pass.
|
||||
`;
|
||||
|
||||
const REVIEW_AS_BRIEF = `---
|
||||
type: trekreview
|
||||
task: "Review delivered trekreview v1.0"
|
||||
slug: trekreview
|
||||
project_dir: .claude/projects/2026-05-01-trekreview/
|
||||
findings:
|
||||
- 0123456789abcdef0123456789abcdef01234567
|
||||
- fedcba9876543210fedcba9876543210fedcba98
|
||||
---
|
||||
|
||||
# Review brief
|
||||
|
||||
## Intent
|
||||
|
||||
Adversarial review of delivered trekreview v1.0.
|
||||
|
||||
## Goal
|
||||
|
||||
Find what was missed.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- All BLOCKER findings get a fix-plan.
|
||||
`;
|
||||
|
||||
// signals blocks
|
||||
const GOOD_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: opus
|
||||
`;
|
||||
const BAD_PHASE_SIGNALS = `phase_signals:
|
||||
- phase: nonsense
|
||||
effort: high
|
||||
`;
|
||||
const BAD_EFFORT_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: turbo
|
||||
`;
|
||||
const BAD_MODEL_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: gpt4
|
||||
`;
|
||||
|
||||
// convenience builders
|
||||
const v21 = (s) => s.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
const withLine = (s, line) => s.replace('source: interview\n', `source: interview\n${line}`);
|
||||
|
||||
// ---- the canonical in-scope set + a class-balanced pair for each -------------
|
||||
|
||||
const IN_SCOPE_CODES = [
|
||||
'BRIEF_MISSING_FIELD',
|
||||
'BRIEF_BAD_STATUS',
|
||||
'BRIEF_STATE_INCOHERENT',
|
||||
'BRIEF_WRONG_TYPE',
|
||||
'BRIEF_BAD_FINDINGS_TYPE',
|
||||
'BRIEF_MISSING_SECTION',
|
||||
'BRIEF_INVALID_FRAMING',
|
||||
'BRIEF_MISSING_FRAMING',
|
||||
'BRIEF_V51_MISSING_SIGNALS',
|
||||
'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE',
|
||||
'BRIEF_INVALID_PHASE_SIGNALS',
|
||||
'BRIEF_INVALID_PHASE_SIGNAL_PHASE',
|
||||
'BRIEF_INVALID_EFFORT',
|
||||
'BRIEF_INVALID_MODEL',
|
||||
];
|
||||
|
||||
const CASES = {
|
||||
BRIEF_MISSING_FIELD: {
|
||||
fail: GOOD_BRIEF.replace(/^research_topics: 2\n/m, ''),
|
||||
pass: GOOD_BRIEF,
|
||||
},
|
||||
BRIEF_BAD_STATUS: {
|
||||
fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: maybe'),
|
||||
pass: GOOD_BRIEF,
|
||||
},
|
||||
BRIEF_STATE_INCOHERENT: {
|
||||
fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: skipped'),
|
||||
pass: GOOD_BRIEF,
|
||||
},
|
||||
BRIEF_WRONG_TYPE: {
|
||||
fail: GOOD_BRIEF.replace('type: trekbrief', 'type: notabrief'),
|
||||
pass: GOOD_BRIEF,
|
||||
},
|
||||
BRIEF_BAD_FINDINGS_TYPE: {
|
||||
fail: REVIEW_AS_BRIEF.replace(/findings:\n - 0123[\s\S]*?- fedcba[0-9a-f]+/, 'findings: not-an-array'),
|
||||
pass: REVIEW_AS_BRIEF,
|
||||
},
|
||||
BRIEF_MISSING_SECTION: {
|
||||
fail: GOOD_BRIEF.replace(/## Intent\n\nWhy this matters\.\n\n/, ''),
|
||||
pass: GOOD_BRIEF,
|
||||
},
|
||||
BRIEF_INVALID_FRAMING: {
|
||||
fail: withLine(GOOD_BRIEF, 'framing: sideways\n'),
|
||||
pass: GOOD_BRIEF_22,
|
||||
},
|
||||
BRIEF_MISSING_FRAMING: {
|
||||
fail: GOOD_BRIEF_22.replace('framing: new-direction\n', ''),
|
||||
pass: GOOD_BRIEF_22,
|
||||
},
|
||||
BRIEF_V51_MISSING_SIGNALS: {
|
||||
fail: v21(GOOD_BRIEF),
|
||||
pass: GOOD_BRIEF, // 2.0 — gate inactive
|
||||
},
|
||||
BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE: {
|
||||
fail: withLine(v21(GOOD_BRIEF), `phase_signals_partial: true\n${GOOD_SIGNALS}`),
|
||||
pass: withLine(v21(GOOD_BRIEF), 'phase_signals_partial: true\n'),
|
||||
},
|
||||
BRIEF_INVALID_PHASE_SIGNALS: {
|
||||
fail: withLine(GOOD_BRIEF, 'phase_signals: notalist\n'),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_PHASE_SIGNAL_PHASE: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_PHASE_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_EFFORT: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_EFFORT_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_MODEL: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_MODEL_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
};
|
||||
|
||||
test('gate coverage — CASES keys exactly match IN_SCOPE_CODES (no silent gap)', () => {
|
||||
assert.deepEqual(Object.keys(CASES).sort(), [...IN_SCOPE_CODES].sort());
|
||||
});
|
||||
|
||||
test('gate coverage — every in-scope BRIEF_* BLOCKER raises on its fail case', () => {
|
||||
for (const code of IN_SCOPE_CODES) {
|
||||
const r = validateBriefContent(CASES[code].fail, { strict: true });
|
||||
assert.ok(
|
||||
r.errors.find((e) => e.code === code),
|
||||
`${code}: fail case did NOT raise it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('gate coverage — every in-scope BRIEF_* BLOCKER is absent on its (valid) pass case', () => {
|
||||
for (const code of IN_SCOPE_CODES) {
|
||||
const r = validateBriefContent(CASES[code].pass, { strict: true });
|
||||
assert.ok(
|
||||
!r.errors.find((e) => e.code === code),
|
||||
`${code}: pass case wrongly raised it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`,
|
||||
);
|
||||
assert.equal(r.valid, true, `${code}: pass case should be a fully valid brief; errors=${JSON.stringify(r.errors)}`);
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue