diff --git a/tests/validators/brief-gate-coverage.test.mjs b/tests/validators/brief-gate-coverage.test.mjs new file mode 100644 index 0000000..e8602d8 --- /dev/null +++ b/tests/validators/brief-gate-coverage.test.mjs @@ -0,0 +1,244 @@ +// tests/validators/brief-gate-coverage.test.mjs +// SKAL-1·4a — two-sided (class-balanced) coverage for every error-severity +// BRIEF_* BLOCKER the brief-validator can emit. +// +// brief-validator.test.mjs already tests many codes; several were FAILURE-only +// (only the raising case asserted) and three were untested. This meta-test pins +// a class-balanced pair { fail, pass } for EACH in-scope BLOCKER and a parity +// guard so a code can't sit in IN_SCOPE_CODES without a paired case. It removes +// the one-sided-optimization risk programmatically, not by inspection. +// +// In scope = ERROR-severity BRIEF_* codes the validator pushes to errors[] +// (BLOCKERs). EXCLUDED: warning-severity codes (BRIEF_TLDR_TOO_LONG, +// BRIEF_VERSION_FORMAT, BRIEF_VERSION_BELOW_MINIMUM, BRIEF_PARTIAL_SKIPPED) — a +// "passing" counterpart to a warning is not a BLOCKER concern — and the +// file-level BRIEF_NOT_FOUND / BRIEF_READ_ERROR which live in validateBrief() +// (filesystem), not validateBriefContent(). +// +// LIMITATION: brief-validator.mjs does not export its code set (codes are string +// literals inside issue(...)), so IN_SCOPE_CODES is HAND-MAINTAINED. A NEW +// validator BLOCKER must be added here manually; the parity guard catches +// list/case drift, not validator drift. (Operator chose the hand-maintained list +// over exporting BRIEF_ERROR_CODES from the forbidden_paths-guarded validator.) + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; + +// ---- valid bases (mirror tests/validators/brief-validator.test.mjs) ---------- + +const GOOD_BRIEF = `--- +type: trekbrief +brief_version: "2.0" +created: 2026-04-30 +task: "Add JWT auth to API" +slug: jwt-auth +project_dir: .claude/projects/2026-04-30-jwt-auth/ +research_topics: 2 +research_status: pending +auto_research: false +interview_turns: 5 +source: interview +--- + +# Task: JWT auth + +## Intent + +Why this matters. + +## Goal + +What success looks like. + +## Success Criteria + +- All tests pass. +`; + +const GOOD_BRIEF_22 = `--- +type: trekbrief +brief_version: "2.2" +created: 2026-06-18 +task: "Add JWT auth to API" +slug: jwt-auth +project_dir: .claude/projects/2026-06-18-jwt-auth/ +research_topics: 0 +research_status: complete +auto_research: false +interview_turns: 5 +source: interview +framing: new-direction +phase_signals_partial: true +--- + +# Task: JWT auth + +## TL;DR + +Net-new JWT auth; no prior brief to anchor against. + +## Intent + +Why this matters. + +## Goal + +What success looks like. + +## Success Criteria + +- All tests pass. +`; + +const REVIEW_AS_BRIEF = `--- +type: trekreview +task: "Review delivered trekreview v1.0" +slug: trekreview +project_dir: .claude/projects/2026-05-01-trekreview/ +findings: + - 0123456789abcdef0123456789abcdef01234567 + - fedcba9876543210fedcba9876543210fedcba98 +--- + +# Review brief + +## Intent + +Adversarial review of delivered trekreview v1.0. + +## Goal + +Find what was missed. + +## Success Criteria + +- All BLOCKER findings get a fix-plan. +`; + +// signals blocks +const GOOD_SIGNALS = `phase_signals: + - phase: plan + effort: high + model: opus +`; +const BAD_PHASE_SIGNALS = `phase_signals: + - phase: nonsense + effort: high +`; +const BAD_EFFORT_SIGNALS = `phase_signals: + - phase: plan + effort: turbo +`; +const BAD_MODEL_SIGNALS = `phase_signals: + - phase: plan + effort: high + model: gpt4 +`; + +// convenience builders +const v21 = (s) => s.replace('brief_version: "2.0"', 'brief_version: "2.1"'); +const withLine = (s, line) => s.replace('source: interview\n', `source: interview\n${line}`); + +// ---- the canonical in-scope set + a class-balanced pair for each ------------- + +const IN_SCOPE_CODES = [ + 'BRIEF_MISSING_FIELD', + 'BRIEF_BAD_STATUS', + 'BRIEF_STATE_INCOHERENT', + 'BRIEF_WRONG_TYPE', + 'BRIEF_BAD_FINDINGS_TYPE', + 'BRIEF_MISSING_SECTION', + 'BRIEF_INVALID_FRAMING', + 'BRIEF_MISSING_FRAMING', + 'BRIEF_V51_MISSING_SIGNALS', + 'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE', + 'BRIEF_INVALID_PHASE_SIGNALS', + 'BRIEF_INVALID_PHASE_SIGNAL_PHASE', + 'BRIEF_INVALID_EFFORT', + 'BRIEF_INVALID_MODEL', +]; + +const CASES = { + BRIEF_MISSING_FIELD: { + fail: GOOD_BRIEF.replace(/^research_topics: 2\n/m, ''), + pass: GOOD_BRIEF, + }, + BRIEF_BAD_STATUS: { + fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: maybe'), + pass: GOOD_BRIEF, + }, + BRIEF_STATE_INCOHERENT: { + fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: skipped'), + pass: GOOD_BRIEF, + }, + BRIEF_WRONG_TYPE: { + fail: GOOD_BRIEF.replace('type: trekbrief', 'type: notabrief'), + pass: GOOD_BRIEF, + }, + BRIEF_BAD_FINDINGS_TYPE: { + fail: REVIEW_AS_BRIEF.replace(/findings:\n - 0123[\s\S]*?- fedcba[0-9a-f]+/, 'findings: not-an-array'), + pass: REVIEW_AS_BRIEF, + }, + BRIEF_MISSING_SECTION: { + fail: GOOD_BRIEF.replace(/## Intent\n\nWhy this matters\.\n\n/, ''), + pass: GOOD_BRIEF, + }, + BRIEF_INVALID_FRAMING: { + fail: withLine(GOOD_BRIEF, 'framing: sideways\n'), + pass: GOOD_BRIEF_22, + }, + BRIEF_MISSING_FRAMING: { + fail: GOOD_BRIEF_22.replace('framing: new-direction\n', ''), + pass: GOOD_BRIEF_22, + }, + BRIEF_V51_MISSING_SIGNALS: { + fail: v21(GOOD_BRIEF), + pass: GOOD_BRIEF, // 2.0 — gate inactive + }, + BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE: { + fail: withLine(v21(GOOD_BRIEF), `phase_signals_partial: true\n${GOOD_SIGNALS}`), + pass: withLine(v21(GOOD_BRIEF), 'phase_signals_partial: true\n'), + }, + BRIEF_INVALID_PHASE_SIGNALS: { + fail: withLine(GOOD_BRIEF, 'phase_signals: notalist\n'), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_PHASE_SIGNAL_PHASE: { + fail: withLine(v21(GOOD_BRIEF), BAD_PHASE_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_EFFORT: { + fail: withLine(v21(GOOD_BRIEF), BAD_EFFORT_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_MODEL: { + fail: withLine(v21(GOOD_BRIEF), BAD_MODEL_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, +}; + +test('gate coverage — CASES keys exactly match IN_SCOPE_CODES (no silent gap)', () => { + assert.deepEqual(Object.keys(CASES).sort(), [...IN_SCOPE_CODES].sort()); +}); + +test('gate coverage — every in-scope BRIEF_* BLOCKER raises on its fail case', () => { + for (const code of IN_SCOPE_CODES) { + const r = validateBriefContent(CASES[code].fail, { strict: true }); + assert.ok( + r.errors.find((e) => e.code === code), + `${code}: fail case did NOT raise it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`, + ); + } +}); + +test('gate coverage — every in-scope BRIEF_* BLOCKER is absent on its (valid) pass case', () => { + for (const code of IN_SCOPE_CODES) { + const r = validateBriefContent(CASES[code].pass, { strict: true }); + assert.ok( + !r.errors.find((e) => e.code === code), + `${code}: pass case wrongly raised it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`, + ); + assert.equal(r.valid, true, `${code}: pass case should be a fully valid brief; errors=${JSON.stringify(r.errors)}`); + } +});