244 lines
6.9 KiB
JavaScript
244 lines
6.9 KiB
JavaScript
// tests/validators/brief-gate-coverage.test.mjs
|
|
// SKAL-1·4a — two-sided (class-balanced) coverage for every error-severity
|
|
// BRIEF_* BLOCKER the brief-validator can emit.
|
|
//
|
|
// brief-validator.test.mjs already tests many codes; several were FAILURE-only
|
|
// (only the raising case asserted) and three were untested. This meta-test pins
|
|
// a class-balanced pair { fail, pass } for EACH in-scope BLOCKER and a parity
|
|
// guard so a code can't sit in IN_SCOPE_CODES without a paired case. It removes
|
|
// the one-sided-optimization risk programmatically, not by inspection.
|
|
//
|
|
// In scope = ERROR-severity BRIEF_* codes the validator pushes to errors[]
|
|
// (BLOCKERs). EXCLUDED: warning-severity codes (BRIEF_TLDR_TOO_LONG,
|
|
// BRIEF_VERSION_FORMAT, BRIEF_VERSION_BELOW_MINIMUM, BRIEF_PARTIAL_SKIPPED) — a
|
|
// "passing" counterpart to a warning is not a BLOCKER concern — and the
|
|
// file-level BRIEF_NOT_FOUND / BRIEF_READ_ERROR which live in validateBrief()
|
|
// (filesystem), not validateBriefContent().
|
|
//
|
|
// LIMITATION: brief-validator.mjs does not export its code set (codes are string
|
|
// literals inside issue(...)), so IN_SCOPE_CODES is HAND-MAINTAINED. A NEW
|
|
// validator BLOCKER must be added here manually; the parity guard catches
|
|
// list/case drift, not validator drift. (Operator chose the hand-maintained list
|
|
// over exporting BRIEF_ERROR_CODES from the forbidden_paths-guarded validator.)
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
|
|
|
|
// ---- valid bases (mirror tests/validators/brief-validator.test.mjs) ----------
|
|
|
|
const GOOD_BRIEF = `---
|
|
type: trekbrief
|
|
brief_version: "2.0"
|
|
created: 2026-04-30
|
|
task: "Add JWT auth to API"
|
|
slug: jwt-auth
|
|
project_dir: .claude/projects/2026-04-30-jwt-auth/
|
|
research_topics: 2
|
|
research_status: pending
|
|
auto_research: false
|
|
interview_turns: 5
|
|
source: interview
|
|
---
|
|
|
|
# Task: JWT auth
|
|
|
|
## Intent
|
|
|
|
Why this matters.
|
|
|
|
## Goal
|
|
|
|
What success looks like.
|
|
|
|
## Success Criteria
|
|
|
|
- All tests pass.
|
|
`;
|
|
|
|
const GOOD_BRIEF_22 = `---
|
|
type: trekbrief
|
|
brief_version: "2.2"
|
|
created: 2026-06-18
|
|
task: "Add JWT auth to API"
|
|
slug: jwt-auth
|
|
project_dir: .claude/projects/2026-06-18-jwt-auth/
|
|
research_topics: 0
|
|
research_status: complete
|
|
auto_research: false
|
|
interview_turns: 5
|
|
source: interview
|
|
framing: new-direction
|
|
phase_signals_partial: true
|
|
---
|
|
|
|
# Task: JWT auth
|
|
|
|
## TL;DR
|
|
|
|
Net-new JWT auth; no prior brief to anchor against.
|
|
|
|
## Intent
|
|
|
|
Why this matters.
|
|
|
|
## Goal
|
|
|
|
What success looks like.
|
|
|
|
## Success Criteria
|
|
|
|
- All tests pass.
|
|
`;
|
|
|
|
const REVIEW_AS_BRIEF = `---
|
|
type: trekreview
|
|
task: "Review delivered trekreview v1.0"
|
|
slug: trekreview
|
|
project_dir: .claude/projects/2026-05-01-trekreview/
|
|
findings:
|
|
- 0123456789abcdef0123456789abcdef01234567
|
|
- fedcba9876543210fedcba9876543210fedcba98
|
|
---
|
|
|
|
# Review brief
|
|
|
|
## Intent
|
|
|
|
Adversarial review of delivered trekreview v1.0.
|
|
|
|
## Goal
|
|
|
|
Find what was missed.
|
|
|
|
## Success Criteria
|
|
|
|
- All BLOCKER findings get a fix-plan.
|
|
`;
|
|
|
|
// signals blocks
|
|
const GOOD_SIGNALS = `phase_signals:
|
|
- phase: plan
|
|
effort: high
|
|
model: opus
|
|
`;
|
|
const BAD_PHASE_SIGNALS = `phase_signals:
|
|
- phase: nonsense
|
|
effort: high
|
|
`;
|
|
const BAD_EFFORT_SIGNALS = `phase_signals:
|
|
- phase: plan
|
|
effort: turbo
|
|
`;
|
|
const BAD_MODEL_SIGNALS = `phase_signals:
|
|
- phase: plan
|
|
effort: high
|
|
model: gpt4
|
|
`;
|
|
|
|
// convenience builders
|
|
const v21 = (s) => s.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
|
const withLine = (s, line) => s.replace('source: interview\n', `source: interview\n${line}`);
|
|
|
|
// ---- the canonical in-scope set + a class-balanced pair for each -------------
|
|
|
|
const IN_SCOPE_CODES = [
|
|
'BRIEF_MISSING_FIELD',
|
|
'BRIEF_BAD_STATUS',
|
|
'BRIEF_STATE_INCOHERENT',
|
|
'BRIEF_WRONG_TYPE',
|
|
'BRIEF_BAD_FINDINGS_TYPE',
|
|
'BRIEF_MISSING_SECTION',
|
|
'BRIEF_INVALID_FRAMING',
|
|
'BRIEF_MISSING_FRAMING',
|
|
'BRIEF_V51_MISSING_SIGNALS',
|
|
'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE',
|
|
'BRIEF_INVALID_PHASE_SIGNALS',
|
|
'BRIEF_INVALID_PHASE_SIGNAL_PHASE',
|
|
'BRIEF_INVALID_EFFORT',
|
|
'BRIEF_INVALID_MODEL',
|
|
];
|
|
|
|
const CASES = {
|
|
BRIEF_MISSING_FIELD: {
|
|
fail: GOOD_BRIEF.replace(/^research_topics: 2\n/m, ''),
|
|
pass: GOOD_BRIEF,
|
|
},
|
|
BRIEF_BAD_STATUS: {
|
|
fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: maybe'),
|
|
pass: GOOD_BRIEF,
|
|
},
|
|
BRIEF_STATE_INCOHERENT: {
|
|
fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: skipped'),
|
|
pass: GOOD_BRIEF,
|
|
},
|
|
BRIEF_WRONG_TYPE: {
|
|
fail: GOOD_BRIEF.replace('type: trekbrief', 'type: notabrief'),
|
|
pass: GOOD_BRIEF,
|
|
},
|
|
BRIEF_BAD_FINDINGS_TYPE: {
|
|
fail: REVIEW_AS_BRIEF.replace(/findings:\n - 0123[\s\S]*?- fedcba[0-9a-f]+/, 'findings: not-an-array'),
|
|
pass: REVIEW_AS_BRIEF,
|
|
},
|
|
BRIEF_MISSING_SECTION: {
|
|
fail: GOOD_BRIEF.replace(/## Intent\n\nWhy this matters\.\n\n/, ''),
|
|
pass: GOOD_BRIEF,
|
|
},
|
|
BRIEF_INVALID_FRAMING: {
|
|
fail: withLine(GOOD_BRIEF, 'framing: sideways\n'),
|
|
pass: GOOD_BRIEF_22,
|
|
},
|
|
BRIEF_MISSING_FRAMING: {
|
|
fail: GOOD_BRIEF_22.replace('framing: new-direction\n', ''),
|
|
pass: GOOD_BRIEF_22,
|
|
},
|
|
BRIEF_V51_MISSING_SIGNALS: {
|
|
fail: v21(GOOD_BRIEF),
|
|
pass: GOOD_BRIEF, // 2.0 — gate inactive
|
|
},
|
|
BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE: {
|
|
fail: withLine(v21(GOOD_BRIEF), `phase_signals_partial: true\n${GOOD_SIGNALS}`),
|
|
pass: withLine(v21(GOOD_BRIEF), 'phase_signals_partial: true\n'),
|
|
},
|
|
BRIEF_INVALID_PHASE_SIGNALS: {
|
|
fail: withLine(GOOD_BRIEF, 'phase_signals: notalist\n'),
|
|
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
|
},
|
|
BRIEF_INVALID_PHASE_SIGNAL_PHASE: {
|
|
fail: withLine(v21(GOOD_BRIEF), BAD_PHASE_SIGNALS),
|
|
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
|
},
|
|
BRIEF_INVALID_EFFORT: {
|
|
fail: withLine(v21(GOOD_BRIEF), BAD_EFFORT_SIGNALS),
|
|
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
|
},
|
|
BRIEF_INVALID_MODEL: {
|
|
fail: withLine(v21(GOOD_BRIEF), BAD_MODEL_SIGNALS),
|
|
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
|
},
|
|
};
|
|
|
|
test('gate coverage — CASES keys exactly match IN_SCOPE_CODES (no silent gap)', () => {
|
|
assert.deepEqual(Object.keys(CASES).sort(), [...IN_SCOPE_CODES].sort());
|
|
});
|
|
|
|
test('gate coverage — every in-scope BRIEF_* BLOCKER raises on its fail case', () => {
|
|
for (const code of IN_SCOPE_CODES) {
|
|
const r = validateBriefContent(CASES[code].fail, { strict: true });
|
|
assert.ok(
|
|
r.errors.find((e) => e.code === code),
|
|
`${code}: fail case did NOT raise it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
test('gate coverage — every in-scope BRIEF_* BLOCKER is absent on its (valid) pass case', () => {
|
|
for (const code of IN_SCOPE_CODES) {
|
|
const r = validateBriefContent(CASES[code].pass, { strict: true });
|
|
assert.ok(
|
|
!r.errors.find((e) => e.code === code),
|
|
`${code}: pass case wrongly raised it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`,
|
|
);
|
|
assert.equal(r.valid, true, `${code}: pass case should be a fully valid brief; errors=${JSON.stringify(r.errors)}`);
|
|
}
|
|
});
|