From 8b7a849a762b6f182d6088d316dd657201d1f7a4 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 16:56:26 +0200 Subject: [PATCH] test(validators): cover fable accept + unknown-model reject in both gate layers --- tests/fixtures/brief-effort-fable.md | 45 +++++++++++++++++++++++ tests/lib/phase-signal-resolver.test.mjs | 6 ++- tests/validators/brief-validator.test.mjs | 21 ++++++++++- 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/brief-effort-fable.md diff --git a/tests/fixtures/brief-effort-fable.md b/tests/fixtures/brief-effort-fable.md new file mode 100644 index 0000000..b1d50fb --- /dev/null +++ b/tests/fixtures/brief-effort-fable.md @@ -0,0 +1,45 @@ +--- +type: trekbrief +brief_version: "2.1" +created: 2026-07-02 +task: "Fixture: high-effort all phases on fable (v5.9 allowlist test)" +slug: brief-effort-fable +project_dir: .claude/projects/2026-07-02-brief-effort-fable/ +research_topics: 0 +research_status: complete +auto_research: false +interview_turns: 4 +source: fixture +phase_signals: + - phase: research + effort: high + model: fable + - phase: plan + effort: high + model: fable + - phase: execute + effort: high + model: fable + - phase: review + effort: high + model: fable +--- + +# Task: High-effort fable fixture + +## Intent + +Test fixture for the v5.9 fable model tier — all 4 phases at the +high effort tier with explicit fable model overrides. Mirrors +brief-effort-high.md with `model: opus` replaced by `model: fable`. + +## Goal + +Resolver returns `{effort: 'high', model: 'fable'}` for each of the 4 +PHASE_SIGNAL_PHASES. + +## Success Criteria + +- Validator passes with no BRIEF_INVALID_MODEL. +- resolvePhaseSignal(fm, phase).effort === 'high' for all 4 phases. +- resolvePhaseSignal(fm, phase).model === 'fable' for all 4 phases. diff --git a/tests/lib/phase-signal-resolver.test.mjs b/tests/lib/phase-signal-resolver.test.mjs index bfb1872..8799b2e 100644 --- a/tests/lib/phase-signal-resolver.test.mjs +++ b/tests/lib/phase-signal-resolver.test.mjs @@ -50,7 +50,7 @@ test('resolvePhaseSignal — defensive: null/non-object input returns null', () test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in-depth gate)', () => { // MAJOR fix (S4): line that copies `model` must gate against the same - // allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus']), + // allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus','fable']), // mirroring how effort is gated against EFFORT_LEVELS. A brief that slipped // validation (hand-edited, validation skipped) must not hand a junk model // string to a command that then spawns an agent with `model: `. @@ -70,15 +70,17 @@ test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in- assert.ok(!('model' in review), 'model key absent for haiku'); }); -test('resolvePhaseSignal — keeps valid models (sonnet, opus) after gating', () => { +test('resolvePhaseSignal — keeps valid models (sonnet, opus, fable) after gating', () => { const fm = { phase_signals: [ { phase: 'research', effort: 'low', model: 'sonnet' }, { phase: 'execute', effort: 'high', model: 'opus' }, + { phase: 'review', effort: 'high', model: 'fable' }, ], }; assert.equal(resolvePhaseSignal(fm, 'research').model, 'sonnet'); assert.equal(resolvePhaseSignal(fm, 'execute').model, 'opus'); + assert.equal(resolvePhaseSignal(fm, 'review').model, 'fable'); }); test('resolvePhaseSignalFromFile + CLI shim — writes JSON to stdout, exit 0', () => { diff --git a/tests/validators/brief-validator.test.mjs b/tests/validators/brief-validator.test.mjs index 3a05928..314435f 100644 --- a/tests/validators/brief-validator.test.mjs +++ b/tests/validators/brief-validator.test.mjs @@ -1,7 +1,7 @@ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; @@ -255,6 +255,25 @@ test('validateBrief — v5.1.1: UNQUOTED brief_version 2.1 WITH phase_signals is assert.ok(!r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS')); }); +// --- v5.9 — fable model tier (BASE_ALLOWED_MODELS widened to three values) --- + +test('validateBrief — v5.9: fable phase_signals fixture accepted (no BRIEF_INVALID_MODEL)', () => { + const t = readFileSync(new URL('../fixtures/brief-effort-fable.md', import.meta.url), 'utf-8'); + const r = validateBriefContent(t, { strict: true }); + assert.equal(r.valid, true, JSON.stringify(r.errors)); + assert.ok(!r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL')); +}); + +test('validateBrief — v5.9: unknown model gpt5 in phase_signals rejected with BRIEF_INVALID_MODEL', () => { + const t = GOOD_BRIEF + .replace('brief_version: "2.0"', 'brief_version: "2.1"') + .replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK.replace('model: opus', 'model: gpt5')}`); + const r = validateBriefContent(t, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'), + `expected BRIEF_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`); +}); + // --- v5.5 — framing enforcement + obligatory TL;DR (gated at brief_version ≥ 2.2) --- // Operator decision (S6, option A1): framing + TL;DR are hard BLOCKERs for briefs // declaring brief_version ≥ 2.2; existing 2.0/2.1 briefs stay valid (forward-compat,