test(validators): cover fable accept + unknown-model reject in both gate layers

This commit is contained in:
Kjell Tore Guttormsen 2026-07-02 16:56:26 +02:00
commit 8b7a849a76
3 changed files with 69 additions and 3 deletions

45
tests/fixtures/brief-effort-fable.md vendored Normal file
View file

@ -0,0 +1,45 @@
---
type: trekbrief
brief_version: "2.1"
created: 2026-07-02
task: "Fixture: high-effort all phases on fable (v5.9 allowlist test)"
slug: brief-effort-fable
project_dir: .claude/projects/2026-07-02-brief-effort-fable/
research_topics: 0
research_status: complete
auto_research: false
interview_turns: 4
source: fixture
phase_signals:
- phase: research
effort: high
model: fable
- phase: plan
effort: high
model: fable
- phase: execute
effort: high
model: fable
- phase: review
effort: high
model: fable
---
# Task: High-effort fable fixture
## Intent
Test fixture for the v5.9 fable model tier — all 4 phases at the
high effort tier with explicit fable model overrides. Mirrors
brief-effort-high.md with `model: opus` replaced by `model: fable`.
## Goal
Resolver returns `{effort: 'high', model: 'fable'}` for each of the 4
PHASE_SIGNAL_PHASES.
## Success Criteria
- Validator passes with no BRIEF_INVALID_MODEL.
- resolvePhaseSignal(fm, phase).effort === 'high' for all 4 phases.
- resolvePhaseSignal(fm, phase).model === 'fable' for all 4 phases.

View file

@ -50,7 +50,7 @@ test('resolvePhaseSignal — defensive: null/non-object input returns null', ()
test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in-depth gate)', () => {
// MAJOR fix (S4): line that copies `model` must gate against the same
// allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus']),
// allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus','fable']),
// mirroring how effort is gated against EFFORT_LEVELS. A brief that slipped
// validation (hand-edited, validation skipped) must not hand a junk model
// string to a command that then spawns an agent with `model: <junk>`.
@ -70,15 +70,17 @@ test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in-
assert.ok(!('model' in review), 'model key absent for haiku');
});
test('resolvePhaseSignal — keeps valid models (sonnet, opus) after gating', () => {
test('resolvePhaseSignal — keeps valid models (sonnet, opus, fable) after gating', () => {
const fm = {
phase_signals: [
{ phase: 'research', effort: 'low', model: 'sonnet' },
{ phase: 'execute', effort: 'high', model: 'opus' },
{ phase: 'review', effort: 'high', model: 'fable' },
],
};
assert.equal(resolvePhaseSignal(fm, 'research').model, 'sonnet');
assert.equal(resolvePhaseSignal(fm, 'execute').model, 'opus');
assert.equal(resolvePhaseSignal(fm, 'review').model, 'fable');
});
test('resolvePhaseSignalFromFile + CLI shim — writes JSON to stdout, exit 0', () => {

View file

@ -1,7 +1,7 @@
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { execFileSync } from 'node:child_process';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { validateBriefContent } from '../../lib/validators/brief-validator.mjs';
@ -255,6 +255,25 @@ test('validateBrief — v5.1.1: UNQUOTED brief_version 2.1 WITH phase_signals is
assert.ok(!r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'));
});
// --- v5.9 — fable model tier (BASE_ALLOWED_MODELS widened to three values) ---
test('validateBrief — v5.9: fable phase_signals fixture accepted (no BRIEF_INVALID_MODEL)', () => {
const t = readFileSync(new URL('../fixtures/brief-effort-fable.md', import.meta.url), 'utf-8');
const r = validateBriefContent(t, { strict: true });
assert.equal(r.valid, true, JSON.stringify(r.errors));
assert.ok(!r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'));
});
test('validateBrief — v5.9: unknown model gpt5 in phase_signals rejected with BRIEF_INVALID_MODEL', () => {
const t = GOOD_BRIEF
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
.replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK.replace('model: opus', 'model: gpt5')}`);
const r = validateBriefContent(t, { strict: true });
assert.equal(r.valid, false);
assert.ok(r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'),
`expected BRIEF_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`);
});
// --- v5.5 — framing enforcement + obligatory TL;DR (gated at brief_version ≥ 2.2) ---
// Operator decision (S6, option A1): framing + TL;DR are hard BLOCKERs for briefs
// declaring brief_version ≥ 2.2; existing 2.0/2.1 briefs stay valid (forward-compat,