chore(cleanup): R-b R3 — profiles, phase signals and the effort dialog removed, one fixed path per command
The four profiles, the resolver, the profile validator and docs/profiles.md go; /trekbrief Phase 3.5 and the brief's phase_signals checks go (an older brief's key is ignored); MANIFEST_PROFILE_DRIFT and profile_used go. Every Profile, Composition rule, Sequencing gate and High-effort section leaves the commands. Decision 209: /trekresearch runs the full swarm with contrarian-researcher as wave 2 every run; /trekreview and the coordinator run all four passes with no high-effort normalization. Voyage agents are spawned with no model (their own model: opus applies); the two built-in spawns pass model "opus". --profile stays in the parser as a retired flag that swallows its value. §Model & effort moves to docs/architecture.md; the sequencing-gate eval case goes. Files removed: 44 (43 from the R3 list + tests/commands/trekreview.test.mjs, whose every test pinned removed text). Files edited: 43. Tests: 2467 (2466 pass / 0 fail / 1 skipped) before, 2309 (2309 pass / 0 fail / 0 skipped) after. Durable check: all three groups green, none pending. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
32621dcb63
commit
2a35d067d9
87 changed files with 225 additions and 4967 deletions
|
|
@ -27,7 +27,6 @@ project_dir: .claude/projects/2026-10-06-dark-mode-toggle/
|
|||
research_topics: 0
|
||||
research_status: complete
|
||||
framing: new-direction
|
||||
phase_signals_partial: true
|
||||
${extra}---
|
||||
|
||||
# Task
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ auto_research: false
|
|||
interview_turns: 5
|
||||
source: interview
|
||||
framing: new-direction
|
||||
phase_signals_partial: true
|
||||
---
|
||||
|
||||
# Task: JWT auth
|
||||
|
|
@ -116,28 +115,7 @@ Find what was missed.
|
|||
- All BLOCKER findings get a fix-plan.
|
||||
`;
|
||||
|
||||
// signals blocks
|
||||
const GOOD_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: opus
|
||||
`;
|
||||
const BAD_PHASE_SIGNALS = `phase_signals:
|
||||
- phase: nonsense
|
||||
effort: high
|
||||
`;
|
||||
const BAD_EFFORT_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: turbo
|
||||
`;
|
||||
const BAD_MODEL_SIGNALS = `phase_signals:
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: gpt4
|
||||
`;
|
||||
|
||||
// convenience builders
|
||||
const v21 = (s) => s.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
const withLine = (s, line) => s.replace('source: interview\n', `source: interview\n${line}`);
|
||||
|
||||
// ---- the canonical in-scope set + a class-balanced pair for each -------------
|
||||
|
|
@ -151,12 +129,6 @@ const IN_SCOPE_CODES = [
|
|||
'BRIEF_MISSING_SECTION',
|
||||
'BRIEF_INVALID_FRAMING',
|
||||
'BRIEF_MISSING_FRAMING',
|
||||
'BRIEF_V51_MISSING_SIGNALS',
|
||||
'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE',
|
||||
'BRIEF_INVALID_PHASE_SIGNALS',
|
||||
'BRIEF_INVALID_PHASE_SIGNAL_PHASE',
|
||||
'BRIEF_INVALID_EFFORT',
|
||||
'BRIEF_INVALID_MODEL',
|
||||
];
|
||||
|
||||
const CASES = {
|
||||
|
|
@ -192,30 +164,6 @@ const CASES = {
|
|||
fail: GOOD_BRIEF_22.replace('framing: new-direction\n', ''),
|
||||
pass: GOOD_BRIEF_22,
|
||||
},
|
||||
BRIEF_V51_MISSING_SIGNALS: {
|
||||
fail: v21(GOOD_BRIEF),
|
||||
pass: GOOD_BRIEF, // 2.0 — gate inactive
|
||||
},
|
||||
BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE: {
|
||||
fail: withLine(v21(GOOD_BRIEF), `phase_signals_partial: true\n${GOOD_SIGNALS}`),
|
||||
pass: withLine(v21(GOOD_BRIEF), 'phase_signals_partial: true\n'),
|
||||
},
|
||||
BRIEF_INVALID_PHASE_SIGNALS: {
|
||||
fail: withLine(GOOD_BRIEF, 'phase_signals: notalist\n'),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_PHASE_SIGNAL_PHASE: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_PHASE_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_EFFORT: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_EFFORT_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
BRIEF_INVALID_MODEL: {
|
||||
fail: withLine(v21(GOOD_BRIEF), BAD_MODEL_SIGNALS),
|
||||
pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS),
|
||||
},
|
||||
};
|
||||
|
||||
test('gate coverage — CASES keys exactly match IN_SCOPE_CODES (no silent gap)', () => {
|
||||
|
|
|
|||
|
|
@ -157,127 +157,10 @@ test('validateBrief — wrong-type error message includes accepted set', () => {
|
|||
assert.ok(/trekreview/.test(wrongType.message));
|
||||
});
|
||||
|
||||
// --- v5.1 — phase_signals additive field + sequencing gate ---
|
||||
|
||||
const SIGNALS_BLOCK = `phase_signals:
|
||||
- phase: research
|
||||
effort: standard
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: opus
|
||||
- phase: execute
|
||||
effort: low
|
||||
model: sonnet
|
||||
- phase: review
|
||||
effort: standard
|
||||
`;
|
||||
|
||||
test('validateBrief — v5.1 well-formed phase_signals accepted', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK}`);
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
});
|
||||
|
||||
test('validateBrief — pre-v5.1 brief without phase_signals accepted (backward-compat)', () => {
|
||||
const r = validateBriefContent(GOOD_BRIEF, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'));
|
||||
});
|
||||
|
||||
test('validateBrief — v5.1+ brief missing phase_signals + partial emits BRIEF_V51_MISSING_SIGNALS', () => {
|
||||
const t = GOOD_BRIEF.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'));
|
||||
});
|
||||
|
||||
test('validateBrief — v5.1+ brief with phase_signals_partial: true accepted', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', 'source: interview\nphase_signals_partial: true\n');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
});
|
||||
|
||||
test('validateBrief — phase_signals + phase_signals_partial both set rejected (mutually exclusive)', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', `source: interview\nphase_signals_partial: true\n${SIGNALS_BLOCK}`);
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE'));
|
||||
});
|
||||
|
||||
test('validateBrief — phase_signals with unknown phase rejected', () => {
|
||||
const BAD_SIGNALS = `phase_signals:
|
||||
- phase: nonsense
|
||||
effort: standard
|
||||
`;
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', `source: interview\n${BAD_SIGNALS}`);
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'BRIEF_INVALID_PHASE_SIGNAL_PHASE'));
|
||||
});
|
||||
|
||||
// --- v5.1.1 regression: YAML-number bypass closed ---
|
||||
// Findings 3c834097 + df1435a2: v5.1.0 shipped with an unquoted `brief_version: 2.1`
|
||||
// template. parseScalar coerces unquoted "2.1" to Number 2.1, and the original gate
|
||||
// guarded `typeof === 'string'`, silently bypassing the sequencing check. v5.1.1
|
||||
// coerces via String() so both shapes trigger the gate.
|
||||
|
||||
test('validateBrief — v5.1.1: UNQUOTED brief_version 2.1 without signals triggers gate', () => {
|
||||
const t = GOOD_BRIEF.replace('brief_version: "2.0"', 'brief_version: 2.1');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(
|
||||
r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'),
|
||||
`gate must fire for unquoted brief_version: 2.1 (YAML Number); errors=${JSON.stringify(r.errors)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('validateBrief — v5.1.1: QUOTED brief_version "2.1" without signals triggers gate (regression guard)', () => {
|
||||
const t = GOOD_BRIEF.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'));
|
||||
});
|
||||
|
||||
test('validateBrief — v5.1.1: UNQUOTED brief_version 2.1 WITH phase_signals is valid (positive case)', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: 2.1')
|
||||
.replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK}`);
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS'));
|
||||
});
|
||||
|
||||
// --- v5.9 — fable model tier (BASE_ALLOWED_MODELS widened to three values) ---
|
||||
|
||||
test('validateBrief — v5.9: fable phase_signals fixture accepted (no BRIEF_INVALID_MODEL)', () => {
|
||||
const t = readFileSync(new URL('../fixtures/brief-effort-fable.md', import.meta.url), 'utf-8');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'));
|
||||
});
|
||||
|
||||
test('validateBrief — v5.9: unknown model gpt5 in phase_signals rejected with BRIEF_INVALID_MODEL', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK.replace('model: opus', 'model: gpt5')}`);
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'),
|
||||
`expected BRIEF_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`);
|
||||
});
|
||||
|
||||
// --- v5.5 — framing enforcement + obligatory TL;DR (gated at brief_version ≥ 2.2) ---
|
||||
// Operator decision (S6, option A1): framing + TL;DR are hard BLOCKERs for briefs
|
||||
// declaring brief_version ≥ 2.2; existing 2.0/2.1 briefs stay valid (forward-compat,
|
||||
// mirroring the phase_signals ≥ 2.1 precedent). The framing ENUM check fires on any
|
||||
// declaring brief_version ≥ 2.2; existing 2.0/2.1 briefs stay valid (forward-compat).
|
||||
// The framing ENUM check fires on any
|
||||
// version when the field is present but malformed.
|
||||
|
||||
const GOOD_BRIEF_22 = `---
|
||||
|
|
@ -293,7 +176,6 @@ auto_research: false
|
|||
interview_turns: 5
|
||||
source: interview
|
||||
framing: new-direction
|
||||
phase_signals_partial: true
|
||||
---
|
||||
|
||||
# Task: JWT auth
|
||||
|
|
@ -315,6 +197,18 @@ What success looks like.
|
|||
- All tests pass.
|
||||
`;
|
||||
|
||||
// An older brief may still carry a phase_signals block (removed in cleanup R-b,
|
||||
// decision 191): the key is ignored, with no error and no warning.
|
||||
|
||||
test('validateBrief — a 2.2 brief that still carries phase_signals validates with no error or warning about it', () => {
|
||||
const t = GOOD_BRIEF_22.replace('framing: new-direction\n',
|
||||
'framing: new-direction\nphase_signals:\n - phase: plan\n effort: high\n - phase: brief\n effort: xhigh\n');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.deepEqual(r.errors, []);
|
||||
assert.deepEqual(r.warnings.filter(w => /signal/i.test(w.code + ' ' + w.message)), []);
|
||||
});
|
||||
|
||||
test('validateBrief — v5.5 well-formed 2.2 brief (framing + TL;DR) accepted', () => {
|
||||
const r = validateBriefContent(GOOD_BRIEF_22, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
|
|
@ -367,8 +261,7 @@ test('validateBrief — v5.5 TL;DR exceeding 5 lines emits BRIEF_TLDR_TOO_LONG w
|
|||
|
||||
test('validateBrief — v5.5 backward-compat: 2.1 brief without framing/TL;DR stays valid', () => {
|
||||
const t = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', 'source: interview\nphase_signals_partial: true\n');
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
const r = validateBriefContent(t, { strict: true });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.find(e => e.code === 'BRIEF_MISSING_FRAMING'));
|
||||
|
|
@ -385,8 +278,7 @@ test('validateBrief — v5.5 trekreview brief not subject to framing/TL;DR gate'
|
|||
// S18 — opt-in --min-brief-version gate. Warns (never blocks) when a brief
|
||||
// declares a version below the caller's floor, sidestepping framing enforcement.
|
||||
const BRIEF_21 = GOOD_BRIEF
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"')
|
||||
.replace('source: interview\n', 'source: interview\nphase_signals_partial: true\n');
|
||||
.replace('brief_version: "2.0"', 'brief_version: "2.1"');
|
||||
|
||||
test('validateBrief — S18 min-version: older brief below floor warns but does not block', () => {
|
||||
const r = validateBriefContent(BRIEF_21, { strict: true, minBriefVersion: '2.2' });
|
||||
|
|
@ -457,7 +349,7 @@ const codes = (list) => list.map((e) => e.code);
|
|||
function briefWith(sections) {
|
||||
return [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
...sections,
|
||||
].join('\n');
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ auto_research: false
|
|||
interview_turns: 4
|
||||
source: interview
|
||||
framing: new-direction
|
||||
phase_signals_partial: true
|
||||
---
|
||||
|
||||
# Task: Dark-mode toggle
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
// tests/validators/plan-validator-profile-drift.test.mjs
|
||||
// SC #20 — MANIFEST_PROFILE_DRIFT warning per brief Assumptions block 7.
|
||||
// In strict mode, plan-validator must emit a warning (NOT an error) when a
|
||||
// step manifest's profile_used differs from the plan's frontmatter profile.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
|
||||
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(__dirname, '..', '..');
|
||||
|
||||
function loadFixture(name) {
|
||||
return readFileSync(resolve(ROOT, 'tests/fixtures', name), 'utf-8');
|
||||
}
|
||||
|
||||
const DRIFT_FIXTURE = loadFixture('plan-profile-drift.md');
|
||||
|
||||
test('drift detected in strict mode — emits MANIFEST_PROFILE_DRIFT warning, not error', () => {
|
||||
const r = validatePlanContent(DRIFT_FIXTURE, { strict: true });
|
||||
assert.equal(r.valid, true, `plan must remain valid; errors: ${JSON.stringify(r.errors)}`);
|
||||
const drift = r.warnings.filter((w) => w.code === 'MANIFEST_PROFILE_DRIFT');
|
||||
assert.equal(drift.length, 1, `expected 1 drift warning, got ${drift.length}: ${JSON.stringify(drift)}`);
|
||||
assert.match(drift[0].message, /step 2/, `warning message must reference step 2: ${drift[0].message}`);
|
||||
assert.match(drift[0].message, /premium/, 'warning must include offending profile_used value');
|
||||
assert.match(drift[0].message, /economy/, 'warning must include plan-level profile value');
|
||||
});
|
||||
|
||||
test('drift NOT detected in soft mode — strict gate honored', () => {
|
||||
const r = validatePlanContent(DRIFT_FIXTURE, { strict: false });
|
||||
const drift = r.warnings.filter((w) => w.code === 'MANIFEST_PROFILE_DRIFT');
|
||||
assert.equal(
|
||||
drift.length,
|
||||
0,
|
||||
'MANIFEST_PROFILE_DRIFT must only emit in strict mode (per brief assumption 7)',
|
||||
);
|
||||
});
|
||||
|
||||
test('matching profile — no drift warning emitted', () => {
|
||||
// Same fixture body, but rewrite step 2 profile_used to match plan profile.
|
||||
// Use /g to catch both the doc-comment mention and the actual manifest entry.
|
||||
const matching = DRIFT_FIXTURE.replace(/profile_used: premium/g, 'profile_used: economy');
|
||||
const r = validatePlanContent(matching, { strict: true });
|
||||
assert.equal(r.valid, true);
|
||||
const drift = r.warnings.filter((w) => w.code === 'MANIFEST_PROFILE_DRIFT');
|
||||
assert.equal(
|
||||
drift.length,
|
||||
0,
|
||||
`no drift expected when all step profile_used match plan profile; got ${JSON.stringify(drift)}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('plan without frontmatter profile — no drift warnings emitted', () => {
|
||||
// If plan-level profile is absent, step-level profile_used can be anything
|
||||
// without triggering drift (drift is only meaningful relative to a baseline).
|
||||
const noProfile = DRIFT_FIXTURE.replace(/profile: economy\n/, '');
|
||||
const r = validatePlanContent(noProfile, { strict: true });
|
||||
const drift = r.warnings.filter((w) => w.code === 'MANIFEST_PROFILE_DRIFT');
|
||||
assert.equal(
|
||||
drift.length,
|
||||
0,
|
||||
'no plan-level profile means no baseline; drift detection must be silent',
|
||||
);
|
||||
});
|
||||
|
|
@ -1,276 +0,0 @@
|
|||
// tests/validators/profile-validator.test.mjs
|
||||
// SC #1, #2, #3: profile-validator validates lib/profiles/{economy,balanced,premium,fable}.yaml
|
||||
// (innebygde profiler) plus rejects invalid models and invalid enum types.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
import {
|
||||
validateProfile,
|
||||
validateProfileContent,
|
||||
PROFILE_REQUIRED_FIELDS,
|
||||
PROFILE_REQUIRED_PHASES,
|
||||
BASE_ALLOWED_MODELS,
|
||||
} from '../../lib/validators/profile-validator.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = join(__dirname, '..', '..');
|
||||
|
||||
// SC #1: alle 4 innebygde profiler grønne
|
||||
|
||||
for (const profileName of ['economy', 'balanced', 'premium', 'fable']) {
|
||||
test(`SC #1: lib/profiles/${profileName}.yaml validates clean`, () => {
|
||||
const r = validateProfile(join(REPO_ROOT, 'lib', 'profiles', `${profileName}.yaml`));
|
||||
assert.equal(r.valid, true,
|
||||
`expected valid, got errors: ${JSON.stringify(r.errors, null, 2)}`);
|
||||
assert.equal(r.errors.length, 0);
|
||||
// Spot-check: all 6 phases present
|
||||
const phases = r.parsed.frontmatter.phase_models.map(p => p.phase);
|
||||
for (const required of PROFILE_REQUIRED_PHASES) {
|
||||
assert.ok(phases.includes(required), `${profileName} missing phase: ${required}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// SC #2: PROFILE_INVALID_MODEL fired when phase_models[N].model not in allowlist
|
||||
|
||||
test('SC #2: profile-invalid-model.yaml rejected with PROFILE_INVALID_MODEL at phase_models[2].model', () => {
|
||||
const r = validateProfile(join(REPO_ROOT, 'tests', 'fixtures', 'profile-invalid-model.yaml'));
|
||||
assert.equal(r.valid, false);
|
||||
const found = r.errors.find(e => e.code === 'PROFILE_INVALID_MODEL');
|
||||
assert.ok(found, `expected PROFILE_INVALID_MODEL, got: ${JSON.stringify(r.errors)}`);
|
||||
assert.equal(found.location, 'phase_models[2].model',
|
||||
`expected location phase_models[2].model, got ${found.location}`);
|
||||
assert.match(found.message, /gpt-4/);
|
||||
});
|
||||
|
||||
// SC #3: PROFILE_INVALID_ENUM for wrong-type values
|
||||
|
||||
test('SC #3: profile-invalid-enum.yaml rejected with PROFILE_INVALID_ENUM (external_research_enabled is string)', () => {
|
||||
const r = validateProfile(join(REPO_ROOT, 'tests', 'fixtures', 'profile-invalid-enum.yaml'));
|
||||
assert.equal(r.valid, false);
|
||||
const found = r.errors.find(e => e.code === 'PROFILE_INVALID_ENUM' && /external_research_enabled/.test(e.message));
|
||||
assert.ok(found, `expected PROFILE_INVALID_ENUM for external_research_enabled, got: ${JSON.stringify(r.errors)}`);
|
||||
assert.match(found.message, /boolean/);
|
||||
});
|
||||
|
||||
// VOYAGE_ALLOW_HAIKU env-var allows haiku model
|
||||
|
||||
test('VOYAGE_ALLOW_HAIKU=1 allows haiku in phase_models', () => {
|
||||
const haikuProfile = `---
|
||||
profile_version: "1.0"
|
||||
name: haiku-allowed
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: haiku
|
||||
- phase: research
|
||||
model: sonnet
|
||||
- phase: plan
|
||||
model: opus
|
||||
- phase: execute
|
||||
model: sonnet
|
||||
- phase: review
|
||||
model: opus
|
||||
- phase: continue
|
||||
model: sonnet
|
||||
parallel_agents_min: 2
|
||||
parallel_agents_max: 4
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
---
|
||||
`;
|
||||
// Default env: haiku rejected
|
||||
const denied = validateProfileContent(haikuProfile, { env: { /* no VOYAGE_ALLOW_HAIKU */ } });
|
||||
assert.equal(denied.valid, false);
|
||||
const haikuErr = denied.errors.find(e => e.code === 'PROFILE_INVALID_MODEL' && /haiku/.test(e.message));
|
||||
assert.ok(haikuErr, `expected haiku rejection: ${JSON.stringify(denied.errors)}`);
|
||||
assert.match(haikuErr.message, /VOYAGE_ALLOW_HAIKU/);
|
||||
|
||||
// With opt-in: haiku accepted
|
||||
const allowed = validateProfileContent(haikuProfile, { env: { VOYAGE_ALLOW_HAIKU: '1' } });
|
||||
assert.equal(allowed.valid, true,
|
||||
`expected valid with VOYAGE_ALLOW_HAIKU=1, got: ${JSON.stringify(allowed.errors)}`);
|
||||
});
|
||||
|
||||
// Fable tier (v5.9): fable accepted under DEFAULT env (no opt-in flag),
|
||||
// unknown models still rejected — the allowlist gate must demonstrably fire.
|
||||
|
||||
test('fable accepted in phase_models under default env (no env flag)', () => {
|
||||
const fableProfile = `---
|
||||
profile_version: "1.0"
|
||||
name: fable-inline
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: fable
|
||||
- phase: research
|
||||
model: fable
|
||||
- phase: plan
|
||||
model: fable
|
||||
- phase: execute
|
||||
model: fable
|
||||
- phase: review
|
||||
model: fable
|
||||
- phase: continue
|
||||
model: fable
|
||||
parallel_agents_min: 6
|
||||
parallel_agents_max: 8
|
||||
external_research_enabled: true
|
||||
brief_reviewer_iter_cap: 3
|
||||
---
|
||||
`;
|
||||
const r = validateProfileContent(fableProfile, { env: { /* default: no flags */ } });
|
||||
assert.equal(r.valid, true,
|
||||
`expected fable accepted under default env, got: ${JSON.stringify(r.errors)}`);
|
||||
assert.equal(r.errors.length, 0);
|
||||
});
|
||||
|
||||
test('unknown model gpt5 rejected with PROFILE_INVALID_MODEL under default env', () => {
|
||||
const gpt5Profile = `---
|
||||
profile_version: "1.0"
|
||||
name: gpt5-inline
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: gpt5
|
||||
- phase: research
|
||||
model: sonnet
|
||||
- phase: plan
|
||||
model: opus
|
||||
- phase: execute
|
||||
model: sonnet
|
||||
- phase: review
|
||||
model: opus
|
||||
- phase: continue
|
||||
model: sonnet
|
||||
parallel_agents_min: 2
|
||||
parallel_agents_max: 4
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
---
|
||||
`;
|
||||
const r = validateProfileContent(gpt5Profile, { env: { /* default: no flags */ } });
|
||||
assert.equal(r.valid, false);
|
||||
const found = r.errors.find(e => e.code === 'PROFILE_INVALID_MODEL' && /gpt5/.test(e.message));
|
||||
assert.ok(found, `expected PROFILE_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`);
|
||||
});
|
||||
|
||||
// BASE_ALLOWED_MODELS allowlist drift-pin (mirrors the PROFILE_REQUIRED_FIELDS pin)
|
||||
|
||||
test('BASE_ALLOWED_MODELS export drift-pin', () => {
|
||||
assert.deepEqual(
|
||||
[...BASE_ALLOWED_MODELS],
|
||||
['sonnet', 'opus', 'fable'],
|
||||
'BASE_ALLOWED_MODELS contract drift — pin contract',
|
||||
);
|
||||
});
|
||||
|
||||
// Required fields presence
|
||||
|
||||
test('PROFILE_MISSING_FIELD when name absent', () => {
|
||||
const r = validateProfileContent(`---
|
||||
profile_version: "1.0"
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: sonnet
|
||||
- phase: research
|
||||
model: sonnet
|
||||
- phase: plan
|
||||
model: opus
|
||||
- phase: execute
|
||||
model: sonnet
|
||||
- phase: review
|
||||
model: opus
|
||||
- phase: continue
|
||||
model: sonnet
|
||||
parallel_agents_min: 2
|
||||
parallel_agents_max: 4
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
---
|
||||
`);
|
||||
assert.equal(r.valid, false);
|
||||
const found = r.errors.find(e => e.code === 'PROFILE_MISSING_FIELD' && /name/.test(e.message));
|
||||
assert.ok(found, `expected PROFILE_MISSING_FIELD for name, got: ${JSON.stringify(r.errors)}`);
|
||||
});
|
||||
|
||||
// PROFILE_NOT_FOUND for missing file
|
||||
|
||||
test('PROFILE_NOT_FOUND for non-existent file', () => {
|
||||
const r = validateProfile('/tmp/does-not-exist-profile-xyz.yaml');
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.errors.find(e => e.code === 'PROFILE_NOT_FOUND'));
|
||||
});
|
||||
|
||||
// REQUIRED_FIELDS frontmatter contract drift-pin
|
||||
|
||||
test('PROFILE_REQUIRED_FIELDS export drift-pin', () => {
|
||||
assert.deepEqual(
|
||||
[...PROFILE_REQUIRED_FIELDS].sort(),
|
||||
['brief_reviewer_iter_cap', 'external_research_enabled', 'name',
|
||||
'parallel_agents_max', 'parallel_agents_min', 'phase_models'].sort(),
|
||||
'PROFILE_REQUIRED_FIELDS contract drift — pin contract',
|
||||
);
|
||||
});
|
||||
|
||||
test('PROFILE_REQUIRED_PHASES export drift-pin', () => {
|
||||
assert.deepEqual(
|
||||
[...PROFILE_REQUIRED_PHASES].sort(),
|
||||
['brief', 'research', 'plan', 'execute', 'review', 'continue'].sort(),
|
||||
'PROFILE_REQUIRED_PHASES contract drift — pin contract',
|
||||
);
|
||||
});
|
||||
|
||||
// S34 (V30) — economy carries an explicit `experimental` marker.
|
||||
// The cross-tier Jaccard floor (0.55) rests on parked-synthetic fixtures;
|
||||
// empirical Step-17 calibration is deferred to v4.2 ($60-120 LLM budget,
|
||||
// unauthorized). Until a real calibration lands, `economy` is self-declared
|
||||
// experimental in the profile DATA — not only in prose — so the status
|
||||
// survives a docs rewrite. See tests/synthetic/profile-jaccard-calibration.md.
|
||||
|
||||
test('S34: economy.yaml declares experimental: true (uncalibrated Jaccard floor)', () => {
|
||||
const r = validateProfile(join(REPO_ROOT, 'lib', 'profiles', 'economy.yaml'));
|
||||
assert.equal(r.valid, true, `economy.yaml should validate: ${JSON.stringify(r.errors)}`);
|
||||
assert.equal(r.parsed.frontmatter.experimental, true,
|
||||
'economy.yaml must carry experimental: true until the Jaccard floor is empirically calibrated (S34/V30)');
|
||||
});
|
||||
|
||||
test('S34: premium + balanced are NOT flagged experimental', () => {
|
||||
for (const name of ['premium', 'balanced']) {
|
||||
const r = validateProfile(join(REPO_ROOT, 'lib', 'profiles', `${name}.yaml`));
|
||||
assert.equal(r.valid, true, `${name}.yaml should validate: ${JSON.stringify(r.errors)}`);
|
||||
assert.notEqual(r.parsed.frontmatter.experimental, true,
|
||||
`${name}.yaml must not be flagged experimental`);
|
||||
}
|
||||
});
|
||||
|
||||
test('S34: validator type-checks experimental as boolean (PROFILE_INVALID_ENUM)', () => {
|
||||
const bad = `---
|
||||
profile_version: "1.0"
|
||||
name: bad-experimental
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: sonnet
|
||||
- phase: research
|
||||
model: sonnet
|
||||
- phase: plan
|
||||
model: sonnet
|
||||
- phase: execute
|
||||
model: sonnet
|
||||
- phase: review
|
||||
model: sonnet
|
||||
- phase: continue
|
||||
model: sonnet
|
||||
parallel_agents_min: 2
|
||||
parallel_agents_max: 3
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
experimental: "yes"
|
||||
---
|
||||
`;
|
||||
const r = validateProfileContent(bad);
|
||||
assert.equal(r.valid, false, 'experimental: "yes" (string) must be rejected');
|
||||
const found = r.errors.find(e => e.code === 'PROFILE_INVALID_ENUM' && /experimental/.test(e.message));
|
||||
assert.ok(found, `expected PROFILE_INVALID_ENUM for experimental, got: ${JSON.stringify(r.errors)}`);
|
||||
assert.match(found.message, /boolean/);
|
||||
});
|
||||
|
|
@ -171,7 +171,7 @@ test('runner: a failing verification is never verified end to end', () => {
|
|||
function brief(criteria) {
|
||||
return [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
'## Success Criteria', '', ...criteria, '',
|
||||
].join('\n');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue