chore(cleanup): R-b R3 — profiles, phase signals and the effort dialog removed, one fixed path per command
The four profiles, the resolver, the profile validator and docs/profiles.md go; /trekbrief Phase 3.5 and the brief's phase_signals checks go (an older brief's key is ignored); MANIFEST_PROFILE_DRIFT and profile_used go. Every Profile, Composition rule, Sequencing gate and High-effort section leaves the commands. Decision 209: /trekresearch runs the full swarm with contrarian-researcher as wave 2 every run; /trekreview and the coordinator run all four passes with no high-effort normalization. Voyage agents are spawned with no model (their own model: opus applies); the two built-in spawns pass model "opus". --profile stays in the parser as a retired flag that swallows its value. §Model & effort moves to docs/architecture.md; the sequencing-gate eval case goes. Files removed: 44 (43 from the R3 list + tests/commands/trekreview.test.mjs, whose every test pinned removed text). Files edited: 43. Tests: 2467 (2466 pass / 0 fail / 1 skipped) before, 2309 (2309 pass / 0 fail / 0 skipped) after. Durable check: all three groups green, none pending. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
32621dcb63
commit
2a35d067d9
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
87 changed files with 225 additions and 4967 deletions
|
|
@ -281,3 +281,8 @@ test('trekreview --answers: the parser accepts it as a boolean, and the usage li
|
|||
assert.equal(usage.status, 1, usage.stdout + usage.stderr);
|
||||
assert.match(usage.stderr, /Usage: \/trekreview --project <dir> [^\n]*\[--answers\]/, `the usage line names --answers: ${usage.stderr}`);
|
||||
});
|
||||
|
||||
test('retired --profile swallows its value: --profile economy leaves positional untouched', () => {
|
||||
const r = parseArgs('--profile economy .claude/projects/x', 'trekcontinue');
|
||||
assert.deepEqual(r.positional, ['.claude/projects/x'], `the retired flag's value is no positional: ${JSON.stringify(r)}`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -169,10 +169,9 @@ test('[#1] sibling: commands/trekexecute.md: `gates_mode` changes nothing in `/t
|
|||
'trekexecute does not say gates_mode changes nothing there and the one stop is the merge into main');
|
||||
});
|
||||
|
||||
test('[#1] sibling: docs/architecture.md does not call --gates open the executor\'s low effort', () => {
|
||||
test('[#1] sibling: docs/architecture.md does not name --gates open', () => {
|
||||
const text = read('docs/architecture.md');
|
||||
assert.doesNotMatch(text, /--gates open/);
|
||||
assert.match(text, /\(`\/trekexecute` low effort = one session at a time\)/);
|
||||
});
|
||||
|
||||
test('[#1] sibling: docs/command-modes.md gives every --gates row the four values and no strictness policy', () => {
|
||||
|
|
|
|||
|
|
@ -124,8 +124,7 @@ test('computeVerdict — counts each severity tier', () => {
|
|||
|
||||
test('reasonablenessFilter — citation-less is REFUTED, unknown rule_key is UNVERIFIED, severity mismatch corrected', () => {
|
||||
// Contract change (fail-closed): only `no-citation` refutes. An ad-hoc
|
||||
// rule_key is a real defect wearing the wrong label — v5.1.1 high-effort mode
|
||||
// already keeps those, normalised to PLAN_EXECUTE_DRIFT.
|
||||
// rule_key is a real defect wearing the wrong label.
|
||||
const r = reasonablenessFilter([
|
||||
{ file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'BLOCKER' }, // unknown → unverified
|
||||
{ file: '', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // no file → drop
|
||||
|
|
|
|||
|
|
@ -833,7 +833,7 @@ test('runCriteria: a nested `node --test` on a failing test is red even when NOD
|
|||
function briefWith(sections) {
|
||||
return [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
...sections,
|
||||
].join('\n');
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@
|
|||
//
|
||||
// Two kinds of pin live here (devil's-advocate audit §Top changes #8, S19):
|
||||
// • STRUCTURAL cross-file invariants — derive one side from a source-of-truth
|
||||
// (file counts, dimension counts from `### N.` headers, profile tables from
|
||||
// the yaml, version-sync) and survive harmless rewording. These are the
|
||||
// (file counts, dimension counts from `### N.` headers, version-sync) and
|
||||
// survive harmless rewording. These are the
|
||||
// load-bearing ones; keep and grow them.
|
||||
// • PROSE / EXISTENCE pins — assert a literal string or file fact. Most still
|
||||
// guard a real wiring (CLI shims, annotate.mjs, anti-stale CHANGELOG,
|
||||
|
|
@ -22,7 +22,6 @@ import { readFileSync, readdirSync } from 'node:fs';
|
|||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseDocument } from '../../lib/util/frontmatter.mjs';
|
||||
import { resolveProfile, loadProfile } from '../../lib/profiles/resolver.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
|
|
@ -310,27 +309,6 @@ test('S23: Phase-9 dedup hand-off uses --stdin and the reviewers emit a json fin
|
|||
}
|
||||
});
|
||||
|
||||
// --- S20 — anti-false-claim: operations.md custom-profile prose must match findProfilePath ---
|
||||
//
|
||||
// findProfilePath (lib/profiles/resolver.mjs:57) resolves built-in FIRST, then
|
||||
// repo-root voyage-profiles/, then ~/.claude/voyage-profiles/. Custom profiles
|
||||
// live in voyage-profiles/ (NOT lib/profiles/), must use NEW names, and CANNOT
|
||||
// shadow a built-in. The old prose claimed the opposite (lib/profiles/<custom>;
|
||||
// "override built-ins of the same name"; "alphabetical … precedence") — guard it.
|
||||
test('operations.md custom-profile prose matches findProfilePath resolution order [S20]', () => {
|
||||
const ops = read('docs/operations.md');
|
||||
for (const bad of ['lib/profiles/<custom>', 'alphabetical', 'override built-ins of the same name']) {
|
||||
assert.ok(
|
||||
!ops.includes(bad),
|
||||
`docs/operations.md: false custom-profile claim "${bad}" contradicts findProfilePath — fix the prose, not this test`,
|
||||
);
|
||||
}
|
||||
assert.ok(
|
||||
ops.includes('voyage-profiles/'),
|
||||
'docs/operations.md must point custom profiles at voyage-profiles/ (not lib/profiles/)',
|
||||
);
|
||||
});
|
||||
|
||||
// --- S24 — anti-false-claim: operations.md + command-modes.md autonomy-gate truth-pass ---
|
||||
//
|
||||
// autonomy-gate.mjs is the --gates flag parser: --gates [true|false|closed|open]
|
||||
|
|
@ -371,113 +349,6 @@ test('operations.md + command-modes.md describe the real autonomy-gate, not a fa
|
|||
}
|
||||
});
|
||||
|
||||
// --- v4.1 Step 21 — pin --profile + phase_models on the 6 commands ---
|
||||
//
|
||||
// CLAUDE.md / README.md pinning is deferred to Step 22 (post-write of
|
||||
// those documents). Step 21 only verifies command-file content, which
|
||||
// was written in Step 7 (Wave 3).
|
||||
|
||||
const PIPELINE_COMMANDS = [
|
||||
'trekbrief.md',
|
||||
'trekresearch.md',
|
||||
'trekplan.md',
|
||||
'trekexecute.md',
|
||||
'trekreview.md',
|
||||
'trekcontinue.md',
|
||||
];
|
||||
|
||||
test('every pipeline command-file documents the --profile flag (SC #20)', () => {
|
||||
for (const f of PIPELINE_COMMANDS) {
|
||||
const text = read(`commands/${f}`);
|
||||
assert.match(
|
||||
text,
|
||||
/--profile\b/,
|
||||
`commands/${f}: --profile flag is required documentation in v4.1`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('command-files mentioning model profiles use canonical name `phase_models`', () => {
|
||||
// Reject legacy / brainstormed alternatives that would confuse readers.
|
||||
const FORBIDDEN = ['model_per_phase', 'phase_to_model', 'profile_phase_models'];
|
||||
for (const f of PIPELINE_COMMANDS) {
|
||||
const text = read(`commands/${f}`);
|
||||
for (const bad of FORBIDDEN) {
|
||||
assert.ok(
|
||||
!text.includes(bad),
|
||||
`commands/${f}: forbidden alias "${bad}" — canonical name is "phase_models"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('at least one pipeline command-file references `phase_models` canonical name', () => {
|
||||
// Sanity: not every command has to enumerate phase_models inline (e.g.
|
||||
// trekbrief and trekcontinue may only mention --profile), but ≥ 1
|
||||
// command-file must spell out the canonical name so the regression test
|
||||
// pins drift.
|
||||
let mentioned = 0;
|
||||
for (const f of PIPELINE_COMMANDS) {
|
||||
if (read(`commands/${f}`).includes('phase_models')) mentioned += 1;
|
||||
}
|
||||
assert.ok(
|
||||
mentioned >= 1,
|
||||
`expected ≥ 1 command-file to mention canonical name "phase_models", got ${mentioned}`,
|
||||
);
|
||||
});
|
||||
|
||||
// --- v4.1 Step 22 — post-write CLAUDE.md / README.md pinning ---
|
||||
//
|
||||
// Plan-critic Blocker 2 fix: Step 21 only pinned commands/*.md (which
|
||||
// are written in Step 7 / Wave 3). Step 22 writes the top-level docs
|
||||
// and extends pinning here so doc-consistency stays green AFTER Step 22.
|
||||
|
||||
test('CLAUDE.md documents --profile flag', () => {
|
||||
const md = read('CLAUDE.md');
|
||||
assert.match(
|
||||
md,
|
||||
/--profile\b/,
|
||||
'CLAUDE.md must document the --profile flag (v4.1 SC #20)',
|
||||
);
|
||||
});
|
||||
|
||||
test('canonical name `phase_models` documented in docs/profiles.md', () => {
|
||||
// v5.x doc consolidation (67f6dd5) moved profile/phase detail OUT of
|
||||
// CLAUDE.md and into docs/. docs/profiles.md is now the canonical home for
|
||||
// phase_models; CLAUDE.md only links to it via "Reference docs". Assert the
|
||||
// canonical name lives in the canonical doc and no legacy alias crept in.
|
||||
// (was: read('CLAUDE.md') — v4.1 SC #20.)
|
||||
const md = read('docs/profiles.md');
|
||||
assert.match(
|
||||
md,
|
||||
/phase_models/,
|
||||
'docs/profiles.md must use canonical name "phase_models" (v4.1 SC #20)',
|
||||
);
|
||||
for (const bad of ['model_per_phase', 'phase_to_model', 'profile_phase_models']) {
|
||||
assert.ok(
|
||||
!md.includes(bad),
|
||||
`docs/profiles.md must NOT use legacy alias "${bad}"`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('README.md documents --profile flag for all 6 commands', () => {
|
||||
// SG1: README flag-table coverage is gating for SC #20. README is the
|
||||
// primary discovery surface for new users.
|
||||
const md = read('README.md');
|
||||
// Top-level Profile system section is required so the flag is
|
||||
// discoverable independent of per-command tables.
|
||||
assert.match(md, /## Profile system/, 'README.md missing top-level "## Profile system" section');
|
||||
// Every per-command Modes table must include --profile (count of
|
||||
// --profile occurrences should be ≥ 6 — one per command + Profile
|
||||
// system section).
|
||||
const profileMentions = (md.match(/--profile\b/g) || []).length;
|
||||
assert.ok(
|
||||
profileMentions >= 6,
|
||||
`README.md must mention --profile ≥ 6 times (one per command + section), got ${profileMentions}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('CHANGELOG.md has v4.1.0 entry', () => {
|
||||
const cl = read('CHANGELOG.md');
|
||||
assert.match(
|
||||
|
|
@ -487,19 +358,6 @@ test('CHANGELOG.md has v4.1.0 entry', () => {
|
|||
);
|
||||
});
|
||||
|
||||
test('docs/profiles.md exists and documents Custom.yaml authoring', () => {
|
||||
const dp = read('docs/profiles.md');
|
||||
assert.ok(dp.length > 1000, 'docs/profiles.md must be substantive (> 1000 chars)');
|
||||
// Must document custom-profile authoring (Step 22 manifest must_contain
|
||||
// pattern: "Custom.yaml" — case-insensitive match handled here as
|
||||
// /[Cc]ustom[. ]/ to allow either "custom.yaml" or "Custom profile" prose).
|
||||
assert.match(
|
||||
dp,
|
||||
/[Cc]ustom\.yaml|[Cc]ustom profile|<custom>\.yaml/,
|
||||
'docs/profiles.md must document custom profile authoring',
|
||||
);
|
||||
});
|
||||
|
||||
test('commands/trekplan.md Phase 8 seals Opus-4.7 schema-drift defense', () => {
|
||||
const cmd = read('commands/trekplan.md');
|
||||
// Locate Phase 8 section
|
||||
|
|
@ -714,7 +572,7 @@ test('operational files no longer reference trekrevise (v5.0.0 removal)', () =>
|
|||
}
|
||||
});
|
||||
|
||||
// --- v5.1 — phase_signals + brief_version 2.1 ---
|
||||
// --- v5.1 — brief_version 2.1 ---
|
||||
|
||||
test('v5.5 — templates/trekbrief-template.md declares brief_version: "2.2" (quoted)', () => {
|
||||
const t = read('templates/trekbrief-template.md');
|
||||
|
|
@ -722,67 +580,11 @@ test('v5.5 — templates/trekbrief-template.md declares brief_version: "2.2" (qu
|
|||
'trekbrief-template.md must declare brief_version: "2.2" (quoted) — unquoted parses as Number and bypasses sequencing gate; v5.5 bumped 2.1→2.2 for framing enforcement');
|
||||
});
|
||||
|
||||
test('v5.1 — templates/trekbrief-template.md contains phase_signals: block', () => {
|
||||
const t = read('templates/trekbrief-template.md');
|
||||
assert.match(t, /^phase_signals:$/m,
|
||||
'trekbrief-template.md must contain a phase_signals: block in frontmatter');
|
||||
});
|
||||
|
||||
test('v5.1 — HANDOVER-CONTRACTS.md schema row includes phase_signals + phase_signals_partial', () => {
|
||||
const t = read('docs/HANDOVER-CONTRACTS.md');
|
||||
assert.ok(t.includes('| `phase_signals` |'),
|
||||
'HANDOVER-CONTRACTS must add a phase_signals row to the Handover 1 schema table');
|
||||
assert.ok(t.includes('| `phase_signals_partial` |'),
|
||||
'HANDOVER-CONTRACTS must add a phase_signals_partial row to the Handover 1 schema table');
|
||||
});
|
||||
|
||||
test('v5.1 — phase_signals documented in docs/HANDOVER-CONTRACTS.md', () => {
|
||||
// v5.x doc consolidation (67f6dd5) moved phase_signals out of CLAUDE.md.
|
||||
// phase_signals is a Handover 1 (brief) field, so docs/HANDOVER-CONTRACTS.md
|
||||
// is its canonical home (the schema rows are pinned separately above).
|
||||
// CLAUDE.md only links to it. (was: read('CLAUDE.md') — v5.1.)
|
||||
const t = read('docs/HANDOVER-CONTRACTS.md');
|
||||
assert.ok(t.includes('phase_signals'),
|
||||
'docs/HANDOVER-CONTRACTS.md must document phase_signals (v5.1)');
|
||||
});
|
||||
|
||||
test('v5.1 — voyage README.md mentions phase_signals', () => {
|
||||
const t = read('README.md');
|
||||
assert.ok(t.includes('phase_signals'),
|
||||
'voyage README.md must mention phase_signals (v5.1 "What\'s new" bullet)');
|
||||
});
|
||||
|
||||
// --- v5.1.1 — High-effort behavior sub-section per command (Step 10) ---
|
||||
|
||||
test('v5.1.1 — commands/trekplan.md contains ### High-effort behavior (v5.1.1) sub-section', () => {
|
||||
const t = read('commands/trekplan.md');
|
||||
assert.match(t, /^### High-effort behavior \(v5\.1\.1\)$/m,
|
||||
'trekplan.md must contain ### High-effort behavior (v5.1.1) sub-section under Composition rule (Adversarial Pass 2, unavailable/skipped since v5.10.1)');
|
||||
});
|
||||
|
||||
test('v5.1.1 — commands/trekresearch.md contains ### High-effort behavior (v5.1.1) sub-section', () => {
|
||||
const t = read('commands/trekresearch.md');
|
||||
assert.match(t, /^### High-effort behavior \(v5\.1\.1\)$/m,
|
||||
'trekresearch.md must contain ### High-effort behavior (v5.1.1) sub-section under Composition rule (contrarian-researcher always-on)');
|
||||
});
|
||||
|
||||
test('v5.1.1 — commands/trekreview.md contains ### High-effort behavior (v5.1.1) sub-section', () => {
|
||||
const t = read('commands/trekreview.md');
|
||||
assert.match(t, /^### High-effort behavior \(v5\.1\.1\)$/m,
|
||||
'trekreview.md must contain ### High-effort behavior (v5.1.1) sub-section under Composition rule (skip Pass 3 + coordinator normalization)');
|
||||
});
|
||||
|
||||
test('v5.1.1 — commands/trekexecute.md contains ### High-effort behavior (v5.1.1) sub-section', () => {
|
||||
const t = read('commands/trekexecute.md');
|
||||
assert.match(t, /^### High-effort behavior \(v5\.1\.1\)$/m,
|
||||
'trekexecute.md must contain ### High-effort behavior (v5.1.1) sub-section under Composition rule (high effort changes nothing in /trekexecute)');
|
||||
});
|
||||
|
||||
// --- v5.4 — brief.md formalized as PUBLIC CONTRACT (Handover 1) ---
|
||||
// S5: the brief schema is the only public producer↔Voyage integration boundary
|
||||
// (Trinity asymmetry invariant). These pin the formalization as enforced doc-truth,
|
||||
// not prose — same pattern as the v5.1 Handover-1 pins above. Freeze, not promote:
|
||||
// phase_signals stays optional; brief_version 2.1 is the frozen baseline (S3 decision).
|
||||
// brief_version 2.1 is the frozen baseline (S3 decision).
|
||||
|
||||
test('v5.4 — HANDOVER-CONTRACTS.md Handover 1 heading is labeled (PUBLIC CONTRACT)', () => {
|
||||
const t = read('docs/HANDOVER-CONTRACTS.md');
|
||||
|
|
@ -808,10 +610,8 @@ test('v5.5 — brief_version 2.2 is current in the public-contract schema table
|
|||
'Handover 1 schema table must mark brief_version "2.2" as current');
|
||||
});
|
||||
|
||||
test('v5.5 — phase_signals stays optional: not promoted to required', () => {
|
||||
test('v5.5 — the Versioning note cites no phantom v5.4 and records the 2.1 baseline', () => {
|
||||
const t = read('docs/HANDOVER-CONTRACTS.md');
|
||||
assert.ok(!t.includes('v5.4 may promote'),
|
||||
'the speculative "v5.4 may promote phase_signals to required" line must be resolved (S3 froze the shape)');
|
||||
assert.ok(!/v5\.4/.test(t),
|
||||
'the contract doc must not cite the phantom v5.4 release (v5.2–5.4 never shipped; S16 → v5.5.0)');
|
||||
assert.ok(t.includes('established `2.1` as the public-contract baseline'),
|
||||
|
|
@ -851,103 +651,6 @@ test('v5.5 — voyage README.md mentions framing enforcement / brief_version 2.2
|
|||
'voyage README.md must carry a "What\'s new" note for framing enforcement (brief_version 2.2)');
|
||||
});
|
||||
|
||||
// --- S15 — default-profile name + per-phase model claims (cost-claim truth-pass) ---
|
||||
// The resolver's hardcoded default (lib/profiles/resolver.mjs) is the single
|
||||
// source of truth for which built-in profile a fresh user gets. README,
|
||||
// docs/profiles.md, and docs/operations.md must all mark THAT profile as the
|
||||
// default and must not mark any other built-in as default. Guards the
|
||||
// code-vs-docs contradiction the S14 audit surfaced (Addendum finding #3).
|
||||
const PROFILE_DOCS = ['README.md', 'docs/profiles.md', 'docs/operations.md'];
|
||||
|
||||
test('S15: default-profile name is consistent across resolver + all profile docs', () => {
|
||||
// Empty env so the operator's VOYAGE_PROFILE does not mask the hardcoded default.
|
||||
const { profile: def, profile_source } = resolveProfile({}, {});
|
||||
assert.equal(profile_source, 'default', 'resolveProfile({}, {}) must report source=default');
|
||||
assert.equal(def, 'premium', 'resolver hardcoded default is premium (operator decision 2026-05-13, commit 40d8742)');
|
||||
|
||||
const OTHERS = ['economy', 'balanced', 'premium', 'fable'].filter((p) => p !== def);
|
||||
for (const doc of PROFILE_DOCS) {
|
||||
const body = read(doc);
|
||||
assert.ok(
|
||||
body.includes(`\`${def}\` (default)`),
|
||||
`${doc}: profile table must mark \`${def}\` as "(default)" to match resolver`,
|
||||
);
|
||||
for (const other of OTHERS) {
|
||||
assert.ok(
|
||||
!body.includes(`\`${other}\` (default)`),
|
||||
`${doc}: must NOT mark \`${other}\` as "(default)" — resolver default is ${def}`,
|
||||
);
|
||||
assert.ok(
|
||||
!body.includes(`Default \`${other}\``),
|
||||
`${doc}: lookup-order must not say "Default \`${other}\`" — resolver default is ${def}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('S15: profile tables encode each built-in yaml phase_models exactly', () => {
|
||||
// Column order in every profile table: Profile | Brief | Research | Plan | Execute | Review | Continue | Use case
|
||||
const PHASES = ['brief', 'research', 'plan', 'execute', 'review', 'continue'];
|
||||
for (const name of ['economy', 'balanced', 'premium', 'fable']) {
|
||||
const pm = loadProfile(name).phase_models; // {brief:'opus', ...}
|
||||
const expected = PHASES.map((ph) => pm[ph]);
|
||||
for (const doc of PROFILE_DOCS) {
|
||||
const row = read(doc)
|
||||
.split('\n')
|
||||
.find((l) => new RegExp(`^\\|\\s*\`${name}\``).test(l));
|
||||
assert.ok(row, `${doc}: profile table is missing a \`${name}\` row`);
|
||||
const cells = row.split('|').map((c) => c.trim());
|
||||
// cells[0] is '' (leading pipe), cells[1] is the name, cells[2..7] are the 6 models
|
||||
const docModels = cells.slice(2, 8);
|
||||
assert.deepEqual(
|
||||
docModels,
|
||||
expected,
|
||||
`${doc}: \`${name}\` row models ${JSON.stringify(docModels)} must equal lib/profiles/${name}.yaml ${JSON.stringify(expected)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// --- S34 (V30) — economy is self-declared experimental until the cross-tier
|
||||
// Jaccard floor (0.55) is empirically calibrated (Step-17 calibration deferred
|
||||
// to v4.2). The status must be visible in BOTH the profile data
|
||||
// (lib/profiles/economy.yaml `experimental: true`) and every profile doc's
|
||||
// economy row. Fix the SOURCE doc/profile, not these pins.
|
||||
test('S34: economy profile row is flagged experimental across all profile docs', () => {
|
||||
for (const doc of PROFILE_DOCS) {
|
||||
const body = read(doc);
|
||||
const row = body.split('\n').find((l) => /^\|\s*`economy`/.test(l));
|
||||
assert.ok(row, `${doc}: profile table missing an \`economy\` row`);
|
||||
assert.match(
|
||||
row,
|
||||
/experimental/i,
|
||||
`${doc}: the \`economy\` table row must flag it experimental ` +
|
||||
`(uncalibrated Jaccard floor, S34/V30) — matches lib/profiles/economy.yaml experimental: true`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('S34: economy experimental flag agrees with parked-synthetic calibration status', () => {
|
||||
// economy.yaml experimental:true is honest ONLY while calibration is still
|
||||
// parked-synthetic. When empirical calibration lands (status: empirical),
|
||||
// drop experimental:true in the SAME change — this pin enforces the linkage.
|
||||
const eco = parseDocument(read('lib/profiles/economy.yaml')).parsed.frontmatter;
|
||||
const cal = parseDocument(read('tests/synthetic/profile-jaccard-calibration.md')).parsed.frontmatter;
|
||||
if (cal.status === 'parked-synthetic') {
|
||||
assert.equal(
|
||||
eco.experimental,
|
||||
true,
|
||||
'while calibration is parked-synthetic, economy.yaml must stay experimental: true',
|
||||
);
|
||||
} else {
|
||||
assert.notEqual(
|
||||
eco.experimental,
|
||||
true,
|
||||
'calibration is no longer parked-synthetic — drop economy experimental: true in the same change',
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('S15: README cost prose does not resurrect the false Sonnet-swarm claim', () => {
|
||||
// All 24 sub-agents are model: opus (operator-pinned, commit 40d8742) and the
|
||||
// model is uniform per phase — there is no "Opus orchestrates / Sonnet runs
|
||||
|
|
@ -962,7 +665,7 @@ test('S15: README cost prose does not resurrect the false Sonnet-swarm claim', (
|
|||
for (const phrase of BANNED) {
|
||||
assert.ok(
|
||||
!md.includes(phrase),
|
||||
`README.md must not claim "${phrase}" — sub-agents are opus-pinned; default profile runs Opus on every phase`,
|
||||
`README.md must not claim "${phrase}" — sub-agents are opus-pinned`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
|
@ -1070,7 +773,7 @@ test('S32: trekbrief Phase 3 delegates Q&A turn-taking to AskUserQuestion (V01
|
|||
);
|
||||
const p3Start = text.indexOf('## Phase 3 — Completeness loop');
|
||||
assert.ok(p3Start >= 0, 'trekbrief.md must have a "## Phase 3 — Completeness loop" section');
|
||||
const p3End = text.indexOf('## Phase 3.5', p3Start);
|
||||
const p3End = text.indexOf('## Phase 4', p3Start);
|
||||
const p3 = text.slice(p3Start, p3End > p3Start ? p3End : undefined);
|
||||
assert.ok(
|
||||
/Use `AskUserQuestion` for every question/.test(p3),
|
||||
|
|
@ -1125,29 +828,6 @@ test('S38: forward-guard — no product-facing doc asserts main-context relief u
|
|||
);
|
||||
});
|
||||
|
||||
// ── v5.9 (fable-tier step 6) — composed-resolver wiring pin ────────────────
|
||||
// The four pipeline commands must resolve {effort, model} via the composed
|
||||
// CLI (`resolver.mjs --resolve-phase-model`, brief > profile > default). A
|
||||
// direct `phase-signal-resolver.mjs --brief` invocation in a command Bash
|
||||
// block is the brief-only CLI: it silently drops the profile layer (the
|
||||
// AP2-1 footgun — `--profile <x>` would never reach sub-agent spawns again).
|
||||
// If this pin fails, re-wire the command to the composed CLI — do not relax
|
||||
// the pin.
|
||||
|
||||
for (const cmd of ['trekresearch', 'trekplan', 'trekreview', 'trekexecute']) {
|
||||
test(`v5.9: commands/${cmd}.md invokes the composed resolver, not the brief-only CLI`, () => {
|
||||
const text = read(`commands/${cmd}.md`);
|
||||
assert.ok(
|
||||
text.includes('--resolve-phase-model'),
|
||||
`commands/${cmd}.md must invoke resolver.mjs --resolve-phase-model (composed brief > profile > default)`,
|
||||
);
|
||||
assert.ok(
|
||||
!/phase-signal-resolver\.mjs --brief/.test(text),
|
||||
`commands/${cmd}.md must not invoke the brief-only phase-signal-resolver CLI — the profile layer would be silently dropped`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// ── v5.9 (fable-tier step 7) — orchestrator model-pin ABSENCE pin ───────────
|
||||
// Command frontmatter deliberately omits `model:` — omission is the only
|
||||
// session-inheritance spelling documented by BOTH official surfaces (AP2-2:
|
||||
|
|
@ -1221,11 +901,7 @@ test('v5.9: no commands/*.md frontmatter carries a model: key (session inheritan
|
|||
// class as bumping a version string inside a measurement doc.
|
||||
//
|
||||
// BLIND SPOT, stated rather than hidden: THIS file cannot be in the live-surface
|
||||
// list, because the S95 block below is necessarily full of the word. So the two
|
||||
// v5.1.1 assertion messages above (which describe the high-effort rules) are not
|
||||
// covered by the scanner and were found stale by review, not by npm test — they
|
||||
// match on the heading regex, so a wrong message never fails. When the high-effort
|
||||
// rules change, re-read those messages by hand.
|
||||
// list, because the S95 block below is necessarily full of the word.
|
||||
|
||||
// Live = anything that can still steer a run. Explicit, so a new file forces a decision.
|
||||
const GEMINI_LIVE_SURFACES = [
|
||||
|
|
@ -1275,20 +951,10 @@ test('S95: known-positive control — the historical records still DO carry gemi
|
|||
);
|
||||
});
|
||||
|
||||
test('S95: trekplan high-effort Pass 2 degrades explicitly instead of failing', () => {
|
||||
const t = read('commands/trekplan.md');
|
||||
const section = t.split('\n### High-effort behavior (v5.1.1)\n')[1];
|
||||
assert.ok(section, 'trekplan.md must still carry the High-effort behavior sub-section');
|
||||
const head = section.split('\n## ')[0];
|
||||
assert.match(
|
||||
head, /unavailable, skipped/,
|
||||
'the dropped Pass 2 must read "unavailable, skipped" explicitly — a removed pass that is silent is indistinguishable from one that failed',
|
||||
);
|
||||
});
|
||||
|
||||
// End-state defect D-07: since v5.9.0 no command pins `model:`, so the orchestrator follows
|
||||
// the session model, and sub-agent models are resolved at the spawn sites (brief signal >
|
||||
// profile > agent frontmatter). No user-facing doc may claim otherwise. Fix the SOURCE.
|
||||
// the session model; a Voyage agent's spawn passes no model (its own `model: opus` applies)
|
||||
// and a built-in agent's spawn passes `model: "opus"`. No user-facing doc may claim
|
||||
// otherwise. Fix the SOURCE.
|
||||
test('D-07: no doc claims an opus orchestrator or ships a recipe for model: lines no command has', () => {
|
||||
const readme = read('README.md');
|
||||
assert.ok(
|
||||
|
|
@ -1304,15 +970,19 @@ test('D-07: no doc claims an opus orchestrator or ships a recipe for model: line
|
|||
}
|
||||
});
|
||||
|
||||
test('D-07: trekresearch launch rules inject the resolved model instead of a blanket opus', () => {
|
||||
test('D-07: trekresearch launch rules leave a Voyage agent on its own pin, with no blanket opus rule', () => {
|
||||
const t = read('commands/trekresearch.md');
|
||||
const rules = (t.split('\n### Launch rules\n')[1] || '').split('\n## ')[0];
|
||||
assert.ok(rules.length > 0, 'trekresearch.md must still carry the Launch rules sub-section');
|
||||
assert.ok(
|
||||
!/model: "opus" for all sub-agents/i.test(rules),
|
||||
'a blanket opus rule contradicts the file\'s own resolution (brief > profile > agent frontmatter, wired in v5.9.0)',
|
||||
'a blanket opus rule contradicts the file\'s own rule: a Voyage agent keeps its own pin, only a built-in agent gets opus',
|
||||
);
|
||||
assert.match(rules, /phase_signal_result\.model/, 'the launch rules must name the resolved model the spawn sites inject');
|
||||
const flatRules = rules.replace(/\s+/g, ' ');
|
||||
assert.ok(flatRules.includes("Pass no model at a Voyage agent's spawn: the agent's own frontmatter `model: opus` applies."),
|
||||
'the launch rules must say a Voyage agent\'s spawn passes no model');
|
||||
assert.ok(t.replace(/\s+/g, ' ').includes('Spawn it with `model: "opus"`: a built-in agent has no Voyage pin.'),
|
||||
'a built-in follow-up agent is spawned with model opus');
|
||||
});
|
||||
|
||||
// End-state defect D-03: a trekplan's `## Verification` is where the brief's success
|
||||
|
|
@ -1447,7 +1117,7 @@ test('trekexecute Phase 7 says NO_CRITERIA out loud instead of failing silently'
|
|||
// what is true now; historical records keep what was true when written and are never
|
||||
// rewritten. When an alias moves again, re-verify against the docs and fix the SOURCE.
|
||||
const ALIAS_LIVE_SURFACES = [
|
||||
'CLAUDE.md', 'README.md', 'docs/profiles.md', 'docs/operations.md',
|
||||
'CLAUDE.md', 'README.md', 'docs/architecture.md', 'docs/operations.md',
|
||||
...listMd('commands').map((f) => `commands/${f}`),
|
||||
...listMd('agents').map((f) => `agents/${f}`),
|
||||
];
|
||||
|
|
@ -1487,10 +1157,10 @@ test('model aliases: known-positive control — the same query still finds every
|
|||
assert.deepEqual(silent, [], `no known-positive left for: ${silent.join(', ')}`);
|
||||
});
|
||||
|
||||
test('model aliases: profiles.md states the current targets, defaults and session inheritance', () => {
|
||||
const t = read('docs/profiles.md');
|
||||
const axes = (t.split('\n## Model & effort axes\n')[1] || '').split('\n## ')[0];
|
||||
assert.ok(axes.length > 0, 'docs/profiles.md must still carry the Model & effort axes section');
|
||||
test('model aliases: architecture.md states the current targets, defaults and session inheritance', () => {
|
||||
const t = read('docs/architecture.md');
|
||||
const axes = (t.split('\n## Model & effort\n')[1] || '').split('\n## ')[0];
|
||||
assert.ok(axes.length > 0, 'docs/architecture.md must still carry the Model & effort section');
|
||||
assert.match(axes, /Opus 5\.5[^\n]*\n?[^\n]*`medium`/, 'opus -> Opus 5.5, default effort `medium`');
|
||||
assert.match(axes, /Sonnet 5\.5/, 'sonnet -> Sonnet 5.5');
|
||||
assert.match(axes, /Fable 5\.1[\s\S]{0,120}`high`/, 'fable -> Fable 5.1, default effort `high`');
|
||||
|
|
@ -1499,7 +1169,8 @@ test('model aliases: profiles.md states the current targets, defaults and sessio
|
|||
assert.ok(!/overrides effort entirely/.test(axes),
|
||||
'`MAX_THINKING_TOKENS=0` has no effect on Opus 5.5, Sonnet 5.5 or the Fable models');
|
||||
assert.match(axes, /maxEffortLevel/, 'effort is capped by `maxEffortLevel` / an organization cap, not by `availableModels`');
|
||||
for (const f of ['CLAUDE.md', 'README.md']) {
|
||||
// README's alias sentence went with its §Cost profile (cleanup R-b).
|
||||
for (const f of ['CLAUDE.md']) {
|
||||
for (const target of ['Opus 5.5', 'Sonnet 5.5', 'Fable 5.1']) {
|
||||
assert.ok(read(f).includes(target), `${f} must name the current alias target ${target}`);
|
||||
}
|
||||
|
|
@ -1527,7 +1198,7 @@ const RETRIEVAL_MEDIUM = /retrieval(?: agents)?\s*(?:→|->|at)\s*`medium`/i;
|
|||
|
||||
test('effort floor: no live doc claims retrieval agents run at medium', () => {
|
||||
const hits = [];
|
||||
for (const f of ['docs/profiles.md', 'CLAUDE.md', 'README.md']) {
|
||||
for (const f of ['docs/architecture.md', 'CLAUDE.md', 'README.md']) {
|
||||
read(f).split('\n').forEach((l, i) => { if (RETRIEVAL_MEDIUM.test(l)) hits.push(`${f}:${i + 1}`); });
|
||||
}
|
||||
assert.deepEqual(hits, [], 'the four retrieval agents pin `high` now');
|
||||
|
|
@ -1540,29 +1211,10 @@ test('effort floor: known-positive control — the query finds the old phrases,
|
|||
assert.doesNotMatch('both with default reasoning effort `medium`', RETRIEVAL_MEDIUM);
|
||||
});
|
||||
|
||||
test('effort floor: profiles.md names the four retrieval agents as high', () => {
|
||||
const axes = (read('docs/profiles.md').split('\n## Model & effort axes\n')[1] || '').split('\n## ')[0];
|
||||
test('effort floor: architecture.md names the four retrieval agents as high', () => {
|
||||
const axes = (read('docs/architecture.md').split('\n## Model & effort\n')[1] || '').split('\n## ')[0];
|
||||
const bullet = axes.split('\n- ').find((b) => ['task-finder', 'git-historian', 'dependency-tracer', 'architecture-mapper'].every((a) => b.includes(a)));
|
||||
assert.ok(bullet, 'one bullet in §Model & effort axes names all four retrieval agents');
|
||||
assert.ok(bullet, 'one bullet in §Model & effort names all four retrieval agents');
|
||||
assert.match(bullet, /`high`/);
|
||||
assert.doesNotMatch(bullet, /`medium`/);
|
||||
});
|
||||
|
||||
// The resolver returns `effort` only when the brief's phase signal sets one;
|
||||
// no command says it always does.
|
||||
const RESOLVER_COMMANDS = ['trekplan', 'trekexecute', 'trekresearch', 'trekreview'];
|
||||
|
||||
test('resolver: no command says the resolver always returns {effort, model, source}', () => {
|
||||
for (const f of readdirSync(join(ROOT, 'commands')).filter((x) => x.endsWith('.md'))) {
|
||||
assert.doesNotMatch(read(`commands/${f}`), /\{effort, model, source\}/, f);
|
||||
}
|
||||
});
|
||||
|
||||
test('resolver: each resolver-using command says `effort` comes only from the brief signal, else standard', () => {
|
||||
for (const c of RESOLVER_COMMANDS) {
|
||||
const t = read(`commands/${c}.md`);
|
||||
const hits = t.match(/\{model, source\}[^\n]*\n?[^\n]*effort[^\n]*only when the brief's phase signal sets one/g) ?? [];
|
||||
assert.ok(hits.length >= 1, `${c}: no statement that effort is only present with a brief signal`);
|
||||
assert.match(t, /without it[^\n]*\n?[^\n]*`?standard`?/i, c);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -91,43 +91,8 @@ test('extension does NOT break REQUIRED_KEYS contract', () => {
|
|||
}
|
||||
});
|
||||
|
||||
// v4.1 Step 3 — OPTIONAL_STRING_KEYS dispatch (profile_used)
|
||||
// v4.1 Step 3 — OPTIONAL_STRING_KEYS dispatch; its one key went in cleanup R-b.
|
||||
|
||||
test('OPTIONAL_STRING_KEYS exports profile_used', () => {
|
||||
assert.deepEqual(
|
||||
[...OPTIONAL_STRING_KEYS].sort(),
|
||||
['profile_used'].sort(),
|
||||
'OPTIONAL_STRING_KEYS export drift — pin contract',
|
||||
);
|
||||
});
|
||||
|
||||
test('profile_used: economy parses successfully (SC #10 forward-compat)', () => {
|
||||
const r = parseManifest(bodyWithExtras(' profile_used: economy'));
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.equal(r.parsed.profile_used, 'economy');
|
||||
});
|
||||
|
||||
test('profile_used: numeric rejected with MANIFEST_OPTIONAL_TYPE', () => {
|
||||
const r = parseManifest(bodyWithExtras(' profile_used: 42'));
|
||||
assert.equal(r.valid, false);
|
||||
const found = r.errors.find(e => e.code === 'MANIFEST_OPTIONAL_TYPE');
|
||||
assert.ok(found, `expected MANIFEST_OPTIONAL_TYPE, got: ${JSON.stringify(r.errors)}`);
|
||||
assert.match(found.message, /profile_used/);
|
||||
assert.match(found.message, /string/);
|
||||
});
|
||||
|
||||
test('absence of profile_used: field is NOT in parsed (NOT defaulted, unlike boolean)', () => {
|
||||
const r = parseManifest(bodyOnlyRequired());
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
// Absence semantics differ from boolean: parsed should NOT contain the key
|
||||
assert.equal('profile_used' in r.parsed, false,
|
||||
'profile_used must NOT be auto-defaulted when absent — string-key semantics');
|
||||
});
|
||||
|
||||
test('profile_used works alongside boolean optional keys (skip_commit_check + memory_write)', () => {
|
||||
const r = parseManifest(bodyWithExtras(' skip_commit_check: true\n memory_write: true\n profile_used: balanced'));
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
assert.equal(r.parsed.skip_commit_check, true);
|
||||
assert.equal(r.parsed.memory_write, true);
|
||||
assert.equal(r.parsed.profile_used, 'balanced');
|
||||
test('OPTIONAL_STRING_KEYS exports no key', () => {
|
||||
assert.deepEqual([...OPTIONAL_STRING_KEYS], [], 'OPTIONAL_STRING_KEYS export drift — pin contract');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,112 +0,0 @@
|
|||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { writeFileSync, unlinkSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { resolvePhaseSignal, resolvePhaseSignalFromFile } from '../../lib/profiles/phase-signal-resolver.mjs';
|
||||
|
||||
const FULL_SIGNALS_FM = {
|
||||
phase_signals: [
|
||||
{ phase: 'research', effort: 'low', model: 'sonnet' },
|
||||
{ phase: 'plan', effort: 'standard' },
|
||||
{ phase: 'execute', effort: 'high', model: 'opus' },
|
||||
{ phase: 'review', effort: 'standard', model: 'sonnet' },
|
||||
],
|
||||
};
|
||||
|
||||
test('resolvePhaseSignal — returns {effort, model} for all 4 phases on full-signals brief', () => {
|
||||
for (const phase of ['research', 'plan', 'execute', 'review']) {
|
||||
const r = resolvePhaseSignal(FULL_SIGNALS_FM, phase);
|
||||
assert.ok(r && typeof r === 'object', `phase=${phase} should resolve non-null`);
|
||||
assert.ok(typeof r.effort === 'string', `phase=${phase} should have effort`);
|
||||
}
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — returns null when brief has no phase_signals', () => {
|
||||
const r = resolvePhaseSignal({ task: 'x' }, 'plan');
|
||||
assert.equal(r, null);
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — returns partial {effort} with model undefined when signal omits model', () => {
|
||||
const r = resolvePhaseSignal(FULL_SIGNALS_FM, 'plan');
|
||||
assert.equal(r.effort, 'standard');
|
||||
assert.equal(r.model, undefined);
|
||||
assert.ok(!('model' in r), 'model key should be absent when not in signal');
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — returns null when phase is not in PHASE_SIGNAL_PHASES', () => {
|
||||
assert.equal(resolvePhaseSignal(FULL_SIGNALS_FM, 'brief'), null);
|
||||
assert.equal(resolvePhaseSignal(FULL_SIGNALS_FM, 'continue'), null);
|
||||
assert.equal(resolvePhaseSignal(FULL_SIGNALS_FM, 'nonsense'), null);
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — defensive: null/non-object input returns null', () => {
|
||||
assert.equal(resolvePhaseSignal(null, 'plan'), null);
|
||||
assert.equal(resolvePhaseSignal(undefined, 'plan'), null);
|
||||
assert.equal(resolvePhaseSignal('string', 'plan'), null);
|
||||
assert.equal(resolvePhaseSignal({ phase_signals: 'not-array' }, 'plan'), null);
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in-depth gate)', () => {
|
||||
// MAJOR fix (S4): line that copies `model` must gate against the same
|
||||
// allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus','fable']),
|
||||
// mirroring how effort is gated against EFFORT_LEVELS. A brief that slipped
|
||||
// validation (hand-edited, validation skipped) must not hand a junk model
|
||||
// string to a command that then spawns an agent with `model: <junk>`.
|
||||
const fm = {
|
||||
phase_signals: [
|
||||
{ phase: 'plan', effort: 'high', model: 'gpt-4' }, // invalid model
|
||||
{ phase: 'review', effort: 'standard', model: 'haiku' }, // not in BASE_ALLOWED_MODELS
|
||||
],
|
||||
};
|
||||
const plan = resolvePhaseSignal(fm, 'plan');
|
||||
assert.equal(plan.effort, 'high', 'valid effort still resolves');
|
||||
assert.equal(plan.model, undefined, 'invalid model "gpt-4" must be dropped');
|
||||
assert.ok(!('model' in plan), 'model key absent when value not in allowlist');
|
||||
|
||||
const review = resolvePhaseSignal(fm, 'review');
|
||||
assert.equal(review.model, undefined, 'haiku must be dropped (not in BASE_ALLOWED_MODELS)');
|
||||
assert.ok(!('model' in review), 'model key absent for haiku');
|
||||
});
|
||||
|
||||
test('resolvePhaseSignal — keeps valid models (sonnet, opus, fable) after gating', () => {
|
||||
const fm = {
|
||||
phase_signals: [
|
||||
{ phase: 'research', effort: 'low', model: 'sonnet' },
|
||||
{ phase: 'execute', effort: 'high', model: 'opus' },
|
||||
{ phase: 'review', effort: 'high', model: 'fable' },
|
||||
],
|
||||
};
|
||||
assert.equal(resolvePhaseSignal(fm, 'research').model, 'sonnet');
|
||||
assert.equal(resolvePhaseSignal(fm, 'execute').model, 'opus');
|
||||
assert.equal(resolvePhaseSignal(fm, 'review').model, 'fable');
|
||||
});
|
||||
|
||||
test('resolvePhaseSignalFromFile + CLI shim — writes JSON to stdout, exit 0', () => {
|
||||
const fixture = join(tmpdir(), `phase-signal-test-${process.pid}.md`);
|
||||
writeFileSync(fixture, `---
|
||||
type: trekbrief
|
||||
brief_version: "2.1"
|
||||
phase_signals:
|
||||
- phase: plan
|
||||
effort: high
|
||||
model: opus
|
||||
---
|
||||
# x
|
||||
`);
|
||||
try {
|
||||
// Programmatic invocation
|
||||
const r = resolvePhaseSignalFromFile(fixture, 'plan');
|
||||
assert.deepEqual(r, { effort: 'high', model: 'opus' });
|
||||
// CLI shim
|
||||
const helperPath = new URL('../../lib/profiles/phase-signal-resolver.mjs', import.meta.url).pathname;
|
||||
const out = execFileSync('node', [helperPath, '--brief', fixture, '--phase', 'plan', '--json'], {
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
const parsed = JSON.parse(out.trim());
|
||||
assert.deepEqual(parsed, { effort: 'high', model: 'opus' });
|
||||
} finally {
|
||||
try { unlinkSync(fixture); } catch { /* swallow */ }
|
||||
}
|
||||
});
|
||||
|
|
@ -151,12 +151,13 @@ function paraAfter(rel, marker) {
|
|||
return flat(lines.slice(start, i).join('\n'));
|
||||
}
|
||||
|
||||
// ---- spawn model: phase_signal_result.model, the agent's own model: as the fallback ----
|
||||
// ---- spawn model: a Voyage agent gets no model:, a built-in agent gets opus ----
|
||||
|
||||
const PASS_MODEL = /Pass `model: \{phase_signal_result\.model\}` at this spawn; without it \(the `--brief` branch, where the resolver does not run, or a resolver that printed nothing\) the agent's own `model:` applies\./;
|
||||
const NO_MODEL = "This spawn passes no `model:`: the agent's own `model: opus` applies.";
|
||||
const BUILT_IN = 'Spawn it with `model: "opus"`: a built-in agent has no Voyage pin.';
|
||||
|
||||
test('spawn model: commands/trekplan.md pins no spawn to `model: "opus"`', () => {
|
||||
assert.doesNotMatch(PLAN_CMD, /model: "opus"/);
|
||||
test('spawn model: commands/trekplan.md pins `model: "opus"` once, at the built-in Explore deep-dive', () => {
|
||||
assert.equal(PLAN_CMD.split('model: "opus"').length - 1, 1, 'expected exactly one `model: "opus"` in commands/trekplan.md');
|
||||
});
|
||||
|
||||
for (const [where, marker] of [
|
||||
|
|
@ -164,32 +165,25 @@ for (const [where, marker] of [
|
|||
['### Always launch (Phase 5)', '### Always launch (all codebase sizes):'],
|
||||
['research-scout (Phase 5)', 'launch **research-scout** in'],
|
||||
['plan-critic and scope-guardian (Phase 9)', 'it traces into `## Verification`.'],
|
||||
['session-decomposer (Phase 1.5)', 'Launch the **session-decomposer** agent:'],
|
||||
]) {
|
||||
test(`spawn model: ${where}: the spawn takes phase_signal_result.model, the agent's own model: as the fallback`, () => {
|
||||
test(`spawn model: ${where}: the spawn passes no model:, the agent's own model: opus applies`, () => {
|
||||
const part = paraAfter('commands/trekplan.md', marker);
|
||||
assert.match(part, PASS_MODEL, `${where}: missing the resolved-model paragraph after "${marker}"`);
|
||||
assert.ok(part.includes(NO_MODEL), `${where}: missing the no-model sentence after "${marker}": ${part}`);
|
||||
assert.doesNotMatch(part, /model: "opus"/);
|
||||
});
|
||||
}
|
||||
|
||||
test('spawn model: session-decomposer (Phase 1.5): --decompose has no resolved model, so the spawn passes no model:', () => {
|
||||
const part = paraAfter('commands/trekplan.md', 'Launch the **session-decomposer** agent:');
|
||||
assert.ok(part.includes('Phase 1 runs the model resolver only for `--project`, so `--decompose` has no resolved model: this spawn '
|
||||
+ 'passes no `model:`, and the agent\'s own `model:` applies.'), `the session-decomposer paragraph does not say it passes no model: ${part}`);
|
||||
assert.ok(!part.includes('phase_signal_result'), 'the session-decomposer spawn still names phase_signal_result');
|
||||
assert.doesNotMatch(part, /model: "opus"/);
|
||||
});
|
||||
|
||||
test('spawn model: convention-scanner (Phase 5): the spawn takes phase_signal_result.model, the frontmatter as the fallback', () => {
|
||||
test('spawn model: convention-scanner (Phase 5): the spawn passes no model:, its own model: opus applies', () => {
|
||||
const part = paraAfter('commands/trekplan.md', 'use the `convention-scanner` plugin agent');
|
||||
assert.match(part, /\(model: phase_signal_result\.model, the agent's frontmatter as the fallback\)/);
|
||||
assert.ok(part.includes('(no `model:`; its own `model: opus` applies)'), part);
|
||||
assert.doesNotMatch(part, /model: "opus"/);
|
||||
});
|
||||
|
||||
test('spawn model: deep-dives (Phase 6): the spawn takes phase_signal_result.model, else opus', () => {
|
||||
test('spawn model: deep-dives (Phase 6): the built-in Explore agent is spawned with model opus', () => {
|
||||
const part = paraAfter('commands/trekplan.md', 'spawn a targeted deep-dive agent');
|
||||
assert.match(part, /\(model: phase_signal_result\.model, else opus — Explore is Claude Code's built-in agent and has no Voyage frontmatter to fall back to, subagent_type: "Explore"\)/);
|
||||
assert.doesNotMatch(part, /model: "opus"/);
|
||||
assert.ok(part.includes('(subagent_type: "Explore", Claude Code\'s built-in agent)'), part);
|
||||
assert.ok(part.includes(BUILT_IN), part);
|
||||
});
|
||||
|
||||
// ---- Phase 4 sizing: no turn budget, and the count it runs ---------------------
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@
|
|||
// can be checked offline: that each grader can see what it claims to see.
|
||||
//
|
||||
// - the new case for veikart steg 1's gate exists, and its fixture really
|
||||
// reaches the intent gate (passes the sequencing gate, has no marker)
|
||||
// - case 4 (plan-halts-without-phase-signals) can only be stopped by the
|
||||
// sequencing gate: its fixture carries a current intent approval
|
||||
// reaches the intent gate (passes the brief validator, has no marker)
|
||||
// - review-requires-project: the error line is written by CODE (the arg
|
||||
// parser), and the grader reads the parser's code from the trace
|
||||
// - grader weaknesses the PM re-measurement found (2026-09-23):
|
||||
|
|
@ -77,14 +75,13 @@ test('proevesett: a case exists for the steg 1 intent gate, with its graders', (
|
|||
const re = new RegExp(n.pattern);
|
||||
assert.match(inTrace({ valid: false, errors: [{ code: 'BRIEF_INTENT_NOT_APPROVED' }] }, 2), re);
|
||||
assert.doesNotMatch(COMMAND_PROSE, re, 'the pattern must not be satisfiable by quoting the command prose');
|
||||
assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
|
||||
const noAgent = grader(`evals/${GATE_CASE}/graders/no-agent.md`);
|
||||
assert.deepEqual([noAgent.type, noAgent.tool, noAgent.min, noAgent.max], ['tool_used', 'Agent', '0', '0']);
|
||||
const noPlan = grader(`evals/${GATE_CASE}/graders/no-plan-file.md`);
|
||||
assert.deepEqual([noPlan.type, noPlan.path, noPlan.exists], ['file_exists', '**/plan.md', 'false']);
|
||||
});
|
||||
|
||||
test('proevesett: the gate case fixture passes the sequencing gate and stops at the intent gate', () => {
|
||||
test('proevesett: the gate case fixture passes the brief validator and stops at the intent gate', () => {
|
||||
const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
|
||||
try {
|
||||
const s = spawnSync('bash', [join(EVALS, GATE_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
|
||||
|
|
@ -98,54 +95,6 @@ test('proevesett: the gate case fixture passes the sequencing gate and stops at
|
|||
} finally { rmSync(ws, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// ---- case 4: only the sequencing gate may stop it ----
|
||||
// Measured 2026-09-23: with no intent marker the child sometimes ran the
|
||||
// intent gate first and halted there (PM: 6 of 12 on dc9b480, 7 of 10 on
|
||||
// 66e1fa1). The halt was right, but the case then measured the wrong gate.
|
||||
|
||||
const SEQ_CASE = 'plan-halts-without-phase-signals';
|
||||
|
||||
test('plan-halts-without-phase-signals: the fixture carries a current intent approval and fails only the sequencing gate', () => {
|
||||
const ws = mkdtempSync(join(tmpdir(), 'proevesett-'));
|
||||
try {
|
||||
const s = spawnSync('bash', [join(EVALS, SEQ_CASE, 'scaffold.sh')], { cwd: ws, encoding: 'utf8' });
|
||||
assert.equal(s.status, 0, s.stderr);
|
||||
const brief = join(ws, 'proj', 'brief.md');
|
||||
const g = spawnSync('node', [join(ROOT, 'lib/validators/intent-approval.mjs'), '--check', '--json', brief], { encoding: 'utf8' });
|
||||
assert.equal(g.status, 0, `the intent gate must pass so only the sequencing gate can stop the run: ${g.stdout}`);
|
||||
const v = spawnSync('node', [join(ROOT, 'lib/validators/brief-validator.mjs'), '--soft', '--json', brief], { encoding: 'utf8' });
|
||||
assert.deepEqual(JSON.parse(v.stdout).errors.map((e) => e.code), ['BRIEF_V51_MISSING_SIGNALS']);
|
||||
} finally { rmSync(ws, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('plan-halts-without-phase-signals: a halt at the intent gate fails the case', () => {
|
||||
assert.ok(graders(SEQ_CASE).includes('no-intent-halt.md'), graders(SEQ_CASE).join(','));
|
||||
const g = grader(`evals/${SEQ_CASE}/graders/no-intent-halt.md`);
|
||||
assert.deepEqual([g.type, g.target, g.match], ['regex', 'trace', 'not_contains']);
|
||||
const re = new RegExp(g.pattern);
|
||||
for (const code of ['BRIEF_INTENT_NOT_APPROVED', 'BRIEF_INTENT_APPROVAL_STALE', 'BRIEF_INTENT_APPROVAL_INVALID']) {
|
||||
assert.match(inTrace({ valid: false, errors: [{ code }] }, 2), re, `${code} must fail the case`);
|
||||
}
|
||||
assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
|
||||
assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
|
||||
assert.doesNotMatch(COMMAND_PROSE, re, 'the command prose must not trip the grader');
|
||||
});
|
||||
|
||||
// Known-positive, measured 2026-09-23: with the check in brief-validator.mjs
|
||||
// disabled, the child still halted on the prose rule and named phase_signals
|
||||
// (3 of 3 passed). The halt alone does not show that the validator fired.
|
||||
test('plan-halts-without-phase-signals: the validator\'s own code must be in the trace', () => {
|
||||
assert.ok(graders(SEQ_CASE).includes('names-code.md'), graders(SEQ_CASE).join(','));
|
||||
const g = grader(`evals/${SEQ_CASE}/graders/names-code.md`);
|
||||
assert.deepEqual([g.type, g.target], ['regex', 'trace']);
|
||||
const re = new RegExp(g.pattern);
|
||||
assert.match(inTrace({ valid: false, errors: [{ code: 'BRIEF_V51_MISSING_SIGNALS' }] }, 2), re);
|
||||
assert.doesNotMatch(inTrace({ valid: true, errors: [], warnings: [] }, 2), re);
|
||||
assert.doesNotMatch(inTrace({ valid: false, errors: [{ code: 'BRIEF_INTENT_NOT_APPROVED' }] }, 2), re);
|
||||
assert.doesNotMatch(COMMAND_PROSE, re, 'the pattern must not be satisfiable by quoting the command prose');
|
||||
assert.doesNotMatch(read('lib/validators/brief-validator.mjs'), re, 'nor by reading the validator source');
|
||||
});
|
||||
|
||||
// ---- case 5: the missing-flag error comes from code ----
|
||||
|
||||
test('review-requires-project: the arg parser, run as trekreview.md spells it, prints the error (exit 1)', () => {
|
||||
|
|
|
|||
|
|
@ -1,244 +0,0 @@
|
|||
// tests/lib/profile-application.test.mjs
|
||||
// SC #5-#9 + backward-compat edge-case for lib/profiles/resolver.mjs.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
loadProfile,
|
||||
resolveProfile,
|
||||
resolveTrekcontinueProfile,
|
||||
validateProfileFile,
|
||||
findProfilePath,
|
||||
} from '../../lib/profiles/resolver.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = join(__dirname, '..', '..');
|
||||
|
||||
// SC #5: loadProfile returns matrix-match for all 6 phase_models
|
||||
|
||||
test('SC #5: loadProfile("economy") returns flattened phase_models with all 6 phases', () => {
|
||||
const p = loadProfile('economy');
|
||||
assert.equal(p.name, 'economy');
|
||||
assert.equal(p.phase_models.brief, 'sonnet');
|
||||
assert.equal(p.phase_models.research, 'sonnet');
|
||||
assert.equal(p.phase_models.plan, 'sonnet');
|
||||
assert.equal(p.phase_models.execute, 'sonnet');
|
||||
assert.equal(p.phase_models.review, 'sonnet');
|
||||
assert.equal(p.phase_models.continue, 'sonnet');
|
||||
assert.equal(p.parallel_agents_min, 2);
|
||||
assert.equal(p.parallel_agents_max, 3);
|
||||
assert.equal(p.external_research_enabled, false);
|
||||
assert.equal(p.brief_reviewer_iter_cap, 1);
|
||||
});
|
||||
|
||||
test('SC #5: loadProfile("balanced") returns mixed phase_models', () => {
|
||||
const p = loadProfile('balanced');
|
||||
assert.equal(p.phase_models.plan, 'opus');
|
||||
assert.equal(p.phase_models.review, 'opus');
|
||||
assert.equal(p.phase_models.brief, 'sonnet');
|
||||
assert.equal(p.phase_models.execute, 'sonnet');
|
||||
});
|
||||
|
||||
test('SC #5: loadProfile("premium") returns all-opus', () => {
|
||||
const p = loadProfile('premium');
|
||||
for (const phase of ['brief', 'research', 'plan', 'execute', 'review', 'continue']) {
|
||||
assert.equal(p.phase_models[phase], 'opus', `premium ${phase} should be opus`);
|
||||
}
|
||||
});
|
||||
|
||||
test('SC #5: loadProfile("fable") returns all-fable (BUILTIN_NAMES canary)', () => {
|
||||
// Throws PROFILE_NOT_FOUND if 'fable' regresses out of BUILTIN_NAMES —
|
||||
// the silent-fallback-to-premium failure mode this pin exists to catch.
|
||||
const p = loadProfile('fable');
|
||||
assert.equal(p.name, 'fable');
|
||||
for (const phase of ['brief', 'research', 'plan', 'execute', 'review', 'continue']) {
|
||||
assert.equal(p.phase_models[phase], 'fable', `fable ${phase} should be fable`);
|
||||
}
|
||||
assert.equal(p.parallel_agents_min, 6);
|
||||
assert.equal(p.parallel_agents_max, 8);
|
||||
assert.equal(p.external_research_enabled, true);
|
||||
assert.equal(p.brief_reviewer_iter_cap, 3);
|
||||
});
|
||||
|
||||
test('SC #5: loadProfile throws PROFILE_NOT_FOUND for unknown profile', () => {
|
||||
try {
|
||||
loadProfile('does-not-exist-xyz');
|
||||
assert.fail('expected throw');
|
||||
} catch (e) {
|
||||
assert.equal(e.cause, 'PROFILE_NOT_FOUND');
|
||||
assert.match(e.message, /not found/);
|
||||
assert.ok(Array.isArray(e.attempted), 'should expose attempted paths');
|
||||
}
|
||||
});
|
||||
|
||||
// SC #6: env-var fallback flag > env > default
|
||||
|
||||
test('SC #6: resolveProfile flag > env > default', () => {
|
||||
// flag wins
|
||||
const r1 = resolveProfile({ flags: { '--profile': 'balanced' } }, { VOYAGE_PROFILE: 'economy' });
|
||||
assert.equal(r1.profile, 'balanced');
|
||||
assert.equal(r1.profile_source, 'flag');
|
||||
|
||||
// env wins when no flag
|
||||
const r2 = resolveProfile({ flags: {} }, { VOYAGE_PROFILE: 'economy' });
|
||||
assert.equal(r2.profile, 'economy');
|
||||
assert.equal(r2.profile_source, 'env');
|
||||
|
||||
// default when neither
|
||||
const r3 = resolveProfile({ flags: {} }, {});
|
||||
assert.equal(r3.profile, 'premium');
|
||||
assert.equal(r3.profile_source, 'default');
|
||||
});
|
||||
|
||||
// SC #7: performance — loadProfile 1000 iter < 50ms average (allowing some headroom)
|
||||
|
||||
test('SC #7: loadProfile 1000-iter performance < 50ms average', () => {
|
||||
const iterations = 1000;
|
||||
const start = performance.now();
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
loadProfile('economy');
|
||||
}
|
||||
const elapsed = performance.now() - start;
|
||||
const avgMs = elapsed / iterations;
|
||||
assert.ok(avgMs < 50, `loadProfile too slow: ${avgMs.toFixed(3)}ms average over ${iterations} iter`);
|
||||
});
|
||||
|
||||
// SC #8: custom.yaml from repo-root trumps ~/.claude/
|
||||
|
||||
test('SC #8: custom profile from <cwd>/voyage-profiles/<name>.yaml takes precedence over ~/.claude/', () => {
|
||||
const tmpRepo = mkdtempSync(join(tmpdir(), 'voyage-resolver-repo-'));
|
||||
const tmpHome = mkdtempSync(join(tmpdir(), 'voyage-resolver-home-'));
|
||||
try {
|
||||
// Place custom profile in repo and home — repo should win
|
||||
mkdirSync(join(tmpRepo, 'voyage-profiles'), { recursive: true });
|
||||
mkdirSync(join(tmpHome, '.claude', 'voyage-profiles'), { recursive: true });
|
||||
|
||||
writeFileSync(join(tmpRepo, 'voyage-profiles', 'mycustom.yaml'),
|
||||
`---
|
||||
profile_version: "1.0"
|
||||
name: mycustom-repo
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: sonnet
|
||||
- phase: research
|
||||
model: sonnet
|
||||
- phase: plan
|
||||
model: sonnet
|
||||
- phase: execute
|
||||
model: sonnet
|
||||
- phase: review
|
||||
model: sonnet
|
||||
- phase: continue
|
||||
model: sonnet
|
||||
parallel_agents_min: 1
|
||||
parallel_agents_max: 2
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
---
|
||||
`);
|
||||
writeFileSync(join(tmpHome, '.claude', 'voyage-profiles', 'mycustom.yaml'),
|
||||
`---
|
||||
profile_version: "1.0"
|
||||
name: mycustom-home
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: opus
|
||||
- phase: research
|
||||
model: opus
|
||||
- phase: plan
|
||||
model: opus
|
||||
- phase: execute
|
||||
model: opus
|
||||
- phase: review
|
||||
model: opus
|
||||
- phase: continue
|
||||
model: opus
|
||||
parallel_agents_min: 1
|
||||
parallel_agents_max: 2
|
||||
external_research_enabled: true
|
||||
brief_reviewer_iter_cap: 3
|
||||
---
|
||||
`);
|
||||
|
||||
const found = findProfilePath('mycustom', { cwd: tmpRepo, home: tmpHome });
|
||||
assert.ok(found.path, `expected to find mycustom; attempted: ${found.attempted.join(', ')}`);
|
||||
assert.ok(found.path.startsWith(tmpRepo),
|
||||
`expected repo-rot win (path under ${tmpRepo}), got: ${found.path}`);
|
||||
|
||||
const p = loadProfile('mycustom', { cwd: tmpRepo, home: tmpHome });
|
||||
assert.equal(p.name, 'mycustom-repo', 'repo profile should win');
|
||||
assert.equal(p.phase_models.brief, 'sonnet');
|
||||
} finally {
|
||||
rmSync(tmpRepo, { recursive: true, force: true });
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('SC #8: missing profile error message includes both attempted paths', () => {
|
||||
const tmpRepo = mkdtempSync(join(tmpdir(), 'voyage-resolver-empty-'));
|
||||
const tmpHome = mkdtempSync(join(tmpdir(), 'voyage-resolver-emptyhome-'));
|
||||
try {
|
||||
try {
|
||||
loadProfile('not-a-real-profile', { cwd: tmpRepo, home: tmpHome });
|
||||
assert.fail('expected throw');
|
||||
} catch (e) {
|
||||
assert.equal(e.cause, 'PROFILE_NOT_FOUND');
|
||||
// Both attempted paths should be in the error message for diagnostic clarity
|
||||
const msg = e.message;
|
||||
assert.match(msg, /voyage-profiles\/not-a-real-profile\.yaml/);
|
||||
assert.match(msg, /\.claude\/voyage-profiles\/not-a-real-profile\.yaml/);
|
||||
}
|
||||
} finally {
|
||||
rmSync(tmpRepo, { recursive: true, force: true });
|
||||
rmSync(tmpHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// SC #9: resolveTrekcontinueProfile inheritance from plan-frontmatter
|
||||
|
||||
test('SC #9: resolveTrekcontinueProfile inherits from plan-frontmatter (profile: balanced)', () => {
|
||||
const planPath = join(REPO_ROOT, 'tests', 'fixtures', 'plan-with-profile.md');
|
||||
const r = resolveTrekcontinueProfile(planPath, { flags: {} });
|
||||
assert.equal(r.profile, 'balanced');
|
||||
assert.equal(r.profile_source, 'inheritance');
|
||||
});
|
||||
|
||||
test('SC #9: resolveTrekcontinueProfile flag overrides plan-frontmatter (advisory)', () => {
|
||||
const planPath = join(REPO_ROOT, 'tests', 'fixtures', 'plan-with-profile.md');
|
||||
const advisories = [];
|
||||
const fakeConsole = { error: (m) => advisories.push(m) };
|
||||
const r = resolveTrekcontinueProfile(planPath,
|
||||
{ flags: { '--profile': 'economy' } },
|
||||
{ console: fakeConsole });
|
||||
assert.equal(r.profile, 'economy');
|
||||
assert.equal(r.profile_source, 'flag');
|
||||
assert.equal(advisories.length, 1, 'expected one advisory message');
|
||||
assert.match(advisories[0], /balanced.*economy/);
|
||||
assert.match(advisories[0], /\[voyage\]/);
|
||||
});
|
||||
|
||||
// Backward-compat edge-case: v4.0-style plan WITHOUT profile field
|
||||
|
||||
test('Backward-compat: resolveTrekcontinueProfile on v4.0 plan without profile field returns default premium', () => {
|
||||
const planPath = join(REPO_ROOT, 'tests', 'fixtures', 'plan-without-profile.md');
|
||||
const r = resolveTrekcontinueProfile(planPath, { flags: {} });
|
||||
assert.equal(r.profile, 'premium');
|
||||
assert.equal(r.profile_source, 'default');
|
||||
});
|
||||
|
||||
test('Backward-compat: resolveTrekcontinueProfile with non-existent plan path returns default premium', () => {
|
||||
const r = resolveTrekcontinueProfile('/tmp/does-not-exist-plan-xyz.md', { flags: {} });
|
||||
assert.equal(r.profile, 'premium');
|
||||
assert.equal(r.profile_source, 'default');
|
||||
});
|
||||
|
||||
// validateProfileFile re-export sanity
|
||||
|
||||
test('validateProfileFile re-exports validateProfile (locked-interface compat)', () => {
|
||||
const r = validateProfileFile(join(REPO_ROOT, 'lib', 'profiles', 'economy.yaml'));
|
||||
assert.equal(r.valid, true);
|
||||
});
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
// tests/lib/profile-flag-coverage.test.mjs
|
||||
// SC #4 (docs side): every command file must document --profile + VOYAGE_PROFILE.
|
||||
// /trekcontinue.md must additionally describe profile-arv (inheritance) policy.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const COMMANDS_DIR = join(__dirname, '..', '..', 'commands');
|
||||
|
||||
const COMMAND_FILES = [
|
||||
'trekbrief.md',
|
||||
'trekresearch.md',
|
||||
'trekplan.md',
|
||||
'trekexecute.md',
|
||||
'trekreview.md',
|
||||
'trekcontinue.md',
|
||||
];
|
||||
|
||||
for (const filename of COMMAND_FILES) {
|
||||
test(`${filename} documents --profile flag`, () => {
|
||||
const content = readFileSync(join(COMMANDS_DIR, filename), 'utf-8');
|
||||
assert.match(content, /--profile/,
|
||||
`${filename} must contain --profile flag documentation`);
|
||||
});
|
||||
|
||||
test(`${filename} mentions VOYAGE_PROFILE env-var`, () => {
|
||||
const content = readFileSync(join(COMMANDS_DIR, filename), 'utf-8');
|
||||
assert.match(content, /VOYAGE_PROFILE/,
|
||||
`${filename} must mention VOYAGE_PROFILE env-var (resolution order)`);
|
||||
});
|
||||
}
|
||||
|
||||
test('trekcontinue.md documents inheritance policy (profile arves fra plan-frontmatter)', () => {
|
||||
const content = readFileSync(join(COMMANDS_DIR, 'trekcontinue.md'), 'utf-8');
|
||||
assert.match(content, /inheritance/,
|
||||
'trekcontinue.md must describe profile-arv (inheritance) policy from plan-frontmatter');
|
||||
});
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
// tests/lib/profile-resolver.test.mjs
|
||||
// v5.1.1 SC5 — non-interference cases for resolvePhaseModel().
|
||||
// Verifies the new highest-priority lookup step (brief.phase_signals[phase].model)
|
||||
// wins over --profile flag and VOYAGE_PROFILE env; falls through cleanly when
|
||||
// no brief signal is present.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { resolvePhaseModel } from '../../lib/profiles/resolver.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO_ROOT = join(__dirname, '..', '..');
|
||||
const FIXTURE = (name) => join(REPO_ROOT, 'tests', 'fixtures', name);
|
||||
|
||||
test('resolvePhaseModel — Case 1: brief signal wins over VOYAGE_PROFILE env', () => {
|
||||
// brief-effort-low.md pins all 4 phases to model: sonnet.
|
||||
// env says premium (would normally select opus). Brief must win.
|
||||
const r = resolvePhaseModel('research', FIXTURE('brief-effort-low.md'), [], { VOYAGE_PROFILE: 'premium' });
|
||||
assert.equal(r.model, 'sonnet', `brief signal should beat env; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.source, 'brief-signal');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — Case 2: brief signal wins over --profile flag', () => {
|
||||
// brief-effort-high.md pins all 4 phases to model: opus.
|
||||
// flag says economy (would normally select sonnet). Brief must win.
|
||||
const r = resolvePhaseModel('execute', FIXTURE('brief-effort-high.md'), ['--profile', 'economy'], {});
|
||||
assert.equal(r.model, 'opus', `brief signal should beat flag; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.source, 'brief-signal');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — Case 3: no phase_signals → fallthrough to --profile flag', () => {
|
||||
// brief-without-phase-signals fixture lacks phase_signals entirely.
|
||||
// --profile balanced is set. Should return balanced.phase_models.plan (= opus per yaml).
|
||||
const r = resolvePhaseModel('plan', FIXTURE('brief-without-phase-signals.md'), ['--profile', 'balanced'], {});
|
||||
assert.equal(r.model, 'opus', `balanced.plan should be opus; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.source, 'flag');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — Case 4: phase not in PHASE_SIGNAL_PHASES falls through gracefully', () => {
|
||||
// brief-effort-high.md has signals for the 4 supported phases.
|
||||
// Asking for 'continue' (not in PHASE_SIGNAL_PHASES) must fall through.
|
||||
// --profile premium is set, so continue resolves to premium.phase_models.continue (= opus).
|
||||
const r = resolvePhaseModel('continue', FIXTURE('brief-effort-high.md'), ['--profile', 'premium'], {});
|
||||
assert.equal(r.model, 'opus', `premium.continue should be opus; got ${JSON.stringify(r)}`);
|
||||
assert.ok(r.source !== 'brief-signal', 'continue must not resolve via brief-signal');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — Case 5 (defensive): missing brief file falls through cleanly', () => {
|
||||
// Non-existent path. Must not throw; must fall through to flag/env/default.
|
||||
const r = resolvePhaseModel('plan', '/nonexistent/brief.md', ['--profile', 'economy'], {});
|
||||
assert.equal(r.model, 'sonnet', 'economy.plan should be sonnet on fallthrough');
|
||||
assert.equal(r.source, 'flag');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — Case 6 (defensive): null briefPath falls through to default', () => {
|
||||
// null briefPath, no flag, no env → default = premium.
|
||||
const r = resolvePhaseModel('plan', null, [], {});
|
||||
assert.equal(r.model, 'opus', 'premium.plan default = opus');
|
||||
assert.equal(r.source, 'default');
|
||||
});
|
||||
|
||||
// v5.9 — fable profile composition (brief SC 4) + effort passthrough coherence
|
||||
|
||||
test('resolvePhaseModel — --profile fable: all six phases resolve model fable (no brief signal)', () => {
|
||||
for (const phase of ['brief', 'research', 'plan', 'execute', 'review', 'continue']) {
|
||||
const r = resolvePhaseModel(phase, null, ['--profile', 'fable'], {});
|
||||
assert.equal(r.model, 'fable', `fable.${phase} should be fable; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.source, 'flag');
|
||||
}
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — brief signal (opus) beats --profile fable (composition precedence pin)', () => {
|
||||
// brief-effort-high.md pins execute to model: opus. Brief must beat the fable profile.
|
||||
const r = resolvePhaseModel('execute', FIXTURE('brief-effort-high.md'), ['--profile', 'fable'], {});
|
||||
assert.equal(r.model, 'opus', `brief signal should beat fable profile; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.source, 'brief-signal');
|
||||
});
|
||||
|
||||
test('resolvePhaseModel — composed output carries {effort, model} atomically (v5.9 passthrough)', () => {
|
||||
// brief-effort-high.md: execute → {effort: high, model: opus}. The composed
|
||||
// resolver must return BOTH fields from one call — the split-CLI design this
|
||||
// passthrough replaced could drift effort and model apart.
|
||||
const r = resolvePhaseModel('execute', FIXTURE('brief-effort-high.md'), [], {});
|
||||
assert.equal(r.effort, 'high', `effort must pass through; got ${JSON.stringify(r)}`);
|
||||
assert.equal(r.model, 'opus');
|
||||
assert.equal(r.source, 'brief-signal');
|
||||
});
|
||||
|
|
@ -9,8 +9,8 @@
|
|||
// files (never tracked) and this test. A term is a plain substring, except a
|
||||
// flag, which must stand as its own token (`--engine`, `--profile`).
|
||||
//
|
||||
// When this test fails, remove the mention; add an EXEMPT entry only for a
|
||||
// line that must keep the term, with the reason.
|
||||
// Every group always runs. When this test fails, remove the mention; add an
|
||||
// EXEMPT entry only for a line that must keep the term, with the reason.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
|
|
@ -45,13 +45,25 @@ export const GROUPS = {
|
|||
],
|
||||
};
|
||||
|
||||
// Groups whose removal stage has not run yet. Each stage of R-b empties its
|
||||
// own entry; REMOVED_SURFACES_ALL=1 checks every group (the red proof).
|
||||
const PENDING = new Set(['R3']);
|
||||
|
||||
// Named exceptions: "path:line" -> reason. Each must still hit (a stale
|
||||
// exemption fails below).
|
||||
const EXEMPT = new Map([
|
||||
['docs/HANDOVER-CONTRACTS.md:81', 'public contract: an older brief\'s key is ignored (decisions 179/191)'],
|
||||
['docs/proevesett.md:139', 'dated v1/v2 measurement record of the fixed test set, kept unedited'],
|
||||
['docs/proevesett.md:143', 'dated v1/v2 measurement record of the fixed test set, kept unedited'],
|
||||
['docs/proevesett.md:146', 'dated v1/v2 measurement record of the fixed test set, kept unedited'],
|
||||
['lib/parsers/arg-parser.mjs:16', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:20', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:25', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:30', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:36', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:41', 'retired flag swallows its value'],
|
||||
['lib/parsers/arg-parser.mjs:46', 'retired flag swallows its value'],
|
||||
['tests/lib/arg-parser.test.mjs:285', 'retired flag swallows its value'],
|
||||
['tests/lib/arg-parser.test.mjs:286', 'retired flag swallows its value'],
|
||||
['tests/validators/brief-validator.test.mjs:200', 'older brief stays valid (decision 191)'],
|
||||
['tests/validators/brief-validator.test.mjs:203', 'older brief stays valid (decision 191)'],
|
||||
['tests/validators/brief-validator.test.mjs:205', 'older brief stays valid (decision 191)'],
|
||||
]);
|
||||
|
||||
const matches = (term, line) => (typeof term === 'string' ? line.includes(term) : term.test(line));
|
||||
|
|
@ -108,10 +120,7 @@ test('scope leaves out history, local files and this test', () => {
|
|||
});
|
||||
|
||||
for (const group of Object.keys(GROUPS)) {
|
||||
test(`${group}: no tracked file names a removed surface`, (t) => {
|
||||
if (PENDING.has(group) && process.env.REMOVED_SURFACES_ALL !== '1') {
|
||||
return t.skip(`${group} is not removed yet`);
|
||||
}
|
||||
test(`${group}: no tracked file names a removed surface`, () => {
|
||||
const all = tracked();
|
||||
assert.ok(all.length > 100, `git ls-files returned ${all.length} files; refusing to pass on an empty scan`);
|
||||
const hits = scan(ROOT, all, GROUPS[group]).filter(h => !EXEMPT.has(h));
|
||||
|
|
|
|||
|
|
@ -1043,7 +1043,7 @@ test('review 630 r9 / f8087ca4 criteria-runner.mjs:172: a check refused for its
|
|||
// (2) A brief SC with such a comment is BRIEF_SC_NOT_RUNNABLE, not prose-only.
|
||||
const brief = (sc) => [
|
||||
'---', 'type: trekbrief', 'brief_version: "2.2"', 'task: t', 'slug: s',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', 'phase_signals_partial: true', '---', '',
|
||||
'research_topics: 0', 'research_status: skipped', 'framing: new-direction', '---', '',
|
||||
'## TL;DR', '', 'A toy.', '', '## Intent', '', 'Why.', '', '## Goal', '', 'What.', '',
|
||||
'## Success Criteria', '', sc, '',
|
||||
].join('\n');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue