voyage/tests/lib/fix-round-4-5a-prose.test.mjs

402 lines
23 KiB
JavaScript

// tests/lib/fix-round-4-5a-prose.test.mjs
// Prose pins for fix round 4-5a (the serious findings, plan
// .claude/projects/2026-10-20-fix-round-serious-findings): the fixes that are
// text a session or a helper reads. The name is the brief's;
// lib/util/test-census.mjs counts it as a behaviour test (it does not match
// PIN_FILE_RE). Every test carries its id tag in brackets, and every tagged
// test fails on an assertion at ab6b32c (the round's start point).
// #23 code-correctness-reviewer cites a "MANDATORY OVERRIDE rule above"
// that is no longer there; the sentence says the rule itself.
// #24 research-scout says "Try Tavily first" but its tools held no Tavily
// tool; its tools line now holds both, in docs-researcher's order.
// #111 A reviewer drops a finding that feels off-tier only when it can say
// why it is wrong, and then as one Declined to Judge line with that
// reason; the skill's line agrees; an unknown rule_key is routed to
// `unverified`, not dropped (lib/review/coordinator-contract.mjs).
// Fix the SOURCE a pin guards, not the pin.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { existsSync, readFileSync, readdirSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { findSection } from '../../lib/parsers/plan-schema.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
/** A repo file's text; the file must exist. */
function read(rel) {
assert.ok(existsSync(join(ROOT, rel)), `missing file ${rel}`);
return readFileSync(join(ROOT, rel), 'utf8');
}
/** Whitespace (line breaks too) collapsed to one space. */
const flat = (s) => s.replace(/\s+/g, ' ').trim();
/**
* The text of a heading's section, up to the next heading of the same or a
* higher level outside a fence. The heading must exist.
*/
function section(text, heading) {
const level = heading.match(/^#+/)[0].length;
const s = findSection(text, heading.replace(/^#+ /, ''), level);
assert.ok(s, `missing ${heading}`);
const lines = s.text.split('\n');
let fenced = false;
for (let i = 1; i < lines.length; i++) {
if (/^\s{0,3}(`{3,}|~{3,})/.test(lines[i])) fenced = !fenced;
const m = !fenced && lines[i].match(/^(#+) /);
if (m && m[1].length < level) return lines.slice(0, i).join('\n');
}
return s.text;
}
/** From the line holding `marker` to the first line where `stop` is true, flattened. */
function partAt(rel, marker, stop) {
const lines = read(rel).split('\n');
const start = lines.findIndex((l) => l.includes(marker));
assert.ok(start !== -1, `${rel} must hold "${marker}"`);
let end = lines.length;
for (let i = start + 1; i < lines.length; i++) if (stop(lines[i])) { end = i; break; }
return flat(lines.slice(start, end).join(' '));
}
/** The single line in `text` that runs `lib/<mod>` (and holds `verb`, when given). */
function line(text, mod, verb = '') {
const esc = mod.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
const re = new RegExp(`^\\s*node \\$\\{CLAUDE_PLUGIN_ROOT\\}/lib/${esc}\\b`);
const hits = text.split('\n').filter((l) => re.test(l) && l.includes(verb));
assert.equal(hits.length, 1, `expected exactly one \`${mod}${verb ? ` ${verb}` : ''}\` line; found ${hits.length}`);
return hits[0].trim();
}
/** Run a command line from the prose with the plugin root filled in. */
function bash(cmd, cwd) {
return spawnSync('bash', ['-c', cmd.split('${CLAUDE_PLUGIN_ROOT}').join(ROOT)], { cwd, encoding: 'utf8' });
}
// --- #23 ---------------------------------------------------------------------------
test('[#23] no agent file refers to a MANDATORY OVERRIDE rule', () => {
const hits = readdirSync(join(ROOT, 'agents'))
.filter((f) => f.endsWith('.md'))
.filter((f) => read(`agents/${f}`).includes('MANDATORY OVERRIDE'));
assert.deepEqual(hits, [], 'an agent still points at a MANDATORY OVERRIDE rule that is not there');
const d5 = flat(section(read('agents/code-correctness-reviewer.md'), '### 5. Placeholder code — `PLACEHOLDER_IN_CODE` (MAJOR)'));
assert.match(d5, /Never accept "not implemented" placeholders as fine "for now"/);
});
// --- #24 ---------------------------------------------------------------------------
test('[#24] research-scout holds both Tavily tools, in docs-researcher\'s order, and still tries Tavily first', () => {
const text = read('agents/research-scout.md');
const fm = readFrontmatterFields(text);
assert.ok(fm.ok, fm.error);
assert.deepEqual(fm.fields.tools.value,
['WebSearch', 'WebFetch', 'Read', 'mcp__tavily__tavily_search', 'mcp__tavily__tavily_research']);
assert.match(text, /\*\*Try Tavily first\*\*/);
});
// --- #111 --------------------------------------------------------------------------
/** No bare "drop it" choice; a drop needs a reason, written as one Declined to Judge line. */
function dropsOnlyWithTrace(part, where) {
assert.doesNotMatch(part, /either drop (it|the finding)|or drop (it|the finding)\s*\.|drop (it|the finding) \(it was wrong\)/i,
`${where}: an off-tier finding may still be dropped with no trace`);
assert.match(part, /drop[^.]{0,60}only when you can say why it is wrong[^.]{0,120}Declined to Judge[^.]{0,60}with that reason/i,
`${where}: a drop must carry its reason on a Declined to Judge line`);
assert.match(part, /catalogue('s)? (tier|severity)/i, `${where}: the catalogue tier stays the default`);
}
const toBlank = (l) => l.trim() === '';
test('[#111] code-correctness-reviewer dimension 4: a nice-to-have test gap is dropped only with a Declined to Judge line', () => {
dropsOnlyWithTrace(partAt('agents/code-correctness-reviewer.md', 'When the brief explicitly asks for tests', toBlank),
'code-correctness-reviewer dimension 4');
});
test('[#111] code-correctness-reviewer severity rules: an off-tier finding is dropped only with a Declined to Judge line', () => {
dropsOnlyWithTrace(partAt('agents/code-correctness-reviewer.md', 'If a finding feels off-tier', toBlank),
'code-correctness-reviewer ## Severity rules');
});
test('[#111] code-correctness-reviewer rules: "No silent severity downgrades" drops only with a Declined to Judge line', () => {
dropsOnlyWithTrace(partAt('agents/code-correctness-reviewer.md', '**No silent severity downgrades.**', (l) => /^- /.test(l) || toBlank(l)),
'code-correctness-reviewer ## Rules');
});
test('[#111] brief-conformance-reviewer severity rules: a finding is dropped only with a Declined to Judge line', () => {
dropsOnlyWithTrace(partAt('agents/brief-conformance-reviewer.md', 'If a finding feels less severe than its catalogue tier', toBlank),
'brief-conformance-reviewer ## Severity rules');
});
test('[#111] reviewing-adversarially: keep the tier, or drop as wrong with its reason on a Declined to Judge line', () => {
const s5 = flat(section(read('skills/reviewing-adversarially/SKILL.md'), '### 5. No downgrade without a reason'));
assert.match(s5, /keep the tier, or drop the finding as wrong with its reason on a Declined to Judge line; never soften it in between/);
assert.doesNotMatch(s5, /keep the tier or drop the finding as wrong;/);
});
/** The stale "unknown rule keys are dropped" line now says they are routed to `unverified`. */
function routedNotDropped(part, where) {
assert.doesNotMatch(part, /dropped by (the coordinator's reasonableness filter|Pass 3)/i, `${where}: still says unknown rule keys are dropped`);
assert.match(part, /routed to `unverified`/, `${where}: must say an unknown rule key is routed to \`unverified\``);
}
const toBullet = (l) => /^- /.test(l) || toBlank(l) || /^#/.test(l);
test('[#111] code-correctness-reviewer: an unknown rule_key is routed to `unverified`, not dropped', () => {
routedNotDropped(partAt('agents/code-correctness-reviewer.md', '`rule_key` MUST be in the catalogue', toBullet),
'code-correctness-reviewer ## JSON output rules');
});
test('[#111] brief-conformance-reviewer: an unknown rule_key is routed to `unverified`, not dropped', () => {
routedNotDropped(partAt('agents/brief-conformance-reviewer.md', '`rule_key` MUST be in the catalogue', toBullet),
'brief-conformance-reviewer ## JSON output rules');
});
test('[#111] review-coordinator input: a rule_key outside the catalogue is routed to `unverified` by Pass 3', () => {
routedNotDropped(partAt('agents/review-coordinator.md', '**Rule catalogue** — `lib/review/rule-catalogue.mjs`', toBullet),
'review-coordinator ## Input');
});
// Steps 2-24 use these helpers; referenced here so the file states its interface.
test('helpers: line and bash are the shared line-runner pair', () => {
assert.equal(typeof line, 'function');
assert.equal(typeof bash, 'function');
assert.equal(line('x\n node ${CLAUDE_PLUGIN_ROOT}/lib/util/a.mjs go\n', 'util/a.mjs', 'go'), 'node ${CLAUDE_PLUGIN_ROOT}/lib/util/a.mjs go');
const r = bash('printf %s "${CLAUDE_PLUGIN_ROOT}"', ROOT);
assert.equal(r.status, 0);
assert.equal(r.stdout, ROOT);
});
// --- #2 ----------------------------------------------------------------------------
// planning-orchestrator.md is a reference document that nothing runs; the text
// that sent a reader to it ("re-run", "must emit", "should have prevented",
// Consumer/Producer) now names /trekplan.
test('[#2] /trekexecute never sends anyone to re-run the planning reference document', () => {
const text = flat(read('commands/trekexecute.md'));
assert.doesNotMatch(text, /re-run planning-orchestrator/i, 'still says to re-run planning-orchestrator');
assert.doesNotMatch(text, /planning-orchestrator must/i, 'still says planning-orchestrator must emit or include');
assert.doesNotMatch(text, /planning-orchestrator should have prevented/i, 'still says planning-orchestrator should have prevented this');
assert.match(text, /Re-run \/trekplan — the plan is not executable\./);
assert.match(text, /Fix: re-run \/trekplan — the plan must use v1\.7 format/);
assert.match(text, /Fix: re-run \/trekplan — every step needs a Manifest YAML block/);
assert.match(text, /\/trekplan's validator should have prevented this/);
});
test('[#2] sibling: commands/trekbrief.md names /trekplan as what finishes', () => {
const text = flat(read('commands/trekbrief.md'));
assert.doesNotMatch(text, /When the planning-orchestrator finishes/);
assert.match(text, /When \/trekplan finishes/);
});
test('[#2] sibling: agents/brief-reviewer.md names /trekplan Phase 4b as its reader', () => {
const text = flat(read('agents/brief-reviewer.md'));
assert.doesNotMatch(text, /`planning-orchestrator` Phase 1b/);
assert.match(text, /\/trekplan Phase 4b/);
});
test('[#2] sibling: docs/HANDOVER-CONTRACTS.md has /trekplan, not planning-orchestrator, as Consumer, Producer and the one that must run', () => {
const text = read('docs/HANDOVER-CONTRACTS.md');
assert.doesNotMatch(text, /^\*\*(Consumer|Producer):\*\*[^\n]*planning-orchestrator/m, 'planning-orchestrator is still a Consumer or Producer');
assert.doesNotMatch(text, /planning-orchestrator\)? \*\*must\*\* run/, 'planning-orchestrator is still the one that must run the validator');
const lines = text.split('\n');
for (const want of [
'**Consumer:** `/trekplan` Phase 1 (project-dir auto-discovery) + `/trekplan` Phase 3 and Phase 7 (consumes findings as context).',
'**Consumer:** `/trekplan` Phase 1 (architecture-discovery) + `/trekplan` Phase 7 (cross-reference architecture-note as priors during synthesis).',
'**Producer:** `/trekplan` Phase 8 (plan synthesis, then the schema self-check via `plan-validator --strict`).',
]) assert.ok(lines.includes(want), `missing line: ${want}`);
assert.match(text, /`\/trekplan` Phase 8 \*\*must\*\* run `plan-validator --strict` before handing the plan to plan-critic\./);
});
test('[#2] sibling: docs/architecture.md names /trekplan Phase 8 as the plan-validator caller', () => {
const text = read('docs/architecture.md');
assert.doesNotMatch(text, /`planning-orchestrator` Phase 5\.5/);
assert.match(text, /^- `\/trekplan` Phase 8 → `plan-validator --strict` \(replaces 3 `grep -cE` calls\)$/m);
});
// --- #26 ---------------------------------------------------------------------------
test('[#26] session-decomposer says to re-run /trekplan, not the planning reference document', () => {
const text = flat(read('agents/session-decomposer.md'));
assert.doesNotMatch(text, /Re-run planning-orchestrator/i);
assert.match(text, /Re-run \/trekplan\./);
});
test('[#26] the three reference documents hold read-only tools', () => {
for (const name of ['planning-orchestrator', 'research-orchestrator', 'review-orchestrator']) {
const fm = readFrontmatterFields(read(`agents/${name}.md`));
assert.ok(fm.ok, fm.error);
assert.deepEqual(fm.fields.tools.value, ['Read', 'Glob', 'Grep'], `${name}: tools must be Read, Glob, Grep`);
assert.equal(fm.fields.model.value, 'opus', `${name}: model stays opus`);
}
});
// --- #52 ---------------------------------------------------------------------------
// /trekplan's "execute" answer began implementing the plan in the planning
// session, past /trekexecute's approval check (K2), test lock and step helpers.
// It now prints the /trekexecute line and stops.
/** The `/trekexecute` line /trekplan prints, in both forms. */
function printsTrekexecuteLine(part, where) {
assert.match(part, /`\/trekexecute --project \{project_dir\}`/, `${where}: must print the /trekexecute --project line`);
assert.match(part, /`\/trekexecute \{plan_path\}`/, `${where}: must print the /trekexecute {plan_path} line`);
}
/**
* No text that has /trekplan build the plan itself: no "Begin implementing",
* and no "in this session" other than the one prohibition sentence.
*/
function neverImplementsHere(part, where) {
const rest = part.split('Do not implement in this session.').join('');
assert.doesNotMatch(rest, /Begin implementing/i, `${where}: still begins implementing`);
assert.doesNotMatch(rest, /in this session/i, `${where}: still implements in this session`);
}
test('[#52] Phase 11 "execute": prints the /trekexecute line and stops, never implements in this session', () => {
const block = flat(section(read('commands/trekplan.md'), '### "execute" / "go" / "start"'));
neverImplementsHere(block, 'Phase 11 "execute"');
assert.match(block, /^### "execute" \/ "go" \/ "start" Do not implement in this session\./);
printsTrekexecuteLine(block, 'Phase 11 "execute"');
assert.match(block, /without a project directory — and stop\./);
assert.match(block, /`\/trekexecute` is what checks the plan's approval \(K2\), locks the tests, runs one step helper per step and the review\./);
});
test('[#52] Phase 11 "team": no team path, the same /trekexecute line, then stop', () => {
const block = flat(section(read('commands/trekplan.md'), '### "execute with team" / "team"'));
neverImplementsHere(block, 'Phase 11 "team"');
assert.doesNotMatch(block, /proceed exactly as for "execute"/, 'Phase 11 "team": still proceeds as "execute" did');
assert.match(block, /There is no team path\. Print the same `\/trekexecute` line and stop; parallel work is `--decompose` followed by `\/trekexecute`'s session orders\./);
});
test('[#52] Phase 10 menu: "execute" is for the /trekexecute line, and a stopped gate withholds it', () => {
const p10 = flat(section(read('commands/trekplan.md'), '## Phase 10 — Present and refine'));
assert.doesNotMatch(p10, /start implementing/i, 'Phase 10 still offers or withholds "start implementing"');
assert.match(p10, /- Say \*\*"execute"\*\* for the `\/trekexecute` line to run ← only when the gate exited 0/);
assert.match(p10, /`Execution is blocked: \{the gate's reasons\}` and do not print the `\/trekexecute` line, whatever is said next, until the gate exits 0\./);
});
test('[#52] sibling: plan mode step 3 prints the /trekexecute line and stops', () => {
const pm = flat(section(read('commands/trekplan.md'), '### In plan mode'));
assert.doesNotMatch(pm, /Then continue with the operator's choice \(Phase 11\)\./, 'plan mode still continues with the operator\'s choice');
assert.match(pm, /Then print the `\/trekexecute` line \(Phase 11\) and stop — leaving plan mode is not an order to implement\./);
});
test('[#52] hard rule: "No premature execution" holds, and nothing in Phases 10-11 implements the plan', () => {
const text = read('commands/trekplan.md');
const rules = flat(section(text, '## Hard rules'));
assert.match(rules, /\*\*No premature execution\*\*: Do not modify any project files until the user explicitly approves the plan, and never while `plan-gate\.mjs` exits non-zero on it\./);
for (const h of ['## Phase 10 — Present and refine', '## Phase 11 — Handoff']) {
const part = flat(section(text, h));
neverImplementsHere(part, h);
assert.doesNotMatch(part, /start implementing|Follow the plan exactly|Mark each step complete/i, `${h}: still has /trekplan build the plan`);
}
});
test('[#52] sibling: agents/planning-orchestrator.md says "execute" gives the /trekexecute line', () => {
const text = read('agents/planning-orchestrator.md');
assert.doesNotMatch(text, /Say "execute" to implement/, 'planning-orchestrator still says "execute" implements');
assert.ok(text.split('\n').includes('- Say "execute" for the /trekexecute line'), 'missing line: - Say "execute" for the /trekexecute line');
});
// --- #48 / #12 / #57 ---------------------------------------------------------------
// The planning and research spawn sites pinned `model: "opus"`, past the
// resolved model (brief signal > profile > agent frontmatter); every spawn now
// takes `phase_signal_result.model`. No `maxTurns` budget is promised: the
// Agent tool takes none, so "scale turns by size" was a promise nothing kept.
/**
* From the line holding `marker` (it must be the only one) to the end of the
* blank-line-separated paragraph after the marker's own paragraph, flattened.
*/
function paraAfter(rel, marker) {
const lines = read(rel).split('\n');
const hits = lines.filter((l) => l.includes(marker)).length;
assert.equal(hits, 1, `${rel}: expected exactly one line holding "${marker}"; found ${hits}`);
const start = lines.findIndex((l) => l.includes(marker));
let i = start;
while (i < lines.length && lines[i].trim() !== '') i++;
while (i < lines.length && lines[i].trim() === '') i++;
while (i < lines.length && lines[i].trim() !== '') i++;
return flat(lines.slice(start, i).join('\n'));
}
const PASS_MODEL = /Pass `model: \{phase_signal_result\.model\}` at this spawn; without it \(the `--brief` branch, where the resolver does not run, or a resolver that printed nothing\) the agent's own `model:` applies\./;
test('[#48] commands/trekplan.md pins no spawn to `model: "opus"`', () => {
assert.doesNotMatch(read('commands/trekplan.md'), /model: "opus"/);
});
for (const [where, marker] of [
['session-decomposer (Phase 1.5)', 'Launch the **session-decomposer** agent:'],
['brief-reviewer (Phase 4b)', 'Launch the **brief-reviewer** agent.'],
['### Always launch (Phase 5)', '### Always launch (all codebase sizes):'],
['research-scout (Phase 5)', 'launch **research-scout** in'],
['plan-critic and scope-guardian (Phase 9)', 'it traces into `## Verification`.'],
]) {
test(`[#48] ${where}: the spawn takes phase_signal_result.model, the agent's own model: as the fallback`, () => {
const part = paraAfter('commands/trekplan.md', marker);
assert.match(part, PASS_MODEL, `${where}: missing the resolved-model paragraph after "${marker}"`);
assert.doesNotMatch(part, /model: "opus"/);
});
}
test('[#48] convention-scanner (Phase 5): the spawn takes phase_signal_result.model, the frontmatter as the fallback', () => {
const part = paraAfter('commands/trekplan.md', 'use the `convention-scanner` plugin agent');
assert.match(part, /\(model: phase_signal_result\.model, the agent's frontmatter as the fallback\)/);
assert.doesNotMatch(part, /model: "opus"/);
});
test('[#48] deep-dives (Phase 6): the spawn takes phase_signal_result.model, else opus', () => {
const part = paraAfter('commands/trekplan.md', 'spawn a targeted deep-dive agent');
assert.match(part, /\(model: phase_signal_result\.model, else opus — Explore is Claude Code's built-in agent and has no Voyage frontmatter to fall back to, subagent_type: "Explore"\)/);
assert.doesNotMatch(part, /model: "opus"/);
});
test('[#48] sibling: agents/planning-orchestrator.md:137 takes phase_signal_result.model', () => {
const text = read('agents/planning-orchestrator.md');
assert.doesNotMatch(text, /model: "opus"/);
const part = partAt('agents/planning-orchestrator.md', '**Convention Scanner** — use the `convention-scanner` plugin agent', toBlank);
assert.match(part, /\(model: phase_signal_result\.model, the agent's frontmatter as the fallback\)/);
});
test('[#12] trekresearch Phase 5 follow-ups take phase_signal_result.model, else opus', () => {
const text = read('commands/trekresearch.md');
const p5 = flat(section(text, '## Phase 5 — Targeted follow-ups'));
assert.doesNotMatch(p5, /model: "opus"/);
const part = partAt('commands/trekresearch.md', 'launch a targeted follow-up agent', toBlank);
assert.match(part, /launch a targeted follow-up agent \(model: phase_signal_result\.model, else opus — a follow-up may be one of Claude Code's built-in agents, which have no Voyage frontmatter to fall back to; see Launch rules\)/);
assert.match(flat(text), /captured as `phase_signal_result` and used at Agent-spawn sites in Phase 4 and Phase 5 to inject the resolved model/);
});
test('[#57] commands/trekresearch.md promises no maxTurns budget', () => {
const text = read('commands/trekresearch.md');
assert.doesNotMatch(text, /maxTurns/);
assert.match(text, /- convention-scanner: medium\+ codebases only \(50\+ files\)/);
});
test('[#57] sibling: commands/trekplan.md promises no maxTurns budget, and Phase 4 calibrates no turns', () => {
const text = read('commands/trekplan.md');
assert.doesNotMatch(text, /maxTurns/);
const p4 = flat(section(text, '## Phase 4 — Codebase sizing'));
assert.doesNotMatch(p4, /calibrate agent turns/);
assert.match(p4, /Determine codebase scale to decide whether convention-scanner runs\./);
assert.match(text, /^\*\*All agents run for all codebase sizes\.\*\*$/m);
});
test('[#57] sibling: agents/planning-orchestrator.md promises no maxTurns budget and scales no turns', () => {
const text = read('agents/planning-orchestrator.md');
assert.doesNotMatch(text, /maxTurns/);
assert.doesNotMatch(text, /Scale turns/i);
assert.match(text, /^\*\*All agents run for all codebase sizes\.\*\*$/m);
assert.match(text, /^- \*\*Adaptive:\*\* All agents run for all sizes\.$/m);
});
test('keep: /trekexecute\'s controller and its model note stay pinned to opus', () => {
const text = read('commands/trekexecute.md');
assert.match(text, /`subagent_type: "general-purpose"`, `model: "opus"`, no `name`/);
assert.match(flat(text), /controller's `model: "opus"` — so `phase_signal_result\.model` still has no spawn site here/);
});