open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

merge: main (7a skills workshop, 9 rettepakke) into wave2-skills-research

Browse source 
Conflict: CLAUDE.md command table. Kept both sides: this branch's
/trekresearch row (the external agents preload researching-sources) and
main's /trekplan row (open minors listed for the operator).

researching-sources adapted to contract S1, now that 7a's form check and
routing test run over it:
- SKILL.md takes the fixed headings (Overview, When to use, Process, Red
  flags, Enforced by, Credits); the five rules and the confidence scale move
  under Process unchanged; Red flags lists the thoughts the baseline runs
  showed; "text only" becomes "prose only" as the form check reads it.
- The description adds "official" documentation and "looking up online what
  an error message means or what causes it": the error-lookup case ranked
  #3 behind debugging. That prompt also said "our tests fail", which is
  debugging's job; it now asks only what the error means and where that is
  documented. No threshold changed.
- Eval cases carry the routing tags (skill-trigger / skill-no-trigger /
  skill-effect, skill=, owner= task-finder and trekplan, from=), 7a's
  Skill grader pattern, arm: both on the no-trigger graders, and the
  judge-model line on the effect case.
- The `## Skills` row in README and CLAUDE.md.
- tests/lib/skills-research-prose-pins.test.mjs drops its two form pins
  (skill-lint is the authority now): 14 -> 12.

npm test on the merged tree: 2055 = 2053 pass / 0 fail / 2 skipped
(main 2043 + 12).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-09 19:34:38 +02:00
commit e226c33916
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
67 changed files with 2732 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

251
tests/lib/skill-lint.test.mjs Normal file
View file

@ -0,0 +1,251 @@
// tests/lib/skill-lint.test.mjs
// The skill form check (S1): lib/skills/skill-lint.mjs. Model-free, part of
// `npm test`. Each rule is shown biting on a skill built for it (known
// positive) and passing a skill that keeps the form (known negative); the last
// tests run the check over the skills Voyage ships.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { lintSkill, lintSkillsRoot, REQUIRED_HEADINGS, EXEMPTIONS } from '../../lib/skills/skill-lint.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'skill-lint.mjs');
const BODY = [
'# Example', '',
'## Overview', 'One line.', '',
'## When to use', '- when the example applies', '- not when it does not', '',
'## Process', '1. Do the thing.', '',
'## Red flags', '- skipping the thing', '',
'## Enforced by', '- prose only: nothing checks this.', '',
'## Credits', '- Voyage\'s own content.', '',
].join('\n');
const GOOD_FM = [
'---',
'name: good-skill',
'description: Checks an example the same way every time. Use when an example needs checking.',
'---',
].join('\n');
/** A plugin-shaped temp root with skills/<name>/SKILL.md; returns the root. */
function plugin(skills) {
const root = mkdtempSync(join(tmpdir(), 'skill-lint-'));
for (const [name, text] of Object.entries(skills)) {
mkdirSync(join(root, 'skills', name), { recursive: true });
writeFileSync(join(root, 'skills', name, 'SKILL.md'), text);
}
return root;
}
const codes = (r) => r.errors.map((e) => e.code).sort();
function withSkill(name, text, fn) {
const root = plugin({ [name]: text });
try { return fn(lintSkill(join(root, 'skills', name), { pluginRoot: root }), root); }
finally { rmSync(root, { recursive: true, force: true }); }
}
// ---- the test that fails first (ordre 7a) ----
test('skill-lint: a skill without "Use when" and with an unquoted colon is red', () => {
const text = [
'---',
'name: bad-skill',
'description: Checks things: all of them, every time.',
'---',
BODY,
].join('\n');
withSkill('bad-skill', text, (r) => {
assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), codes(r).join(','));
assert.ok(codes(r).includes('SKILL_DESC_NO_USE_WHEN'), codes(r).join(','));
});
});
test('skill-lint: a skill that keeps the form is green (known negative)', () => {
withSkill('good-skill', `${GOOD_FM}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
// ---- one rule at a time ----
test('skill-lint: name must equal the directory name and be kebab-case', () => {
withSkill('other-dir', `${GOOD_FM}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_MISMATCH')));
const fm = GOOD_FM.replace('name: good-skill', 'name: Good_Skill');
withSkill('Good_Skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
const reserved = GOOD_FM.replace('name: good-skill', 'name: claude-helper');
withSkill('claude-helper', `${reserved}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
});
test('skill-lint: description is required, at most 1 024 characters', () => {
const none = GOOD_FM.replace(/^description:.*$/m, 'license: MIT');
withSkill('good-skill', `${none}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_MISSING')));
const long = `Checks an example. Use when ${'an example needs checking and '.repeat(40)}done.`;
assert.ok(long.length > 1024);
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${long}`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_TOO_LONG')));
const exact = `Use when x. ${'y'.repeat(1024 - 12)}`;
assert.equal(exact.length, 1024);
const fm2 = GOOD_FM.replace(/^description:.*$/m, `description: ${exact}`);
withSkill('good-skill', `${fm2}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_TOO_LONG')));
});
test('skill-lint: description is in the third person', () => {
for (const d of [
'I check an example. Use when an example needs checking.',
'Checks your example. Use when an example needs checking.',
'Use when you need an example checked.',
'We check examples. Use when an example needs checking.',
]) {
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${d}`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON'), d));
}
const ok = GOOD_FM.replace(/^description:.*$/m, 'description: Checks an example, i.e. one case. Use when the user asks for a check.');
withSkill('good-skill', `${ok}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON')));
});
test('skill-lint: YAML traps are caught: unquoted colon, comment marker, tab, unclosed quote, leading indicator', () => {
for (const line of [
'description: Use when x: y happens.',
'description: Use when x happens #1.',
'description:\tUse when x happens.',
'description: "Use when x happens.',
'description: `Use` when x happens.',
'description: @Use when x happens.',
]) {
const fm = GOOD_FM.replace(/^description:.*$/m, line);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), line));
}
// Quoted, the colon is safe.
const quoted = GOOD_FM.replace(/^description:.*$/m, 'description: "Checks x: y. Use when x happens."');
withSkill('good-skill', `${quoted}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: a multi-line block-scalar description is read whole', () => {
const fm = ['---', 'name: good-skill', 'description: |', ' Checks an example the same way: every time.',
' Use when an example needs checking.', '---'].join('\n');
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
const f = readFrontmatterFields(`${fm}\n${BODY}`);
assert.equal(f.fields.description.value, 'Checks an example the same way: every time.\nUse when an example needs checking.\n');
const folded = readFrontmatterFields(['---', 'description: >-', ' one', ' two', '---', ''].join('\n'));
assert.equal(folded.fields.description.value, 'one two');
});
test('skill-lint: disable-model-invocation in any true spelling is refused', () => {
for (const v of ['true', 'True', 'yes', 'on', '1']) {
const fm = GOOD_FM.replace('---\nname', `---\ndisable-model-invocation: ${v}\nname`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_MODEL_INVOCATION_DISABLED'), v));
}
const off = GOOD_FM.replace('---\nname', '---\ndisable-model-invocation: false\nname');
withSkill('good-skill', `${off}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: only Claude Code\'s own frontmatter fields are accepted', () => {
const fm = GOOD_FM.replace('---\nname', '---\ntrigger: always\nname');
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_UNKNOWN_FIELD')));
const known = GOOD_FM.replace('---\nname', '---\nlicense: MIT\nuser-invocable: true\nname');
withSkill('good-skill', `${known}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: SKILL.md must stay under 500 lines', () => {
const pad = Array.from({ length: 500 }, (_, i) => `- line ${i}`).join('\n');
withSkill('good-skill', `${GOOD_FM}\n${BODY}\n${pad}\n`, (r) => assert.ok(codes(r).includes('SKILL_TOO_LONG')));
});
test('skill-lint: the fixed headings must all be there, in order', () => {
assert.deepEqual(REQUIRED_HEADINGS, ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits']);
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('## Red flags', '## Warnings')}`,
(r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
const swapped = BODY.replace('## Overview\nOne line.', '## Process\n1. x.').replace('## Process\n1. Do the thing.', '## Overview\nOne line.');
withSkill('good-skill', `${GOOD_FM}\n${swapped}`, (r) => assert.ok(codes(r).includes('SKILL_HEADING_ORDER')));
// A heading inside a fenced code block does not count.
const fenced = BODY.replace('## Credits', '```\n## Credits\n```');
withSkill('good-skill', `${GOOD_FM}\n${fenced}`, (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
});
test('skill-lint: "Enforced by" names a check that exists, or says prose only', () => {
const none = BODY.replace('- prose only: nothing checks this.', '- something, somewhere.');
withSkill('good-skill', `${GOOD_FM}\n${none}`, (r) => assert.ok(codes(r).includes('SKILL_ENFORCED_BY_VAGUE')));
const real = BODY.replace('- prose only: nothing checks this.', '- `lib/check.mjs` stops it.');
const root = plugin({ 'good-skill': `${GOOD_FM}\n${real}` });
try {
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
mkdirSync(join(root, 'lib'));
writeFileSync(join(root, 'lib', 'check.mjs'), '');
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('skill-lint: files the skill points to must exist', () => {
const linked = BODY.replace('One line.', 'One line. See [the detail](reference/detail.md).');
const root = plugin({ 'good-skill': `${GOOD_FM}\n${linked}` });
try {
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
mkdirSync(join(root, 'skills', 'good-skill', 'reference'));
writeFileSync(join(root, 'skills', 'good-skill', 'reference', 'detail.md'), '# detail\n');
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('skill-lint: paths under "Credits" name the source repository, not this plugin', () => {
const credited = BODY.replace("- Voyage's own content.", '- owner/repo @abc1234, `skills/tdd/SKILL.md`: MIT.');
withSkill('good-skill', `${GOOD_FM}\n${credited}`, (r) => assert.deepEqual(r.errors, []));
// The same path anywhere else must exist.
const elsewhere = BODY.replace('One line.', 'One line, see `skills/tdd/SKILL.md`.');
withSkill('good-skill', `${GOOD_FM}\n${elsewhere}`, (r) => assert.ok(codes(r).includes('SKILL_REF_MISSING')));
});
test('skill-lint: no shell injection (inline or fenced), since Voyage advises turning it off', () => {
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', 'Status: !`git status`')}`,
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```!\ngit status\n```')}`,
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
// A plain fenced bash block is fine; Claude reads it, nothing runs it on load.
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```bash\nnpm test\n```')}`,
(r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: an exemption lives in the check, never in the skill', () => {
assert.ok(EXEMPTIONS instanceof Map);
for (const [name, entry] of EXEMPTIONS) {
assert.ok(Array.isArray(entry.codes) && entry.codes.length > 0, `${name}: exempt codes`);
assert.ok(typeof entry.reason === 'string' && entry.reason.length > 10, `${name}: a reason`);
}
const self = `${GOOD_FM}\n<!-- skill-lint-exempt: SKILL_MISSING_HEADING -->\n${BODY}`;
withSkill('good-skill', self, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
const meta = GOOD_FM.replace('---\nname', '---\nmetadata:\n lint-exempt: true\nname');
withSkill('good-skill', `${meta}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
});
test('skill-lint CLI: 0 clean, 1 findings, 2 could not run', () => {
const good = plugin({ 'good-skill': `${GOOD_FM}\n${BODY}` });
const bad = plugin({ 'bad-skill': `---\nname: bad-skill\ndescription: Checks: things.\n---\n${BODY}` });
try {
const g = spawnSync('node', [CLI, '--root', good], { encoding: 'utf8' });
assert.equal(g.status, 0, g.stdout + g.stderr);
const b = spawnSync('node', [CLI, '--root', bad, '--json'], { encoding: 'utf8' });
assert.equal(b.status, 1, b.stdout + b.stderr);
assert.ok(JSON.parse(b.stdout).skills['bad-skill'].errors.length > 0);
const n = spawnSync('node', [CLI, '--root', join(good, 'nope')], { encoding: 'utf8' });
assert.equal(n.status, 2, n.stdout + n.stderr);
} finally {
rmSync(good, { recursive: true, force: true });
rmSync(bad, { recursive: true, force: true });
}
});
// ---- the skills Voyage ships ----
test('skill-lint: every skill Voyage ships passes the form check', () => {
const r = lintSkillsRoot(ROOT);
assert.ok(Object.keys(r.skills).length >= 2, `found ${Object.keys(r.skills).join(',')}`);
for (const [name, res] of Object.entries(r.skills)) {
assert.deepEqual(res.errors, [], `${name}: ${JSON.stringify(res.errors, null, 2)}`);
}
});

215
tests/lib/skill-routing.test.mjs Normal file
View file

@ -0,0 +1,215 @@
// tests/lib/skill-routing.test.mjs
// The model-free routing test (P10, "Bygg nå"): lib/skills/routing.mjs ranks
// the descriptions of every skill, the 7 commands and the 21 spawnable agents
// against prompts, with no model. A collision between two descriptions fails;
// each skill needs 3 prompts where it should fire (top 3) and 2 where it
// should not, and for those a named owner must rank higher. The thresholds are
// pinned here: they are never lowered.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
THRESHOLDS, tokenize, buildCorpus, rank, collisions, loadEntries, loadCases, checkRouting,
} from '../../lib/skills/routing.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');
const skillMd = (name, description) =>
`---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`;
/** Plugin-shaped temp root: skills, commands, agents, evals. */
function plugin({ skills = {}, commands = {}, agents = {}, cases = {} }) {
const root = mkdtempSync(join(tmpdir(), 'skill-routing-'));
for (const [n, d] of Object.entries(skills)) {
mkdirSync(join(root, 'skills', n), { recursive: true });
writeFileSync(join(root, 'skills', n, 'SKILL.md'), skillMd(n, d));
}
mkdirSync(join(root, 'commands'), { recursive: true });
for (const [n, d] of Object.entries(commands)) {
writeFileSync(join(root, 'commands', `${n}.md`), `---\nname: ${n}\ndescription: ${d}\n---\n`);
}
mkdirSync(join(root, 'agents'), { recursive: true });
for (const [n, d] of Object.entries(agents)) {
writeFileSync(join(root, 'agents', `${n}.md`), `---\nname: ${n}\ndescription: |\n ${d}\nmodel: opus\n---\n`);
}
for (const [n, { tags, prompt }] of Object.entries(cases)) {
mkdirSync(join(root, 'evals', n, 'graders'), { recursive: true });
writeFileSync(join(root, 'evals', n, 'prompt.md'),
`---\ndescription: "case ${n}"\ntags: [${tags.join(', ')}]\nruns: 1\n---\n\n${prompt}\n`);
}
return root;
}
function withPlugin(spec, fn) {
const root = plugin(spec);
try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); }
}
const codes = (r) => r.errors.map((e) => e.code);
// ---- the test that fails first (ordre 7a) ----
test('routing: two test skills with near-identical descriptions are red (collision)', () => {
withPlugin({
skills: {
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
},
}, (root) => {
const r = checkRouting(root);
assert.ok(codes(r).includes('ROUTING_COLLISION'), JSON.stringify(r.errors));
const c = r.errors.find((e) => e.code === 'ROUTING_COLLISION');
assert.deepEqual([c.a, c.b].sort(), ['skill:order-imports', 'skill:tidy-imports']);
});
});
test('routing: distinct descriptions do not collide (known negative)', () => {
const corpus = buildCorpus([
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file. Use when imports are out of order.' },
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk. Use when the log directory grows.' },
]);
assert.deepEqual(collisions(corpus).filter((c) => c.level === 'error'), []);
});
// ---- the pieces ----
test('routing: thresholds are pinned and never lowered', () => {
assert.deepEqual({ ...THRESHOLDS }, {
COLLISION_ERROR: 0.75,
COLLISION_WARN: 0.5,
POSITIVE_TOP_K: 3,
MIN_POSITIVE: 3,
MIN_NEGATIVE: 2,
MIN_EFFECT: 1,
MIN_RANK1_RATE: 1,
});
assert.ok(Object.isFrozen(THRESHOLDS));
});
test('routing: the tokenizer folds case, drops stop words and light-stems', () => {
assert.deepEqual(tokenize('Use when the Tests are FAILING'), tokenize('test fail'));
assert.deepEqual(tokenize('a an the of'), []);
});
test('routing: rank puts the matching description first, and a no-overlap prompt scores 0', () => {
const corpus = buildCorpus([
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file.' },
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk.' },
]);
assert.equal(rank('the log files fill the disk', corpus)[0].id, 'skill:rotate-logs');
assert.ok(rank('zebra quantum', corpus).every((r) => r.score === 0));
});
test('routing: the reader takes block-scalar agent descriptions and skips orchestrator reference docs', () => {
withPlugin({
agents: { 'plan-critic': 'Use this agent when a plan needs adversarial review.' },
commands: { trekplan: 'Deep implementation planning from a task brief.' },
}, (root) => {
writeFileSync(join(root, 'agents', 'planning-orchestrator.md'),
'---\nname: planning-orchestrator\ndescription: Reference document, not a spawnable capability.\n---\n');
const ids = loadEntries(root).map((e) => e.id).sort();
assert.deepEqual(ids, ['agent:plan-critic', 'command:trekplan']);
assert.equal(loadEntries(root).find((e) => e.id === 'agent:plan-critic').description.trim(),
'Use this agent when a plan needs adversarial review.');
});
});
test('routing: cases come from evals/ tags; a skill below 3 / 2 / 1 is red', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: {
'skill-tidy-imports-fires-1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines in main.js' },
},
}, (root) => {
const cases = loadCases(root);
assert.equal(cases.length, 1);
assert.deepEqual([cases[0].kind, cases[0].skill, cases[0].from], ['trigger', 'tidy-imports', 'test']);
assert.ok(codes(checkRouting(root)).includes('ROUTING_BELOW_MINIMUM'));
});
});
test('routing: a positive outside the top 3 is red; a negative won by the skill itself is red', () => {
const spec = {
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
commands: {
'rotate-logs': 'Compresses and deletes old log files on disk.',
'scan-ports': 'Lists the open network ports of a host.',
'count-words': 'Counts the words of a text document.',
},
cases: {
'p1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines' },
'p2': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'import lines are out of order' },
'p3': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'compress the old logs on disk and list ports and count words' },
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'delete old log files' },
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'sort the import lines of the log file' },
'e1': { tags: ['skill-effect', 'skill=tidy-imports', 'from=test'], prompt: 'x' },
},
};
withPlugin(spec, (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_POSITIVE_MISSED' && e.case === 'p3'), JSON.stringify(r.errors));
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n2'), JSON.stringify(r.errors));
assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
});
});
test('routing: a negative must name an owner that exists and outranks the skill', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: {
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'delete old log files' },
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=nobody', 'from=test'], prompt: 'delete old log files' },
},
}, (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_NO_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n2'), JSON.stringify(r.errors));
});
});
test('routing: every case names the event it came from (P10)', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: { 'p1': { tags: ['skill-trigger', 'skill=tidy-imports'], prompt: 'sort the import lines' } },
}, (root) => assert.ok(codes(checkRouting(root)).includes('ROUTING_CASE_NO_SOURCE')));
});
test('routing CLI: 0 clean, 1 findings, 2 could not run', () => {
withPlugin({
skills: {
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
},
}, (root) => {
const b = spawnSync('node', [CLI, '--root', root, '--json'], { encoding: 'utf8' });
assert.equal(b.status, 1, b.stdout + b.stderr);
assert.ok(JSON.parse(b.stdout).errors.length > 0);
const n = spawnSync('node', [CLI, '--root', join(root, 'nope')], { encoding: 'utf8' });
assert.equal(n.status, 2, n.stdout + n.stderr);
});
});
// ---- Voyage's own catalogue ----
test('routing: Voyage\'s catalogue holds every skill, the 7 commands and the 21 spawnable agents', () => {
const entries = loadEntries(ROOT);
const by = (k) => entries.filter((e) => e.kind === k);
assert.equal(by('command').length, 7, by('command').map((e) => e.name).join(','));
assert.equal(by('agent').length, 21, by('agent').map((e) => e.name).join(','));
assert.ok(by('skill').length >= 2, by('skill').map((e) => e.name).join(','));
for (const e of entries) assert.ok(e.description.trim().length > 20, `${e.id}: description read as "${e.description}"`);
assert.equal(new Set(entries.map((e) => e.name)).size, entries.length, 'names must be unique across kinds');
});
test('routing: Voyage\'s catalogue is green: no collision, every case routes, rank-1 floor held', () => {
const r = checkRouting(ROOT);
assert.deepEqual(r.errors, [], JSON.stringify(r.errors, null, 2));
assert.ok(r.stats.positives >= 6 && r.stats.negatives >= 4, JSON.stringify(r.stats));
});

38
tests/lib/skills-research-prose-pins.test.mjs
View file

@ -2,10 +2,10 @@
// Wave 2, order 7b: the `researching-sources` skill and its wiring.
//
// What is pinned, and why:
// • the skill's form (name = folder, "Use when", <= 1 024 characters, under
// 500 lines, no `disable-model-invocation`) until order 7a's form check
// covers every skill — then that check is the authority and the form pins
// here can go;
// • the skill's form is not pinned here: order 7a's form check
// (lib/skills/skill-lint.mjs) and routing test (lib/skills/routing.mjs)
// run over every skill in `npm test` and are the authority (contract S1);
// • the five rules, and the two measured red-baseline lines, stay in it;
// • every web-search agent preloads the skill through its `skills:` line, and
// /trekresearch names it where it starts them ("commands are the trunk");
// • the shared source rules that moved into the skill are gone from the
@ -57,10 +57,6 @@ function listField(fm, key) {
return [raw];
}
function unquote(s) {
return s.startsWith('"') ? JSON.parse(s) : s;
}
function section(text, startRe, endRe) {
const start = text.search(startRe);
assert.ok(start >= 0, `section ${startRe} not found`);
@ -69,25 +65,7 @@ function section(text, startRe, endRe) {
return end < 0 ? rest : rest.slice(0, end + 1);
}
// --- the skill's form -------------------------------------------------------
test('researching-sources: name matches its folder and the description is a trigger', () => {
const fm = frontmatter(read(SKILL));
assert.equal(field(fm, 'name'), 'researching-sources');
const desc = unquote(field(fm, 'description'));
assert.ok(desc.length <= 1024, `description is ${desc.length} characters, max 1 024`);
assert.match(desc, /\bUse when\b/, 'description must say "Use when …"');
assert.doesNotMatch(desc, /\b(I|you|we)\b/i, 'description is written in the third person');
assert.doesNotMatch(field(fm, 'description'), /^[^"].*: /, 'an unquoted description must not carry ": " (YAML trap)');
});
test('researching-sources: model-invocable, under 500 lines, no shell blocks', () => {
const text = read(SKILL);
assert.doesNotMatch(frontmatter(text), /disable-model-invocation:\s*true/,
'a Voyage skill is never user-only: preloading into agents needs model invocation');
assert.ok(text.split('\n').length < 500, 'SKILL.md must stay under 500 lines');
assert.doesNotMatch(text, /```(?:bash|sh|zsh|shell)\b/, 'no shell commands in the skill');
});
// --- the skill's content ----------------------------------------------------
test('researching-sources: carries the five rules the order names', () => {
const text = read(SKILL);
@ -102,13 +80,13 @@ test('researching-sources: carries the five rules the order names', () => {
assert.match(text, /never raises a\s+confidence rating/i, 'an empty search never raises a rating (the measured red baseline)');
});
test('researching-sources: "Enforced by" names the structure-only check and says the rest is text only', () => {
test('researching-sources: "Enforced by" names the structure-only check and says the rest is prose only', () => {
const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
assert.match(enforced, /research-validator\.mjs/);
assert.match(enforced, /structure/i);
assert.match(enforced, /query-privacy-gate\.mjs/);
assert.match(enforced, /off by default/i, 'the privacy gate runs only in the default-off steps; say so');
assert.match(enforced, /\*\*text only\*\*/);
assert.match(enforced, /\*\*prose only\*\*/);
});
test('researching-sources: credits carry source, commit and licence', () => {
@ -201,7 +179,7 @@ function norm(s) {
}
test('one confidence scale: the skill table and /trekresearch triangulation define the same four ratings', () => {
const table = section(read(SKILL), /^## Confidence scale/m, /^## /m);
const table = section(read(SKILL), /^### Confidence scale/m, /\n##+ /);
const skillRows = [...table.matchAll(/^\| \*\*(\w+)\*\* \| (.+) \|$/gm)].map((m) => [m[1], norm(m[2])]);
const tri = section(read('commands/trekresearch.md'), /^## Phase 6 — Triangulation/m, /^## Phase 7/m);
const cmdRows = [...tri.matchAll(/^ {3}- \*\*(\w+)\*\* — (.+)$/gm)].map((m) => [m[1], norm(m[2])]);

150
tests/lib/skills-workshop-prose-pins.test.mjs Normal file
View file

@ -0,0 +1,150 @@
// tests/lib/skills-workshop-prose-pins.test.mjs
// Pins for the skill workshop (ordre 7a, contracts S1 and S2): every skill is
// asked for by a command or an agent ("the commands are the trunk"), none is
// closed to the model, the step helper preloads test-first and debugging, each
// skill names what enforces it and credits what it borrowed, the eval cases
// keep the 3 / 2 / 1 form, and the docs carry the ## Skills section.
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { findSkillUsers } from '../../lib/skills/skill-lint.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
import { loadCases } from '../../lib/skills/routing.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
const SKILLS = readdirSync(join(ROOT, 'skills')).filter((d) => existsSync(join(ROOT, 'skills', d, 'SKILL.md'))).sort();
test('pin: Voyage ships the test-first and debugging skills (S2)', () => {
for (const s of ['test-first', 'debugging']) assert.ok(SKILLS.includes(s), `skills/${s}/SKILL.md missing`);
});
// ---- "the commands are the trunk": a skill nobody asks for is red ----
test('pin: findSkillUsers sees a skills: line and a voyage:<name> mention, and reports an unused skill', () => {
const root = mkdtempSync(join(tmpdir(), 'skill-users-'));
try {
for (const s of ['used-by-agent', 'used-by-command', 'unused']) {
mkdirSync(join(root, 'skills', s), { recursive: true });
writeFileSync(join(root, 'skills', s, 'SKILL.md'), `---\nname: ${s}\ndescription: x\n---\n`);
}
mkdirSync(join(root, 'agents'));
mkdirSync(join(root, 'commands'));
writeFileSync(join(root, 'agents', 'a.md'), '---\nname: a\nskills:\n - voyage:used-by-agent\n---\nbody\n');
writeFileSync(join(root, 'commands', 'c.md'), '---\nname: c\n---\nCall the Skill tool with `voyage:used-by-command`.\n');
const users = findSkillUsers(root);
assert.deepEqual(users['used-by-agent'], ['agents/a.md']);
assert.deepEqual(users['used-by-command'], ['commands/c.md']);
assert.deepEqual(users.unused, []);
// A bare word in prose is not a request for the skill.
writeFileSync(join(root, 'commands', 'd.md'), '---\nname: d\n---\nThis mentions unused in passing.\n');
assert.deepEqual(findSkillUsers(root).unused, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('pin: every skill in skills/ is asked for by at least one command or agent', () => {
const users = findSkillUsers(ROOT);
for (const s of SKILLS) assert.ok(users[s] && users[s].length > 0, `skills/${s}: no command or agent names it`);
});
test('pin: no Voyage skill sets disable-model-invocation (it would block preloading and command calls)', () => {
for (const s of SKILLS) {
const f = readFrontmatterFields(read(`skills/${s}/SKILL.md`)).fields['disable-model-invocation'];
assert.ok(!f || !/^(true|yes|on|1)$/i.test(String(f.value).trim()), `skills/${s}: disable-model-invocation is set`);
}
});
// ---- S2: the step helper preloads both ----
test('pin: step-implementer preloads voyage:test-first and voyage:debugging, without the Skill tool', () => {
const fm = readFrontmatterFields(read('agents/step-implementer.md')).fields;
assert.deepEqual(fm.skills.value, ['voyage:test-first', 'voyage:debugging']);
// Preloading uses the skills field, not the Skill tool (sub-agents.md,
// "Preload skills into subagents"): the tool list stays as it was.
assert.deepEqual(fm.tools.value, ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep']);
});
// ---- each skill says what enforces it, and credits what it borrowed ----
test('pin: debugging says plainly that the root-cause claim is prose only', () => {
const t = read('skills/debugging/SKILL.md');
const enforced = t.split('## Enforced by')[1].split('## Credits')[0];
assert.match(enforced, /prose only: the claim that the root cause was found/);
assert.match(enforced, /lib\/verification\/red-proof\.mjs/);
});
test('pin: test-first says the proof of red counts a crash as red, so the assertion red is prose only', () => {
const enforced = read('skills/test-first/SKILL.md').split('## Enforced by')[1].split('## Credits')[0];
assert.match(enforced, /a crash also passes it/);
for (const p of ['lib/verification/red-proof.mjs', 'lib/util/test-lock.mjs', 'lib/verification/test-integrity.mjs', 'lib/verification/done-line.mjs']) {
assert.ok(enforced.includes(p), `test-first must name ${p}`);
}
});
test('pin: each skill credits its sources with commit and licence', () => {
const want = {
'test-first': ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
debugging: ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
};
for (const [s, sources] of Object.entries(want)) {
const credits = read(`skills/${s}/SKILL.md`).split('## Credits')[1];
for (const src of sources) assert.ok(credits.includes(src), `${s}: credit ${src}`);
assert.equal((credits.match(/\bMIT\b/g) || []).length, sources.length, `${s}: one licence per source`);
}
});
// ---- S1: the eval case form (3 should fire / 2 should not / 1 effect) ----
test('pin: each skill has 3 trigger, 2 no-trigger and 1 effect case, and the graders read the Skill tool', () => {
const cases = loadCases(ROOT);
for (const s of SKILLS) {
const mine = cases.filter((c) => c.skill === s);
const n = (k) => mine.filter((c) => c.kind === k);
assert.ok(n('trigger').length >= 3 && n('no-trigger').length >= 2 && n('effect').length >= 1, `${s}: ${mine.length} cases`);
for (const c of [...n('trigger'), ...n('no-trigger')]) {
const dir = join(ROOT, 'evals', c.case, 'graders');
const g = readdirSync(dir).map((f) => readFrontmatterFields(readFileSync(join(dir, f), 'utf8')).fields);
const skillGrader = g.find((x) => x.tool && x.tool.value === 'Skill');
assert.ok(skillGrader, `${c.case}: needs a tool_used Skill grader`);
const re = new RegExp(skillGrader.input_match.value);
assert.match(JSON.stringify({ skill: `voyage:${s}` }), re, `${c.case}: must see voyage:${s}`);
assert.match(JSON.stringify({ skill: s }), re, `${c.case}: must see the bare name`);
for (const other of SKILLS.filter((o) => o !== s)) assert.doesNotMatch(JSON.stringify({ skill: `voyage:${other}` }), re);
if (c.kind === 'no-trigger') {
assert.deepEqual([skillGrader.min.value, skillGrader.max.value, skillGrader.arm.value], ['0', '0', 'both'], c.case);
}
}
for (const c of n('effect')) {
const desc = readFrontmatterFields(read(`evals/${c.case}/prompt.md`)).fields.description.value;
assert.match(desc, /judge model explicitly \(never Haiku\)/, `${c.case}: the judge model is the operator's explicit choice`);
}
}
});
// ---- docs: ## Skills, the authoring guide ----
test('pin: README and CLAUDE.md carry a ## Skills section naming both skills and the workshop checks', () => {
for (const f of ['README.md', 'CLAUDE.md']) {
const t = read(f);
const sec = t.split(/^## Skills$/m)[1];
assert.ok(sec, `${f}: ## Skills missing`);
const body = sec.split(/^## /m)[0];
for (const s of ['test-first', 'debugging', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'docs/skills-authoring.md']) {
assert.ok(body.includes(s), `${f} ## Skills must name ${s}`);
}
}
});
test('pin: docs/skills-authoring.md teaches the red baseline and stays public-safe', () => {
const t = read('docs/skills-authoring.md');
for (const s of ['## Red baseline first', 'control', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'skill-trigger', 'skill-no-trigger', 'skill-effect', 'from=']) {
assert.ok(t.includes(s), `skills-authoring.md must cover ${s}`);
}
assert.doesNotMatch(t, /\/Users\/|\/private\/tmp|\.local\.md/, 'no local paths in a public doc');
});

130
tests/lib/trial2-minor-prose-pins.test.mjs Normal file
View file

@ -0,0 +1,130 @@
// tests/lib/trial2-minor-prose-pins.test.mjs
// Prose pins for the minor points of the second voyage-lekerepo trial
// (2026-10-08, report parts B and C) whose fix is text a session reads.
// Counted as doc pins by lib/util/test-census.mjs (the file name).
// B8 minors left after the last critic round have a way to be fixed
// (operator decision 2026-10-08: listed in the summary, the operator picks).
// B9 K10 `Regression:` is described where plans are written and reviewed.
// B10 K11 the four `On failure:` words are named where plans are written.
// B12 /trekbrief Step 4c no longer claims a foreground, blocking spawn.
// C13 the codebase-size count in /trekplan Phase 4 counts .mjs/.cjs/.mts/.cts.
// C17 git-historian leaves the brief's frozen files alone and names the
// remote from `git remote -v`.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { findSection } from '../../lib/parsers/plan-schema.mjs';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
/** The text of a level-2 section, up to the next `## ` heading outside a fence. */
function section(text, heading) {
const s = findSection(text, heading.replace(/^## /, ''), 2);
assert.ok(s, `missing ${heading}`);
return s.text;
}
// --- B8 ------------------------------------------------------------------------
test('B8: a PASS with minors sends them to the Phase 10 summary, numbered, for the operator to pick', () => {
const plan = read('commands/trekplan.md');
const p9 = section(plan, '## Phase 9 — Adversarial review (bounded critic loop)');
assert.match(p9, /\*\*PASS\*\*[\s\S]{0,600}minors[\s\S]{0,300}Phase 10/i);
const p10 = section(plan, '## Phase 10 — Present and refine');
assert.match(p10, /\*\*Open minors:\*\*/);
assert.match(p10, /numbered/i);
});
test('B8: fixing picked minors after the last round runs one more round with --extra-round, the pick being the yes', () => {
const p10 = section(read('commands/trekplan.md'), '## Phase 10 — Present and refine');
assert.match(p10, /--extra-round/);
assert.match(p10, /pick(ed|s)?\s+(is|counts as)\s+(the|their)\s+yes/i);
});
// --- B9 / B10 --------------------------------------------------------------------
test('B9: /trekplan Phase 8 names `Regression:` as an allowed optional field and says what it is for', () => {
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
assert.match(p8, /`Regression:`/);
assert.match(p8, /PLAN_STEP_REGRESSION_NOT_RUNNABLE/);
assert.match(p8, /\*\*Regression:\*\* `npm test`/, 'the canonical example shows the field');
assert.match(p8, /before (the|its) checkpoint/i);
});
test('B9: the plan template shows `Regression:` as optional', () => {
const t = read('templates/plan-template.md');
assert.match(t, /- \*\*Regression:\*\* `[^\n]*\*\(optional\)\*/);
assert.match(t, /`Regression:` \(ONE backticked command/);
});
test('B9: plan-critic knows the field and does not demand the suite in `Verify:`', () => {
const c = read('agents/plan-critic.md');
assert.match(c, /`Regression:`/);
assert.match(c, /(never|do\s+not)\s+(demand|require)\s+`npm test`\s+in\s+`Verify:`/i);
});
test('B9/B10: Handover 4 documents `Regression:` (K10) and the `On failure:` words (K11)', () => {
const h4 = section(read('docs/HANDOVER-CONTRACTS.md'), '## Handover 4 — `plan.md` → execute');
for (const code of ['PLAN_STEP_REGRESSION_NOT_RUNNABLE', 'PLAN_STEP_ON_FAILURE_UNKNOWN']) assert.match(h4, new RegExp(code));
assert.match(h4, /K10/);
assert.match(h4, /K11/);
assert.match(h4, /revert[^\n]*retry[^\n]*skip[^\n]*escalate/);
});
test('B10: /trekplan Phase 8 says `On failure:` opens with one of the four words', () => {
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
assert.match(p8, /On failure:\*\*[^\n]*(opens|starts) with/i);
assert.match(p8, /PLAN_STEP_ON_FAILURE_UNKNOWN/);
});
// --- B12 -------------------------------------------------------------------------
test('B12: /trekbrief Step 4c does not call the brief-reviewer spawn foreground or blocking, and waits for it', () => {
const t = read('commands/trekbrief.md');
const start = t.indexOf('**Step 4c — Launch brief-reviewer**');
const end = t.indexOf('**Step 4d — Parse JSON scores**');
assert.ok(start !== -1 && end > start);
const step = t.slice(start, end);
assert.doesNotMatch(step, /foreground, blocking/);
assert.match(step, /wait\s+for\s+(its|the agent's)\s+(result|completion)/i);
assert.match(step, /(background|asynchronous)/i, 'says how the spawn actually runs');
});
// --- C13 -------------------------------------------------------------------------
test('C13: the Phase 4 size count counts .mjs, .cjs, .mts and .cts files', () => {
const p4 = section(read('commands/trekplan.md'), '## Phase 4 — Codebase sizing');
const cmd = p4.match(/```\n(find [^\n]+)\n```/);
assert.ok(cmd, 'the find command stands in a fence of its own');
const dir = mkdtempSync(join(tmpdir(), 'voyage-sizing-'));
try {
for (const f of ['a.mjs', 'b.cjs', 'c.mts', 'd.cts', 'e.ts', 'skip.txt']) writeFileSync(join(dir, f), 'x\n');
const r = spawnSync('/bin/sh', ['-c', cmd[1]], { cwd: dir, encoding: 'utf8' });
assert.equal(r.status, 0, r.stderr);
assert.equal(r.stdout.trim(), '5');
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// --- C17 -------------------------------------------------------------------------
test('C17: git-historian reads the brief, never proposes changing a file it freezes', () => {
const g = read('agents/git-historian.md');
assert.match(g, /brief/i);
assert.match(g, /(never|do not) (propose|suggest|recommend)[^\n]*(change|edit)[^\n]*(frozen|unchanged)|(frozen|unchanged)[^\n]*(never|do not) (propose|suggest|recommend)/i);
const p5 = section(read('commands/trekplan.md'), '## Phase 5 — Parallel exploration (specialized agents + research)');
assert.match(p5, /\*\*git-historian\*\*[\s\S]{0,400}\{brief_path\}/, 'the spawn hands it the brief');
});
test('C17: git-historian names the remote from `git remote -v`, never by guess', () => {
const g = read('agents/git-historian.md');
assert.match(g, /`git remote -v`/);
assert.match(g, /local (path|repository|bare)/i);
});

253
tests/lib/trial2-minor.test.mjs Normal file
View file

@ -0,0 +1,253 @@
// tests/lib/trial2-minor.test.mjs
// The minor points of the second voyage-lekerepo trial (2026-10-08, report
// parts B and C) that are behaviour, not prose. Each test was red before its
// fix.
// B9 K10 — optional step field `Regression:`: one runnable command, or the
// validator refuses it.
// B10 K11 — `On failure:` opens with revert | retry | skip | escalate, or the
// validator warns.
// B11 two findings on the same file:line with the same content merge (the
// trial's pair measured Jaccard 0.44 against a 0.7 threshold).
// C14 `--evidence` shows the first output line on a PASS row, not "—".
// C16 a file:line cited in a reviewer's prose is checked against the file.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
import { onFailureAction, ON_FAILURE_ACTIONS } from '../../lib/parsers/plan-schema.mjs';
import { dedupFindings, DEFAULT_THRESHOLD } from '../../lib/review/plan-review-dedup.mjs';
import { formatCriteriaEvidence } from '../../lib/verification/criteria-runner.mjs';
import { validateFindings, validateReviewerOutput } from '../../lib/review/findings-schema.mjs';
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
const codes = (list) => list.map((e) => e.code);
// Step 1's Verify line in the builder's plan; the new field goes right under it.
const STEP1_VERIFY = '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass';
const withRegression = (line, plan = buildPlan()) => plan.replace(STEP1_VERIFY, `${STEP1_VERIFY}\n${line}`);
const withOnFailure = (text, plan = buildPlan()) =>
plan.replace(/^- \*\*On failure:\*\* .*$/m, `- **On failure:** ${text}`);
// --- B9: K10 `Regression:` ----------------------------------------------------
test('K10: `Regression: npm test | tee x` is refused — one command, unpiped', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x` → expected: exit 0'));
assert.equal(r.valid, false);
const e = r.errors.find((x) => x.code === 'PLAN_STEP_REGRESSION_NOT_RUNNABLE');
assert.ok(e, codes(r.errors).join(','));
assert.match(e.message, /Step 1/);
assert.match(e.message, /shell operator/);
});
test('K10: `Regression: npm test` is accepted', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test` → expected: exit 0'));
assert.deepEqual(codes(r.errors), []);
assert.ok(!codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
test('K10: without the field the plan validates exactly as before', () => {
const r = validatePlanContent(buildPlan());
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.deepEqual(codes(r.warnings), []);
});
test('K10: `npm test && bash tests/e2e.sh`, a bare word, or no backticks is refused', () => {
for (const line of [
'- **Regression:** `npm test && bash tests/e2e.sh`',
'- **Regression:** `./run-all-tests`',
'- **Regression:** npm test',
'- **Regression:**',
]) {
const r = validatePlanContent(withRegression(line));
assert.ok(codes(r.errors).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), `${line}: ${codes(r.errors).join(',')}`);
}
});
test('K10: --soft reports an unrunnable Regression as a warning, not an error', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x`'), { strict: false });
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.ok(codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
// --- B10: K11 `On failure:` ---------------------------------------------------
test('K11: `On failure: try again` warns, and the plan stays valid', () => {
const r = validatePlanContent(withOnFailure('try again'));
assert.equal(r.valid, true, codes(r.errors).join(','));
const w = r.warnings.find((x) => x.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN');
assert.ok(w, codes(r.warnings).join(','));
assert.match(w.message, /Step 1/);
for (const word of ON_FAILURE_ACTIONS) assert.match(w.hint, new RegExp(word));
});
test('K11: `On failure: retry once, then escalate` gives no warning', () => {
const r = validatePlanContent(withOnFailure('retry once, then escalate'));
assert.ok(!codes(r.warnings).includes('PLAN_STEP_ON_FAILURE_UNKNOWN'), codes(r.warnings).join(','));
});
test('K11: the four words, in any case and with markdown around them, are read; a longer word is not', () => {
assert.deepEqual([...ON_FAILURE_ACTIONS], ['revert', 'retry', 'skip', 'escalate']);
assert.equal(onFailureAction('revert — `git checkout -- x`'), 'revert');
assert.equal(onFailureAction('Escalate: stop the run'), 'escalate');
assert.equal(onFailureAction('**skip** this step'), 'skip');
assert.equal(onFailureAction('`retry` once'), 'retry');
assert.equal(onFailureAction('retrying is fine'), null);
assert.equal(onFailureAction('try again'), null);
assert.equal(onFailureAction(''), null);
});
// --- B11: same file:line, same content ---------------------------------------
// The two findings from trial 2's round 1, verbatim (plan.md:248).
const PLAN_MD = '.claude/projects/2026-10-08-slug-sep-option/plan.md';
const CRITIC_248 = {
severity: 'minor', rule_key: 'verification/contract-brief-form-missing', file: PLAN_MD, line: 248,
text: "Verification runs `bash tests/e2e.sh` in the brief's own form but not `bash tests/contract.sh`, the form brief SC4 names.",
};
const GUARDIAN_248 = {
severity: 'minor', rule_key: 'gap-verification-command-form', file: PLAN_MD, line: 248,
text: "Brief SC4 runs 'bash tests/contract.sh'; Verification lists only '/bin/bash tests/contract.sh' (unlike e2e, where both forms are listed) - add the brief's form",
};
const sources = (a, b) => [
{ agent: 'plan-critic', payload: { findings: a } },
{ agent: 'scope-guardian', payload: { findings: b } },
];
test('B11: the two trial-2 findings on plan.md:248 merge into one, raised by both', () => {
const r = dedupFindings(sources([CRITIC_248], [GUARDIAN_248]));
assert.equal(r.findings.length, 1, JSON.stringify(r.findings, null, 2));
assert.deepEqual(r.findings[0].raised_by, ['plan-critic', 'scope-guardian']);
assert.equal(r.dedup_stats.anchor_dups, 1);
});
test('B11: two different findings on the same line stay two', () => {
const other = {
severity: 'minor', rule_key: 'verification/no-negative-case', file: PLAN_MD, line: 248,
text: 'No criterion feeds an invalid separator such as a letter and checks the exit code.',
};
const r = dedupFindings(sources([CRITIC_248], [other]));
assert.equal(r.findings.length, 2);
});
test('B11: a short finding held by a longer one on the same line stays its own (independent review)', () => {
const at = (text, severity = 'minor') => ({ severity, rule_key: `k-${text.length}`, file: PLAN_MD, line: 40, text });
for (const [a, b] of [
['The token is checked.', 'The token is not checked before the write.'],
['Step 3 test is missing.', 'Step 3 is missing a checkpoint; the test file already exists.'],
['Verify runs npm test.', 'Verify must not run npm test, it is green before the step.'],
]) {
assert.equal(dedupFindings(sources([at(a)], [at(b)])).findings.length, 2, `${a} | ${b}`);
}
});
test('B11: one reviewer\'s two findings on one line are never merged by the anchor rule', () => {
const r = dedupFindings([{ agent: 'plan-critic', payload: { findings: [CRITIC_248, { ...GUARDIAN_248, rule_key: 'other' }] } }]);
assert.equal(r.findings.length, 2);
assert.equal(r.dedup_stats.anchor_dups, 0);
});
test('B11: the same content on another line is not merged by the anchor rule; the threshold elsewhere is unchanged', () => {
assert.equal(DEFAULT_THRESHOLD, 0.7);
const r = dedupFindings(sources([CRITIC_248], [{ ...GUARDIAN_248, line: 249 }]));
assert.equal(r.findings.length, 2);
const noLine = dedupFindings(sources([{ ...CRITIC_248, line: 0 }], [{ ...GUARDIAN_248, line: 0 }]));
assert.equal(noLine.findings.length, 2, 'line 0 is file-scoped, not an anchor');
});
// --- C14: --evidence on a PASS row ---------------------------------------------
const evidenceRep = (results) => ({
source: 'brief.md',
heading: '## Success Criteria',
error: null,
results,
summary: { passed: 1, failed: 0, blocked: 0, unrunnable: 0, total: 1, ok: true },
});
test('C14: --evidence shows the first line of output on a passed criterion', () => {
const block = formatCriteriaEvidence(evidenceRep([
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '\n# tests 12\n# pass 12\n' },
]));
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| # tests 12 \|/);
});
test('C14: a passed criterion with no output still says "—"', () => {
const block = formatCriteriaEvidence(evidenceRep([
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '' },
]));
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| — \|/);
});
// --- C16: file:line in prose --------------------------------------------------
function repoWith33() {
const dir = mkdtempSync(join(tmpdir(), 'voyage-prose-cite-'));
mkdirSync(join(dir, 'tests'));
writeFileSync(join(dir, 'tests', 'x.sh'), Array.from({ length: 33 }, (_, i) => `echo ${i + 1}`).join('\n') + '\n');
return dir;
}
const finding = (over = {}) => ({
id: '0123456789abcdef0123456789abcdef01234567',
severity: 'MAJOR', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'tests/x.sh', line: 3,
title: 'A title', detail: 'A detail.', recommended_action: 'Do one thing.', ...over,
});
test('C16: a finding whose prose cites `tests/x.sh:119` in a 33-line file is refused', () => {
const dir = repoWith33();
try {
const r = validateFindings(
{ reviewer: 'code-correctness-reviewer', findings: [finding({ detail: 'The check at `tests/x.sh:119` relies on git.' })] },
{ repoRoot: dir },
);
assert.equal(r.valid, false);
const e = r.errors.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
assert.ok(e, JSON.stringify(r.errors));
assert.match(e.message, /tests\/x\.sh:119/);
assert.match(e.message, /33 lines/);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('C16: the same citation in prose outside the JSON block is a warning, never silent (a JSON-only re-ask cannot fix prose)', () => {
const dir = repoWith33();
try {
const raw = [
'### Risk noted but not raised as a finding',
'',
'`tests/x.sh:119` relies on `git` and on a fixed commit.',
'',
'```json',
JSON.stringify({ reviewer: 'code-correctness-reviewer', findings: [] }),
'```',
].join('\n');
const r = validateReviewerOutput(raw, { repoRoot: dir });
assert.equal(r.valid, true, JSON.stringify(r.errors));
const w = r.warnings.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
assert.ok(w, JSON.stringify(r.warnings));
assert.match(w.message, /reviewer prose cites `?tests\/x\.sh:119/);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('C16: a line that exists, a range ending in the file, a file absent from the tree and no repo root all pass', () => {
const dir = repoWith33();
try {
const ok = (detail, opts = { repoRoot: dir }) =>
validateFindings({ reviewer: 'r', findings: [finding({ detail })] }, opts).valid;
assert.equal(ok('See tests/x.sh:33 and tests/x.sh:10-12.'), true);
assert.equal(ok('See src/absent.mjs:900.'), true);
assert.equal(ok('See tests/x.sh:119.', {}), true);
assert.equal(ok('See tests/x.sh:30-40.'), false, 'a range that runs past the end is out of range');
assert.equal(ok('Served on http://localhost:8080 and example.com:443.'), true);
assert.equal(ok('On Windows: C:\\work\\tests\\x.sh:119 and tests\\x.sh:119.'), true, 'a Windows path is not the root\'s file');
assert.equal(ok('See tests/x.sh:30\u201440.'), false, 'an em-dash range is read as a range');
} finally {
rmSync(dir, { recursive: true, force: true });
}
});