open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

merge: main (7a skills workshop, 9 rettepakke, 7b researching-sources) into wave2-utgivelse

Browse source 
No conflicts. npm test on the merged tree: 2085 tests, 2083 pass, 0 fail,
2 skipped (main 2055 + this branch's 30). K2 and K7 tests green against
the real modules; the verdict gate and approval log CLIs run end to end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-09 19:50:19 +02:00
commit 7ccdff4629
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
78 changed files with 3308 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

251
tests/lib/skill-lint.test.mjs Normal file
View file

@ -0,0 +1,251 @@
// tests/lib/skill-lint.test.mjs
// The skill form check (S1): lib/skills/skill-lint.mjs. Model-free, part of
// `npm test`. Each rule is shown biting on a skill built for it (known
// positive) and passing a skill that keeps the form (known negative); the last
// tests run the check over the skills Voyage ships.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { lintSkill, lintSkillsRoot, REQUIRED_HEADINGS, EXEMPTIONS } from '../../lib/skills/skill-lint.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'skill-lint.mjs');
const BODY = [
'# Example', '',
'## Overview', 'One line.', '',
'## When to use', '- when the example applies', '- not when it does not', '',
'## Process', '1. Do the thing.', '',
'## Red flags', '- skipping the thing', '',
'## Enforced by', '- prose only: nothing checks this.', '',
'## Credits', '- Voyage\'s own content.', '',
].join('\n');
const GOOD_FM = [
'---',
'name: good-skill',
'description: Checks an example the same way every time. Use when an example needs checking.',
'---',
].join('\n');
/** A plugin-shaped temp root with skills/<name>/SKILL.md; returns the root. */
function plugin(skills) {
const root = mkdtempSync(join(tmpdir(), 'skill-lint-'));
for (const [name, text] of Object.entries(skills)) {
mkdirSync(join(root, 'skills', name), { recursive: true });
writeFileSync(join(root, 'skills', name, 'SKILL.md'), text);
}
return root;
}
const codes = (r) => r.errors.map((e) => e.code).sort();
function withSkill(name, text, fn) {
const root = plugin({ [name]: text });
try { return fn(lintSkill(join(root, 'skills', name), { pluginRoot: root }), root); }
finally { rmSync(root, { recursive: true, force: true }); }
}
// ---- the test that fails first (ordre 7a) ----
test('skill-lint: a skill without "Use when" and with an unquoted colon is red', () => {
const text = [
'---',
'name: bad-skill',
'description: Checks things: all of them, every time.',
'---',
BODY,
].join('\n');
withSkill('bad-skill', text, (r) => {
assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), codes(r).join(','));
assert.ok(codes(r).includes('SKILL_DESC_NO_USE_WHEN'), codes(r).join(','));
});
});
test('skill-lint: a skill that keeps the form is green (known negative)', () => {
withSkill('good-skill', `${GOOD_FM}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
// ---- one rule at a time ----
test('skill-lint: name must equal the directory name and be kebab-case', () => {
withSkill('other-dir', `${GOOD_FM}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_MISMATCH')));
const fm = GOOD_FM.replace('name: good-skill', 'name: Good_Skill');
withSkill('Good_Skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
const reserved = GOOD_FM.replace('name: good-skill', 'name: claude-helper');
withSkill('claude-helper', `${reserved}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
});
test('skill-lint: description is required, at most 1 024 characters', () => {
const none = GOOD_FM.replace(/^description:.*$/m, 'license: MIT');
withSkill('good-skill', `${none}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_MISSING')));
const long = `Checks an example. Use when ${'an example needs checking and '.repeat(40)}done.`;
assert.ok(long.length > 1024);
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${long}`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_TOO_LONG')));
const exact = `Use when x. ${'y'.repeat(1024 - 12)}`;
assert.equal(exact.length, 1024);
const fm2 = GOOD_FM.replace(/^description:.*$/m, `description: ${exact}`);
withSkill('good-skill', `${fm2}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_TOO_LONG')));
});
test('skill-lint: description is in the third person', () => {
for (const d of [
'I check an example. Use when an example needs checking.',
'Checks your example. Use when an example needs checking.',
'Use when you need an example checked.',
'We check examples. Use when an example needs checking.',
]) {
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${d}`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON'), d));
}
const ok = GOOD_FM.replace(/^description:.*$/m, 'description: Checks an example, i.e. one case. Use when the user asks for a check.');
withSkill('good-skill', `${ok}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON')));
});
test('skill-lint: YAML traps are caught: unquoted colon, comment marker, tab, unclosed quote, leading indicator', () => {
for (const line of [
'description: Use when x: y happens.',
'description: Use when x happens #1.',
'description:\tUse when x happens.',
'description: "Use when x happens.',
'description: `Use` when x happens.',
'description: @Use when x happens.',
]) {
const fm = GOOD_FM.replace(/^description:.*$/m, line);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), line));
}
// Quoted, the colon is safe.
const quoted = GOOD_FM.replace(/^description:.*$/m, 'description: "Checks x: y. Use when x happens."');
withSkill('good-skill', `${quoted}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: a multi-line block-scalar description is read whole', () => {
const fm = ['---', 'name: good-skill', 'description: |', ' Checks an example the same way: every time.',
' Use when an example needs checking.', '---'].join('\n');
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
const f = readFrontmatterFields(`${fm}\n${BODY}`);
assert.equal(f.fields.description.value, 'Checks an example the same way: every time.\nUse when an example needs checking.\n');
const folded = readFrontmatterFields(['---', 'description: >-', ' one', ' two', '---', ''].join('\n'));
assert.equal(folded.fields.description.value, 'one two');
});
test('skill-lint: disable-model-invocation in any true spelling is refused', () => {
for (const v of ['true', 'True', 'yes', 'on', '1']) {
const fm = GOOD_FM.replace('---\nname', `---\ndisable-model-invocation: ${v}\nname`);
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_MODEL_INVOCATION_DISABLED'), v));
}
const off = GOOD_FM.replace('---\nname', '---\ndisable-model-invocation: false\nname');
withSkill('good-skill', `${off}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: only Claude Code\'s own frontmatter fields are accepted', () => {
const fm = GOOD_FM.replace('---\nname', '---\ntrigger: always\nname');
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_UNKNOWN_FIELD')));
const known = GOOD_FM.replace('---\nname', '---\nlicense: MIT\nuser-invocable: true\nname');
withSkill('good-skill', `${known}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: SKILL.md must stay under 500 lines', () => {
const pad = Array.from({ length: 500 }, (_, i) => `- line ${i}`).join('\n');
withSkill('good-skill', `${GOOD_FM}\n${BODY}\n${pad}\n`, (r) => assert.ok(codes(r).includes('SKILL_TOO_LONG')));
});
test('skill-lint: the fixed headings must all be there, in order', () => {
assert.deepEqual(REQUIRED_HEADINGS, ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits']);
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('## Red flags', '## Warnings')}`,
(r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
const swapped = BODY.replace('## Overview\nOne line.', '## Process\n1. x.').replace('## Process\n1. Do the thing.', '## Overview\nOne line.');
withSkill('good-skill', `${GOOD_FM}\n${swapped}`, (r) => assert.ok(codes(r).includes('SKILL_HEADING_ORDER')));
// A heading inside a fenced code block does not count.
const fenced = BODY.replace('## Credits', '```\n## Credits\n```');
withSkill('good-skill', `${GOOD_FM}\n${fenced}`, (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
});
test('skill-lint: "Enforced by" names a check that exists, or says prose only', () => {
const none = BODY.replace('- prose only: nothing checks this.', '- something, somewhere.');
withSkill('good-skill', `${GOOD_FM}\n${none}`, (r) => assert.ok(codes(r).includes('SKILL_ENFORCED_BY_VAGUE')));
const real = BODY.replace('- prose only: nothing checks this.', '- `lib/check.mjs` stops it.');
const root = plugin({ 'good-skill': `${GOOD_FM}\n${real}` });
try {
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
mkdirSync(join(root, 'lib'));
writeFileSync(join(root, 'lib', 'check.mjs'), '');
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('skill-lint: files the skill points to must exist', () => {
const linked = BODY.replace('One line.', 'One line. See [the detail](reference/detail.md).');
const root = plugin({ 'good-skill': `${GOOD_FM}\n${linked}` });
try {
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
mkdirSync(join(root, 'skills', 'good-skill', 'reference'));
writeFileSync(join(root, 'skills', 'good-skill', 'reference', 'detail.md'), '# detail\n');
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('skill-lint: paths under "Credits" name the source repository, not this plugin', () => {
const credited = BODY.replace("- Voyage's own content.", '- owner/repo @abc1234, `skills/tdd/SKILL.md`: MIT.');
withSkill('good-skill', `${GOOD_FM}\n${credited}`, (r) => assert.deepEqual(r.errors, []));
// The same path anywhere else must exist.
const elsewhere = BODY.replace('One line.', 'One line, see `skills/tdd/SKILL.md`.');
withSkill('good-skill', `${GOOD_FM}\n${elsewhere}`, (r) => assert.ok(codes(r).includes('SKILL_REF_MISSING')));
});
test('skill-lint: no shell injection (inline or fenced), since Voyage advises turning it off', () => {
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', 'Status: !`git status`')}`,
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```!\ngit status\n```')}`,
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
// A plain fenced bash block is fine; Claude reads it, nothing runs it on load.
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```bash\nnpm test\n```')}`,
(r) => assert.deepEqual(r.errors, []));
});
test('skill-lint: an exemption lives in the check, never in the skill', () => {
assert.ok(EXEMPTIONS instanceof Map);
for (const [name, entry] of EXEMPTIONS) {
assert.ok(Array.isArray(entry.codes) && entry.codes.length > 0, `${name}: exempt codes`);
assert.ok(typeof entry.reason === 'string' && entry.reason.length > 10, `${name}: a reason`);
}
const self = `${GOOD_FM}\n<!-- skill-lint-exempt: SKILL_MISSING_HEADING -->\n${BODY}`;
withSkill('good-skill', self, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
const meta = GOOD_FM.replace('---\nname', '---\nmetadata:\n lint-exempt: true\nname');
withSkill('good-skill', `${meta}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
});
test('skill-lint CLI: 0 clean, 1 findings, 2 could not run', () => {
const good = plugin({ 'good-skill': `${GOOD_FM}\n${BODY}` });
const bad = plugin({ 'bad-skill': `---\nname: bad-skill\ndescription: Checks: things.\n---\n${BODY}` });
try {
const g = spawnSync('node', [CLI, '--root', good], { encoding: 'utf8' });
assert.equal(g.status, 0, g.stdout + g.stderr);
const b = spawnSync('node', [CLI, '--root', bad, '--json'], { encoding: 'utf8' });
assert.equal(b.status, 1, b.stdout + b.stderr);
assert.ok(JSON.parse(b.stdout).skills['bad-skill'].errors.length > 0);
const n = spawnSync('node', [CLI, '--root', join(good, 'nope')], { encoding: 'utf8' });
assert.equal(n.status, 2, n.stdout + n.stderr);
} finally {
rmSync(good, { recursive: true, force: true });
rmSync(bad, { recursive: true, force: true });
}
});
// ---- the skills Voyage ships ----
test('skill-lint: every skill Voyage ships passes the form check', () => {
const r = lintSkillsRoot(ROOT);
assert.ok(Object.keys(r.skills).length >= 2, `found ${Object.keys(r.skills).join(',')}`);
for (const [name, res] of Object.entries(r.skills)) {
assert.deepEqual(res.errors, [], `${name}: ${JSON.stringify(res.errors, null, 2)}`);
}
});

215
tests/lib/skill-routing.test.mjs Normal file
View file

@ -0,0 +1,215 @@
// tests/lib/skill-routing.test.mjs
// The model-free routing test (P10, "Bygg nå"): lib/skills/routing.mjs ranks
// the descriptions of every skill, the 7 commands and the 21 spawnable agents
// against prompts, with no model. A collision between two descriptions fails;
// each skill needs 3 prompts where it should fire (top 3) and 2 where it
// should not, and for those a named owner must rank higher. The thresholds are
// pinned here: they are never lowered.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
THRESHOLDS, tokenize, buildCorpus, rank, collisions, loadEntries, loadCases, checkRouting,
} from '../../lib/skills/routing.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');
const skillMd = (name, description) =>
`---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`;
/** Plugin-shaped temp root: skills, commands, agents, evals. */
function plugin({ skills = {}, commands = {}, agents = {}, cases = {} }) {
const root = mkdtempSync(join(tmpdir(), 'skill-routing-'));
for (const [n, d] of Object.entries(skills)) {
mkdirSync(join(root, 'skills', n), { recursive: true });
writeFileSync(join(root, 'skills', n, 'SKILL.md'), skillMd(n, d));
}
mkdirSync(join(root, 'commands'), { recursive: true });
for (const [n, d] of Object.entries(commands)) {
writeFileSync(join(root, 'commands', `${n}.md`), `---\nname: ${n}\ndescription: ${d}\n---\n`);
}
mkdirSync(join(root, 'agents'), { recursive: true });
for (const [n, d] of Object.entries(agents)) {
writeFileSync(join(root, 'agents', `${n}.md`), `---\nname: ${n}\ndescription: |\n ${d}\nmodel: opus\n---\n`);
}
for (const [n, { tags, prompt }] of Object.entries(cases)) {
mkdirSync(join(root, 'evals', n, 'graders'), { recursive: true });
writeFileSync(join(root, 'evals', n, 'prompt.md'),
`---\ndescription: "case ${n}"\ntags: [${tags.join(', ')}]\nruns: 1\n---\n\n${prompt}\n`);
}
return root;
}
function withPlugin(spec, fn) {
const root = plugin(spec);
try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); }
}
const codes = (r) => r.errors.map((e) => e.code);
// ---- the test that fails first (ordre 7a) ----
test('routing: two test skills with near-identical descriptions are red (collision)', () => {
withPlugin({
skills: {
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
},
}, (root) => {
const r = checkRouting(root);
assert.ok(codes(r).includes('ROUTING_COLLISION'), JSON.stringify(r.errors));
const c = r.errors.find((e) => e.code === 'ROUTING_COLLISION');
assert.deepEqual([c.a, c.b].sort(), ['skill:order-imports', 'skill:tidy-imports']);
});
});
test('routing: distinct descriptions do not collide (known negative)', () => {
const corpus = buildCorpus([
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file. Use when imports are out of order.' },
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk. Use when the log directory grows.' },
]);
assert.deepEqual(collisions(corpus).filter((c) => c.level === 'error'), []);
});
// ---- the pieces ----
test('routing: thresholds are pinned and never lowered', () => {
assert.deepEqual({ ...THRESHOLDS }, {
COLLISION_ERROR: 0.75,
COLLISION_WARN: 0.5,
POSITIVE_TOP_K: 3,
MIN_POSITIVE: 3,
MIN_NEGATIVE: 2,
MIN_EFFECT: 1,
MIN_RANK1_RATE: 1,
});
assert.ok(Object.isFrozen(THRESHOLDS));
});
test('routing: the tokenizer folds case, drops stop words and light-stems', () => {
assert.deepEqual(tokenize('Use when the Tests are FAILING'), tokenize('test fail'));
assert.deepEqual(tokenize('a an the of'), []);
});
test('routing: rank puts the matching description first, and a no-overlap prompt scores 0', () => {
const corpus = buildCorpus([
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file.' },
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk.' },
]);
assert.equal(rank('the log files fill the disk', corpus)[0].id, 'skill:rotate-logs');
assert.ok(rank('zebra quantum', corpus).every((r) => r.score === 0));
});
test('routing: the reader takes block-scalar agent descriptions and skips orchestrator reference docs', () => {
withPlugin({
agents: { 'plan-critic': 'Use this agent when a plan needs adversarial review.' },
commands: { trekplan: 'Deep implementation planning from a task brief.' },
}, (root) => {
writeFileSync(join(root, 'agents', 'planning-orchestrator.md'),
'---\nname: planning-orchestrator\ndescription: Reference document, not a spawnable capability.\n---\n');
const ids = loadEntries(root).map((e) => e.id).sort();
assert.deepEqual(ids, ['agent:plan-critic', 'command:trekplan']);
assert.equal(loadEntries(root).find((e) => e.id === 'agent:plan-critic').description.trim(),
'Use this agent when a plan needs adversarial review.');
});
});
test('routing: cases come from evals/ tags; a skill below 3 / 2 / 1 is red', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: {
'skill-tidy-imports-fires-1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines in main.js' },
},
}, (root) => {
const cases = loadCases(root);
assert.equal(cases.length, 1);
assert.deepEqual([cases[0].kind, cases[0].skill, cases[0].from], ['trigger', 'tidy-imports', 'test']);
assert.ok(codes(checkRouting(root)).includes('ROUTING_BELOW_MINIMUM'));
});
});
test('routing: a positive outside the top 3 is red; a negative won by the skill itself is red', () => {
const spec = {
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
commands: {
'rotate-logs': 'Compresses and deletes old log files on disk.',
'scan-ports': 'Lists the open network ports of a host.',
'count-words': 'Counts the words of a text document.',
},
cases: {
'p1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines' },
'p2': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'import lines are out of order' },
'p3': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'compress the old logs on disk and list ports and count words' },
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'delete old log files' },
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'sort the import lines of the log file' },
'e1': { tags: ['skill-effect', 'skill=tidy-imports', 'from=test'], prompt: 'x' },
},
};
withPlugin(spec, (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_POSITIVE_MISSED' && e.case === 'p3'), JSON.stringify(r.errors));
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n2'), JSON.stringify(r.errors));
assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
});
});
test('routing: a negative must name an owner that exists and outranks the skill', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: {
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'delete old log files' },
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=nobody', 'from=test'], prompt: 'delete old log files' },
},
}, (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_NO_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n2'), JSON.stringify(r.errors));
});
});
test('routing: every case names the event it came from (P10)', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
cases: { 'p1': { tags: ['skill-trigger', 'skill=tidy-imports'], prompt: 'sort the import lines' } },
}, (root) => assert.ok(codes(checkRouting(root)).includes('ROUTING_CASE_NO_SOURCE')));
});
test('routing CLI: 0 clean, 1 findings, 2 could not run', () => {
withPlugin({
skills: {
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
},
}, (root) => {
const b = spawnSync('node', [CLI, '--root', root, '--json'], { encoding: 'utf8' });
assert.equal(b.status, 1, b.stdout + b.stderr);
assert.ok(JSON.parse(b.stdout).errors.length > 0);
const n = spawnSync('node', [CLI, '--root', join(root, 'nope')], { encoding: 'utf8' });
assert.equal(n.status, 2, n.stdout + n.stderr);
});
});
// ---- Voyage's own catalogue ----
test('routing: Voyage\'s catalogue holds every skill, the 7 commands and the 21 spawnable agents', () => {
const entries = loadEntries(ROOT);
const by = (k) => entries.filter((e) => e.kind === k);
assert.equal(by('command').length, 7, by('command').map((e) => e.name).join(','));
assert.equal(by('agent').length, 21, by('agent').map((e) => e.name).join(','));
assert.ok(by('skill').length >= 2, by('skill').map((e) => e.name).join(','));
for (const e of entries) assert.ok(e.description.trim().length > 20, `${e.id}: description read as "${e.description}"`);
assert.equal(new Set(entries.map((e) => e.name)).size, entries.length, 'names must be unique across kinds');
});
test('routing: Voyage\'s catalogue is green: no collision, every case routes, rank-1 floor held', () => {
const r = checkRouting(ROOT);
assert.deepEqual(r.errors, [], JSON.stringify(r.errors, null, 2));
assert.ok(r.stats.positives >= 6 && r.stats.negatives >= 4, JSON.stringify(r.stats));
});

212
tests/lib/skills-research-prose-pins.test.mjs Normal file
View file

@ -0,0 +1,212 @@
// tests/lib/skills-research-prose-pins.test.mjs
// Wave 2, order 7b: the `researching-sources` skill and its wiring.
//
// What is pinned, and why:
// • the skill's form is not pinned here: order 7a's form check
// (lib/skills/skill-lint.mjs) and routing test (lib/skills/routing.mjs)
// run over every skill in `npm test` and are the authority (contract S1);
// • the five rules, and the two measured red-baseline lines, stay in it;
// • every web-search agent preloads the skill through its `skills:` line, and
// /trekresearch names it where it starts them ("commands are the trunk");
// • the shared source rules that moved into the skill are gone from the
// agents — including the two sentences that contradicted it (an empty
// counter-evidence search "increases confidence"; fall back "silently");
// • one confidence scale: the skill's table and /trekresearch triangulation
// carry the same four definitions;
// • the skill says plainly what enforces it and that the rest is text only.
//
// The behaviour itself (reporting an injected instruction; not counting an
// empty search as support) was measured with helpers in the authoring session
// and is not something a model-free test can see. The eval cases under
// evals/skill-researching-sources-* carry it for the operator's own runs.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync, readdirSync, existsSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf-8');
const SKILL = 'skills/researching-sources/SKILL.md';
const WEB_AGENTS = [
'docs-researcher',
'community-researcher',
'security-researcher',
'contrarian-researcher',
'research-scout',
];
function frontmatter(text) {
const m = text.match(/^---\n([\s\S]*?)\n---\n/);
assert.ok(m, 'file must open with a frontmatter block');
return m[1];
}
function field(fm, key) {
const m = fm.match(new RegExp(`^${key}:\\s*(.*)$`, 'm'));
return m ? m[1].trim() : null;
}
function listField(fm, key) {
const raw = field(fm, key);
if (raw === null) return [];
if (raw.startsWith('[')) return JSON.parse(raw);
return [raw];
}
function section(text, startRe, endRe) {
const start = text.search(startRe);
assert.ok(start >= 0, `section ${startRe} not found`);
const rest = text.slice(start);
const end = rest.slice(1).search(endRe);
return end < 0 ? rest : rest.slice(0, end + 1);
}
// --- the skill's content ----------------------------------------------------
test('researching-sources: carries the five rules the order names', () => {
const text = read(SKILL);
for (const re of [
/official sources? first/i,
/no citation, no finding/i,
/absence of data is not a finding/i,
/data, never instructions/i,
/outbound queries carry nothing local/i,
]) assert.match(text, re, `rule missing: ${re}`);
assert.match(text, /\*\*Report it\.\*\*/, 'an injected instruction is reported, not only ignored (the measured red baseline)');
assert.match(text, /never raises a\s+confidence rating/i, 'an empty search never raises a rating (the measured red baseline)');
});
test('researching-sources: "Enforced by" names the structure-only check and says the rest is prose only', () => {
const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
assert.match(enforced, /research-validator\.mjs/);
assert.match(enforced, /structure/i);
assert.match(enforced, /query-privacy-gate\.mjs/);
assert.match(enforced, /off by default/i, 'the privacy gate runs only in the default-off steps; say so');
assert.match(enforced, /\*\*prose only\*\*/);
});
test('researching-sources: credits carry source, commit and licence', () => {
const credits = section(read(SKILL), /^## Credits/m, /^## /m);
assert.match(credits, /source-driven-development/);
assert.match(credits, /9d0c60d/);
assert.match(credits, /SKILL\.md:97-114/);
assert.match(credits, /diagnosing-superpowers/);
assert.match(credits, /8ca22db/);
assert.equal((credits.match(/\bMIT\b/g) || []).length, 2, 'both borrowed sources are MIT and say so');
});
// --- wiring: agents preload it, the command names it -----------------------
test('every web-search agent preloads researching-sources through its skills: line', () => {
for (const a of WEB_AGENTS) {
const fm = frontmatter(read(`agents/${a}.md`));
const skills = listField(fm, 'skills');
assert.ok(
skills.some((s) => s === 'voyage:researching-sources' || s === 'researching-sources'),
`agents/${a}.md skills: must list researching-sources (got ${JSON.stringify(skills)})`,
);
}
});
test('the web-search agent list is complete: every agent with WebSearch or WebFetch is in it', () => {
const withWeb = readdirSync(join(ROOT, 'agents'))
.filter((f) => f.endsWith('.md'))
.filter((f) => /^tools:.*\bWeb(Search|Fetch)\b/m.test(frontmatter(read(`agents/${f}`))))
.map((f) => f.replace(/\.md$/, ''))
.sort();
assert.deepEqual(withWeb, [...WEB_AGENTS].sort());
});
test('/trekresearch names researching-sources where it starts the external agents', () => {
const ext = section(read('commands/trekresearch.md'), /^### External agents/m, /^### Launch rules/m);
assert.match(ext, /`researching-sources` skill/);
assert.match(ext, /voyage:researching-sources/);
assert.match(ext, /Name it in every external spawn prompt/);
});
// --- the copied text is gone ------------------------------------------------
// Measured 08.10 on 4ee711a: these sentences carried the shared source rules in
// the agents. Each now lives in the skill; a copy coming back is drift.
const MOVED = [
['docs-researcher', 'Never invent documentation.'],
['docs-researcher', 'Always include source URLs.'],
['docs-researcher', 'Date everything.'],
['docs-researcher', 'Flag version mismatches.'],
['docs-researcher', 'Fall back to WebSearch silently'],
['research-scout', 'Never invent documentation.'],
['research-scout', 'Always include source URLs.'],
['research-scout', 'Date everything.'],
['research-scout', 'Fall back to WebSearch silently'],
['community-researcher', 'Date your sources.'],
['community-researcher', 'Flag if a "problem" has since been fixed.'],
['security-researcher', 'Distinguish absence of data from absence of vulnerabilities.'],
['security-researcher', 'Every security concern raised must have a verifiable source.'],
['contrarian-researcher', 'that IS a finding'],
['contrarian-researcher', 'This increases confidence in the original conclusion'],
['contrarian-researcher', 'it is a valuable finding'],
['contrarian-researcher', 'Every counter-argument needs a real source.'],
['contrarian-researcher', 'Check recency.'],
];
test('the source rules that moved into the skill are no longer copied into the agents', () => {
const back = MOVED.filter(([a, s]) => read(`agents/${a}.md`).includes(s));
assert.deepEqual(back, [], 'copied rule text is back in an agent — it belongs to the skill');
});
test('contrarian-researcher: an empty counter-evidence search is reported, and is not support', () => {
const text = read('agents/contrarian-researcher.md');
assert.match(text, /absence of counter-evidence is not support/);
assert.match(text, /does not raise\s+confidence in the original conclusion/);
});
test('each web-search agent keeps its own role and points at the skill for the shared rules', () => {
for (const a of WEB_AGENTS) {
const body = read(`agents/${a}.md`).replace(/^---\n[\s\S]*?\n---\n/, '');
assert.match(body, /preloaded\s+`researching-sources` skill/, `agents/${a}.md must point at the preloaded skill`);
assert.match(body, /This\s+file holds only your own role\./, `agents/${a}.md must say the rest is its own role`);
}
});
// --- one confidence scale ---------------------------------------------------
function norm(s) {
return s.replace(/\*\*/g, '').replace(/\s+/g, ' ').trim().replace(/\.$/, '').toLowerCase();
}
test('one confidence scale: the skill table and /trekresearch triangulation define the same four ratings', () => {
const table = section(read(SKILL), /^### Confidence scale/m, /\n##+ /);
const skillRows = [...table.matchAll(/^\| \*\*(\w+)\*\* \| (.+) \|$/gm)].map((m) => [m[1], norm(m[2])]);
const tri = section(read('commands/trekresearch.md'), /^## Phase 6 — Triangulation/m, /^## Phase 7/m);
const cmdRows = [...tri.matchAll(/^ {3}- \*\*(\w+)\*\* — (.+)$/gm)].map((m) => [m[1], norm(m[2])]);
assert.deepEqual(skillRows.map((r) => r[0]), ['high', 'medium', 'low', 'contradictory']);
assert.deepEqual(cmdRows, skillRows, 'Phase 6 must define the ratings word for word as the skill does');
});
test('one confidence scale: the deep-research adapter and the brief template accept the same four ratings', () => {
const cmd = read('commands/trekresearch.md');
assert.match(cmd, /`### \{Dimension\} -- Confidence: \{high\|medium\|low\|contradictory\}`/);
assert.match(read('templates/research-brief-template.md'), /Confidence: \{high \| medium \| low \| contradictory\}/);
});
// --- the eval cases ---------------------------------------------------------
test('researching-sources eval cases: 3 should trigger, 2 should not, 1 effect case', () => {
const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => d.startsWith('skill-researching-sources-'));
const hits = dirs.filter((d) => d.includes('-hit-'));
const misses = dirs.filter((d) => d.includes('-miss-'));
const effect = dirs.filter((d) => d.includes('-effect-'));
assert.ok(hits.length >= 3, `need >= 3 should-trigger cases, have ${hits.length}`);
assert.ok(misses.length >= 2, `need >= 2 should-not-trigger cases, have ${misses.length}`);
assert.equal(effect.length, 1);
for (const d of hits) assert.match(read(`evals/${d}/graders/skill-loaded.md`), /tool: Skill[\s\S]*min: 1/);
for (const d of misses) assert.match(read(`evals/${d}/graders/skill-not-loaded.md`), /min: 0\nmax: 0/);
for (const d of dirs) {
assert.ok(existsSync(join(ROOT, 'evals', d, 'prompt.md')), `${d}/prompt.md`);
assert.doesNotMatch(read(`evals/${d}/prompt.md`), /\bproevesett\b/, 'skill cases are not part of the fixed test set');
}
});

150
tests/lib/skills-workshop-prose-pins.test.mjs Normal file
View file

@ -0,0 +1,150 @@
// tests/lib/skills-workshop-prose-pins.test.mjs
// Pins for the skill workshop (ordre 7a, contracts S1 and S2): every skill is
// asked for by a command or an agent ("the commands are the trunk"), none is
// closed to the model, the step helper preloads test-first and debugging, each
// skill names what enforces it and credits what it borrowed, the eval cases
// keep the 3 / 2 / 1 form, and the docs carry the ## Skills section.
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { findSkillUsers } from '../../lib/skills/skill-lint.mjs';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
import { loadCases } from '../../lib/skills/routing.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
const SKILLS = readdirSync(join(ROOT, 'skills')).filter((d) => existsSync(join(ROOT, 'skills', d, 'SKILL.md'))).sort();
test('pin: Voyage ships the test-first and debugging skills (S2)', () => {
for (const s of ['test-first', 'debugging']) assert.ok(SKILLS.includes(s), `skills/${s}/SKILL.md missing`);
});
// ---- "the commands are the trunk": a skill nobody asks for is red ----
test('pin: findSkillUsers sees a skills: line and a voyage:<name> mention, and reports an unused skill', () => {
const root = mkdtempSync(join(tmpdir(), 'skill-users-'));
try {
for (const s of ['used-by-agent', 'used-by-command', 'unused']) {
mkdirSync(join(root, 'skills', s), { recursive: true });
writeFileSync(join(root, 'skills', s, 'SKILL.md'), `---\nname: ${s}\ndescription: x\n---\n`);
}
mkdirSync(join(root, 'agents'));
mkdirSync(join(root, 'commands'));
writeFileSync(join(root, 'agents', 'a.md'), '---\nname: a\nskills:\n - voyage:used-by-agent\n---\nbody\n');
writeFileSync(join(root, 'commands', 'c.md'), '---\nname: c\n---\nCall the Skill tool with `voyage:used-by-command`.\n');
const users = findSkillUsers(root);
assert.deepEqual(users['used-by-agent'], ['agents/a.md']);
assert.deepEqual(users['used-by-command'], ['commands/c.md']);
assert.deepEqual(users.unused, []);
// A bare word in prose is not a request for the skill.
writeFileSync(join(root, 'commands', 'd.md'), '---\nname: d\n---\nThis mentions unused in passing.\n');
assert.deepEqual(findSkillUsers(root).unused, []);
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('pin: every skill in skills/ is asked for by at least one command or agent', () => {
const users = findSkillUsers(ROOT);
for (const s of SKILLS) assert.ok(users[s] && users[s].length > 0, `skills/${s}: no command or agent names it`);
});
test('pin: no Voyage skill sets disable-model-invocation (it would block preloading and command calls)', () => {
for (const s of SKILLS) {
const f = readFrontmatterFields(read(`skills/${s}/SKILL.md`)).fields['disable-model-invocation'];
assert.ok(!f || !/^(true|yes|on|1)$/i.test(String(f.value).trim()), `skills/${s}: disable-model-invocation is set`);
}
});
// ---- S2: the step helper preloads both ----
test('pin: step-implementer preloads voyage:test-first and voyage:debugging, without the Skill tool', () => {
const fm = readFrontmatterFields(read('agents/step-implementer.md')).fields;
assert.deepEqual(fm.skills.value, ['voyage:test-first', 'voyage:debugging']);
// Preloading uses the skills field, not the Skill tool (sub-agents.md,
// "Preload skills into subagents"): the tool list stays as it was.
assert.deepEqual(fm.tools.value, ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep']);
});
// ---- each skill says what enforces it, and credits what it borrowed ----
test('pin: debugging says plainly that the root-cause claim is prose only', () => {
const t = read('skills/debugging/SKILL.md');
const enforced = t.split('## Enforced by')[1].split('## Credits')[0];
assert.match(enforced, /prose only: the claim that the root cause was found/);
assert.match(enforced, /lib\/verification\/red-proof\.mjs/);
});
test('pin: test-first says the proof of red counts a crash as red, so the assertion red is prose only', () => {
const enforced = read('skills/test-first/SKILL.md').split('## Enforced by')[1].split('## Credits')[0];
assert.match(enforced, /a crash also passes it/);
for (const p of ['lib/verification/red-proof.mjs', 'lib/util/test-lock.mjs', 'lib/verification/test-integrity.mjs', 'lib/verification/done-line.mjs']) {
assert.ok(enforced.includes(p), `test-first must name ${p}`);
}
});
test('pin: each skill credits its sources with commit and licence', () => {
const want = {
'test-first': ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
debugging: ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
};
for (const [s, sources] of Object.entries(want)) {
const credits = read(`skills/${s}/SKILL.md`).split('## Credits')[1];
for (const src of sources) assert.ok(credits.includes(src), `${s}: credit ${src}`);
assert.equal((credits.match(/\bMIT\b/g) || []).length, sources.length, `${s}: one licence per source`);
}
});
// ---- S1: the eval case form (3 should fire / 2 should not / 1 effect) ----
test('pin: each skill has 3 trigger, 2 no-trigger and 1 effect case, and the graders read the Skill tool', () => {
const cases = loadCases(ROOT);
for (const s of SKILLS) {
const mine = cases.filter((c) => c.skill === s);
const n = (k) => mine.filter((c) => c.kind === k);
assert.ok(n('trigger').length >= 3 && n('no-trigger').length >= 2 && n('effect').length >= 1, `${s}: ${mine.length} cases`);
for (const c of [...n('trigger'), ...n('no-trigger')]) {
const dir = join(ROOT, 'evals', c.case, 'graders');
const g = readdirSync(dir).map((f) => readFrontmatterFields(readFileSync(join(dir, f), 'utf8')).fields);
const skillGrader = g.find((x) => x.tool && x.tool.value === 'Skill');
assert.ok(skillGrader, `${c.case}: needs a tool_used Skill grader`);
const re = new RegExp(skillGrader.input_match.value);
assert.match(JSON.stringify({ skill: `voyage:${s}` }), re, `${c.case}: must see voyage:${s}`);
assert.match(JSON.stringify({ skill: s }), re, `${c.case}: must see the bare name`);
for (const other of SKILLS.filter((o) => o !== s)) assert.doesNotMatch(JSON.stringify({ skill: `voyage:${other}` }), re);
if (c.kind === 'no-trigger') {
assert.deepEqual([skillGrader.min.value, skillGrader.max.value, skillGrader.arm.value], ['0', '0', 'both'], c.case);
}
}
for (const c of n('effect')) {
const desc = readFrontmatterFields(read(`evals/${c.case}/prompt.md`)).fields.description.value;
assert.match(desc, /judge model explicitly \(never Haiku\)/, `${c.case}: the judge model is the operator's explicit choice`);
}
}
});
// ---- docs: ## Skills, the authoring guide ----
test('pin: README and CLAUDE.md carry a ## Skills section naming both skills and the workshop checks', () => {
for (const f of ['README.md', 'CLAUDE.md']) {
const t = read(f);
const sec = t.split(/^## Skills$/m)[1];
assert.ok(sec, `${f}: ## Skills missing`);
const body = sec.split(/^## /m)[0];
for (const s of ['test-first', 'debugging', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'docs/skills-authoring.md']) {
assert.ok(body.includes(s), `${f} ## Skills must name ${s}`);
}
}
});
test('pin: docs/skills-authoring.md teaches the red baseline and stays public-safe', () => {
const t = read('docs/skills-authoring.md');
for (const s of ['## Red baseline first', 'control', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'skill-trigger', 'skill-no-trigger', 'skill-effect', 'from=']) {
assert.ok(t.includes(s), `skills-authoring.md must cover ${s}`);
}
assert.doesNotMatch(t, /\/Users\/|\/private\/tmp|\.local\.md/, 'no local paths in a public doc');
});

130
tests/lib/trial2-minor-prose-pins.test.mjs Normal file
View file

@ -0,0 +1,130 @@
// tests/lib/trial2-minor-prose-pins.test.mjs
// Prose pins for the minor points of the second voyage-lekerepo trial
// (2026-10-08, report parts B and C) whose fix is text a session reads.
// Counted as doc pins by lib/util/test-census.mjs (the file name).
// B8 minors left after the last critic round have a way to be fixed
// (operator decision 2026-10-08: listed in the summary, the operator picks).
// B9 K10 `Regression:` is described where plans are written and reviewed.
// B10 K11 the four `On failure:` words are named where plans are written.
// B12 /trekbrief Step 4c no longer claims a foreground, blocking spawn.
// C13 the codebase-size count in /trekplan Phase 4 counts .mjs/.cjs/.mts/.cts.
// C17 git-historian leaves the brief's frozen files alone and names the
// remote from `git remote -v`.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { findSection } from '../../lib/parsers/plan-schema.mjs';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
/** The text of a level-2 section, up to the next `## ` heading outside a fence. */
function section(text, heading) {
const s = findSection(text, heading.replace(/^## /, ''), 2);
assert.ok(s, `missing ${heading}`);
return s.text;
}
// --- B8 ------------------------------------------------------------------------
test('B8: a PASS with minors sends them to the Phase 10 summary, numbered, for the operator to pick', () => {
const plan = read('commands/trekplan.md');
const p9 = section(plan, '## Phase 9 — Adversarial review (bounded critic loop)');
assert.match(p9, /\*\*PASS\*\*[\s\S]{0,600}minors[\s\S]{0,300}Phase 10/i);
const p10 = section(plan, '## Phase 10 — Present and refine');
assert.match(p10, /\*\*Open minors:\*\*/);
assert.match(p10, /numbered/i);
});
test('B8: fixing picked minors after the last round runs one more round with --extra-round, the pick being the yes', () => {
const p10 = section(read('commands/trekplan.md'), '## Phase 10 — Present and refine');
assert.match(p10, /--extra-round/);
assert.match(p10, /pick(ed|s)?\s+(is|counts as)\s+(the|their)\s+yes/i);
});
// --- B9 / B10 --------------------------------------------------------------------
test('B9: /trekplan Phase 8 names `Regression:` as an allowed optional field and says what it is for', () => {
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
assert.match(p8, /`Regression:`/);
assert.match(p8, /PLAN_STEP_REGRESSION_NOT_RUNNABLE/);
assert.match(p8, /\*\*Regression:\*\* `npm test`/, 'the canonical example shows the field');
assert.match(p8, /before (the|its) checkpoint/i);
});
test('B9: the plan template shows `Regression:` as optional', () => {
const t = read('templates/plan-template.md');
assert.match(t, /- \*\*Regression:\*\* `[^\n]*\*\(optional\)\*/);
assert.match(t, /`Regression:` \(ONE backticked command/);
});
test('B9: plan-critic knows the field and does not demand the suite in `Verify:`', () => {
const c = read('agents/plan-critic.md');
assert.match(c, /`Regression:`/);
assert.match(c, /(never|do\s+not)\s+(demand|require)\s+`npm test`\s+in\s+`Verify:`/i);
});
test('B9/B10: Handover 4 documents `Regression:` (K10) and the `On failure:` words (K11)', () => {
const h4 = section(read('docs/HANDOVER-CONTRACTS.md'), '## Handover 4 — `plan.md` → execute');
for (const code of ['PLAN_STEP_REGRESSION_NOT_RUNNABLE', 'PLAN_STEP_ON_FAILURE_UNKNOWN']) assert.match(h4, new RegExp(code));
assert.match(h4, /K10/);
assert.match(h4, /K11/);
assert.match(h4, /revert[^\n]*retry[^\n]*skip[^\n]*escalate/);
});
test('B10: /trekplan Phase 8 says `On failure:` opens with one of the four words', () => {
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
assert.match(p8, /On failure:\*\*[^\n]*(opens|starts) with/i);
assert.match(p8, /PLAN_STEP_ON_FAILURE_UNKNOWN/);
});
// --- B12 -------------------------------------------------------------------------
test('B12: /trekbrief Step 4c does not call the brief-reviewer spawn foreground or blocking, and waits for it', () => {
const t = read('commands/trekbrief.md');
const start = t.indexOf('**Step 4c — Launch brief-reviewer**');
const end = t.indexOf('**Step 4d — Parse JSON scores**');
assert.ok(start !== -1 && end > start);
const step = t.slice(start, end);
assert.doesNotMatch(step, /foreground, blocking/);
assert.match(step, /wait\s+for\s+(its|the agent's)\s+(result|completion)/i);
assert.match(step, /(background|asynchronous)/i, 'says how the spawn actually runs');
});
// --- C13 -------------------------------------------------------------------------
test('C13: the Phase 4 size count counts .mjs, .cjs, .mts and .cts files', () => {
const p4 = section(read('commands/trekplan.md'), '## Phase 4 — Codebase sizing');
const cmd = p4.match(/```\n(find [^\n]+)\n```/);
assert.ok(cmd, 'the find command stands in a fence of its own');
const dir = mkdtempSync(join(tmpdir(), 'voyage-sizing-'));
try {
for (const f of ['a.mjs', 'b.cjs', 'c.mts', 'd.cts', 'e.ts', 'skip.txt']) writeFileSync(join(dir, f), 'x\n');
const r = spawnSync('/bin/sh', ['-c', cmd[1]], { cwd: dir, encoding: 'utf8' });
assert.equal(r.status, 0, r.stderr);
assert.equal(r.stdout.trim(), '5');
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// --- C17 -------------------------------------------------------------------------
test('C17: git-historian reads the brief, never proposes changing a file it freezes', () => {
const g = read('agents/git-historian.md');
assert.match(g, /brief/i);
assert.match(g, /(never|do not) (propose|suggest|recommend)[^\n]*(change|edit)[^\n]*(frozen|unchanged)|(frozen|unchanged)[^\n]*(never|do not) (propose|suggest|recommend)/i);
const p5 = section(read('commands/trekplan.md'), '## Phase 5 — Parallel exploration (specialized agents + research)');
assert.match(p5, /\*\*git-historian\*\*[\s\S]{0,400}\{brief_path\}/, 'the spawn hands it the brief');
});
test('C17: git-historian names the remote from `git remote -v`, never by guess', () => {
const g = read('agents/git-historian.md');
assert.match(g, /`git remote -v`/);
assert.match(g, /local (path|repository|bare)/i);
});

253
tests/lib/trial2-minor.test.mjs Normal file
View file

@ -0,0 +1,253 @@
// tests/lib/trial2-minor.test.mjs
// The minor points of the second voyage-lekerepo trial (2026-10-08, report
// parts B and C) that are behaviour, not prose. Each test was red before its
// fix.
// B9 K10 — optional step field `Regression:`: one runnable command, or the
// validator refuses it.
// B10 K11 — `On failure:` opens with revert | retry | skip | escalate, or the
// validator warns.
// B11 two findings on the same file:line with the same content merge (the
// trial's pair measured Jaccard 0.44 against a 0.7 threshold).
// C14 `--evidence` shows the first output line on a PASS row, not "—".
// C16 a file:line cited in a reviewer's prose is checked against the file.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
import { onFailureAction, ON_FAILURE_ACTIONS } from '../../lib/parsers/plan-schema.mjs';
import { dedupFindings, DEFAULT_THRESHOLD } from '../../lib/review/plan-review-dedup.mjs';
import { formatCriteriaEvidence } from '../../lib/verification/criteria-runner.mjs';
import { validateFindings, validateReviewerOutput } from '../../lib/review/findings-schema.mjs';
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
const codes = (list) => list.map((e) => e.code);
// Step 1's Verify line in the builder's plan; the new field goes right under it.
const STEP1_VERIFY = '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass';
const withRegression = (line, plan = buildPlan()) => plan.replace(STEP1_VERIFY, `${STEP1_VERIFY}\n${line}`);
const withOnFailure = (text, plan = buildPlan()) =>
plan.replace(/^- \*\*On failure:\*\* .*$/m, `- **On failure:** ${text}`);
// --- B9: K10 `Regression:` ----------------------------------------------------
test('K10: `Regression: npm test | tee x` is refused — one command, unpiped', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x` → expected: exit 0'));
assert.equal(r.valid, false);
const e = r.errors.find((x) => x.code === 'PLAN_STEP_REGRESSION_NOT_RUNNABLE');
assert.ok(e, codes(r.errors).join(','));
assert.match(e.message, /Step 1/);
assert.match(e.message, /shell operator/);
});
test('K10: `Regression: npm test` is accepted', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test` → expected: exit 0'));
assert.deepEqual(codes(r.errors), []);
assert.ok(!codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
test('K10: without the field the plan validates exactly as before', () => {
const r = validatePlanContent(buildPlan());
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.deepEqual(codes(r.warnings), []);
});
test('K10: `npm test && bash tests/e2e.sh`, a bare word, or no backticks is refused', () => {
for (const line of [
'- **Regression:** `npm test && bash tests/e2e.sh`',
'- **Regression:** `./run-all-tests`',
'- **Regression:** npm test',
'- **Regression:**',
]) {
const r = validatePlanContent(withRegression(line));
assert.ok(codes(r.errors).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), `${line}: ${codes(r.errors).join(',')}`);
}
});
test('K10: --soft reports an unrunnable Regression as a warning, not an error', () => {
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x`'), { strict: false });
assert.equal(r.valid, true, codes(r.errors).join(','));
assert.ok(codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
});
// --- B10: K11 `On failure:` ---------------------------------------------------
test('K11: `On failure: try again` warns, and the plan stays valid', () => {
const r = validatePlanContent(withOnFailure('try again'));
assert.equal(r.valid, true, codes(r.errors).join(','));
const w = r.warnings.find((x) => x.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN');
assert.ok(w, codes(r.warnings).join(','));
assert.match(w.message, /Step 1/);
for (const word of ON_FAILURE_ACTIONS) assert.match(w.hint, new RegExp(word));
});
test('K11: `On failure: retry once, then escalate` gives no warning', () => {
const r = validatePlanContent(withOnFailure('retry once, then escalate'));
assert.ok(!codes(r.warnings).includes('PLAN_STEP_ON_FAILURE_UNKNOWN'), codes(r.warnings).join(','));
});
test('K11: the four words, in any case and with markdown around them, are read; a longer word is not', () => {
assert.deepEqual([...ON_FAILURE_ACTIONS], ['revert', 'retry', 'skip', 'escalate']);
assert.equal(onFailureAction('revert — `git checkout -- x`'), 'revert');
assert.equal(onFailureAction('Escalate: stop the run'), 'escalate');
assert.equal(onFailureAction('**skip** this step'), 'skip');
assert.equal(onFailureAction('`retry` once'), 'retry');
assert.equal(onFailureAction('retrying is fine'), null);
assert.equal(onFailureAction('try again'), null);
assert.equal(onFailureAction(''), null);
});
// --- B11: same file:line, same content ---------------------------------------
// The two findings from trial 2's round 1, verbatim (plan.md:248).
const PLAN_MD = '.claude/projects/2026-10-08-slug-sep-option/plan.md';
const CRITIC_248 = {
severity: 'minor', rule_key: 'verification/contract-brief-form-missing', file: PLAN_MD, line: 248,
text: "Verification runs `bash tests/e2e.sh` in the brief's own form but not `bash tests/contract.sh`, the form brief SC4 names.",
};
const GUARDIAN_248 = {
severity: 'minor', rule_key: 'gap-verification-command-form', file: PLAN_MD, line: 248,
text: "Brief SC4 runs 'bash tests/contract.sh'; Verification lists only '/bin/bash tests/contract.sh' (unlike e2e, where both forms are listed) - add the brief's form",
};
const sources = (a, b) => [
{ agent: 'plan-critic', payload: { findings: a } },
{ agent: 'scope-guardian', payload: { findings: b } },
];
test('B11: the two trial-2 findings on plan.md:248 merge into one, raised by both', () => {
const r = dedupFindings(sources([CRITIC_248], [GUARDIAN_248]));
assert.equal(r.findings.length, 1, JSON.stringify(r.findings, null, 2));
assert.deepEqual(r.findings[0].raised_by, ['plan-critic', 'scope-guardian']);
assert.equal(r.dedup_stats.anchor_dups, 1);
});
test('B11: two different findings on the same line stay two', () => {
const other = {
severity: 'minor', rule_key: 'verification/no-negative-case', file: PLAN_MD, line: 248,
text: 'No criterion feeds an invalid separator such as a letter and checks the exit code.',
};
const r = dedupFindings(sources([CRITIC_248], [other]));
assert.equal(r.findings.length, 2);
});
test('B11: a short finding held by a longer one on the same line stays its own (independent review)', () => {
const at = (text, severity = 'minor') => ({ severity, rule_key: `k-${text.length}`, file: PLAN_MD, line: 40, text });
for (const [a, b] of [
['The token is checked.', 'The token is not checked before the write.'],
['Step 3 test is missing.', 'Step 3 is missing a checkpoint; the test file already exists.'],
['Verify runs npm test.', 'Verify must not run npm test, it is green before the step.'],
]) {
assert.equal(dedupFindings(sources([at(a)], [at(b)])).findings.length, 2, `${a} | ${b}`);
}
});
test('B11: one reviewer\'s two findings on one line are never merged by the anchor rule', () => {
const r = dedupFindings([{ agent: 'plan-critic', payload: { findings: [CRITIC_248, { ...GUARDIAN_248, rule_key: 'other' }] } }]);
assert.equal(r.findings.length, 2);
assert.equal(r.dedup_stats.anchor_dups, 0);
});
test('B11: the same content on another line is not merged by the anchor rule; the threshold elsewhere is unchanged', () => {
assert.equal(DEFAULT_THRESHOLD, 0.7);
const r = dedupFindings(sources([CRITIC_248], [{ ...GUARDIAN_248, line: 249 }]));
assert.equal(r.findings.length, 2);
const noLine = dedupFindings(sources([{ ...CRITIC_248, line: 0 }], [{ ...GUARDIAN_248, line: 0 }]));
assert.equal(noLine.findings.length, 2, 'line 0 is file-scoped, not an anchor');
});
// --- C14: --evidence on a PASS row ---------------------------------------------
const evidenceRep = (results) => ({
source: 'brief.md',
heading: '## Success Criteria',
error: null,
results,
summary: { passed: 1, failed: 0, blocked: 0, unrunnable: 0, total: 1, ok: true },
});
test('C14: --evidence shows the first line of output on a passed criterion', () => {
const block = formatCriteriaEvidence(evidenceRep([
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '\n# tests 12\n# pass 12\n' },
]));
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| # tests 12 \|/);
});
test('C14: a passed criterion with no output still says "—"', () => {
const block = formatCriteriaEvidence(evidenceRep([
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '' },
]));
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| — \|/);
});
// --- C16: file:line in prose --------------------------------------------------
function repoWith33() {
const dir = mkdtempSync(join(tmpdir(), 'voyage-prose-cite-'));
mkdirSync(join(dir, 'tests'));
writeFileSync(join(dir, 'tests', 'x.sh'), Array.from({ length: 33 }, (_, i) => `echo ${i + 1}`).join('\n') + '\n');
return dir;
}
const finding = (over = {}) => ({
id: '0123456789abcdef0123456789abcdef01234567',
severity: 'MAJOR', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'tests/x.sh', line: 3,
title: 'A title', detail: 'A detail.', recommended_action: 'Do one thing.', ...over,
});
test('C16: a finding whose prose cites `tests/x.sh:119` in a 33-line file is refused', () => {
const dir = repoWith33();
try {
const r = validateFindings(
{ reviewer: 'code-correctness-reviewer', findings: [finding({ detail: 'The check at `tests/x.sh:119` relies on git.' })] },
{ repoRoot: dir },
);
assert.equal(r.valid, false);
const e = r.errors.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
assert.ok(e, JSON.stringify(r.errors));
assert.match(e.message, /tests\/x\.sh:119/);
assert.match(e.message, /33 lines/);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('C16: the same citation in prose outside the JSON block is a warning, never silent (a JSON-only re-ask cannot fix prose)', () => {
const dir = repoWith33();
try {
const raw = [
'### Risk noted but not raised as a finding',
'',
'`tests/x.sh:119` relies on `git` and on a fixed commit.',
'',
'```json',
JSON.stringify({ reviewer: 'code-correctness-reviewer', findings: [] }),
'```',
].join('\n');
const r = validateReviewerOutput(raw, { repoRoot: dir });
assert.equal(r.valid, true, JSON.stringify(r.errors));
const w = r.warnings.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
assert.ok(w, JSON.stringify(r.warnings));
assert.match(w.message, /reviewer prose cites `?tests\/x\.sh:119/);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('C16: a line that exists, a range ending in the file, a file absent from the tree and no repo root all pass', () => {
const dir = repoWith33();
try {
const ok = (detail, opts = { repoRoot: dir }) =>
validateFindings({ reviewer: 'r', findings: [finding({ detail })] }, opts).valid;
assert.equal(ok('See tests/x.sh:33 and tests/x.sh:10-12.'), true);
assert.equal(ok('See src/absent.mjs:900.'), true);
assert.equal(ok('See tests/x.sh:119.', {}), true);
assert.equal(ok('See tests/x.sh:30-40.'), false, 'a range that runs past the end is out of range');
assert.equal(ok('Served on http://localhost:8080 and example.com:443.'), true);
assert.equal(ok('On Windows: C:\\work\\tests\\x.sh:119 and tests\\x.sh:119.'), true, 'a Windows path is not the root\'s file');
assert.equal(ok('See tests/x.sh:30\u201440.'), false, 'an em-dash range is read as a range');
} finally {
rmSync(dir, { recursive: true, force: true });
}
});