merge: main (7a skills workshop, 9 rettepakke, 7b researching-sources, 8 release runbook) into wave2-kobling
Union lines resolved: CLAUDE.md command table keeps main's /trekresearch and /trekplan rows and this branch's /trekexecute row. README's architecture block now counts the three skills main ships (it said 2 after 7b added researching-sources). Contracts run against the real ones on the merged tree: K10/K11 (plan validator refuses a piped Regression:, warns on an unknown On failure:; regression-check passes/fails/absent/unrunnable), S2 (skills/test-first and skills/debugging exist, voyage:<name> form as 7a chose), K7 (real review_version 1.1 WARN -> 0, a BLOCK copy -> 1, a 1.0 review with no verdict -> 2). npm test: 2102 / 2100 pass / 0 fail / 2 skipped. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
3c18fdb5f7
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
83 changed files with 4293 additions and 71 deletions
27
tests/fixtures/release-drill/rollback-commands.txt
vendored
Normal file
27
tests/fixtures/release-drill/rollback-commands.txt
vendored
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Commands run in the release + rollback drill (wave-2 order 8, P13), phases
|
||||
# ROLLBACK A and ROLLBACK B, copied from the drill log of 2026-10-08T19:27:15Z.
|
||||
# The drill ran in a sandbox: local bare repos as the remotes, a HOME of its own,
|
||||
# no FORGEJO_TOKEN. Versions are written as the runbook's placeholders:
|
||||
# 6.2.0 = <previous-version>, 6.2.1 = <withdrawn-version>, 6.2.2 = <next-patch>
|
||||
# Commits appear as <release-commit>, <merge-commit>, <revert-commit>; the review
|
||||
# file as <review.md>; drill-only helpers as <drill-helper>. [label] = the repo it ran in.
|
||||
# tests/lib/release-prose-pins.test.mjs: every command in docs/release-runbook.md § 3
|
||||
# must be one of these lines.
|
||||
[voyage] git revert --no-edit <release-commit>
|
||||
[voyage] git revert --no-edit -m 1 <merge-commit>
|
||||
[voyage] node -p require('./.claude-plugin/plugin.json').version
|
||||
[voyage] git push -q origin main
|
||||
[catalog] node scripts/check-versions.mjs
|
||||
[catalog] node scripts/release-plugin.mjs voyage --version <previous-version>
|
||||
[catalog] mkdir -p "$HOME/.claude/runtime/push-approvals" && touch "$HOME/.claude/runtime/push-approvals/$(pwd | sed 's|/|_|g')"
|
||||
[catalog] node scripts/release-plugin.mjs voyage --version <previous-version> --write --commit --push
|
||||
[catalog] node scripts/check-versions.mjs
|
||||
[sandbox] git -C $SANDBOX/remotes/catalog.git show main:.claude-plugin/marketplace.json
|
||||
[voyage] node <drill-helper>/bump.mjs . <previous-version> <next-patch> $SANDBOX/section-<next-patch>.md
|
||||
[voyage] git commit -q -am "chore(release): <next-patch> - withdraws <withdrawn-version>"
|
||||
[voyage] node scripts/release-check.mjs --review <review.md>
|
||||
[voyage] node lib/util/approval-log.mjs append --kind release --subject CHANGELOG.md --sha256-of CHANGELOG.md
|
||||
[voyage] git push -q origin main
|
||||
[catalog] mkdir -p "$HOME/.claude/runtime/push-approvals" && touch "$HOME/.claude/runtime/push-approvals/$(pwd | sed 's|/|_|g')"
|
||||
[catalog] node scripts/release-plugin.mjs voyage --create-tag --write --commit --push
|
||||
[catalog] node scripts/check-versions.mjs
|
||||
146
tests/lib/release-prose-pins.test.mjs
Normal file
146
tests/lib/release-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
// tests/lib/release-prose-pins.test.mjs
|
||||
// Prose pins owned by the release process (wave-2 order 8, P13). Counted as
|
||||
// doc pins by lib/util/test-census.mjs (PIN_FILE_RE matches `prose-pins.test.mjs`).
|
||||
//
|
||||
// The load-bearing pin: the rollback part of docs/release-runbook.md may only
|
||||
// hold commands that were run in the release + rollback drill. The drill's
|
||||
// rollback commands are recorded in tests/fixtures/release-drill/rollback-commands.txt
|
||||
// (copied from the drill log, versions and commits written as the runbook's
|
||||
// placeholders). A command written into § 3 that the drill never ran fails here.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseDoors, releaseSection } from '../../scripts/release-check.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
|
||||
const RUNBOOK = 'docs/release-runbook.md';
|
||||
const FIXTURE = 'tests/fixtures/release-drill/rollback-commands.txt';
|
||||
const APPROVAL = 'node lib/util/approval-log.mjs append --kind release --subject CHANGELOG.md --sha256-of CHANGELOG.md';
|
||||
|
||||
/** The lines from `## <heading>` to the next `## ` heading outside a fenced block. */
|
||||
function part(md, heading) {
|
||||
const lines = md.split('\n');
|
||||
const start = lines.findIndex((l) => l === `## ${heading}`);
|
||||
assert.ok(start !== -1, `${RUNBOOK} must have "## ${heading}"`);
|
||||
let fenced = false;
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('```')) fenced = !fenced;
|
||||
if (!fenced && lines[i].startsWith('## ')) { end = i; break; }
|
||||
}
|
||||
return lines.slice(start, end).join('\n');
|
||||
}
|
||||
|
||||
/** Every command line in the ```sh blocks of a markdown part. */
|
||||
function shCommands(md) {
|
||||
const out = [];
|
||||
const re = /```sh\n([\s\S]*?)```/g;
|
||||
let m;
|
||||
while ((m = re.exec(md)) !== null) {
|
||||
for (const line of m[1].split('\n')) {
|
||||
const t = line.trim();
|
||||
if (t && !t.startsWith('#')) out.push(t);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function drillCommands() {
|
||||
return read(FIXTURE).split('\n')
|
||||
.filter((l) => l.trim() && !l.startsWith('#'))
|
||||
.map((l) => l.replace(/^\[(voyage|catalog)\] /, '').trim());
|
||||
}
|
||||
|
||||
test('runbook: the three parts, in order — before, release, rollback', () => {
|
||||
const md = read(RUNBOOK);
|
||||
const at = ['## 1. Before the release', '## 2. The release', '## 3. Rollback'].map((h) => md.indexOf(`\n${h}`));
|
||||
for (const i of at) assert.ok(i !== -1, `${RUNBOOK} must have each of the three parts`);
|
||||
assert.ok(at[0] < at[1] && at[1] < at[2], 'the parts must come in the order before → release → rollback');
|
||||
});
|
||||
|
||||
test('runbook: every command in the rollback part was run in the drill', () => {
|
||||
const ran = new Set(drillCommands());
|
||||
const written = shCommands(part(read(RUNBOOK), '3. Rollback'));
|
||||
assert.ok(written.length >= 10, `the rollback part must carry the drilled commands (found ${written.length})`);
|
||||
const unrun = written.filter((c) => !ran.has(c));
|
||||
assert.deepEqual(unrun, [], `commands in § 3 that the drill never ran:\n${unrun.join('\n')}`);
|
||||
});
|
||||
|
||||
test('drill fixture: both rollback paths were run — revert + ref back, and the forward patch', () => {
|
||||
const ran = drillCommands();
|
||||
for (const c of [
|
||||
'git revert --no-edit <release-commit>',
|
||||
'git revert --no-edit -m 1 <merge-commit>',
|
||||
'node scripts/release-plugin.mjs voyage --version <previous-version> --write --commit --push',
|
||||
'node scripts/release-plugin.mjs voyage --create-tag --write --commit --push',
|
||||
'node scripts/check-versions.mjs',
|
||||
]) assert.ok(ran.includes(c), `the drill fixture must record: ${c}`);
|
||||
});
|
||||
|
||||
test('runbook § 1: the verdict gate (K7), the release check and the logged yes (K2)', () => {
|
||||
const before = part(read(RUNBOOK), '1. Before the release');
|
||||
assert.ok(before.includes('node lib/review/verdict-gate.mjs <review.md>'), '§ 1 must run the K7 verdict gate');
|
||||
assert.ok(before.includes('node scripts/release-check.mjs --review <review.md>'), '§ 1 must run the release check');
|
||||
assert.ok(before.includes(APPROVAL), '§ 1 must log the release approval with the K2 command the check prints');
|
||||
assert.match(before, /npm test/, '§ 1 must require the suite on the merged tree');
|
||||
});
|
||||
|
||||
test('runbook § 2: push last, then one release-plugin run, then the catalog check, then cleanup', () => {
|
||||
const rel = shCommands(part(read(RUNBOOK), '2. The release'));
|
||||
const order = [
|
||||
'git push origin main',
|
||||
'node scripts/release-plugin.mjs voyage --create-tag --write --commit --push',
|
||||
'node scripts/check-versions.mjs',
|
||||
].map((c) => rel.indexOf(c));
|
||||
for (const i of order) assert.ok(i !== -1, '§ 2 must carry the push, the release run and the catalog check');
|
||||
assert.ok(order[0] < order[1] && order[1] < order[2], '§ 2 order: git push, release-plugin, check-versions');
|
||||
assert.match(part(read(RUNBOOK), '2. The release'), /Cleanup comes last/);
|
||||
});
|
||||
|
||||
test('runbook: the door-type example parses as a release section with one-way and two-way lines', () => {
|
||||
const md = part(read(RUNBOOK), 'Door types');
|
||||
const block = md.match(/```markdown\n([\s\S]*?)```/);
|
||||
assert.ok(block, 'the door types part must show the CHANGELOG form');
|
||||
const section = releaseSection(block[1], '6.3.0');
|
||||
assert.ok(section !== null, 'the example must be a release section');
|
||||
const doors = parseDoors(section);
|
||||
assert.deepEqual(doors.entries.map((e) => e.door).sort(), ['one-way', 'two-way']);
|
||||
assert.deepEqual(doors.bad, []);
|
||||
});
|
||||
|
||||
test('runbook: the release check exits as the script does — 0 ready, 1 not ready, 2 never green', () => {
|
||||
const before = part(read(RUNBOOK), '1. Before the release').replace(/\s+/g, ' ');
|
||||
assert.match(before, /0 = ready, 1 = not ready[^.]*, 2 = could not run — never green/);
|
||||
});
|
||||
|
||||
test('runbook: the eval gate (P10) is named as not built, with where it goes', () => {
|
||||
const p10 = part(read(RUNBOOK), 'Not built yet: the eval gate (P10)');
|
||||
assert.ok(p10.includes('scripts/release-check.mjs'), 'P10 must say which file the gate goes into');
|
||||
});
|
||||
|
||||
test('runbook: public — no local paths', () => {
|
||||
const md = read(RUNBOOK);
|
||||
for (const bad of ['/Users/', '/private/', '/tmp/', '~/repos', 'scratchpad']) {
|
||||
assert.ok(!md.includes(bad), `${RUNBOOK} must not carry a local path (${bad})`);
|
||||
}
|
||||
});
|
||||
|
||||
test('README: a section on releasing and rolling back that points to the runbook', () => {
|
||||
const md = read('README.md');
|
||||
const start = md.indexOf('\n## Releasing and rolling back');
|
||||
assert.ok(start !== -1, 'README must have "## Releasing and rolling back"');
|
||||
const sec = md.slice(start, md.indexOf('\n## ', start + 1));
|
||||
assert.ok(sec.includes('docs/release-runbook.md'), 'the README section must link the runbook');
|
||||
assert.ok(sec.includes('scripts/release-check.mjs'), 'the README section must name the release check');
|
||||
});
|
||||
|
||||
test('CLAUDE.md: the pointer to the runbook and the release check', () => {
|
||||
const md = read('CLAUDE.md');
|
||||
assert.ok(md.includes('docs/release-runbook.md'), 'CLAUDE.md must point to the runbook');
|
||||
assert.ok(md.includes('scripts/release-check.mjs'), 'CLAUDE.md must name the release check');
|
||||
});
|
||||
251
tests/lib/skill-lint.test.mjs
Normal file
251
tests/lib/skill-lint.test.mjs
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
// tests/lib/skill-lint.test.mjs
|
||||
// The skill form check (S1): lib/skills/skill-lint.mjs. Model-free, part of
|
||||
// `npm test`. Each rule is shown biting on a skill built for it (known
|
||||
// positive) and passing a skill that keeps the form (known negative); the last
|
||||
// tests run the check over the skills Voyage ships.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { lintSkill, lintSkillsRoot, REQUIRED_HEADINGS, EXEMPTIONS } from '../../lib/skills/skill-lint.mjs';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const CLI = join(ROOT, 'lib', 'skills', 'skill-lint.mjs');
|
||||
|
||||
const BODY = [
|
||||
'# Example', '',
|
||||
'## Overview', 'One line.', '',
|
||||
'## When to use', '- when the example applies', '- not when it does not', '',
|
||||
'## Process', '1. Do the thing.', '',
|
||||
'## Red flags', '- skipping the thing', '',
|
||||
'## Enforced by', '- prose only: nothing checks this.', '',
|
||||
'## Credits', '- Voyage\'s own content.', '',
|
||||
].join('\n');
|
||||
|
||||
const GOOD_FM = [
|
||||
'---',
|
||||
'name: good-skill',
|
||||
'description: Checks an example the same way every time. Use when an example needs checking.',
|
||||
'---',
|
||||
].join('\n');
|
||||
|
||||
/** A plugin-shaped temp root with skills/<name>/SKILL.md; returns the root. */
|
||||
function plugin(skills) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-lint-'));
|
||||
for (const [name, text] of Object.entries(skills)) {
|
||||
mkdirSync(join(root, 'skills', name), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', name, 'SKILL.md'), text);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
const codes = (r) => r.errors.map((e) => e.code).sort();
|
||||
|
||||
function withSkill(name, text, fn) {
|
||||
const root = plugin({ [name]: text });
|
||||
try { return fn(lintSkill(join(root, 'skills', name), { pluginRoot: root }), root); }
|
||||
finally { rmSync(root, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
// ---- the test that fails first (ordre 7a) ----
|
||||
|
||||
test('skill-lint: a skill without "Use when" and with an unquoted colon is red', () => {
|
||||
const text = [
|
||||
'---',
|
||||
'name: bad-skill',
|
||||
'description: Checks things: all of them, every time.',
|
||||
'---',
|
||||
BODY,
|
||||
].join('\n');
|
||||
withSkill('bad-skill', text, (r) => {
|
||||
assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), codes(r).join(','));
|
||||
assert.ok(codes(r).includes('SKILL_DESC_NO_USE_WHEN'), codes(r).join(','));
|
||||
});
|
||||
});
|
||||
|
||||
test('skill-lint: a skill that keeps the form is green (known negative)', () => {
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
// ---- one rule at a time ----
|
||||
|
||||
test('skill-lint: name must equal the directory name and be kebab-case', () => {
|
||||
withSkill('other-dir', `${GOOD_FM}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_MISMATCH')));
|
||||
const fm = GOOD_FM.replace('name: good-skill', 'name: Good_Skill');
|
||||
withSkill('Good_Skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
|
||||
const reserved = GOOD_FM.replace('name: good-skill', 'name: claude-helper');
|
||||
withSkill('claude-helper', `${reserved}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_NAME_FORM')));
|
||||
});
|
||||
|
||||
test('skill-lint: description is required, at most 1 024 characters', () => {
|
||||
const none = GOOD_FM.replace(/^description:.*$/m, 'license: MIT');
|
||||
withSkill('good-skill', `${none}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_MISSING')));
|
||||
const long = `Checks an example. Use when ${'an example needs checking and '.repeat(40)}done.`;
|
||||
assert.ok(long.length > 1024);
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${long}`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_TOO_LONG')));
|
||||
const exact = `Use when x. ${'y'.repeat(1024 - 12)}`;
|
||||
assert.equal(exact.length, 1024);
|
||||
const fm2 = GOOD_FM.replace(/^description:.*$/m, `description: ${exact}`);
|
||||
withSkill('good-skill', `${fm2}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_TOO_LONG')));
|
||||
});
|
||||
|
||||
test('skill-lint: description is in the third person', () => {
|
||||
for (const d of [
|
||||
'I check an example. Use when an example needs checking.',
|
||||
'Checks your example. Use when an example needs checking.',
|
||||
'Use when you need an example checked.',
|
||||
'We check examples. Use when an example needs checking.',
|
||||
]) {
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, `description: ${d}`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON'), d));
|
||||
}
|
||||
const ok = GOOD_FM.replace(/^description:.*$/m, 'description: Checks an example, i.e. one case. Use when the user asks for a check.');
|
||||
withSkill('good-skill', `${ok}\n${BODY}`, (r) => assert.ok(!codes(r).includes('SKILL_DESC_NOT_THIRD_PERSON')));
|
||||
});
|
||||
|
||||
test('skill-lint: YAML traps are caught: unquoted colon, comment marker, tab, unclosed quote, leading indicator', () => {
|
||||
for (const line of [
|
||||
'description: Use when x: y happens.',
|
||||
'description: Use when x happens #1.',
|
||||
'description:\tUse when x happens.',
|
||||
'description: "Use when x happens.',
|
||||
'description: `Use` when x happens.',
|
||||
'description: @Use when x happens.',
|
||||
]) {
|
||||
const fm = GOOD_FM.replace(/^description:.*$/m, line);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_YAML_HAZARD'), line));
|
||||
}
|
||||
// Quoted, the colon is safe.
|
||||
const quoted = GOOD_FM.replace(/^description:.*$/m, 'description: "Checks x: y. Use when x happens."');
|
||||
withSkill('good-skill', `${quoted}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: a multi-line block-scalar description is read whole', () => {
|
||||
const fm = ['---', 'name: good-skill', 'description: |', ' Checks an example the same way: every time.',
|
||||
' Use when an example needs checking.', '---'].join('\n');
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
const f = readFrontmatterFields(`${fm}\n${BODY}`);
|
||||
assert.equal(f.fields.description.value, 'Checks an example the same way: every time.\nUse when an example needs checking.\n');
|
||||
const folded = readFrontmatterFields(['---', 'description: >-', ' one', ' two', '---', ''].join('\n'));
|
||||
assert.equal(folded.fields.description.value, 'one two');
|
||||
});
|
||||
|
||||
test('skill-lint: disable-model-invocation in any true spelling is refused', () => {
|
||||
for (const v of ['true', 'True', 'yes', 'on', '1']) {
|
||||
const fm = GOOD_FM.replace('---\nname', `---\ndisable-model-invocation: ${v}\nname`);
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_MODEL_INVOCATION_DISABLED'), v));
|
||||
}
|
||||
const off = GOOD_FM.replace('---\nname', '---\ndisable-model-invocation: false\nname');
|
||||
withSkill('good-skill', `${off}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: only Claude Code\'s own frontmatter fields are accepted', () => {
|
||||
const fm = GOOD_FM.replace('---\nname', '---\ntrigger: always\nname');
|
||||
withSkill('good-skill', `${fm}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_UNKNOWN_FIELD')));
|
||||
const known = GOOD_FM.replace('---\nname', '---\nlicense: MIT\nuser-invocable: true\nname');
|
||||
withSkill('good-skill', `${known}\n${BODY}`, (r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: SKILL.md must stay under 500 lines', () => {
|
||||
const pad = Array.from({ length: 500 }, (_, i) => `- line ${i}`).join('\n');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY}\n${pad}\n`, (r) => assert.ok(codes(r).includes('SKILL_TOO_LONG')));
|
||||
});
|
||||
|
||||
test('skill-lint: the fixed headings must all be there, in order', () => {
|
||||
assert.deepEqual(REQUIRED_HEADINGS, ['Overview', 'When to use', 'Process', 'Red flags', 'Enforced by', 'Credits']);
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('## Red flags', '## Warnings')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
|
||||
const swapped = BODY.replace('## Overview\nOne line.', '## Process\n1. x.').replace('## Process\n1. Do the thing.', '## Overview\nOne line.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${swapped}`, (r) => assert.ok(codes(r).includes('SKILL_HEADING_ORDER')));
|
||||
// A heading inside a fenced code block does not count.
|
||||
const fenced = BODY.replace('## Credits', '```\n## Credits\n```');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${fenced}`, (r) => assert.ok(codes(r).includes('SKILL_MISSING_HEADING')));
|
||||
});
|
||||
|
||||
test('skill-lint: "Enforced by" names a check that exists, or says prose only', () => {
|
||||
const none = BODY.replace('- prose only: nothing checks this.', '- something, somewhere.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${none}`, (r) => assert.ok(codes(r).includes('SKILL_ENFORCED_BY_VAGUE')));
|
||||
const real = BODY.replace('- prose only: nothing checks this.', '- `lib/check.mjs` stops it.');
|
||||
const root = plugin({ 'good-skill': `${GOOD_FM}\n${real}` });
|
||||
try {
|
||||
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
|
||||
mkdirSync(join(root, 'lib'));
|
||||
writeFileSync(join(root, 'lib', 'check.mjs'), '');
|
||||
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('skill-lint: files the skill points to must exist', () => {
|
||||
const linked = BODY.replace('One line.', 'One line. See [the detail](reference/detail.md).');
|
||||
const root = plugin({ 'good-skill': `${GOOD_FM}\n${linked}` });
|
||||
try {
|
||||
assert.ok(codes(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root })).includes('SKILL_REF_MISSING'));
|
||||
mkdirSync(join(root, 'skills', 'good-skill', 'reference'));
|
||||
writeFileSync(join(root, 'skills', 'good-skill', 'reference', 'detail.md'), '# detail\n');
|
||||
assert.deepEqual(lintSkill(join(root, 'skills', 'good-skill'), { pluginRoot: root }).errors, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('skill-lint: paths under "Credits" name the source repository, not this plugin', () => {
|
||||
const credited = BODY.replace("- Voyage's own content.", '- owner/repo @abc1234, `skills/tdd/SKILL.md`: MIT.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${credited}`, (r) => assert.deepEqual(r.errors, []));
|
||||
// The same path anywhere else must exist.
|
||||
const elsewhere = BODY.replace('One line.', 'One line, see `skills/tdd/SKILL.md`.');
|
||||
withSkill('good-skill', `${GOOD_FM}\n${elsewhere}`, (r) => assert.ok(codes(r).includes('SKILL_REF_MISSING')));
|
||||
});
|
||||
|
||||
test('skill-lint: no shell injection (inline or fenced), since Voyage advises turning it off', () => {
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', 'Status: !`git status`')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```!\ngit status\n```')}`,
|
||||
(r) => assert.ok(codes(r).includes('SKILL_SHELL_INJECTION')));
|
||||
// A plain fenced bash block is fine; Claude reads it, nothing runs it on load.
|
||||
withSkill('good-skill', `${GOOD_FM}\n${BODY.replace('One line.', '```bash\nnpm test\n```')}`,
|
||||
(r) => assert.deepEqual(r.errors, []));
|
||||
});
|
||||
|
||||
test('skill-lint: an exemption lives in the check, never in the skill', () => {
|
||||
assert.ok(EXEMPTIONS instanceof Map);
|
||||
for (const [name, entry] of EXEMPTIONS) {
|
||||
assert.ok(Array.isArray(entry.codes) && entry.codes.length > 0, `${name}: exempt codes`);
|
||||
assert.ok(typeof entry.reason === 'string' && entry.reason.length > 10, `${name}: a reason`);
|
||||
}
|
||||
const self = `${GOOD_FM}\n<!-- skill-lint-exempt: SKILL_MISSING_HEADING -->\n${BODY}`;
|
||||
withSkill('good-skill', self, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
|
||||
const meta = GOOD_FM.replace('---\nname', '---\nmetadata:\n lint-exempt: true\nname');
|
||||
withSkill('good-skill', `${meta}\n${BODY}`, (r) => assert.ok(codes(r).includes('SKILL_SELF_EXEMPTION')));
|
||||
});
|
||||
|
||||
test('skill-lint CLI: 0 clean, 1 findings, 2 could not run', () => {
|
||||
const good = plugin({ 'good-skill': `${GOOD_FM}\n${BODY}` });
|
||||
const bad = plugin({ 'bad-skill': `---\nname: bad-skill\ndescription: Checks: things.\n---\n${BODY}` });
|
||||
try {
|
||||
const g = spawnSync('node', [CLI, '--root', good], { encoding: 'utf8' });
|
||||
assert.equal(g.status, 0, g.stdout + g.stderr);
|
||||
const b = spawnSync('node', [CLI, '--root', bad, '--json'], { encoding: 'utf8' });
|
||||
assert.equal(b.status, 1, b.stdout + b.stderr);
|
||||
assert.ok(JSON.parse(b.stdout).skills['bad-skill'].errors.length > 0);
|
||||
const n = spawnSync('node', [CLI, '--root', join(good, 'nope')], { encoding: 'utf8' });
|
||||
assert.equal(n.status, 2, n.stdout + n.stderr);
|
||||
} finally {
|
||||
rmSync(good, { recursive: true, force: true });
|
||||
rmSync(bad, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// ---- the skills Voyage ships ----
|
||||
|
||||
test('skill-lint: every skill Voyage ships passes the form check', () => {
|
||||
const r = lintSkillsRoot(ROOT);
|
||||
assert.ok(Object.keys(r.skills).length >= 2, `found ${Object.keys(r.skills).join(',')}`);
|
||||
for (const [name, res] of Object.entries(r.skills)) {
|
||||
assert.deepEqual(res.errors, [], `${name}: ${JSON.stringify(res.errors, null, 2)}`);
|
||||
}
|
||||
});
|
||||
215
tests/lib/skill-routing.test.mjs
Normal file
215
tests/lib/skill-routing.test.mjs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
// tests/lib/skill-routing.test.mjs
|
||||
// The model-free routing test (P10, "Bygg nå"): lib/skills/routing.mjs ranks
|
||||
// the descriptions of every skill, the 7 commands and the 21 spawnable agents
|
||||
// against prompts, with no model. A collision between two descriptions fails;
|
||||
// each skill needs 3 prompts where it should fire (top 3) and 2 where it
|
||||
// should not, and for those a named owner must rank higher. The thresholds are
|
||||
// pinned here: they are never lowered.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
THRESHOLDS, tokenize, buildCorpus, rank, collisions, loadEntries, loadCases, checkRouting,
|
||||
} from '../../lib/skills/routing.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');
|
||||
|
||||
const skillMd = (name, description) =>
|
||||
`---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n`;
|
||||
|
||||
/** Plugin-shaped temp root: skills, commands, agents, evals. */
|
||||
function plugin({ skills = {}, commands = {}, agents = {}, cases = {} }) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-routing-'));
|
||||
for (const [n, d] of Object.entries(skills)) {
|
||||
mkdirSync(join(root, 'skills', n), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', n, 'SKILL.md'), skillMd(n, d));
|
||||
}
|
||||
mkdirSync(join(root, 'commands'), { recursive: true });
|
||||
for (const [n, d] of Object.entries(commands)) {
|
||||
writeFileSync(join(root, 'commands', `${n}.md`), `---\nname: ${n}\ndescription: ${d}\n---\n`);
|
||||
}
|
||||
mkdirSync(join(root, 'agents'), { recursive: true });
|
||||
for (const [n, d] of Object.entries(agents)) {
|
||||
writeFileSync(join(root, 'agents', `${n}.md`), `---\nname: ${n}\ndescription: |\n ${d}\nmodel: opus\n---\n`);
|
||||
}
|
||||
for (const [n, { tags, prompt }] of Object.entries(cases)) {
|
||||
mkdirSync(join(root, 'evals', n, 'graders'), { recursive: true });
|
||||
writeFileSync(join(root, 'evals', n, 'prompt.md'),
|
||||
`---\ndescription: "case ${n}"\ntags: [${tags.join(', ')}]\nruns: 1\n---\n\n${prompt}\n`);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function withPlugin(spec, fn) {
|
||||
const root = plugin(spec);
|
||||
try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
const codes = (r) => r.errors.map((e) => e.code);
|
||||
|
||||
// ---- the test that fails first (ordre 7a) ----
|
||||
|
||||
test('routing: two test skills with near-identical descriptions are red (collision)', () => {
|
||||
withPlugin({
|
||||
skills: {
|
||||
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
|
||||
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
|
||||
},
|
||||
}, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(codes(r).includes('ROUTING_COLLISION'), JSON.stringify(r.errors));
|
||||
const c = r.errors.find((e) => e.code === 'ROUTING_COLLISION');
|
||||
assert.deepEqual([c.a, c.b].sort(), ['skill:order-imports', 'skill:tidy-imports']);
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: distinct descriptions do not collide (known negative)', () => {
|
||||
const corpus = buildCorpus([
|
||||
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk. Use when the log directory grows.' },
|
||||
]);
|
||||
assert.deepEqual(collisions(corpus).filter((c) => c.level === 'error'), []);
|
||||
});
|
||||
|
||||
// ---- the pieces ----
|
||||
|
||||
test('routing: thresholds are pinned and never lowered', () => {
|
||||
assert.deepEqual({ ...THRESHOLDS }, {
|
||||
COLLISION_ERROR: 0.75,
|
||||
COLLISION_WARN: 0.5,
|
||||
POSITIVE_TOP_K: 3,
|
||||
MIN_POSITIVE: 3,
|
||||
MIN_NEGATIVE: 2,
|
||||
MIN_EFFECT: 1,
|
||||
MIN_RANK1_RATE: 1,
|
||||
});
|
||||
assert.ok(Object.isFrozen(THRESHOLDS));
|
||||
});
|
||||
|
||||
test('routing: the tokenizer folds case, drops stop words and light-stems', () => {
|
||||
assert.deepEqual(tokenize('Use when the Tests are FAILING'), tokenize('test fail'));
|
||||
assert.deepEqual(tokenize('a an the of'), []);
|
||||
});
|
||||
|
||||
test('routing: rank puts the matching description first, and a no-overlap prompt scores 0', () => {
|
||||
const corpus = buildCorpus([
|
||||
{ id: 'skill:tidy-imports', name: 'tidy-imports', description: 'Sorts the import lines of a source file.' },
|
||||
{ id: 'skill:rotate-logs', name: 'rotate-logs', description: 'Compresses and deletes old log files on disk.' },
|
||||
]);
|
||||
assert.equal(rank('the log files fill the disk', corpus)[0].id, 'skill:rotate-logs');
|
||||
assert.ok(rank('zebra quantum', corpus).every((r) => r.score === 0));
|
||||
});
|
||||
|
||||
test('routing: the reader takes block-scalar agent descriptions and skips orchestrator reference docs', () => {
|
||||
withPlugin({
|
||||
agents: { 'plan-critic': 'Use this agent when a plan needs adversarial review.' },
|
||||
commands: { trekplan: 'Deep implementation planning from a task brief.' },
|
||||
}, (root) => {
|
||||
writeFileSync(join(root, 'agents', 'planning-orchestrator.md'),
|
||||
'---\nname: planning-orchestrator\ndescription: Reference document, not a spawnable capability.\n---\n');
|
||||
const ids = loadEntries(root).map((e) => e.id).sort();
|
||||
assert.deepEqual(ids, ['agent:plan-critic', 'command:trekplan']);
|
||||
assert.equal(loadEntries(root).find((e) => e.id === 'agent:plan-critic').description.trim(),
|
||||
'Use this agent when a plan needs adversarial review.');
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: cases come from evals/ tags; a skill below 3 / 2 / 1 is red', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: {
|
||||
'skill-tidy-imports-fires-1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines in main.js' },
|
||||
},
|
||||
}, (root) => {
|
||||
const cases = loadCases(root);
|
||||
assert.equal(cases.length, 1);
|
||||
assert.deepEqual([cases[0].kind, cases[0].skill, cases[0].from], ['trigger', 'tidy-imports', 'test']);
|
||||
assert.ok(codes(checkRouting(root)).includes('ROUTING_BELOW_MINIMUM'));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: a positive outside the top 3 is red; a negative won by the skill itself is red', () => {
|
||||
const spec = {
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
commands: {
|
||||
'rotate-logs': 'Compresses and deletes old log files on disk.',
|
||||
'scan-ports': 'Lists the open network ports of a host.',
|
||||
'count-words': 'Counts the words of a text document.',
|
||||
},
|
||||
cases: {
|
||||
'p1': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'sort the import lines' },
|
||||
'p2': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'import lines are out of order' },
|
||||
'p3': { tags: ['skill-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'compress the old logs on disk and list ports and count words' },
|
||||
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'delete old log files' },
|
||||
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=rotate-logs', 'from=test'], prompt: 'sort the import lines of the log file' },
|
||||
'e1': { tags: ['skill-effect', 'skill=tidy-imports', 'from=test'], prompt: 'x' },
|
||||
},
|
||||
};
|
||||
withPlugin(spec, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_POSITIVE_MISSED' && e.case === 'p3'), JSON.stringify(r.errors));
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n2'), JSON.stringify(r.errors));
|
||||
assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: a negative must name an owner that exists and outranks the skill', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: {
|
||||
'n1': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'from=test'], prompt: 'delete old log files' },
|
||||
'n2': { tags: ['skill-no-trigger', 'skill=tidy-imports', 'owner=nobody', 'from=test'], prompt: 'delete old log files' },
|
||||
},
|
||||
}, (root) => {
|
||||
const r = checkRouting(root);
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_NO_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
|
||||
assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n2'), JSON.stringify(r.errors));
|
||||
});
|
||||
});
|
||||
|
||||
test('routing: every case names the event it came from (P10)', () => {
|
||||
withPlugin({
|
||||
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
|
||||
cases: { 'p1': { tags: ['skill-trigger', 'skill=tidy-imports'], prompt: 'sort the import lines' } },
|
||||
}, (root) => assert.ok(codes(checkRouting(root)).includes('ROUTING_CASE_NO_SOURCE')));
|
||||
});
|
||||
|
||||
test('routing CLI: 0 clean, 1 findings, 2 could not run', () => {
|
||||
withPlugin({
|
||||
skills: {
|
||||
'tidy-imports': 'Sorts and groups the import lines of a source file. Use when imports are out of order.',
|
||||
'order-imports': 'Sorts and groups the import lines in a source file. Use when the imports are out of order.',
|
||||
},
|
||||
}, (root) => {
|
||||
const b = spawnSync('node', [CLI, '--root', root, '--json'], { encoding: 'utf8' });
|
||||
assert.equal(b.status, 1, b.stdout + b.stderr);
|
||||
assert.ok(JSON.parse(b.stdout).errors.length > 0);
|
||||
const n = spawnSync('node', [CLI, '--root', join(root, 'nope')], { encoding: 'utf8' });
|
||||
assert.equal(n.status, 2, n.stdout + n.stderr);
|
||||
});
|
||||
});
|
||||
|
||||
// ---- Voyage's own catalogue ----
|
||||
|
||||
test('routing: Voyage\'s catalogue holds every skill, the 7 commands and the 21 spawnable agents', () => {
|
||||
const entries = loadEntries(ROOT);
|
||||
const by = (k) => entries.filter((e) => e.kind === k);
|
||||
assert.equal(by('command').length, 7, by('command').map((e) => e.name).join(','));
|
||||
assert.equal(by('agent').length, 21, by('agent').map((e) => e.name).join(','));
|
||||
assert.ok(by('skill').length >= 2, by('skill').map((e) => e.name).join(','));
|
||||
for (const e of entries) assert.ok(e.description.trim().length > 20, `${e.id}: description read as "${e.description}"`);
|
||||
assert.equal(new Set(entries.map((e) => e.name)).size, entries.length, 'names must be unique across kinds');
|
||||
});
|
||||
|
||||
test('routing: Voyage\'s catalogue is green: no collision, every case routes, rank-1 floor held', () => {
|
||||
const r = checkRouting(ROOT);
|
||||
assert.deepEqual(r.errors, [], JSON.stringify(r.errors, null, 2));
|
||||
assert.ok(r.stats.positives >= 6 && r.stats.negatives >= 4, JSON.stringify(r.stats));
|
||||
});
|
||||
212
tests/lib/skills-research-prose-pins.test.mjs
Normal file
212
tests/lib/skills-research-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
// tests/lib/skills-research-prose-pins.test.mjs
|
||||
// Wave 2, order 7b: the `researching-sources` skill and its wiring.
|
||||
//
|
||||
// What is pinned, and why:
|
||||
// • the skill's form is not pinned here: order 7a's form check
|
||||
// (lib/skills/skill-lint.mjs) and routing test (lib/skills/routing.mjs)
|
||||
// run over every skill in `npm test` and are the authority (contract S1);
|
||||
// • the five rules, and the two measured red-baseline lines, stay in it;
|
||||
// • every web-search agent preloads the skill through its `skills:` line, and
|
||||
// /trekresearch names it where it starts them ("commands are the trunk");
|
||||
// • the shared source rules that moved into the skill are gone from the
|
||||
// agents — including the two sentences that contradicted it (an empty
|
||||
// counter-evidence search "increases confidence"; fall back "silently");
|
||||
// • one confidence scale: the skill's table and /trekresearch triangulation
|
||||
// carry the same four definitions;
|
||||
// • the skill says plainly what enforces it and that the rest is text only.
|
||||
//
|
||||
// The behaviour itself (reporting an injected instruction; not counting an
|
||||
// empty search as support) was measured with helpers in the authoring session
|
||||
// and is not something a model-free test can see. The eval cases under
|
||||
// evals/skill-researching-sources-* carry it for the operator's own runs.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf-8');
|
||||
|
||||
const SKILL = 'skills/researching-sources/SKILL.md';
|
||||
const WEB_AGENTS = [
|
||||
'docs-researcher',
|
||||
'community-researcher',
|
||||
'security-researcher',
|
||||
'contrarian-researcher',
|
||||
'research-scout',
|
||||
];
|
||||
|
||||
function frontmatter(text) {
|
||||
const m = text.match(/^---\n([\s\S]*?)\n---\n/);
|
||||
assert.ok(m, 'file must open with a frontmatter block');
|
||||
return m[1];
|
||||
}
|
||||
|
||||
function field(fm, key) {
|
||||
const m = fm.match(new RegExp(`^${key}:\\s*(.*)$`, 'm'));
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
function listField(fm, key) {
|
||||
const raw = field(fm, key);
|
||||
if (raw === null) return [];
|
||||
if (raw.startsWith('[')) return JSON.parse(raw);
|
||||
return [raw];
|
||||
}
|
||||
|
||||
function section(text, startRe, endRe) {
|
||||
const start = text.search(startRe);
|
||||
assert.ok(start >= 0, `section ${startRe} not found`);
|
||||
const rest = text.slice(start);
|
||||
const end = rest.slice(1).search(endRe);
|
||||
return end < 0 ? rest : rest.slice(0, end + 1);
|
||||
}
|
||||
|
||||
// --- the skill's content ----------------------------------------------------
|
||||
|
||||
test('researching-sources: carries the five rules the order names', () => {
|
||||
const text = read(SKILL);
|
||||
for (const re of [
|
||||
/official sources? first/i,
|
||||
/no citation, no finding/i,
|
||||
/absence of data is not a finding/i,
|
||||
/data, never instructions/i,
|
||||
/outbound queries carry nothing local/i,
|
||||
]) assert.match(text, re, `rule missing: ${re}`);
|
||||
assert.match(text, /\*\*Report it\.\*\*/, 'an injected instruction is reported, not only ignored (the measured red baseline)');
|
||||
assert.match(text, /never raises a\s+confidence rating/i, 'an empty search never raises a rating (the measured red baseline)');
|
||||
});
|
||||
|
||||
test('researching-sources: "Enforced by" names the structure-only check and says the rest is prose only', () => {
|
||||
const enforced = section(read(SKILL), /^## Enforced by/m, /^## /m);
|
||||
assert.match(enforced, /research-validator\.mjs/);
|
||||
assert.match(enforced, /structure/i);
|
||||
assert.match(enforced, /query-privacy-gate\.mjs/);
|
||||
assert.match(enforced, /off by default/i, 'the privacy gate runs only in the default-off steps; say so');
|
||||
assert.match(enforced, /\*\*prose only\*\*/);
|
||||
});
|
||||
|
||||
test('researching-sources: credits carry source, commit and licence', () => {
|
||||
const credits = section(read(SKILL), /^## Credits/m, /^## /m);
|
||||
assert.match(credits, /source-driven-development/);
|
||||
assert.match(credits, /9d0c60d/);
|
||||
assert.match(credits, /SKILL\.md:97-114/);
|
||||
assert.match(credits, /diagnosing-superpowers/);
|
||||
assert.match(credits, /8ca22db/);
|
||||
assert.equal((credits.match(/\bMIT\b/g) || []).length, 2, 'both borrowed sources are MIT and say so');
|
||||
});
|
||||
|
||||
// --- wiring: agents preload it, the command names it -----------------------
|
||||
|
||||
test('every web-search agent preloads researching-sources through its skills: line', () => {
|
||||
for (const a of WEB_AGENTS) {
|
||||
const fm = frontmatter(read(`agents/${a}.md`));
|
||||
const skills = listField(fm, 'skills');
|
||||
assert.ok(
|
||||
skills.some((s) => s === 'voyage:researching-sources' || s === 'researching-sources'),
|
||||
`agents/${a}.md skills: must list researching-sources (got ${JSON.stringify(skills)})`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('the web-search agent list is complete: every agent with WebSearch or WebFetch is in it', () => {
|
||||
const withWeb = readdirSync(join(ROOT, 'agents'))
|
||||
.filter((f) => f.endsWith('.md'))
|
||||
.filter((f) => /^tools:.*\bWeb(Search|Fetch)\b/m.test(frontmatter(read(`agents/${f}`))))
|
||||
.map((f) => f.replace(/\.md$/, ''))
|
||||
.sort();
|
||||
assert.deepEqual(withWeb, [...WEB_AGENTS].sort());
|
||||
});
|
||||
|
||||
test('/trekresearch names researching-sources where it starts the external agents', () => {
|
||||
const ext = section(read('commands/trekresearch.md'), /^### External agents/m, /^### Launch rules/m);
|
||||
assert.match(ext, /`researching-sources` skill/);
|
||||
assert.match(ext, /voyage:researching-sources/);
|
||||
assert.match(ext, /Name it in every external spawn prompt/);
|
||||
});
|
||||
|
||||
// --- the copied text is gone ------------------------------------------------
|
||||
|
||||
// Measured 08.10 on 4ee711a: these sentences carried the shared source rules in
|
||||
// the agents. Each now lives in the skill; a copy coming back is drift.
|
||||
const MOVED = [
|
||||
['docs-researcher', 'Never invent documentation.'],
|
||||
['docs-researcher', 'Always include source URLs.'],
|
||||
['docs-researcher', 'Date everything.'],
|
||||
['docs-researcher', 'Flag version mismatches.'],
|
||||
['docs-researcher', 'Fall back to WebSearch silently'],
|
||||
['research-scout', 'Never invent documentation.'],
|
||||
['research-scout', 'Always include source URLs.'],
|
||||
['research-scout', 'Date everything.'],
|
||||
['research-scout', 'Fall back to WebSearch silently'],
|
||||
['community-researcher', 'Date your sources.'],
|
||||
['community-researcher', 'Flag if a "problem" has since been fixed.'],
|
||||
['security-researcher', 'Distinguish absence of data from absence of vulnerabilities.'],
|
||||
['security-researcher', 'Every security concern raised must have a verifiable source.'],
|
||||
['contrarian-researcher', 'that IS a finding'],
|
||||
['contrarian-researcher', 'This increases confidence in the original conclusion'],
|
||||
['contrarian-researcher', 'it is a valuable finding'],
|
||||
['contrarian-researcher', 'Every counter-argument needs a real source.'],
|
||||
['contrarian-researcher', 'Check recency.'],
|
||||
];
|
||||
|
||||
test('the source rules that moved into the skill are no longer copied into the agents', () => {
|
||||
const back = MOVED.filter(([a, s]) => read(`agents/${a}.md`).includes(s));
|
||||
assert.deepEqual(back, [], 'copied rule text is back in an agent — it belongs to the skill');
|
||||
});
|
||||
|
||||
test('contrarian-researcher: an empty counter-evidence search is reported, and is not support', () => {
|
||||
const text = read('agents/contrarian-researcher.md');
|
||||
assert.match(text, /absence of counter-evidence is not support/);
|
||||
assert.match(text, /does not raise\s+confidence in the original conclusion/);
|
||||
});
|
||||
|
||||
test('each web-search agent keeps its own role and points at the skill for the shared rules', () => {
|
||||
for (const a of WEB_AGENTS) {
|
||||
const body = read(`agents/${a}.md`).replace(/^---\n[\s\S]*?\n---\n/, '');
|
||||
assert.match(body, /preloaded\s+`researching-sources` skill/, `agents/${a}.md must point at the preloaded skill`);
|
||||
assert.match(body, /This\s+file holds only your own role\./, `agents/${a}.md must say the rest is its own role`);
|
||||
}
|
||||
});
|
||||
|
||||
// --- one confidence scale ---------------------------------------------------
|
||||
|
||||
function norm(s) {
|
||||
return s.replace(/\*\*/g, '').replace(/\s+/g, ' ').trim().replace(/\.$/, '').toLowerCase();
|
||||
}
|
||||
|
||||
test('one confidence scale: the skill table and /trekresearch triangulation define the same four ratings', () => {
|
||||
const table = section(read(SKILL), /^### Confidence scale/m, /\n##+ /);
|
||||
const skillRows = [...table.matchAll(/^\| \*\*(\w+)\*\* \| (.+) \|$/gm)].map((m) => [m[1], norm(m[2])]);
|
||||
const tri = section(read('commands/trekresearch.md'), /^## Phase 6 — Triangulation/m, /^## Phase 7/m);
|
||||
const cmdRows = [...tri.matchAll(/^ {3}- \*\*(\w+)\*\* — (.+)$/gm)].map((m) => [m[1], norm(m[2])]);
|
||||
assert.deepEqual(skillRows.map((r) => r[0]), ['high', 'medium', 'low', 'contradictory']);
|
||||
assert.deepEqual(cmdRows, skillRows, 'Phase 6 must define the ratings word for word as the skill does');
|
||||
});
|
||||
|
||||
test('one confidence scale: the deep-research adapter and the brief template accept the same four ratings', () => {
|
||||
const cmd = read('commands/trekresearch.md');
|
||||
assert.match(cmd, /`### \{Dimension\} -- Confidence: \{high\|medium\|low\|contradictory\}`/);
|
||||
assert.match(read('templates/research-brief-template.md'), /Confidence: \{high \| medium \| low \| contradictory\}/);
|
||||
});
|
||||
|
||||
// --- the eval cases ---------------------------------------------------------
|
||||
|
||||
test('researching-sources eval cases: 3 should trigger, 2 should not, 1 effect case', () => {
|
||||
const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => d.startsWith('skill-researching-sources-'));
|
||||
const hits = dirs.filter((d) => d.includes('-hit-'));
|
||||
const misses = dirs.filter((d) => d.includes('-miss-'));
|
||||
const effect = dirs.filter((d) => d.includes('-effect-'));
|
||||
assert.ok(hits.length >= 3, `need >= 3 should-trigger cases, have ${hits.length}`);
|
||||
assert.ok(misses.length >= 2, `need >= 2 should-not-trigger cases, have ${misses.length}`);
|
||||
assert.equal(effect.length, 1);
|
||||
for (const d of hits) assert.match(read(`evals/${d}/graders/skill-loaded.md`), /tool: Skill[\s\S]*min: 1/);
|
||||
for (const d of misses) assert.match(read(`evals/${d}/graders/skill-not-loaded.md`), /min: 0\nmax: 0/);
|
||||
for (const d of dirs) {
|
||||
assert.ok(existsSync(join(ROOT, 'evals', d, 'prompt.md')), `${d}/prompt.md`);
|
||||
assert.doesNotMatch(read(`evals/${d}/prompt.md`), /\bproevesett\b/, 'skill cases are not part of the fixed test set');
|
||||
}
|
||||
});
|
||||
150
tests/lib/skills-workshop-prose-pins.test.mjs
Normal file
150
tests/lib/skills-workshop-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
// tests/lib/skills-workshop-prose-pins.test.mjs
|
||||
// Pins for the skill workshop (ordre 7a, contracts S1 and S2): every skill is
|
||||
// asked for by a command or an agent ("the commands are the trunk"), none is
|
||||
// closed to the model, the step helper preloads test-first and debugging, each
|
||||
// skill names what enforces it and credits what it borrowed, the eval cases
|
||||
// keep the 3 / 2 / 1 form, and the docs carry the ## Skills section.
|
||||
// Counted as doc pins by lib/util/test-census.mjs (file name *-prose-pins).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { existsSync, mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { findSkillUsers } from '../../lib/skills/skill-lint.mjs';
|
||||
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
|
||||
import { loadCases } from '../../lib/skills/routing.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
|
||||
const SKILLS = readdirSync(join(ROOT, 'skills')).filter((d) => existsSync(join(ROOT, 'skills', d, 'SKILL.md'))).sort();
|
||||
|
||||
test('pin: Voyage ships the test-first and debugging skills (S2)', () => {
|
||||
for (const s of ['test-first', 'debugging']) assert.ok(SKILLS.includes(s), `skills/${s}/SKILL.md missing`);
|
||||
});
|
||||
|
||||
// ---- "the commands are the trunk": a skill nobody asks for is red ----
|
||||
|
||||
test('pin: findSkillUsers sees a skills: line and a voyage:<name> mention, and reports an unused skill', () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'skill-users-'));
|
||||
try {
|
||||
for (const s of ['used-by-agent', 'used-by-command', 'unused']) {
|
||||
mkdirSync(join(root, 'skills', s), { recursive: true });
|
||||
writeFileSync(join(root, 'skills', s, 'SKILL.md'), `---\nname: ${s}\ndescription: x\n---\n`);
|
||||
}
|
||||
mkdirSync(join(root, 'agents'));
|
||||
mkdirSync(join(root, 'commands'));
|
||||
writeFileSync(join(root, 'agents', 'a.md'), '---\nname: a\nskills:\n - voyage:used-by-agent\n---\nbody\n');
|
||||
writeFileSync(join(root, 'commands', 'c.md'), '---\nname: c\n---\nCall the Skill tool with `voyage:used-by-command`.\n');
|
||||
const users = findSkillUsers(root);
|
||||
assert.deepEqual(users['used-by-agent'], ['agents/a.md']);
|
||||
assert.deepEqual(users['used-by-command'], ['commands/c.md']);
|
||||
assert.deepEqual(users.unused, []);
|
||||
// A bare word in prose is not a request for the skill.
|
||||
writeFileSync(join(root, 'commands', 'd.md'), '---\nname: d\n---\nThis mentions unused in passing.\n');
|
||||
assert.deepEqual(findSkillUsers(root).unused, []);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('pin: every skill in skills/ is asked for by at least one command or agent', () => {
|
||||
const users = findSkillUsers(ROOT);
|
||||
for (const s of SKILLS) assert.ok(users[s] && users[s].length > 0, `skills/${s}: no command or agent names it`);
|
||||
});
|
||||
|
||||
test('pin: no Voyage skill sets disable-model-invocation (it would block preloading and command calls)', () => {
|
||||
for (const s of SKILLS) {
|
||||
const f = readFrontmatterFields(read(`skills/${s}/SKILL.md`)).fields['disable-model-invocation'];
|
||||
assert.ok(!f || !/^(true|yes|on|1)$/i.test(String(f.value).trim()), `skills/${s}: disable-model-invocation is set`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- S2: the step helper preloads both ----
|
||||
|
||||
test('pin: step-implementer preloads voyage:test-first and voyage:debugging, without the Skill tool', () => {
|
||||
const fm = readFrontmatterFields(read('agents/step-implementer.md')).fields;
|
||||
assert.deepEqual(fm.skills.value, ['voyage:test-first', 'voyage:debugging']);
|
||||
// Preloading uses the skills field, not the Skill tool (sub-agents.md,
|
||||
// "Preload skills into subagents"): the tool list stays as it was.
|
||||
assert.deepEqual(fm.tools.value, ['Read', 'Write', 'Edit', 'Bash', 'Glob', 'Grep']);
|
||||
});
|
||||
|
||||
// ---- each skill says what enforces it, and credits what it borrowed ----
|
||||
|
||||
test('pin: debugging says plainly that the root-cause claim is prose only', () => {
|
||||
const t = read('skills/debugging/SKILL.md');
|
||||
const enforced = t.split('## Enforced by')[1].split('## Credits')[0];
|
||||
assert.match(enforced, /prose only: the claim that the root cause was found/);
|
||||
assert.match(enforced, /lib\/verification\/red-proof\.mjs/);
|
||||
});
|
||||
|
||||
test('pin: test-first says the proof of red counts a crash as red, so the assertion red is prose only', () => {
|
||||
const enforced = read('skills/test-first/SKILL.md').split('## Enforced by')[1].split('## Credits')[0];
|
||||
assert.match(enforced, /a crash also passes it/);
|
||||
for (const p of ['lib/verification/red-proof.mjs', 'lib/util/test-lock.mjs', 'lib/verification/test-integrity.mjs', 'lib/verification/done-line.mjs']) {
|
||||
assert.ok(enforced.includes(p), `test-first must name ${p}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('pin: each skill credits its sources with commit and licence', () => {
|
||||
const want = {
|
||||
'test-first': ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
|
||||
debugging: ['obra/superpowers @8ca22db', 'addyosmani/agent-skills @9d0c60d', 'mattpocock/skills @c55ee46'],
|
||||
};
|
||||
for (const [s, sources] of Object.entries(want)) {
|
||||
const credits = read(`skills/${s}/SKILL.md`).split('## Credits')[1];
|
||||
for (const src of sources) assert.ok(credits.includes(src), `${s}: credit ${src}`);
|
||||
assert.equal((credits.match(/\bMIT\b/g) || []).length, sources.length, `${s}: one licence per source`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- S1: the eval case form (3 should fire / 2 should not / 1 effect) ----
|
||||
|
||||
test('pin: each skill has 3 trigger, 2 no-trigger and 1 effect case, and the graders read the Skill tool', () => {
|
||||
const cases = loadCases(ROOT);
|
||||
for (const s of SKILLS) {
|
||||
const mine = cases.filter((c) => c.skill === s);
|
||||
const n = (k) => mine.filter((c) => c.kind === k);
|
||||
assert.ok(n('trigger').length >= 3 && n('no-trigger').length >= 2 && n('effect').length >= 1, `${s}: ${mine.length} cases`);
|
||||
for (const c of [...n('trigger'), ...n('no-trigger')]) {
|
||||
const dir = join(ROOT, 'evals', c.case, 'graders');
|
||||
const g = readdirSync(dir).map((f) => readFrontmatterFields(readFileSync(join(dir, f), 'utf8')).fields);
|
||||
const skillGrader = g.find((x) => x.tool && x.tool.value === 'Skill');
|
||||
assert.ok(skillGrader, `${c.case}: needs a tool_used Skill grader`);
|
||||
const re = new RegExp(skillGrader.input_match.value);
|
||||
assert.match(JSON.stringify({ skill: `voyage:${s}` }), re, `${c.case}: must see voyage:${s}`);
|
||||
assert.match(JSON.stringify({ skill: s }), re, `${c.case}: must see the bare name`);
|
||||
for (const other of SKILLS.filter((o) => o !== s)) assert.doesNotMatch(JSON.stringify({ skill: `voyage:${other}` }), re);
|
||||
if (c.kind === 'no-trigger') {
|
||||
assert.deepEqual([skillGrader.min.value, skillGrader.max.value, skillGrader.arm.value], ['0', '0', 'both'], c.case);
|
||||
}
|
||||
}
|
||||
for (const c of n('effect')) {
|
||||
const desc = readFrontmatterFields(read(`evals/${c.case}/prompt.md`)).fields.description.value;
|
||||
assert.match(desc, /judge model explicitly \(never Haiku\)/, `${c.case}: the judge model is the operator's explicit choice`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---- docs: ## Skills, the authoring guide ----
|
||||
|
||||
test('pin: README and CLAUDE.md carry a ## Skills section naming both skills and the workshop checks', () => {
|
||||
for (const f of ['README.md', 'CLAUDE.md']) {
|
||||
const t = read(f);
|
||||
const sec = t.split(/^## Skills$/m)[1];
|
||||
assert.ok(sec, `${f}: ## Skills missing`);
|
||||
const body = sec.split(/^## /m)[0];
|
||||
for (const s of ['test-first', 'debugging', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'docs/skills-authoring.md']) {
|
||||
assert.ok(body.includes(s), `${f} ## Skills must name ${s}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('pin: docs/skills-authoring.md teaches the red baseline and stays public-safe', () => {
|
||||
const t = read('docs/skills-authoring.md');
|
||||
for (const s of ['## Red baseline first', 'control', 'lib/skills/skill-lint.mjs', 'lib/skills/routing.mjs', 'skill-trigger', 'skill-no-trigger', 'skill-effect', 'from=']) {
|
||||
assert.ok(t.includes(s), `skills-authoring.md must cover ${s}`);
|
||||
}
|
||||
assert.doesNotMatch(t, /\/Users\/|\/private\/tmp|\.local\.md/, 'no local paths in a public doc');
|
||||
});
|
||||
130
tests/lib/trial2-minor-prose-pins.test.mjs
Normal file
130
tests/lib/trial2-minor-prose-pins.test.mjs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// tests/lib/trial2-minor-prose-pins.test.mjs
|
||||
// Prose pins for the minor points of the second voyage-lekerepo trial
|
||||
// (2026-10-08, report parts B and C) whose fix is text a session reads.
|
||||
// Counted as doc pins by lib/util/test-census.mjs (the file name).
|
||||
// B8 minors left after the last critic round have a way to be fixed
|
||||
// (operator decision 2026-10-08: listed in the summary, the operator picks).
|
||||
// B9 K10 `Regression:` is described where plans are written and reviewed.
|
||||
// B10 K11 the four `On failure:` words are named where plans are written.
|
||||
// B12 /trekbrief Step 4c no longer claims a foreground, blocking spawn.
|
||||
// C13 the codebase-size count in /trekplan Phase 4 counts .mjs/.cjs/.mts/.cts.
|
||||
// C17 git-historian leaves the brief's frozen files alone and names the
|
||||
// remote from `git remote -v`.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { findSection } from '../../lib/parsers/plan-schema.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const read = (p) => readFileSync(join(ROOT, p), 'utf8');
|
||||
|
||||
/** The text of a level-2 section, up to the next `## ` heading outside a fence. */
|
||||
function section(text, heading) {
|
||||
const s = findSection(text, heading.replace(/^## /, ''), 2);
|
||||
assert.ok(s, `missing ${heading}`);
|
||||
return s.text;
|
||||
}
|
||||
|
||||
// --- B8 ------------------------------------------------------------------------
|
||||
|
||||
test('B8: a PASS with minors sends them to the Phase 10 summary, numbered, for the operator to pick', () => {
|
||||
const plan = read('commands/trekplan.md');
|
||||
const p9 = section(plan, '## Phase 9 — Adversarial review (bounded critic loop)');
|
||||
assert.match(p9, /\*\*PASS\*\*[\s\S]{0,600}minors[\s\S]{0,300}Phase 10/i);
|
||||
const p10 = section(plan, '## Phase 10 — Present and refine');
|
||||
assert.match(p10, /\*\*Open minors:\*\*/);
|
||||
assert.match(p10, /numbered/i);
|
||||
});
|
||||
|
||||
test('B8: fixing picked minors after the last round runs one more round with --extra-round, the pick being the yes', () => {
|
||||
const p10 = section(read('commands/trekplan.md'), '## Phase 10 — Present and refine');
|
||||
assert.match(p10, /--extra-round/);
|
||||
assert.match(p10, /pick(ed|s)?\s+(is|counts as)\s+(the|their)\s+yes/i);
|
||||
});
|
||||
|
||||
// --- B9 / B10 --------------------------------------------------------------------
|
||||
|
||||
test('B9: /trekplan Phase 8 names `Regression:` as an allowed optional field and says what it is for', () => {
|
||||
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
|
||||
assert.match(p8, /`Regression:`/);
|
||||
assert.match(p8, /PLAN_STEP_REGRESSION_NOT_RUNNABLE/);
|
||||
assert.match(p8, /\*\*Regression:\*\* `npm test`/, 'the canonical example shows the field');
|
||||
assert.match(p8, /before (the|its) checkpoint/i);
|
||||
});
|
||||
|
||||
test('B9: the plan template shows `Regression:` as optional', () => {
|
||||
const t = read('templates/plan-template.md');
|
||||
assert.match(t, /- \*\*Regression:\*\* `[^\n]*\*\(optional\)\*/);
|
||||
assert.match(t, /`Regression:` \(ONE backticked command/);
|
||||
});
|
||||
|
||||
test('B9: plan-critic knows the field and does not demand the suite in `Verify:`', () => {
|
||||
const c = read('agents/plan-critic.md');
|
||||
assert.match(c, /`Regression:`/);
|
||||
assert.match(c, /(never|do\s+not)\s+(demand|require)\s+`npm test`\s+in\s+`Verify:`/i);
|
||||
});
|
||||
|
||||
test('B9/B10: Handover 4 documents `Regression:` (K10) and the `On failure:` words (K11)', () => {
|
||||
const h4 = section(read('docs/HANDOVER-CONTRACTS.md'), '## Handover 4 — `plan.md` → execute');
|
||||
for (const code of ['PLAN_STEP_REGRESSION_NOT_RUNNABLE', 'PLAN_STEP_ON_FAILURE_UNKNOWN']) assert.match(h4, new RegExp(code));
|
||||
assert.match(h4, /K10/);
|
||||
assert.match(h4, /K11/);
|
||||
assert.match(h4, /revert[^\n]*retry[^\n]*skip[^\n]*escalate/);
|
||||
});
|
||||
|
||||
test('B10: /trekplan Phase 8 says `On failure:` opens with one of the four words', () => {
|
||||
const p8 = section(read('commands/trekplan.md'), '## Phase 8 — Deep planning');
|
||||
assert.match(p8, /On failure:\*\*[^\n]*(opens|starts) with/i);
|
||||
assert.match(p8, /PLAN_STEP_ON_FAILURE_UNKNOWN/);
|
||||
});
|
||||
|
||||
// --- B12 -------------------------------------------------------------------------
|
||||
|
||||
test('B12: /trekbrief Step 4c does not call the brief-reviewer spawn foreground or blocking, and waits for it', () => {
|
||||
const t = read('commands/trekbrief.md');
|
||||
const start = t.indexOf('**Step 4c — Launch brief-reviewer**');
|
||||
const end = t.indexOf('**Step 4d — Parse JSON scores**');
|
||||
assert.ok(start !== -1 && end > start);
|
||||
const step = t.slice(start, end);
|
||||
assert.doesNotMatch(step, /foreground, blocking/);
|
||||
assert.match(step, /wait\s+for\s+(its|the agent's)\s+(result|completion)/i);
|
||||
assert.match(step, /(background|asynchronous)/i, 'says how the spawn actually runs');
|
||||
});
|
||||
|
||||
// --- C13 -------------------------------------------------------------------------
|
||||
|
||||
test('C13: the Phase 4 size count counts .mjs, .cjs, .mts and .cts files', () => {
|
||||
const p4 = section(read('commands/trekplan.md'), '## Phase 4 — Codebase sizing');
|
||||
const cmd = p4.match(/```\n(find [^\n]+)\n```/);
|
||||
assert.ok(cmd, 'the find command stands in a fence of its own');
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-sizing-'));
|
||||
try {
|
||||
for (const f of ['a.mjs', 'b.cjs', 'c.mts', 'd.cts', 'e.ts', 'skip.txt']) writeFileSync(join(dir, f), 'x\n');
|
||||
const r = spawnSync('/bin/sh', ['-c', cmd[1]], { cwd: dir, encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.equal(r.stdout.trim(), '5');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- C17 -------------------------------------------------------------------------
|
||||
|
||||
test('C17: git-historian reads the brief, never proposes changing a file it freezes', () => {
|
||||
const g = read('agents/git-historian.md');
|
||||
assert.match(g, /brief/i);
|
||||
assert.match(g, /(never|do not) (propose|suggest|recommend)[^\n]*(change|edit)[^\n]*(frozen|unchanged)|(frozen|unchanged)[^\n]*(never|do not) (propose|suggest|recommend)/i);
|
||||
const p5 = section(read('commands/trekplan.md'), '## Phase 5 — Parallel exploration (specialized agents + research)');
|
||||
assert.match(p5, /\*\*git-historian\*\*[\s\S]{0,400}\{brief_path\}/, 'the spawn hands it the brief');
|
||||
});
|
||||
|
||||
test('C17: git-historian names the remote from `git remote -v`, never by guess', () => {
|
||||
const g = read('agents/git-historian.md');
|
||||
assert.match(g, /`git remote -v`/);
|
||||
assert.match(g, /local (path|repository|bare)/i);
|
||||
});
|
||||
253
tests/lib/trial2-minor.test.mjs
Normal file
253
tests/lib/trial2-minor.test.mjs
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
// tests/lib/trial2-minor.test.mjs
|
||||
// The minor points of the second voyage-lekerepo trial (2026-10-08, report
|
||||
// parts B and C) that are behaviour, not prose. Each test was red before its
|
||||
// fix.
|
||||
// B9 K10 — optional step field `Regression:`: one runnable command, or the
|
||||
// validator refuses it.
|
||||
// B10 K11 — `On failure:` opens with revert | retry | skip | escalate, or the
|
||||
// validator warns.
|
||||
// B11 two findings on the same file:line with the same content merge (the
|
||||
// trial's pair measured Jaccard 0.44 against a 0.7 threshold).
|
||||
// C14 `--evidence` shows the first output line on a PASS row, not "—".
|
||||
// C16 a file:line cited in a reviewer's prose is checked against the file.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { validatePlanContent } from '../../lib/validators/plan-validator.mjs';
|
||||
import { onFailureAction, ON_FAILURE_ACTIONS } from '../../lib/parsers/plan-schema.mjs';
|
||||
import { dedupFindings, DEFAULT_THRESHOLD } from '../../lib/review/plan-review-dedup.mjs';
|
||||
import { formatCriteriaEvidence } from '../../lib/verification/criteria-runner.mjs';
|
||||
import { validateFindings, validateReviewerOutput } from '../../lib/review/findings-schema.mjs';
|
||||
import { buildPlan } from '../helpers/plan-v18-builder.mjs';
|
||||
|
||||
const codes = (list) => list.map((e) => e.code);
|
||||
|
||||
// Step 1's Verify line in the builder's plan; the new field goes right under it.
|
||||
const STEP1_VERIFY = '- **Verify:** `node --test tests/mod1.test.mjs` → expected: pass';
|
||||
const withRegression = (line, plan = buildPlan()) => plan.replace(STEP1_VERIFY, `${STEP1_VERIFY}\n${line}`);
|
||||
const withOnFailure = (text, plan = buildPlan()) =>
|
||||
plan.replace(/^- \*\*On failure:\*\* .*$/m, `- **On failure:** ${text}`);
|
||||
|
||||
// --- B9: K10 `Regression:` ----------------------------------------------------
|
||||
|
||||
test('K10: `Regression: npm test | tee x` is refused — one command, unpiped', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x` → expected: exit 0'));
|
||||
assert.equal(r.valid, false);
|
||||
const e = r.errors.find((x) => x.code === 'PLAN_STEP_REGRESSION_NOT_RUNNABLE');
|
||||
assert.ok(e, codes(r.errors).join(','));
|
||||
assert.match(e.message, /Step 1/);
|
||||
assert.match(e.message, /shell operator/);
|
||||
});
|
||||
|
||||
test('K10: `Regression: npm test` is accepted', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test` → expected: exit 0'));
|
||||
assert.deepEqual(codes(r.errors), []);
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
|
||||
});
|
||||
|
||||
test('K10: without the field the plan validates exactly as before', () => {
|
||||
const r = validatePlanContent(buildPlan());
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
assert.deepEqual(codes(r.warnings), []);
|
||||
});
|
||||
|
||||
test('K10: `npm test && bash tests/e2e.sh`, a bare word, or no backticks is refused', () => {
|
||||
for (const line of [
|
||||
'- **Regression:** `npm test && bash tests/e2e.sh`',
|
||||
'- **Regression:** `./run-all-tests`',
|
||||
'- **Regression:** npm test',
|
||||
'- **Regression:**',
|
||||
]) {
|
||||
const r = validatePlanContent(withRegression(line));
|
||||
assert.ok(codes(r.errors).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'), `${line}: ${codes(r.errors).join(',')}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('K10: --soft reports an unrunnable Regression as a warning, not an error', () => {
|
||||
const r = validatePlanContent(withRegression('- **Regression:** `npm test | tee x`'), { strict: false });
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
assert.ok(codes(r.warnings).includes('PLAN_STEP_REGRESSION_NOT_RUNNABLE'));
|
||||
});
|
||||
|
||||
// --- B10: K11 `On failure:` ---------------------------------------------------
|
||||
|
||||
test('K11: `On failure: try again` warns, and the plan stays valid', () => {
|
||||
const r = validatePlanContent(withOnFailure('try again'));
|
||||
assert.equal(r.valid, true, codes(r.errors).join(','));
|
||||
const w = r.warnings.find((x) => x.code === 'PLAN_STEP_ON_FAILURE_UNKNOWN');
|
||||
assert.ok(w, codes(r.warnings).join(','));
|
||||
assert.match(w.message, /Step 1/);
|
||||
for (const word of ON_FAILURE_ACTIONS) assert.match(w.hint, new RegExp(word));
|
||||
});
|
||||
|
||||
test('K11: `On failure: retry once, then escalate` gives no warning', () => {
|
||||
const r = validatePlanContent(withOnFailure('retry once, then escalate'));
|
||||
assert.ok(!codes(r.warnings).includes('PLAN_STEP_ON_FAILURE_UNKNOWN'), codes(r.warnings).join(','));
|
||||
});
|
||||
|
||||
test('K11: the four words, in any case and with markdown around them, are read; a longer word is not', () => {
|
||||
assert.deepEqual([...ON_FAILURE_ACTIONS], ['revert', 'retry', 'skip', 'escalate']);
|
||||
assert.equal(onFailureAction('revert — `git checkout -- x`'), 'revert');
|
||||
assert.equal(onFailureAction('Escalate: stop the run'), 'escalate');
|
||||
assert.equal(onFailureAction('**skip** this step'), 'skip');
|
||||
assert.equal(onFailureAction('`retry` once'), 'retry');
|
||||
assert.equal(onFailureAction('retrying is fine'), null);
|
||||
assert.equal(onFailureAction('try again'), null);
|
||||
assert.equal(onFailureAction(''), null);
|
||||
});
|
||||
|
||||
// --- B11: same file:line, same content ---------------------------------------
|
||||
|
||||
// The two findings from trial 2's round 1, verbatim (plan.md:248).
|
||||
const PLAN_MD = '.claude/projects/2026-10-08-slug-sep-option/plan.md';
|
||||
const CRITIC_248 = {
|
||||
severity: 'minor', rule_key: 'verification/contract-brief-form-missing', file: PLAN_MD, line: 248,
|
||||
text: "Verification runs `bash tests/e2e.sh` in the brief's own form but not `bash tests/contract.sh`, the form brief SC4 names.",
|
||||
};
|
||||
const GUARDIAN_248 = {
|
||||
severity: 'minor', rule_key: 'gap-verification-command-form', file: PLAN_MD, line: 248,
|
||||
text: "Brief SC4 runs 'bash tests/contract.sh'; Verification lists only '/bin/bash tests/contract.sh' (unlike e2e, where both forms are listed) - add the brief's form",
|
||||
};
|
||||
const sources = (a, b) => [
|
||||
{ agent: 'plan-critic', payload: { findings: a } },
|
||||
{ agent: 'scope-guardian', payload: { findings: b } },
|
||||
];
|
||||
|
||||
test('B11: the two trial-2 findings on plan.md:248 merge into one, raised by both', () => {
|
||||
const r = dedupFindings(sources([CRITIC_248], [GUARDIAN_248]));
|
||||
assert.equal(r.findings.length, 1, JSON.stringify(r.findings, null, 2));
|
||||
assert.deepEqual(r.findings[0].raised_by, ['plan-critic', 'scope-guardian']);
|
||||
assert.equal(r.dedup_stats.anchor_dups, 1);
|
||||
});
|
||||
|
||||
test('B11: two different findings on the same line stay two', () => {
|
||||
const other = {
|
||||
severity: 'minor', rule_key: 'verification/no-negative-case', file: PLAN_MD, line: 248,
|
||||
text: 'No criterion feeds an invalid separator such as a letter and checks the exit code.',
|
||||
};
|
||||
const r = dedupFindings(sources([CRITIC_248], [other]));
|
||||
assert.equal(r.findings.length, 2);
|
||||
});
|
||||
|
||||
test('B11: a short finding held by a longer one on the same line stays its own (independent review)', () => {
|
||||
const at = (text, severity = 'minor') => ({ severity, rule_key: `k-${text.length}`, file: PLAN_MD, line: 40, text });
|
||||
for (const [a, b] of [
|
||||
['The token is checked.', 'The token is not checked before the write.'],
|
||||
['Step 3 test is missing.', 'Step 3 is missing a checkpoint; the test file already exists.'],
|
||||
['Verify runs npm test.', 'Verify must not run npm test, it is green before the step.'],
|
||||
]) {
|
||||
assert.equal(dedupFindings(sources([at(a)], [at(b)])).findings.length, 2, `${a} | ${b}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('B11: one reviewer\'s two findings on one line are never merged by the anchor rule', () => {
|
||||
const r = dedupFindings([{ agent: 'plan-critic', payload: { findings: [CRITIC_248, { ...GUARDIAN_248, rule_key: 'other' }] } }]);
|
||||
assert.equal(r.findings.length, 2);
|
||||
assert.equal(r.dedup_stats.anchor_dups, 0);
|
||||
});
|
||||
|
||||
test('B11: the same content on another line is not merged by the anchor rule; the threshold elsewhere is unchanged', () => {
|
||||
assert.equal(DEFAULT_THRESHOLD, 0.7);
|
||||
const r = dedupFindings(sources([CRITIC_248], [{ ...GUARDIAN_248, line: 249 }]));
|
||||
assert.equal(r.findings.length, 2);
|
||||
const noLine = dedupFindings(sources([{ ...CRITIC_248, line: 0 }], [{ ...GUARDIAN_248, line: 0 }]));
|
||||
assert.equal(noLine.findings.length, 2, 'line 0 is file-scoped, not an anchor');
|
||||
});
|
||||
|
||||
// --- C14: --evidence on a PASS row ---------------------------------------------
|
||||
|
||||
const evidenceRep = (results) => ({
|
||||
source: 'brief.md',
|
||||
heading: '## Success Criteria',
|
||||
error: null,
|
||||
results,
|
||||
summary: { passed: 1, failed: 0, blocked: 0, unrunnable: 0, total: 1, ok: true },
|
||||
});
|
||||
|
||||
test('C14: --evidence shows the first line of output on a passed criterion', () => {
|
||||
const block = formatCriteriaEvidence(evidenceRep([
|
||||
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '\n# tests 12\n# pass 12\n' },
|
||||
]));
|
||||
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| # tests 12 \|/);
|
||||
});
|
||||
|
||||
test('C14: a passed criterion with no output still says "—"', () => {
|
||||
const block = formatCriteriaEvidence(evidenceRep([
|
||||
{ label: 'SC1', text: 't', command: 'npm test', status: 'passed', exitCode: 0, output: '' },
|
||||
]));
|
||||
assert.match(block, /\| SC1 \| `npm test` \| PASS \| exit 0 \| — \|/);
|
||||
});
|
||||
|
||||
// --- C16: file:line in prose --------------------------------------------------
|
||||
|
||||
function repoWith33() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-prose-cite-'));
|
||||
mkdirSync(join(dir, 'tests'));
|
||||
writeFileSync(join(dir, 'tests', 'x.sh'), Array.from({ length: 33 }, (_, i) => `echo ${i + 1}`).join('\n') + '\n');
|
||||
return dir;
|
||||
}
|
||||
|
||||
const finding = (over = {}) => ({
|
||||
id: '0123456789abcdef0123456789abcdef01234567',
|
||||
severity: 'MAJOR', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'tests/x.sh', line: 3,
|
||||
title: 'A title', detail: 'A detail.', recommended_action: 'Do one thing.', ...over,
|
||||
});
|
||||
|
||||
test('C16: a finding whose prose cites `tests/x.sh:119` in a 33-line file is refused', () => {
|
||||
const dir = repoWith33();
|
||||
try {
|
||||
const r = validateFindings(
|
||||
{ reviewer: 'code-correctness-reviewer', findings: [finding({ detail: 'The check at `tests/x.sh:119` relies on git.' })] },
|
||||
{ repoRoot: dir },
|
||||
);
|
||||
assert.equal(r.valid, false);
|
||||
const e = r.errors.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
|
||||
assert.ok(e, JSON.stringify(r.errors));
|
||||
assert.match(e.message, /tests\/x\.sh:119/);
|
||||
assert.match(e.message, /33 lines/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('C16: the same citation in prose outside the JSON block is a warning, never silent (a JSON-only re-ask cannot fix prose)', () => {
|
||||
const dir = repoWith33();
|
||||
try {
|
||||
const raw = [
|
||||
'### Risk noted but not raised as a finding',
|
||||
'',
|
||||
'`tests/x.sh:119` relies on `git` and on a fixed commit.',
|
||||
'',
|
||||
'```json',
|
||||
JSON.stringify({ reviewer: 'code-correctness-reviewer', findings: [] }),
|
||||
'```',
|
||||
].join('\n');
|
||||
const r = validateReviewerOutput(raw, { repoRoot: dir });
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
const w = r.warnings.find((x) => x.code === 'FINDING_PROSE_LINE_OUT_OF_RANGE');
|
||||
assert.ok(w, JSON.stringify(r.warnings));
|
||||
assert.match(w.message, /reviewer prose cites `?tests\/x\.sh:119/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('C16: a line that exists, a range ending in the file, a file absent from the tree and no repo root all pass', () => {
|
||||
const dir = repoWith33();
|
||||
try {
|
||||
const ok = (detail, opts = { repoRoot: dir }) =>
|
||||
validateFindings({ reviewer: 'r', findings: [finding({ detail })] }, opts).valid;
|
||||
assert.equal(ok('See tests/x.sh:33 and tests/x.sh:10-12.'), true);
|
||||
assert.equal(ok('See src/absent.mjs:900.'), true);
|
||||
assert.equal(ok('See tests/x.sh:119.', {}), true);
|
||||
assert.equal(ok('See tests/x.sh:30-40.'), false, 'a range that runs past the end is out of range');
|
||||
assert.equal(ok('Served on http://localhost:8080 and example.com:443.'), true);
|
||||
assert.equal(ok('On Windows: C:\\work\\tests\\x.sh:119 and tests\\x.sh:119.'), true, 'a Windows path is not the root\'s file');
|
||||
assert.equal(ok('See tests/x.sh:30\u201440.'), false, 'an em-dash range is read as a range');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
303
tests/scripts/release-check.test.mjs
Normal file
303
tests/scripts/release-check.test.mjs
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
// tests/scripts/release-check.test.mjs
|
||||
// P13 — the check before a release (wave-2 order 8). One command answers "may
|
||||
// this HEAD be released?": the suite is green at HEAD, the review verdict for
|
||||
// what is released is not BLOCK (K7) and covers it, and every delivery in the
|
||||
// release says whether it is a one-way or a two-way door.
|
||||
// 0 = ready, 1 = not ready (listed), 2 = could not run (never green).
|
||||
//
|
||||
// Every test builds a throwaway git repo shaped like a release: one commit with
|
||||
// the work, then a release commit that only bumps versions and the CHANGELOG.
|
||||
// The repo's own `npm test` is a one-liner that exits 0 or 1, so the suite
|
||||
// check runs for real without running Voyage's suite inside its own suite.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const CHECK = join(ROOT, 'scripts', 'release-check.mjs');
|
||||
const mod = existsSync(CHECK) ? await import('../../scripts/release-check.mjs') : {};
|
||||
|
||||
const git = (cwd, ...args) => {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
};
|
||||
|
||||
function reviewText({ verdict = 'ALLOW', sha }) {
|
||||
const id = 'a'.repeat(40);
|
||||
const severity = verdict === 'BLOCK' ? 'BLOCKER' : 'MINOR';
|
||||
const fmVerdict = verdict === null ? '' : `verdict: ${verdict}\n`;
|
||||
const json = JSON.stringify({
|
||||
verdict: verdict ?? undefined,
|
||||
counts: { BLOCKER: severity === 'BLOCKER' ? 1 : 0, MAJOR: 0, MINOR: severity === 'MINOR' ? 1 : 0, SUGGESTION: 0 },
|
||||
allow_blocked_by: [],
|
||||
findings: [{ id, severity, rule_key: 'MISSING_TEST', file: 'a.mjs', line: 1, title: 't', detail: 'd', recommended_action: 'Add a test.' }],
|
||||
}, null, 2);
|
||||
return `---
|
||||
type: trekreview
|
||||
review_version: "1.1"
|
||||
task: "t"
|
||||
slug: s
|
||||
project_dir: .claude/projects/2026-10-08-s/
|
||||
brief_path: .claude/projects/2026-10-08-s/brief.md
|
||||
scope_sha_end: ${sha}
|
||||
reviewed_files_count: 1
|
||||
${fmVerdict}findings:
|
||||
- ${id}
|
||||
---
|
||||
|
||||
# Review: t
|
||||
|
||||
## Executive Summary
|
||||
|
||||
x
|
||||
|
||||
## Coverage
|
||||
|
||||
| File | Treatment | Reason |
|
||||
|------|-----------|--------|
|
||||
| a.mjs | deep-review | default |
|
||||
|
||||
## Remediation Summary
|
||||
|
||||
x
|
||||
|
||||
\`\`\`json
|
||||
${json}
|
||||
\`\`\`
|
||||
`;
|
||||
}
|
||||
|
||||
const DOORS = `### Doors
|
||||
|
||||
- two-way: the fixture change — revert it and release a new patch.
|
||||
`;
|
||||
|
||||
/**
|
||||
* A repo with the work in one commit and a release commit for 1.0.1 on top.
|
||||
* Options switch one thing off at a time; the default is a ready release.
|
||||
*/
|
||||
function fixture({
|
||||
suiteExit = 0, doors = DOORS, verdict = 'ALLOW', writeReview = true,
|
||||
reviewSha = 'work', afterRelease = null, dirty = false, version = '1.0.1',
|
||||
} = {}) {
|
||||
const base = mkdtempSync(join(tmpdir(), 'release-check-'));
|
||||
const repo = join(base, 'repo');
|
||||
mkdirSync(join(repo, '.claude-plugin'), { recursive: true });
|
||||
git(repo, 'init', '-q', '-b', 'main');
|
||||
git(repo, 'config', 'user.email', 't@example.invalid');
|
||||
git(repo, 'config', 'user.name', 't');
|
||||
git(repo, 'config', 'commit.gpgsign', 'false');
|
||||
|
||||
const pkg = (v) => JSON.stringify({ name: 'fx', version: v, scripts: { test: `node -e "process.exit(${suiteExit})"` } }, null, 2) + '\n';
|
||||
writeFileSync(join(repo, 'package.json'), pkg('1.0.0'));
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version: '1.0.0' }) + '\n');
|
||||
writeFileSync(join(repo, 'README.md'), '# fx\n\n\n\nText.\n');
|
||||
writeFileSync(join(repo, 'CHANGELOG.md'), '# Changelog\n\n## v1.0.0 — 2026-10-01 — first\n\n- first\n');
|
||||
writeFileSync(join(repo, 'a.mjs'), 'export const a = 1;\n');
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'feat: first');
|
||||
writeFileSync(join(repo, 'a.mjs'), 'export const a = 2;\n');
|
||||
git(repo, 'add', 'a.mjs');
|
||||
git(repo, 'commit', '-q', '-m', 'fix: the work');
|
||||
const workSha = git(repo, 'rev-parse', 'HEAD');
|
||||
|
||||
if (version !== null) {
|
||||
writeFileSync(join(repo, 'package.json'), pkg(version));
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx', version }) + '\n');
|
||||
} else {
|
||||
writeFileSync(join(repo, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'fx' }) + '\n');
|
||||
writeFileSync(join(repo, 'package.json'), JSON.stringify({ name: 'fx', scripts: { test: 'node -e "process.exit(0)"' } }) + '\n');
|
||||
}
|
||||
writeFileSync(join(repo, 'README.md'), `# fx\n\n\n\nText.\n`);
|
||||
writeFileSync(join(repo, 'CHANGELOG.md'),
|
||||
`# Changelog\n\n## v${version ?? '9.9.9'} — 2026-10-08 — fixture\n\n${doors ?? ''}\n### Fixed\n\n- the work\n\n## v1.0.0 — 2026-10-01 — first\n\n- first\n`);
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'chore(release): fixture');
|
||||
|
||||
if (afterRelease) {
|
||||
for (const [rel, text] of Object.entries(afterRelease)) writeFileSync(join(repo, rel), text);
|
||||
git(repo, 'add', '-A');
|
||||
git(repo, 'commit', '-q', '-m', 'after the review');
|
||||
}
|
||||
if (dirty) writeFileSync(join(repo, 'a.mjs'), 'export const a = 3;\n');
|
||||
|
||||
const review = join(base, 'review.md');
|
||||
const sha = reviewSha === 'work' ? workSha : reviewSha;
|
||||
if (writeReview) writeFileSync(review, reviewText({ verdict, sha }));
|
||||
return { base, repo, review, cleanup: () => rmSync(base, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
function run(fx, extra = []) {
|
||||
return spawnSync(process.execPath, [CHECK, '--repo', fx.repo, '--review', fx.review, ...extra], { encoding: 'utf8' });
|
||||
}
|
||||
|
||||
test('release-check — the CLI exists', () => {
|
||||
assert.ok(existsSync(CHECK), 'scripts/release-check.mjs must exist (P13)');
|
||||
});
|
||||
|
||||
test('release-check — everything in place → 0 (ready)', () => {
|
||||
const fx = fixture();
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /READY/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a ready release prints the release approval command (K2)', () => {
|
||||
const fx = fixture();
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /approval-log\.mjs append --kind release --subject CHANGELOG\.md --sha256-of CHANGELOG\.md/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — review verdict BLOCK → 1', () => {
|
||||
const fx = fixture({ verdict: 'BLOCK' });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /BLOCK/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — no review.md → 2 (never green)', () => {
|
||||
const fx = fixture({ writeReview: false });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 2, r.stdout + r.stderr);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a review the verdict gate cannot decide (no verdict) → 2', () => {
|
||||
const fx = fixture({ verdict: null });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 2, r.stdout + r.stderr);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — no door type in the release section → 1', () => {
|
||||
const fx = fixture({ doors: null });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /door/i);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a door line that is neither one-way nor two-way → 1', () => {
|
||||
const fx = fixture({ doors: '### Doors\n\n- two-way: fine\n- maybe: not a door type\n' });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /maybe/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — an empty Doors section → 1', () => {
|
||||
const fx = fixture({ doors: '### Doors\n\n' });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /no door entries/i);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — the suite red at HEAD → 1', () => {
|
||||
const fx = fixture({ suiteExit: 1 });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /suite/i);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — uncommitted changes → 1 (the suite would not measure HEAD)', () => {
|
||||
const fx = fixture({ dirty: true });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /uncommitted/i);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a code change after the reviewed commit → 1 (the review does not cover it)', () => {
|
||||
const fx = fixture({ afterRelease: { 'a.mjs': 'export const a = 99;\n' } });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /a\.mjs/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a README prose change after the reviewed commit → 1 (only the badge may move)', () => {
|
||||
const fx = fixture({ afterRelease: { 'README.md': '# fx\n\n\n\nNew text.\n' } });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /README\.md/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — a review of a commit that is not in this history → 1', () => {
|
||||
const fx = fixture({ reviewSha: 'f'.repeat(40) });
|
||||
try {
|
||||
const r = run(fx);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /not in the history of HEAD/);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — no version to release (no --version, no plugin.json version) → 2', () => {
|
||||
const fx = fixture({ version: null });
|
||||
try {
|
||||
assert.equal(run(fx).status, 2);
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — not a git repo → 2', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'release-check-nogit-'));
|
||||
try {
|
||||
const r = spawnSync(process.execPath, [CHECK, '--repo', dir, '--review', join(dir, 'review.md'), '--version', '1.0.0'], { encoding: 'utf8' });
|
||||
assert.equal(r.status, 2, r.stdout + r.stderr);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('release-check — --json reports every check and the exit', () => {
|
||||
const fx = fixture({ verdict: 'BLOCK' });
|
||||
try {
|
||||
const r = run(fx, ['--json']);
|
||||
assert.equal(r.status, 1);
|
||||
const out = JSON.parse(r.stdout);
|
||||
assert.equal(out.exit, 1);
|
||||
assert.equal(out.version, '1.0.1');
|
||||
assert.deepEqual(out.checks.map((c) => c.id), ['tree', 'suite', 'review', 'coverage', 'door']);
|
||||
assert.equal(out.checks.find((c) => c.id === 'review').status, 'fail');
|
||||
} finally { fx.cleanup(); }
|
||||
});
|
||||
|
||||
test('release-check — parseDoors reads one-way and two-way entries and names the bad ones', () => {
|
||||
assert.equal(typeof mod.parseDoors, 'function');
|
||||
const section = '### Doors\n\n- one-way: plan 1.9 rejects plan 1.7 files\n (users regenerate their plans)\n- two-way: a new check\n- maybe: x\n\n### Added\n\n- one-way: not in Doors\n';
|
||||
const d = mod.parseDoors(section);
|
||||
assert.deepEqual(d.entries.map((e) => e.door), ['one-way', 'two-way']);
|
||||
assert.deepEqual(d.bad, ['- maybe: x']);
|
||||
assert.equal(d.found, true);
|
||||
});
|
||||
|
||||
test('release-check — releaseSection matches the CHANGELOG heading dialects exactly', () => {
|
||||
assert.equal(typeof mod.releaseSection, 'function');
|
||||
const text = '# C\n\n## v1.10.0 — x\n\nten\n\n## [1.1.0] - y\n\none\n\n## v1.1.0-pre\n\npre\n';
|
||||
assert.match(mod.releaseSection(text, '1.1.0'), /one/);
|
||||
assert.doesNotMatch(mod.releaseSection(text, '1.1.0'), /ten|pre/);
|
||||
assert.match(mod.releaseSection(text, '1.10.0'), /ten/);
|
||||
assert.equal(mod.releaseSection(text, '2.0.0'), null);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue