merge: main (wave 4 order A) into order B before B goes into main

Conflicts on the union lines only: CHANGELOG Unreleased (both orders'
entries kept), CLAUDE.md (/trekexecute row from A, /trekreview row from
B), tests/fixtures/jsonl-schemas.md (trekplan line_ref from B,
trekexecute line_ref from A). Suite on the merged tree 2682 (2680/0/2).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-19 18:49:02 +02:00
commit 45eaedc6b4
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
24 changed files with 274 additions and 45 deletions

View file

@ -22,7 +22,7 @@
| trekbrief-stats | ts, task, slug, mode, interview_turns, review_iterations, brief_quality, research_topics, auto_research, auto_result, intent_approved, project_dir, order_id, order_claimed_at, order_reason | commands/trekbrief.md (orchestrator-emit Phase 7; order_* added by lib/stats/stats-append.mjs) | trekbrief.md:1125-1149 | profile, phase_models, profile_source | none |
| trekresearch-stats | ts, question, mode, scope, engine, slug, project_dir, brief_path, dimensions, dimensions_baseline, dimensions_baseline_preserved, effort, conv_turns, empty_turns, unique_sources, agents_local, agents_external, gemini_used, confidence, contradictions, open_questions | lib/stats/stats-append.mjs (trekresearch.md Stats tracking) | trekresearch.md:782-808 | profile, phase_models, parallel_agents, external_research_enabled, profile_source | none |
| trekplan-stats | ts, task, mode, slug, brief_path, project_dir, codebase_size, codebase_files, agents_deployed, deep_dives, research_briefs_used, research_scout_used, critic_verdict, guardian_verdict, outcome | lib/stats/stats-append.mjs (trekplan.md Phase 12) | trekplan.md:1273-1301 | profile, phase_models, parallel_agents, profile_source | none |
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2907-2929 | profile, phase_models, profile_source | none |
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:2909-2931 | profile, phase_models, profile_source | none |
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:636-686 | profile, phase_models, profile_source | none |

View file

@ -0,0 +1,82 @@
// tests/lib/p10-model-free.test.mjs
// P10's model-free part (wave 4, order A1). Every case in evals/ names the
// event it came from (`from=`), not only the skill cases; the routing check
// covers every case and says how many it checked for `from=` and how many it
// routed; and the release gate that would read the last test-set run is
// written down as exempted with a reason, not built.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, readFileSync, existsSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { readFrontmatterFields } from '../../lib/skills/skill-frontmatter.mjs';
import { checkRouting } from '../../lib/skills/routing.mjs';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const CLI = join(ROOT, 'lib', 'skills', 'routing.mjs');
const tagsOf = (file) => {
const f = readFrontmatterFields(readFileSync(file, 'utf8')).fields.tags;
return f && Array.isArray(f.value) ? f.value.map(String) : [];
};
test('P10: every case in evals/ names the event it came from (from=)', () => {
const dirs = readdirSync(join(ROOT, 'evals')).filter((d) => existsSync(join(ROOT, 'evals', d, 'prompt.md'))).sort();
assert.ok(dirs.length >= 45, `${dirs.length} cases`);
const missing = dirs.filter((d) => !tagsOf(join(ROOT, 'evals', d, 'prompt.md')).some((t) => /^from=\S+$/.test(t)));
assert.deepEqual(missing, [], `${missing.length} of ${dirs.length} cases have no from=: ${missing.join(', ')}`);
});
/** A plugin root with one skill, its 3/2/1 cases, and the extra cases given. */
function stubRoot(extra) {
const root = mkdtempSync(join(tmpdir(), 'p10-model-free-'));
mkdirSync(join(root, 'skills', 'tidy-imports'), { recursive: true });
writeFileSync(join(root, 'skills', 'tidy-imports', 'SKILL.md'),
'---\nname: tidy-imports\ndescription: Sorts the import lines of a source file. Use when imports are out of order.\n---\n');
mkdirSync(join(root, 'commands'), { recursive: true });
writeFileSync(join(root, 'commands', 'rotate-logs.md'), '---\nname: rotate-logs\ndescription: Compresses and deletes old log files on disk.\n---\n');
const cases = {
p1: ['skill-trigger, skill=tidy-imports, from=test', 'sort the import lines'],
p2: ['skill-trigger, skill=tidy-imports, from=test', 'import lines are out of order'],
p3: ['skill-trigger, skill=tidy-imports, from=test', 'sort imports in the source file'],
n1: ['skill-no-trigger, skill=tidy-imports, owner=rotate-logs, from=test', 'delete old log files'],
n2: ['skill-no-trigger, skill=tidy-imports, owner=rotate-logs, from=test', 'compress the logs on disk'],
e1: ['skill-effect, skill=tidy-imports, from=test', 'x'],
...extra,
};
for (const [n, [tags, prompt]] of Object.entries(cases)) {
mkdirSync(join(root, 'evals', n), { recursive: true });
writeFileSync(join(root, 'evals', n, 'prompt.md'), `---\ndescription: "case ${n}"\ntags: [${tags}]\nruns: 1\n---\n\n${prompt}\n`);
}
return root;
}
test('P10 (R1): a case with no skill tag and no from= is a finding of the routing check', () => {
const root = stubRoot({ 'plain-gate': ['proevesett', '/voyage:trekplan'], 'plain-sourced': ['proevesett, from=test', '/voyage:trekplan'] });
try {
const r = checkRouting(root);
const src = r.errors.filter((e) => e.code === 'ROUTING_CASE_NO_SOURCE').map((e) => e.case);
assert.deepEqual(src, ['plain-gate'], JSON.stringify(r.errors));
assert.ok(!r.errors.some((e) => e.case === 'plain-sourced'), 'a sourced case with no skill tag is not routed');
} finally { rmSync(root, { recursive: true, force: true }); }
});
test('P10 (R1): the report shows the denominators: cases checked for from= and cases routed', () => {
const root = stubRoot({ 'plain-sourced': ['proevesett, from=test', '/voyage:trekplan'] });
try {
const r = checkRouting(root);
assert.deepEqual([r.stats.sourced, r.stats.cases], [7, 6], JSON.stringify(r.stats));
const out = spawnSync('node', [CLI, '--root', root], { encoding: 'utf8' });
assert.match(out.stdout, /7 cases checked for from=, 6 routed/, out.stdout + out.stderr);
} finally { rmSync(root, { recursive: true, force: true }); }
});
// Found by its words, not its heading: the section title is set at the release.
test('P10: the CHANGELOG says the eval gate is exempted, with the reason', () => {
const md = readFileSync(join(ROOT, 'CHANGELOG.md'), 'utf8');
assert.ok(/^- [^\n]*eval gate \(P10\)[^\n]*exempted[^\n]*test set runs again/m.test(md),
'a CHANGELOG line must say the eval gate (P10) is exempted until the test set runs again');
});

View file

@ -118,8 +118,9 @@ test('runbook: the release check exits as the script does — 0 ready, 1 not rea
assert.match(before, /0 = ready, 1 = not ready[^.]*, 2 = could not run — never green/);
});
test('runbook: the eval gate (P10) is named as not built, with where it goes', () => {
const p10 = part(read(RUNBOOK), 'Not built yet: the eval gate (P10)');
test('runbook: the eval gate (P10) is exempted with its reason, and says where it goes', () => {
const p10 = part(read(RUNBOOK), 'Exempted: the eval gate (P10)').replace(/\s+/g, ' ');
assert.match(p10, /waits for the fixed test set to run again/, 'P10 must say why the gate is exempted');
assert.ok(p10.includes('scripts/release-check.mjs'), 'P10 must say which file the gate goes into');
});

View file

@ -384,7 +384,7 @@ test('review 630 r3: routing reads a kind tag only from its own table, never Obj
for (const d of ['skills', 'commands', 'agents', 'evals']) cpSync(join(ROOT, d), join(dir, d), { recursive: true });
const c = join(dir, 'evals', 'zz-proto');
cpSync(join(ROOT, 'evals', 'skill-test-first-quiet-1'), c, { recursive: true });
writeFileSync(join(c, 'prompt.md'), '---\ntags: [constructor, owner=nobody]\n---\n\nWhat time is it?\n');
writeFileSync(join(c, 'prompt.md'), '---\ntags: [constructor, owner=nobody, from=test]\n---\n\nWhat time is it?\n');
const { checkRouting } = awaitImportSync();
const r = checkRouting(dir);
assert.ok(!r.errors.some((e) => String(e.message ?? '').includes('zz-proto') || e.case === 'zz-proto'), JSON.stringify(r.errors));

View file

@ -0,0 +1,59 @@
// tests/lib/run-coupling-prose-pins.test.mjs
// The coupling to Claude Code's built-in `run` skill (wave 4, order A2,
// decisions 54 and 64). The plan's "run the thing" step, the executor's
// `### End to end` and the plan template's run step each say in one sentence
// that the app is seen and driven with `run`, and that the verdict is still
// the `End-to-end:` command, because `run` gives no exit code. Each part is
// read by its heading or marker, never by line number.
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const read = (rel) => readFileSync(join(ROOT, rel), 'utf8');
/** From the line holding `marker` to the first line where `stop` is true. */
function partAt(rel, marker, stop) {
const lines = read(rel).split('\n');
const start = lines.findIndex((l) => l.includes(marker));
assert.ok(start !== -1, `${rel} must hold "${marker}"`);
let end = lines.length;
for (let i = start + 1; i < lines.length; i++) if (stop(lines[i])) { end = i; break; }
return lines.slice(start, end).join(' ').replace(/\s+/g, ' ');
}
const PARTS = [
// An indented paragraph of a numbered list item: it ends at a blank or unindented line.
['commands/trekplan.md', '**The "run the thing" step.**', (l) => l.trim() === '' || /^\S/.test(l)],
['commands/trekexecute.md', '### End to end — "run the thing" (P9 (6))', (l) => l.startsWith('### ')],
['templates/plan-template.md', '### Step 2: {description — the "run the thing" step}', (l) => l.startsWith('### ')],
];
// The template quotes the names instead of backticking them: every backticked
// span on a template step bullet is read as a runnable example
// (tests/lib/skills-criteria-prose-pins.test.mjs), and `run` is no command.
const RUN = '(?:`run`|"run")';
for (const [rel, marker, stop] of PARTS) {
test(`run coupling: ${rel} names Claude Code's run and keeps the verdict on the End-to-end: command`, () => {
const p = partAt(rel, marker, stop);
assert.match(p, new RegExp(`Claude Code's built-in ${RUN} skill`), `${rel}: the run step must name Claude Code's built-in run`);
assert.match(p, /the verdict is still the `?End-to-end:`? command/, `${rel}: the verdict must stay the End-to-end: command`);
assert.match(p, new RegExp(`${RUN} gives no exit code`), `${rel}: the reason must be said`);
});
}
test('run coupling: the run step still says what it runs, through the criteria runner', () => {
const p = partAt(...PARTS[0]);
assert.match(p, /criteria runner's allowlist/);
assert.match(p, /End-to-end: \\`bash tests\/e2e\.sh\\`/);
});
test('run coupling: the writing-criteria quiet case is owned by run, with no stand-in left in its description', () => {
const md = read('evals/skill-writing-criteria-quiet-2/prompt.md');
assert.match(md, /^tags: \[[^\]]*\bowner=run\b[^\]]*\]$/m);
assert.doesNotMatch(md, /stands in/);
assert.doesNotMatch(md, /owner=trekexecute/);
});

View file

@ -174,6 +174,40 @@ test('routing: a negative must name an owner that exists and outranks the skill'
});
});
// ---- R2: an owner from Claude Code's own skills (wave 4, order A2) ----
const R2_SPEC = (prompt, owner = 'run') => ({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },
commands: { 'rotate-logs': 'Compresses and deletes old log files on disk.' },
cases: { n1: { tags: ['skill-no-trigger', 'skill=tidy-imports', `owner=${owner}`, 'from=test'], prompt } },
});
test('routing (R2): owner=run, a built-in Claude Code skill, is no unknown owner, and the report says what was checked', () => {
withPlugin(R2_SPEC('start the app and click through the new screen'), (root) => {
const r = checkRouting(root);
assert.ok(!r.errors.some((e) => e.case === 'n1'), JSON.stringify(r.errors));
const note = (r.notes || []).find((n) => n.case === 'n1');
assert.ok(note, JSON.stringify(r));
assert.match(note.message, /owner outside the corpus: only "not taken" checked/);
const out = spawnSync('node', [CLI, '--root', root], { encoding: 'utf8' });
assert.match(out.stdout, /note ROUTING_OWNER_OUTSIDE_CORPUS: n1: owner=run [^\n]*owner outside the corpus: only "not taken" checked/, out.stdout);
});
});
test('routing (R2): a skill that ranks first is still red when the owner is run', () => {
withPlugin(R2_SPEC('sort the import lines of the source file'), (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_NEGATIVE_WON' && e.case === 'n1'), JSON.stringify(r.errors));
});
});
test('routing (R2): an owner neither in the corpus nor on the built-in list is still unknown', () => {
withPlugin(R2_SPEC('start the app and click through the new screen', 'nosuchthing'), (root) => {
const r = checkRouting(root);
assert.ok(r.errors.some((e) => e.code === 'ROUTING_UNKNOWN_OWNER' && e.case === 'n1'), JSON.stringify(r.errors));
});
});
test('routing: every case names the event it came from (P10)', () => {
withPlugin({
skills: { 'tidy-imports': 'Sorts the import lines of a source file. Use when imports are out of order.' },