From e374a7c0ff67c698cdb914e7cf70b3e3242393d6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 11:51:46 +0200 Subject: [PATCH 01/44] feat(eval): add gold.json golden corpus + loader/validator test --- tests/fixtures/bakeoff-rich/gold.json | 44 +++++++++++++++++ tests/lib/gold-corpus.test.mjs | 68 +++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 tests/fixtures/bakeoff-rich/gold.json create mode 100644 tests/lib/gold-corpus.test.mjs diff --git a/tests/fixtures/bakeoff-rich/gold.json b/tests/fixtures/bakeoff-rich/gold.json new file mode 100644 index 0000000..5e3e711 --- /dev/null +++ b/tests/fixtures/bakeoff-rich/gold.json @@ -0,0 +1,44 @@ +{ + "schema": "voyage-eval-gold/1", + "source": "tests/fixtures/bakeoff-rich/README.md", + "description": "Golden corpus for SKAL-1·4a — the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture, machine-readable. expected_verdict is the review-coordinator Pass-4 outcome (BLOCKER >= 1 -> BLOCK).", + "expected_verdict": "BLOCK", + "findings": [ + { + "file": "lib/handlers/login.mjs", + "line": 17, + "rule_key": "UNIMPLEMENTED_CRITERION", + "severity": "BLOCKER", + "owner_reviewer": "conformance" + }, + { + "file": "lib/auth/jwt.mjs", + "line": 19, + "rule_key": "SECURITY_INJECTION", + "severity": "BLOCKER", + "owner_reviewer": "correctness", + "dual_flaggable": "NON_GOAL_VIOLATED" + }, + { + "file": "lib/auth/refresh.mjs", + "line": 0, + "rule_key": "MISSING_TEST", + "severity": "MAJOR", + "owner_reviewer": "correctness" + }, + { + "file": "lib/handlers/login.mjs", + "line": 13, + "rule_key": "PLAN_EXECUTE_DRIFT", + "severity": "MAJOR", + "owner_reviewer": "conformance" + }, + { + "file": "lib/auth/refresh.mjs", + "line": 10, + "rule_key": "MISSING_ERROR_HANDLING", + "severity": "MINOR", + "owner_reviewer": "correctness" + } + ] +} diff --git a/tests/lib/gold-corpus.test.mjs b/tests/lib/gold-corpus.test.mjs new file mode 100644 index 0000000..3ec9df6 --- /dev/null +++ b/tests/lib/gold-corpus.test.mjs @@ -0,0 +1,68 @@ +// tests/lib/gold-corpus.test.mjs +// SKAL-1·4a — golden corpus loader/validator. +// +// gold.json is the machine-readable form of the 5 seeded findings in +// tests/fixtures/bakeoff-rich/README.md. This test pins its shape and ties +// every rule_key / severity back to the canonical catalogue so the corpus +// can never drift to an invented rule_key or severity tier. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { RULE_KEYS, SEVERITY_VALUES } from '../../lib/review/rule-catalogue.mjs'; +import { FINDING_REQUIRED_FIELDS } from '../../lib/review/findings-schema.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '..', '..'); +const GOLD = join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json'); + +const gold = JSON.parse(readFileSync(GOLD, 'utf-8')); + +test('gold.json — parses and has exactly 5 findings', () => { + assert.ok(Array.isArray(gold.findings), 'findings must be an array'); + assert.equal(gold.findings.length, 5); +}); + +test('gold.json — corpus-level expected_verdict is BLOCK', () => { + assert.equal(gold.expected_verdict, 'BLOCK'); +}); + +test('gold.json — every finding carries the 5 required fields', () => { + // FINDING_REQUIRED_FIELDS (severity, rule_key, file, line) + the eval addition owner_reviewer. + const required = [...FINDING_REQUIRED_FIELDS, 'owner_reviewer']; + for (const [i, f] of gold.findings.entries()) { + for (const field of required) { + assert.ok(field in f, `findings[${i}] missing required field "${field}"`); + } + assert.ok(typeof f.file === 'string' && f.file.length > 0, `findings[${i}].file empty`); + assert.ok(Number.isInteger(f.line) && f.line >= 0, `findings[${i}].line must be an int >= 0`); + } +}); + +test('gold.json — every rule_key is a member of the canonical RULE_CATALOGUE', () => { + for (const [i, f] of gold.findings.entries()) { + assert.ok(RULE_KEYS.has(f.rule_key), `findings[${i}].rule_key "${f.rule_key}" not in catalogue`); + } +}); + +test('gold.json — every severity is a valid catalogue tier', () => { + for (const [i, f] of gold.findings.entries()) { + assert.ok(SEVERITY_VALUES.includes(f.severity), `findings[${i}].severity "${f.severity}" invalid`); + } +}); + +test('gold.json — owner_reviewer is one of the two reviewer roles', () => { + for (const [i, f] of gold.findings.entries()) { + assert.ok( + f.owner_reviewer === 'conformance' || f.owner_reviewer === 'correctness', + `findings[${i}].owner_reviewer "${f.owner_reviewer}" must be conformance|correctness`, + ); + } +}); + +test('gold.json — contains at least one BLOCKER (consistent with expected_verdict BLOCK)', () => { + const hasBlocker = gold.findings.some((f) => f.severity === 'BLOCKER'); + assert.ok(hasBlocker, 'expected_verdict is BLOCK but no BLOCKER finding present'); +}); From 971604d870dbb30157b7d94ed9dcaf0812764db7 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 11:54:48 +0200 Subject: [PATCH 02/44] feat(eval): add review-coordinator contract reference impl + deterministic test --- lib/review/coordinator-contract.mjs | 234 ++++++++++++++++++++++++ tests/lib/coordinator-contract.test.mjs | 163 +++++++++++++++++ 2 files changed, 397 insertions(+) create mode 100644 lib/review/coordinator-contract.mjs create mode 100644 tests/lib/coordinator-contract.test.mjs diff --git a/lib/review/coordinator-contract.mjs b/lib/review/coordinator-contract.mjs new file mode 100644 index 0000000..f2afd8d --- /dev/null +++ b/lib/review/coordinator-contract.mjs @@ -0,0 +1,234 @@ +// lib/review/coordinator-contract.mjs +// SKAL-1·4a — deterministic reference implementation of the review-coordinator +// 4-pass contract (agents/review-coordinator.md §"Your 4-pass process"). +// +// This is a DETERMINISTIC SUBSET, not a full mirror of the LLM coordinator. +// It implements the pure, hermetic passes and DELIBERATELY EXCLUDES the parts +// that need a live filesystem or LLM judgement (which belong to the 4c +// LLM-in-the-loop eval, not this all-agree foundation tier): +// - Pass 2 "Accuracy" file-existence / line-plausibility glob (fs I/O). +// - Pass 2 "Actionability" imperative-verb heuristic — the real coordinator +// uses LLM judgement here; a verb-list approximation would DIVERGE from the +// contract being mirrored, so only the deterministic "recommended_action is +// present-and-non-empty when supplied" half is kept. +// - The doc's 4-tuple `(file,line,rule_key,title)` id recompute — the shipped +// `computeFindingId` is 3-arg `(file,line,rule_key)`; this module follows +// the shipped code and flags the doc divergence (the 4-tuple id is not +// producible by the current helper). +// +// What IS implemented, purely: Pass 1 (triplet dedup → highest-severity-wins +// survivor + conformance tiebreak + detail concat + raised_by provenance), +// Pass 2 succinctness + actionability-presence, Pass 3 reasonableness +// (citation / unknown-rule_key drop, severity-mismatch correction), Pass 4 +// verdict thresholds. No LLM, no network, no time, no randomness. +// +// Reuses: SEVERITY_VALUES / RULE_KEYS / getRule (rule-catalogue.mjs), +// computeFindingId (finding-id.mjs, triplet), validateFindings +// (findings-schema.mjs). Triplet key format mirrors +// scripts/bakeoff-armA-merge.mjs:33; raised_by provenance mirrors +// lib/review/plan-review-dedup.mjs. + +import { SEVERITY_VALUES, RULE_KEYS, getRule } from './rule-catalogue.mjs'; +import { computeFindingId } from '../parsers/finding-id.mjs'; +import { validateFindings } from './findings-schema.mjs'; + +export const JUDGE_TITLE_MAX = 100; +export const JUDGE_DETAIL_MAX = 800; + +/** + * Catalogue-tier rank of a severity: lower number = higher severity. + * BLOCKER=0 … SUGGESTION=3; an unknown severity ranks last. + * @param {string} severity + * @returns {number} + */ +export function severityRank(severity) { + const i = SEVERITY_VALUES.indexOf(severity); + return i === -1 ? SEVERITY_VALUES.length : i; +} + +function isConformance(reviewer) { + return typeof reviewer === 'string' && reviewer.toLowerCase().includes('conformance'); +} + +function tripletKey(f) { + return `${f.file} ${f.line} ${f.rule_key}`; +} + +/** + * Validate each reviewer payload and collect findings from the VALID ones, + * tagging each finding with its source reviewer (mirrors mergeArmA — invalid + * payloads are skipped, not crashed-on). + * @param {Array<{reviewer?: string, findings: object[]}>} reviewerPayloads + * @returns {{ findings: object[], skipped: Array<{reviewer: string|null, error_codes: string[]}> }} + */ +export function ingest(reviewerPayloads) { + const findings = []; + const skipped = []; + for (const payload of reviewerPayloads) { + const r = validateFindings(payload); + if (!r.valid) { + skipped.push({ reviewer: payload?.reviewer ?? null, error_codes: r.errors.map((e) => e.code) }); + continue; + } + for (const f of payload.findings) { + findings.push({ ...f, reviewer: f.reviewer ?? payload.reviewer ?? f.owner_reviewer ?? null }); + } + } + return { findings, skipped }; +} + +/** + * Pass 1 — dedup by (file, line, rule_key) triplet. Survivor = highest + * catalogue severity; severity tie → prefer the conformance reviewer; carries + * raised_by provenance, concatenates other reviewers' attribution into detail, + * and recomputes the id over the triplet. + * @param {object[]} findings + * @returns {object[]} + */ +export function dedupByTriplet(findings) { + const groups = new Map(); + for (const f of findings) { + const key = tripletKey(f); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(f); + } + const out = []; + for (const group of groups.values()) { + let survivor = group[0]; + for (const f of group.slice(1)) { + const higher = severityRank(f.severity) < severityRank(survivor.severity); + const tieToConformance = + severityRank(f.severity) === severityRank(survivor.severity) && + isConformance(f.reviewer) && !isConformance(survivor.reviewer); + if (higher || tieToConformance) survivor = f; + } + const raised_by = [...new Set(group.map((f) => f.reviewer).filter(Boolean))]; + const others = group.filter((f) => f !== survivor); + let detail = survivor.detail; + if (others.length > 0) { + detail = survivor.detail ?? ''; + for (const o of others) { + detail += `\nAlso flagged by ${o.reviewer ?? 'unknown'}: ${o.title ?? o.rule_key}.`; + } + } + const id = computeFindingId(survivor.file, survivor.line, survivor.rule_key); + out.push({ ...survivor, id, ...(detail !== undefined ? { detail } : {}), raised_by }); + } + return out; +} + +/** + * Pass 2 — HubSpot Judge (deterministic subset): drop on succinctness + * (title > 100 or detail > 800 chars) and actionability (recommended_action, + * when present, must be a non-empty string). The imperative-verb test is + * excluded (LLM judgement). + * @param {object[]} findings + * @returns {{ kept: object[], dropped: object[] }} + */ +export function judgeFilter(findings) { + const kept = []; + const dropped = []; + for (const f of findings) { + const titleLen = (f.title ?? '').length; + const detailLen = (f.detail ?? '').length; + let reason = null; + if (titleLen > JUDGE_TITLE_MAX) reason = 'succinctness:title'; + else if (detailLen > JUDGE_DETAIL_MAX) reason = 'succinctness:detail'; + else if ('recommended_action' in f && + (typeof f.recommended_action !== 'string' || f.recommended_action.trim().length === 0)) { + reason = 'actionability:empty'; + } + if (reason) dropped.push({ ...f, suppressed_reason: reason }); + else kept.push(f); + } + return { kept, dropped }; +} + +/** + * Pass 3 — Cloudflare reasonableness (deterministic subset): drop findings + * with no citation (empty file / line < 0) or an unknown rule_key; CORRECT a + * severity that does not match the catalogue tier (a correction, not a drop). + * The fs file-existence glob is excluded (I/O). + * @param {object[]} findings + * @returns {{ kept: object[], dropped: object[] }} + */ +export function reasonablenessFilter(findings) { + const kept = []; + const dropped = []; + for (const f of findings) { + if (typeof f.file !== 'string' || f.file.length === 0 || + (typeof f.line === 'number' && f.line < 0)) { + dropped.push({ ...f, suppressed_reason: 'no-citation' }); + continue; + } + if (!RULE_KEYS.has(f.rule_key)) { + dropped.push({ ...f, suppressed_reason: 'unknown-rule_key' }); + continue; + } + const rule = getRule(f.rule_key); + if (rule && f.severity !== rule.severity) { + kept.push({ ...f, severity: rule.severity, original_severity: f.severity }); + } else { + kept.push(f); + } + } + return { kept, dropped }; +} + +/** + * Pass 4 — compute the verdict from severity counts (after dedup + filtering). + * BLOCKER ≥ 1 → BLOCK; else MAJOR ≥ 1 → WARN; else ALLOW. + * @param {object[]} findings + * @returns {{ verdict: 'BLOCK'|'WARN'|'ALLOW', counts: Record }} + */ +export function computeVerdict(findings) { + const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }; + for (const f of findings) { + if (counts[f.severity] !== undefined) counts[f.severity] += 1; + } + let verdict; + if (counts.BLOCKER >= 1) verdict = 'BLOCK'; + else if (counts.MAJOR >= 1) verdict = 'WARN'; + else verdict = 'ALLOW'; + return { verdict, counts }; +} + +/** + * Run the full deterministic contract: ingest → Pass 1 → Pass 2 → Pass 3 → Pass 4. + * @param {Array<{reviewer?: string, findings: object[]}>} reviewerPayloads + * @returns {{ verdict: string, counts: Record, findings: object[], suppressed: object[], skipped: object[] }} + */ +export function runContract(reviewerPayloads) { + const { findings: ingested, skipped } = ingest(reviewerPayloads); + const deduped = dedupByTriplet(ingested); + const judged = judgeFilter(deduped); + const reasoned = reasonablenessFilter(judged.kept); + const { verdict, counts } = computeVerdict(reasoned.kept); + return { + verdict, + counts, + findings: reasoned.kept, + suppressed: [...judged.dropped, ...reasoned.dropped], + skipped, + }; +} + +// ---- CLI shim ---------------------------------------------------------------- + +if (import.meta.url === `file://${process.argv[1]}`) { + const args = process.argv.slice(2); + const filePath = args.find((a) => !a.startsWith('--')); + if (!filePath) { + process.stderr.write('Usage: coordinator-contract.mjs [--json] \n'); + process.exit(2); + } + const { readFileSync } = await import('node:fs'); + const payloads = JSON.parse(readFileSync(filePath, 'utf-8')); + const result = runContract(Array.isArray(payloads) ? payloads : [payloads]); + if (args.includes('--json')) { + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); + } else { + process.stdout.write(`coordinator-contract: ${result.verdict} (${result.findings.length} findings, ${result.suppressed.length} suppressed)\n`); + } + process.exit(0); +} diff --git a/tests/lib/coordinator-contract.test.mjs b/tests/lib/coordinator-contract.test.mjs new file mode 100644 index 0000000..08daa3b --- /dev/null +++ b/tests/lib/coordinator-contract.test.mjs @@ -0,0 +1,163 @@ +// tests/lib/coordinator-contract.test.mjs +// SKAL-1·4a — deterministic test of the review-coordinator contract subset. +// +// Inline reviewer-JSON fixtures (idiom: tests/validators/brief-validator.test.mjs). +// Hermetic: no LLM, network, time, or randomness. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { + severityRank, + ingest, + dedupByTriplet, + judgeFilter, + reasonablenessFilter, + computeVerdict, + runContract, +} from '../../lib/review/coordinator-contract.mjs'; + +// ---- Pass 1 — dedup -------------------------------------------------------- + +test('dedupByTriplet — genuine cross-reviewer collapse (identical triplet) → 1, raised_by both', () => { + // Both reviewers flag the SAME (file,line,rule_key) triplet — this is the + // real collapse the coordinator performs. + const findings = [ + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header', reviewer: 'correctness' }, + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1 violated', reviewer: 'conformance' }, + ]; + const out = dedupByTriplet(findings); + assert.equal(out.length, 1, 'identical triplets must collapse to one'); + assert.deepEqual([...out[0].raised_by].sort(), ['conformance', 'correctness']); +}); + +test('dedupByTriplet — README #2 two DIFFERENT rule_keys at same file:line do NOT collapse', () => { + // The bakeoff-rich README marks issue #2 dual-flaggable via SECURITY_INJECTION + // AND NON_GOAL_VIOLATED. Those are DIFFERENT triplets → distinct defects → kept. + const findings = [ + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' }, + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', reviewer: 'conformance' }, + ]; + assert.equal(dedupByTriplet(findings).length, 2); +}); + +test('dedupByTriplet — survivor is the highest-severity finding in the group', () => { + const findings = [ + { file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'MINOR', reviewer: 'correctness' }, + { file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' }, + ]; + const out = dedupByTriplet(findings); + assert.equal(out.length, 1); + assert.equal(out[0].severity, 'BLOCKER'); +}); + +test('dedupByTriplet — severity tie breaks toward the conformance reviewer', () => { + const findings = [ + { file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'corr', reviewer: 'correctness' }, + { file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'conf', reviewer: 'conformance' }, + ]; + const out = dedupByTriplet(findings); + assert.equal(out.length, 1); + assert.equal(out[0].reviewer, 'conformance'); +}); + +// ---- severity ranking (4-tier, incl. synthetic SUGGESTION) ----------------- + +test('severityRank — orders the full 4-tier SEVERITY_VALUES incl. SUGGESTION', () => { + assert.ok(severityRank('BLOCKER') < severityRank('MAJOR')); + assert.ok(severityRank('MAJOR') < severityRank('MINOR')); + assert.ok(severityRank('MINOR') < severityRank('SUGGESTION')); +}); + +test('dedupByTriplet — SUGGESTION vs MINOR (synthetic) keeps the MINOR survivor', () => { + // gold corpus has no SUGGESTION; exercise the 4th tier synthetically. + const findings = [ + { file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'SUGGESTION', reviewer: 'correctness' }, + { file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', reviewer: 'correctness' }, + ]; + const out = dedupByTriplet(findings); + assert.equal(out.length, 1); + assert.equal(out[0].severity, 'MINOR'); +}); + +// ---- Pass 4 — verdict thresholds ------------------------------------------- + +test('computeVerdict — BLOCKER>=1 → BLOCK, MAJOR-only → WARN, else ALLOW', () => { + assert.equal(computeVerdict([{ severity: 'BLOCKER' }, { severity: 'MAJOR' }]).verdict, 'BLOCK'); + assert.equal(computeVerdict([{ severity: 'MAJOR' }, { severity: 'MINOR' }]).verdict, 'WARN'); + assert.equal(computeVerdict([{ severity: 'MINOR' }]).verdict, 'ALLOW'); + assert.equal(computeVerdict([{ severity: 'SUGGESTION' }]).verdict, 'ALLOW'); + assert.equal(computeVerdict([]).verdict, 'ALLOW'); +}); + +test('computeVerdict — counts each severity tier', () => { + const { counts } = computeVerdict([ + { severity: 'BLOCKER' }, { severity: 'BLOCKER' }, { severity: 'MAJOR' }, { severity: 'MINOR' }, + ]); + assert.deepEqual(counts, { BLOCKER: 2, MAJOR: 1, MINOR: 1, SUGGESTION: 0 }); +}); + +// ---- Pass 3 — reasonableness ----------------------------------------------- + +test('reasonablenessFilter — drops unknown rule_key + citation-less, corrects severity mismatch', () => { + const r = reasonablenessFilter([ + { file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'BLOCKER' }, // unknown → drop + { file: '', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // no file → drop + { file: 'x.mjs', line: -1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // line < 0 → drop + { file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR' }, // catalogue is MAJOR → correct, keep + ]); + assert.equal(r.kept.length, 1); + assert.equal(r.dropped.length, 3); + assert.equal(r.kept[0].severity, 'MAJOR'); + assert.equal(r.kept[0].original_severity, 'MINOR'); +}); + +// ---- Pass 2 — judge -------------------------------------------------------- + +test('judgeFilter — drops over-long title and empty recommended_action', () => { + const j = judgeFilter([ + { file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'x'.repeat(101) }, // too long → drop + { file: 'x.mjs', line: 2, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok', recommended_action: ' ' }, // empty action → drop + { file: 'x.mjs', line: 3, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok' }, // keep (no action field is fine) + ]); + assert.equal(j.kept.length, 1); + assert.equal(j.dropped.length, 2); +}); + +// ---- ingest ---------------------------------------------------------------- + +test('ingest — collects findings from valid payloads, skips invalid ones', () => { + const { findings, skipped } = ingest([ + { reviewer: 'correctness', findings: [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, + { reviewer: 'bad', findings: [{ line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, // missing file → invalid → skipped + ]); + assert.equal(findings.length, 1); + assert.equal(findings[0].reviewer, 'correctness'); + assert.equal(skipped.length, 1); +}); + +// ---- end-to-end ------------------------------------------------------------ + +test('runContract — two reviewers, dual-flag triplet collapses, verdict BLOCK', () => { + const result = runContract([ + { reviewer: 'correctness', findings: [ + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header' }, + { file: 'lib/auth/refresh.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'no concurrent test' }, + ] }, + { reviewer: 'conformance', findings: [ + { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1' }, // collapses with correctness's + { file: 'lib/handlers/login.mjs', line: 17, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: '200 not 401' }, + ] }, + ]); + assert.equal(result.verdict, 'BLOCK'); + assert.equal(result.findings.length, 3, '4 raw findings, jwt:19 SECURITY_INJECTION collapses → 3'); + const jwt = result.findings.find((f) => f.file === 'lib/auth/jwt.mjs'); + assert.deepEqual([...jwt.raised_by].sort(), ['conformance', 'correctness']); +}); + +test('runContract — deterministic: identical input yields identical output', () => { + const input = [ + { reviewer: 'correctness', findings: [{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 't' }] }, + { reviewer: 'conformance', findings: [{ file: 'b.mjs', line: 2, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: 'u' }] }, + ]; + assert.deepEqual(runContract(input), runContract(input)); +}); From 844492fbcf83be292ce9947162f3e535e7906f12 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 11:57:10 +0200 Subject: [PATCH 03/44] test(eval): two-sided gate coverage for BRIEF_* BLOCKERs --- tests/validators/brief-gate-coverage.test.mjs | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 tests/validators/brief-gate-coverage.test.mjs diff --git a/tests/validators/brief-gate-coverage.test.mjs b/tests/validators/brief-gate-coverage.test.mjs new file mode 100644 index 0000000..e8602d8 --- /dev/null +++ b/tests/validators/brief-gate-coverage.test.mjs @@ -0,0 +1,244 @@ +// tests/validators/brief-gate-coverage.test.mjs +// SKAL-1·4a — two-sided (class-balanced) coverage for every error-severity +// BRIEF_* BLOCKER the brief-validator can emit. +// +// brief-validator.test.mjs already tests many codes; several were FAILURE-only +// (only the raising case asserted) and three were untested. This meta-test pins +// a class-balanced pair { fail, pass } for EACH in-scope BLOCKER and a parity +// guard so a code can't sit in IN_SCOPE_CODES without a paired case. It removes +// the one-sided-optimization risk programmatically, not by inspection. +// +// In scope = ERROR-severity BRIEF_* codes the validator pushes to errors[] +// (BLOCKERs). EXCLUDED: warning-severity codes (BRIEF_TLDR_TOO_LONG, +// BRIEF_VERSION_FORMAT, BRIEF_VERSION_BELOW_MINIMUM, BRIEF_PARTIAL_SKIPPED) — a +// "passing" counterpart to a warning is not a BLOCKER concern — and the +// file-level BRIEF_NOT_FOUND / BRIEF_READ_ERROR which live in validateBrief() +// (filesystem), not validateBriefContent(). +// +// LIMITATION: brief-validator.mjs does not export its code set (codes are string +// literals inside issue(...)), so IN_SCOPE_CODES is HAND-MAINTAINED. A NEW +// validator BLOCKER must be added here manually; the parity guard catches +// list/case drift, not validator drift. (Operator chose the hand-maintained list +// over exporting BRIEF_ERROR_CODES from the forbidden_paths-guarded validator.) + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; + +// ---- valid bases (mirror tests/validators/brief-validator.test.mjs) ---------- + +const GOOD_BRIEF = `--- +type: trekbrief +brief_version: "2.0" +created: 2026-04-30 +task: "Add JWT auth to API" +slug: jwt-auth +project_dir: .claude/projects/2026-04-30-jwt-auth/ +research_topics: 2 +research_status: pending +auto_research: false +interview_turns: 5 +source: interview +--- + +# Task: JWT auth + +## Intent + +Why this matters. + +## Goal + +What success looks like. + +## Success Criteria + +- All tests pass. +`; + +const GOOD_BRIEF_22 = `--- +type: trekbrief +brief_version: "2.2" +created: 2026-06-18 +task: "Add JWT auth to API" +slug: jwt-auth +project_dir: .claude/projects/2026-06-18-jwt-auth/ +research_topics: 0 +research_status: complete +auto_research: false +interview_turns: 5 +source: interview +framing: new-direction +phase_signals_partial: true +--- + +# Task: JWT auth + +## TL;DR + +Net-new JWT auth; no prior brief to anchor against. + +## Intent + +Why this matters. + +## Goal + +What success looks like. + +## Success Criteria + +- All tests pass. +`; + +const REVIEW_AS_BRIEF = `--- +type: trekreview +task: "Review delivered trekreview v1.0" +slug: trekreview +project_dir: .claude/projects/2026-05-01-trekreview/ +findings: + - 0123456789abcdef0123456789abcdef01234567 + - fedcba9876543210fedcba9876543210fedcba98 +--- + +# Review brief + +## Intent + +Adversarial review of delivered trekreview v1.0. + +## Goal + +Find what was missed. + +## Success Criteria + +- All BLOCKER findings get a fix-plan. +`; + +// signals blocks +const GOOD_SIGNALS = `phase_signals: + - phase: plan + effort: high + model: opus +`; +const BAD_PHASE_SIGNALS = `phase_signals: + - phase: nonsense + effort: high +`; +const BAD_EFFORT_SIGNALS = `phase_signals: + - phase: plan + effort: turbo +`; +const BAD_MODEL_SIGNALS = `phase_signals: + - phase: plan + effort: high + model: gpt4 +`; + +// convenience builders +const v21 = (s) => s.replace('brief_version: "2.0"', 'brief_version: "2.1"'); +const withLine = (s, line) => s.replace('source: interview\n', `source: interview\n${line}`); + +// ---- the canonical in-scope set + a class-balanced pair for each ------------- + +const IN_SCOPE_CODES = [ + 'BRIEF_MISSING_FIELD', + 'BRIEF_BAD_STATUS', + 'BRIEF_STATE_INCOHERENT', + 'BRIEF_WRONG_TYPE', + 'BRIEF_BAD_FINDINGS_TYPE', + 'BRIEF_MISSING_SECTION', + 'BRIEF_INVALID_FRAMING', + 'BRIEF_MISSING_FRAMING', + 'BRIEF_V51_MISSING_SIGNALS', + 'BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE', + 'BRIEF_INVALID_PHASE_SIGNALS', + 'BRIEF_INVALID_PHASE_SIGNAL_PHASE', + 'BRIEF_INVALID_EFFORT', + 'BRIEF_INVALID_MODEL', +]; + +const CASES = { + BRIEF_MISSING_FIELD: { + fail: GOOD_BRIEF.replace(/^research_topics: 2\n/m, ''), + pass: GOOD_BRIEF, + }, + BRIEF_BAD_STATUS: { + fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: maybe'), + pass: GOOD_BRIEF, + }, + BRIEF_STATE_INCOHERENT: { + fail: GOOD_BRIEF.replace('research_status: pending', 'research_status: skipped'), + pass: GOOD_BRIEF, + }, + BRIEF_WRONG_TYPE: { + fail: GOOD_BRIEF.replace('type: trekbrief', 'type: notabrief'), + pass: GOOD_BRIEF, + }, + BRIEF_BAD_FINDINGS_TYPE: { + fail: REVIEW_AS_BRIEF.replace(/findings:\n - 0123[\s\S]*?- fedcba[0-9a-f]+/, 'findings: not-an-array'), + pass: REVIEW_AS_BRIEF, + }, + BRIEF_MISSING_SECTION: { + fail: GOOD_BRIEF.replace(/## Intent\n\nWhy this matters\.\n\n/, ''), + pass: GOOD_BRIEF, + }, + BRIEF_INVALID_FRAMING: { + fail: withLine(GOOD_BRIEF, 'framing: sideways\n'), + pass: GOOD_BRIEF_22, + }, + BRIEF_MISSING_FRAMING: { + fail: GOOD_BRIEF_22.replace('framing: new-direction\n', ''), + pass: GOOD_BRIEF_22, + }, + BRIEF_V51_MISSING_SIGNALS: { + fail: v21(GOOD_BRIEF), + pass: GOOD_BRIEF, // 2.0 — gate inactive + }, + BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE: { + fail: withLine(v21(GOOD_BRIEF), `phase_signals_partial: true\n${GOOD_SIGNALS}`), + pass: withLine(v21(GOOD_BRIEF), 'phase_signals_partial: true\n'), + }, + BRIEF_INVALID_PHASE_SIGNALS: { + fail: withLine(GOOD_BRIEF, 'phase_signals: notalist\n'), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_PHASE_SIGNAL_PHASE: { + fail: withLine(v21(GOOD_BRIEF), BAD_PHASE_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_EFFORT: { + fail: withLine(v21(GOOD_BRIEF), BAD_EFFORT_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, + BRIEF_INVALID_MODEL: { + fail: withLine(v21(GOOD_BRIEF), BAD_MODEL_SIGNALS), + pass: withLine(v21(GOOD_BRIEF), GOOD_SIGNALS), + }, +}; + +test('gate coverage — CASES keys exactly match IN_SCOPE_CODES (no silent gap)', () => { + assert.deepEqual(Object.keys(CASES).sort(), [...IN_SCOPE_CODES].sort()); +}); + +test('gate coverage — every in-scope BRIEF_* BLOCKER raises on its fail case', () => { + for (const code of IN_SCOPE_CODES) { + const r = validateBriefContent(CASES[code].fail, { strict: true }); + assert.ok( + r.errors.find((e) => e.code === code), + `${code}: fail case did NOT raise it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`, + ); + } +}); + +test('gate coverage — every in-scope BRIEF_* BLOCKER is absent on its (valid) pass case', () => { + for (const code of IN_SCOPE_CODES) { + const r = validateBriefContent(CASES[code].pass, { strict: true }); + assert.ok( + !r.errors.find((e) => e.code === code), + `${code}: pass case wrongly raised it; errors=${JSON.stringify(r.errors.map((e) => e.code))}`, + ); + assert.equal(r.valid, true, `${code}: pass case should be a fully valid brief; errors=${JSON.stringify(r.errors)}`); + } +}); From 5144e53129ff045be8ea511dd5845b2f3d746764 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 11:57:39 +0200 Subject: [PATCH 04/44] docs(eval): establish eval-corpus frozen-failure home --- docs/eval-corpus/README.md | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/eval-corpus/README.md diff --git a/docs/eval-corpus/README.md b/docs/eval-corpus/README.md new file mode 100644 index 0000000..8f9d675 --- /dev/null +++ b/docs/eval-corpus/README.md @@ -0,0 +1,74 @@ +# Eval corpus — frozen failures for Voyage's own agents + +This directory is the home for the **golden / frozen-failure corpus** that +grounds Voyage's self-evaluation (SKAL-1·4a, the eval-foundation tier). It +follows the Anthropic "collect 20–50 real cases" practice: every time a Voyage +review/coordinator agent **misfires** on a real task, the case is distilled into +a machine-readable record and added here, so the failure can never silently +regress. + +This tier is the **staging ground** for the offline gold-scored output eval +(SKAL-1·4b). It is deliberately **not wired into CI** yet — these are committed +fixtures + a schema, not a scoring run. + +## Seed example + +The first corpus entry is +[`tests/fixtures/bakeoff-rich/gold.json`](../../tests/fixtures/bakeoff-rich/gold.json) +— the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture. +`gold.json` is the **canonical machine-readable form**; the prose table in +`tests/fixtures/bakeoff-rich/README.md` is illustrative. When they disagree, +`gold.json` wins. + +## Record schema (`voyage-eval-gold/1`) + +A corpus file is one JSON object: + +```jsonc +{ + "schema": "voyage-eval-gold/1", + "source": "", + "description": "", + "expected_verdict": "BLOCK | WARN | ALLOW", // review-coordinator Pass-4 outcome + "findings": [ + { + "file": "", + "line": 0, // integer >= 0; 0 = file-scoped + "rule_key": "", + "severity": "BLOCKER | MAJOR | MINOR | SUGGESTION", + "owner_reviewer": "conformance | correctness" + // optional eval-extension fields are permitted, e.g.: + // "dual_flaggable": "" + } + ] +} +``` + +### Hard constraints + +- **`rule_key` must be a member of `RULE_CATALOGUE`** (`lib/review/rule-catalogue.mjs`). + The catalogue is the contract; an invented rule_key is a corpus bug. +- **`severity` must be one of `SEVERITY_VALUES`** (`BLOCKER`, `MAJOR`, `MINOR`, `SUGGESTION`). +- **`expected_verdict`** is the deterministic Pass-4 outcome of + `lib/review/coordinator-contract.mjs::computeVerdict`: `BLOCKER ≥ 1 → BLOCK`, + else `MAJOR ≥ 1 → WARN`, else `ALLOW`. +- Finding-level fields (`file`, `line`, `rule_key`, `severity`) mirror + `FINDING_REQUIRED_FIELDS` (`lib/review/findings-schema.mjs`); `owner_reviewer` + and any extension fields are eval-specific additions (superset). + +## Adding a new frozen failure + +1. Distil the misfire into a `voyage-eval-gold/1` record (one `.json` file here, + or a new entry in an existing corpus file). +2. Confirm every `rule_key` is in the catalogue and `expected_verdict` matches + the Pass-4 computation. +3. Add (or extend) a loader test in the `tests/lib/gold-corpus.test.mjs` shape so + the record's shape + catalogue membership are pinned under `node --test`. + +## Future hardening (not in this tier) + +- SKAL-1·4b: an offline gold-scored output eval that scores recorded agent runs + against these records at `(file, rule_key)` granularity, with committed runs + and a third test-census category. +- A prose↔JSON cross-assertion (parse the fixture README table, diff against + `gold.json`) to mechanically bound the two-source-of-truth drift. From 708ba045718ca3c4139c81e9fa97b5856b932ba2 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:32:10 +0200 Subject: [PATCH 05/44] feat(stats): add pure token-usage parser + cache-aware cost derivation --- lib/stats/token-usage.mjs | 161 ++++++++++++++++++ .../token-usage/sample-transcript.jsonl | 8 + tests/lib/token-usage.test.mjs | 142 +++++++++++++++ 3 files changed, 311 insertions(+) create mode 100644 lib/stats/token-usage.mjs create mode 100644 tests/fixtures/token-usage/sample-transcript.jsonl create mode 100644 tests/lib/token-usage.test.mjs diff --git a/lib/stats/token-usage.mjs b/lib/stats/token-usage.mjs new file mode 100644 index 0000000..72036ab --- /dev/null +++ b/lib/stats/token-usage.mjs @@ -0,0 +1,161 @@ +// lib/stats/token-usage.mjs +// SKAL-2 — pure token-usage parser + cache-aware cost derivation for Voyage +// observability. Captures MAIN-CONTEXT token/cost from the Claude Code +// transcript (transcript_path). Each assistant record carries message.usage +// with a per-REQUEST snapshot. +// +// Verified 2026-06-26 against a real local transcript (216 lines): 70 +// assistant records collapse to 31 distinct requestIds — records duplicate +// per requestId (streamed snapshots), so we dedup by requestId keeping the +// LAST occurrence (GH #28197 streaming-placeholder mitigation), then sum +// across requests. The main transcript carried zero isSidechain:true records. +// +// v1 scope = MAIN-CONTEXT only. Sub-agent (swarm) turns live in separate +// agent-*.jsonl siblings and are a documented v2 follow-on — so the +// main-transcript sum UNDER-counts total Voyage cost. Every record is stamped +// scope:'main-context' so no reader mistakes cost_usd for the session total. +// +// All functions in this module are PURE (no I/O). The impure capture shell +// (captureTokenUsage) is added in Step 4. +// +// Zero npm dependencies. Node stdlib only. + +// Per-Mtok USD prices, resolved 2026-06-26 via the claude-api skill reference: +// base input/output from the model table; cache rates from the prompt-caching +// doc multipliers (cache_read 0.1x, write_5m 1.25x, write_1h 2.0x of input). +export const PRICE_TABLE = Object.freeze({ + 'claude-opus-4-8': Object.freeze({ + input: 5.0, + output: 25.0, + cache_read: 0.5, + cache_write_5m: 6.25, + cache_write_1h: 10.0, + }), +}); + +// Date the PRICE_TABLE values were resolved/verified. Bump when prices change. +export const PRICE_TABLE_VERSION = '2026-06-26'; + +function num(v) { + return typeof v === 'number' && Number.isFinite(v) ? v : 0; +} + +/** + * Single pass over transcript JSONL text. Keeps only main-chain assistant + * records (type==='assistant', isSidechain!==true), dedups by requestId + * keeping the LAST occurrence (per-request usage snapshots duplicate across + * streamed records), and sums the four usage token fields. Malformed lines + * are skipped. + * + * @returns {{tokens_input:number, tokens_output:number, + * tokens_cache_creation:number, tokens_cache_read:number}} + */ +export function parseTranscriptUsage(text) { + const lines = (text || '').split('\n'); + // requestId -> usage (last occurrence wins). Records without a requestId + // get a unique sentinel key so each is still counted exactly once. + const byRequest = new Map(); + let anon = 0; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === '') continue; + let obj; + try { obj = JSON.parse(trimmed); } + catch { continue; } + if (!obj || obj.type !== 'assistant') continue; + if (obj.isSidechain === true) continue; + const usage = obj.message && obj.message.usage; + if (!usage || typeof usage !== 'object') continue; + const key = typeof obj.requestId === 'string' && obj.requestId + ? obj.requestId + : `__anon_${anon++}`; + byRequest.set(key, usage); // last occurrence wins + } + const totals = { + tokens_input: 0, + tokens_output: 0, + tokens_cache_creation: 0, + tokens_cache_read: 0, + }; + for (const usage of byRequest.values()) { + totals.tokens_input += num(usage.input_tokens); + totals.tokens_output += num(usage.output_tokens); + totals.tokens_cache_creation += num(usage.cache_creation_input_tokens); + totals.tokens_cache_read += num(usage.cache_read_input_tokens); + } + return totals; +} + +/** + * Cache-aware USD cost. Lumped cache_creation is priced at the 5m write rate + * (Claude Code's default cache TTL; verified the dominant case on a real + * transcript — the 1h split is a documented v2 refinement). When the model is + * absent from the table we REFUSE to estimate: {cost_usd:null, is_estimate:true}. + * + * @returns {{cost_usd:number|null, is_estimate:boolean}} + */ +export function deriveCost(totals, model, priceTable = PRICE_TABLE) { + const price = priceTable && priceTable[model]; + if (!price) return { cost_usd: null, is_estimate: true }; + const t = totals || {}; + const cost = + (num(t.tokens_input) * price.input + + num(t.tokens_output) * price.output + + num(t.tokens_cache_creation) * price.cache_write_5m + + num(t.tokens_cache_read) * price.cache_read) / 1_000_000; + return { cost_usd: cost, is_estimate: false }; +} + +/** + * Build the flat numeric record. Stamps scope:'main-context' and + * price_table_version. session_id is written for upsert keying but MUST be + * stripped at export (CWE-212, enforced by the field allowlist in Step 2). + * + * @param {{sessionId:string, model:string, totals:object, + * priceTable?:object, now:string}} args + */ +export function buildRecord({ sessionId, model, totals, priceTable = PRICE_TABLE, now }) { + const t = totals || {}; + const { cost_usd, is_estimate } = deriveCost(t, model, priceTable); + return { + ts: now, + session_id: sessionId, + scope: 'main-context', + model, + tokens_input: num(t.tokens_input), + tokens_output: num(t.tokens_output), + tokens_cache_creation: num(t.tokens_cache_creation), + tokens_cache_read: num(t.tokens_cache_read), + cost_usd, + is_estimate, + price_table_version: PRICE_TABLE_VERSION, + }; +} + +/** + * Pure read-modify-write. Parses existing JSONL lines, REPLACES the line whose + * session_id matches the record (dropping any further same-session dupes), + * else APPENDS. Returns the new file text (one line per session, trailing + * newline). One-line-per-session is what keeps cross-session sums from + * double-counting (vs append-and-grow). + */ +export function upsertSessionRecord(existingText, record) { + const lines = (existingText || '').split('\n'); + const out = []; + let replaced = false; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === '') continue; + let obj; + try { obj = JSON.parse(trimmed); } + catch { out.push(trimmed); continue; } // preserve unparseable lines verbatim + if (obj && obj.session_id === record.session_id) { + if (!replaced) { out.push(JSON.stringify(record)); replaced = true; } + // else: duplicate same-session line — drop it + } else { + out.push(trimmed); + } + } + if (!replaced) out.push(JSON.stringify(record)); + return out.join('\n') + '\n'; +} diff --git a/tests/fixtures/token-usage/sample-transcript.jsonl b/tests/fixtures/token-usage/sample-transcript.jsonl new file mode 100644 index 0000000..2ffe083 --- /dev/null +++ b/tests/fixtures/token-usage/sample-transcript.jsonl @@ -0,0 +1,8 @@ +{"type":"user","message":{"role":"user","content":"hi"}} +{"type":"assistant","isSidechain":false,"requestId":"req_A","message":{"model":"claude-opus-4-8","usage":{"input_tokens":1,"output_tokens":1,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}} +{"type":"assistant","isSidechain":false,"requestId":"req_A","message":{"model":"claude-opus-4-8","usage":{"input_tokens":1000,"output_tokens":200,"cache_creation_input_tokens":400,"cache_read_input_tokens":5000}}} +{"type":"assistant","isSidechain":false,"requestId":"req_B","message":{"model":"claude-opus-4-8","usage":{"input_tokens":2000,"output_tokens":300,"cache_creation_input_tokens":600,"cache_read_input_tokens":1000}}} +{"type":"assistant","isSidechain":true,"requestId":"req_SIDE","message":{"model":"claude-opus-4-8","usage":{"input_tokens":9999,"output_tokens":9999,"cache_creation_input_tokens":9999,"cache_read_input_tokens":9999}}} +{"type":"system","content":"some system note with no usage"} +this is not valid json and must be skipped +{"type":"assistant","isSidechain":false,"requestId":"req_B","message":{"model":"claude-opus-4-8","usage":{"input_tokens":2000,"output_tokens":300,"cache_creation_input_tokens":600,"cache_read_input_tokens":1000}}} diff --git a/tests/lib/token-usage.test.mjs b/tests/lib/token-usage.test.mjs new file mode 100644 index 0000000..36e7f8d --- /dev/null +++ b/tests/lib/token-usage.test.mjs @@ -0,0 +1,142 @@ +// tests/lib/token-usage.test.mjs +// SKAL-2 Step 1 — pure token-usage parser + cache-aware cost derivation. +// +// Hermetic: no LLM, network, time, or randomness. Fixture transcript exercises +// the dedup-by-requestId (streaming-placeholder) and isSidechain-exclusion +// paths that were verified against a real local transcript at execute time. +// Idiom: tests/lib/coordinator-contract.test.mjs (pure-module node:test style). + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + parseTranscriptUsage, + deriveCost, + buildRecord, + upsertSessionRecord, + PRICE_TABLE, + PRICE_TABLE_VERSION, +} from '../../lib/stats/token-usage.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = join(HERE, '..', 'fixtures', 'token-usage', 'sample-transcript.jsonl'); + +// ---- parseTranscriptUsage -------------------------------------------------- + +test('parseTranscriptUsage — sums main-chain final-per-requestId; excludes sidechain + placeholder dup', () => { + const text = readFileSync(FIXTURE, 'utf-8'); + const totals = parseTranscriptUsage(text); + // req_A: placeholder {1,1,0,0} then real {1000,200,400,5000} → LAST wins. + // req_B: appears twice identically → dedup → counted once {2000,300,600,1000}. + // req_SIDE (isSidechain:true, all 9999) → EXCLUDED. user/system/malformed → skipped. + assert.deepEqual(totals, { + tokens_input: 3000, // 1000 + 2000 (NOT 1 — placeholder dropped) + tokens_output: 500, // 200 + 300 + tokens_cache_creation: 1000, // 400 + 600 + tokens_cache_read: 6000, // 5000 + 1000 + }); + // Sidechain 9999s must not leak into any field. + assert.ok(totals.tokens_input < 9999, 'sidechain tokens leaked into total'); +}); + +test('parseTranscriptUsage — empty / nullish input → zeroed totals', () => { + const zero = { tokens_input: 0, tokens_output: 0, tokens_cache_creation: 0, tokens_cache_read: 0 }; + assert.deepEqual(parseTranscriptUsage(''), zero); + assert.deepEqual(parseTranscriptUsage(null), zero); + assert.deepEqual(parseTranscriptUsage(undefined), zero); +}); + +test('parseTranscriptUsage — single-pass O(n) shape on a 500-line transcript', () => { + // 500 distinct-requestId main-chain assistant records, each input_tokens:10. + const lines = []; + for (let i = 0; i < 500; i++) { + lines.push(JSON.stringify({ + type: 'assistant', + isSidechain: false, + requestId: `req_${i}`, + message: { model: 'claude-opus-4-8', usage: { input_tokens: 10, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }, + })); + } + const totals = parseTranscriptUsage(lines.join('\n')); + assert.equal(totals.tokens_input, 5000); // 500 × 10 +}); + +// ---- deriveCost ------------------------------------------------------------ + +test('deriveCost — hand-computed value for a known model (cache-aware)', () => { + const totals = { tokens_input: 3000, tokens_output: 500, tokens_cache_creation: 1000, tokens_cache_read: 6000 }; + const { cost_usd, is_estimate } = deriveCost(totals, 'claude-opus-4-8'); + // (3000×5 + 500×25 + 1000×6.25 + 6000×0.5) / 1e6 + // = (15000 + 12500 + 6250 + 3000) / 1e6 = 36750 / 1e6 = 0.03675 + assert.equal(cost_usd, 0.03675); + assert.equal(is_estimate, false); +}); + +test('deriveCost — refuse-to-estimate for an unknown model', () => { + const totals = { tokens_input: 3000, tokens_output: 500, tokens_cache_creation: 1000, tokens_cache_read: 6000 }; + const out = deriveCost(totals, 'claude-unknown-9'); + assert.equal(out.cost_usd, null); + assert.equal(out.is_estimate, true); +}); + +test('PRICE_TABLE — frozen, opus-4-8 carries all five rate fields', () => { + assert.ok(Object.isFrozen(PRICE_TABLE), 'PRICE_TABLE must be frozen (drift-pin)'); + const p = PRICE_TABLE['claude-opus-4-8']; + for (const k of ['input', 'output', 'cache_read', 'cache_write_5m', 'cache_write_1h']) { + assert.equal(typeof p[k], 'number', `missing rate field ${k}`); + } +}); + +// ---- buildRecord ----------------------------------------------------------- + +test('buildRecord — stamps scope:main-context + price_table_version + flat numerics', () => { + const totals = { tokens_input: 3000, tokens_output: 500, tokens_cache_creation: 1000, tokens_cache_read: 6000 }; + const rec = buildRecord({ sessionId: 'S1', model: 'claude-opus-4-8', totals, now: '2026-06-26T12:00:00Z' }); + assert.equal(rec.scope, 'main-context'); + assert.equal(rec.price_table_version, PRICE_TABLE_VERSION); + assert.equal(rec.ts, '2026-06-26T12:00:00Z'); + assert.equal(rec.session_id, 'S1'); + assert.equal(rec.model, 'claude-opus-4-8'); + assert.equal(rec.tokens_input, 3000); + assert.equal(rec.cost_usd, 0.03675); + assert.equal(rec.is_estimate, false); +}); + +test('buildRecord — unknown model → cost_usd null + is_estimate true (honesty contract)', () => { + const rec = buildRecord({ sessionId: 'S9', model: 'mystery', totals: { tokens_input: 1 }, now: 't' }); + assert.equal(rec.cost_usd, null); + assert.equal(rec.is_estimate, true); +}); + +// ---- upsertSessionRecord --------------------------------------------------- + +test('upsertSessionRecord — REPLACES a same-session line (not append)', () => { + const existing = + JSON.stringify({ session_id: 'S1', tokens_input: 1, scope: 'main-context' }) + '\n' + + JSON.stringify({ session_id: 'S2', tokens_input: 2, scope: 'main-context' }) + '\n'; + const rec = buildRecord({ sessionId: 'S1', model: 'claude-opus-4-8', totals: { tokens_input: 999 }, now: 't' }); + const out = upsertSessionRecord(existing, rec); + const recs = out.trim().split('\n').map(JSON.parse); + assert.equal(recs.length, 2, 'replace must NOT add a line'); + const s1 = recs.find(r => r.session_id === 'S1'); + assert.equal(s1.tokens_input, 999, 'S1 must carry the new totals'); + assert.ok(recs.some(r => r.session_id === 'S2'), 'S2 must be preserved'); +}); + +test('upsertSessionRecord — APPENDS a new-session line', () => { + const existing = JSON.stringify({ session_id: 'S1', tokens_input: 1 }) + '\n'; + const rec = buildRecord({ sessionId: 'S3', model: 'claude-opus-4-8', totals: { tokens_input: 7 }, now: 't' }); + const out = upsertSessionRecord(existing, rec); + const recs = out.trim().split('\n').map(JSON.parse); + assert.equal(recs.length, 2, 'new session must append'); + assert.ok(recs.some(r => r.session_id === 'S3' && r.tokens_input === 7)); +}); + +test('upsertSessionRecord — empty file → single record, trailing newline', () => { + const rec = buildRecord({ sessionId: 'S1', model: 'claude-opus-4-8', totals: { tokens_input: 5 }, now: 't' }); + const out = upsertSessionRecord('', rec); + assert.ok(out.endsWith('\n')); + assert.deepEqual(out.trim().split('\n').map(JSON.parse).length, 1); +}); From a9c442c20157df0dc68b5e2303b9377ba840dbb3 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:36:17 +0200 Subject: [PATCH 06/44] feat(exporters): allowlist token-usage schema + assert metric export (CWE-212) --- lib/exporters/field-allowlist.mjs | 12 +++++++ tests/hooks/otel-export-textfile.test.mjs | 34 ++++++++++++++++++++ tests/hooks/otel-export-validators.test.mjs | 35 +++++++++++++++++++++ 3 files changed, 81 insertions(+) diff --git a/lib/exporters/field-allowlist.mjs b/lib/exporters/field-allowlist.mjs index d5226e0..c86bd46 100644 --- a/lib/exporters/field-allowlist.mjs +++ b/lib/exporters/field-allowlist.mjs @@ -76,6 +76,16 @@ const TREKCONTINUE_ALLOWED = Object.freeze(new Set([ 'ts', 'next_session_label', 'status', 'profile', 'profile_source', ])); +// Source: tests/fixtures/jsonl-schemas.md row 9 (token-usage — SKAL-2) +// CWE-212: numeric + low-cardinality-label fields ONLY. DENY BY OMISSION: +// session_id (UUID), transcript_path (filesystem path), cwd (filesystem path) +// are written into the jsonl for upsert keying but MUST NOT reach the exporter. +const TOKEN_USAGE_ALLOWED = Object.freeze(new Set([ + 'ts', 'scope', 'model', + 'tokens_input', 'tokens_output', 'tokens_cache_creation', 'tokens_cache_read', + 'cost_usd', 'is_estimate', 'price_table_version', +])); + // Schema-id → allowlist set const SCHEMA_ALLOWLISTS = Object.freeze({ 'trekbrief': TREKBRIEF_ALLOWED, @@ -87,6 +97,7 @@ const SCHEMA_ALLOWLISTS = Object.freeze({ 'post_bash_stats': POST_BASH_STATS_ALLOWED, // common alt-spelling 'trekreview': TREKREVIEW_ALLOWED, 'trekcontinue': TREKCONTINUE_ALLOWED, + 'token-usage': TOKEN_USAGE_ALLOWED, }); /** @@ -135,4 +146,5 @@ export { TREKEXECUTE_ALLOWED, TREKREVIEW_ALLOWED, TREKCONTINUE_ALLOWED, + TOKEN_USAGE_ALLOWED, }; diff --git a/tests/hooks/otel-export-textfile.test.mjs b/tests/hooks/otel-export-textfile.test.mjs index 3025d94..85d8355 100644 --- a/tests/hooks/otel-export-textfile.test.mjs +++ b/tests/hooks/otel-export-textfile.test.mjs @@ -11,6 +11,8 @@ import { readFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { transformToPrometheus, normalizeMetricName } from '../../lib/exporters/textfile-format.mjs'; +import { transformToOtlpJson } from '../../lib/exporters/otlp-format.mjs'; +import { applyFieldAllowlist } from '../../lib/exporters/field-allowlist.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const FIXTURES = join(__dirname, '..', 'fixtures'); @@ -74,6 +76,38 @@ test('normalizeMetricName: dots/dashes/spaces → underscore, lowercase, voyage_ assert.equal(normalizeMetricName('METRIC NAME'), 'voyage_metric_name'); }); +test('SC#6: allowlisted token-usage record → Prometheus + OTLP metrics; PII absent (end-to-end)', () => { + // Raw record as written to token-usage-stats.jsonl (carries PII for upsert keying). + const raw = { + session_id: 'x', + transcript_path: '/p', + cwd: '/c', + ts: '2026-06-26T12:00:00.000Z', + scope: 'main-context', + model: 'claude-opus-4-8', + tokens_input: 12345, + tokens_output: 6789, + cost_usd: 0.42, + is_estimate: false, + price_table_version: '2026-06-26', + }; + const rec = applyFieldAllowlist(raw, 'token-usage'); + + // Prometheus: numeric fields auto-promote to voyage_token_usage_* metrics. + const prom = transformToPrometheus([rec]); + assert.match(prom, /voyage_token_usage_tokens_input\b/); + assert.match(prom, /voyage_token_usage_cost_usd\b/); + // CWE-212: PII must never appear in exporter output. + assert.ok(!prom.includes('session_id'), 'session_id leaked into Prometheus output'); + assert.ok(!prom.includes('transcript_path'), 'transcript_path leaked into Prometheus output'); + assert.ok(!prom.includes('/p'), 'transcript_path value leaked into Prometheus output'); + + // OTLP parity: dot-separated, dash-preserved naming (NOT the Prometheus underscore form). + const otlp = JSON.stringify(transformToOtlpJson([rec])); + assert.ok(otlp.includes('voyage.token-usage.tokens_input'), 'OTLP token metric missing'); + assert.ok(!otlp.includes('transcript_path'), 'transcript_path leaked into OTLP output'); +}); + test('determinism: identical input produces identical output (sorted keys)', () => { const records = loadJsonl('stats-sample.jsonl'); const out1 = transformToPrometheus(records); diff --git a/tests/hooks/otel-export-validators.test.mjs b/tests/hooks/otel-export-validators.test.mjs index a7cf8e5..6aee4be 100644 --- a/tests/hooks/otel-export-validators.test.mjs +++ b/tests/hooks/otel-export-validators.test.mjs @@ -13,6 +13,7 @@ import { applyFieldAllowlist, POST_BASH_STATS_ALLOWED, EVENT_EMIT_PAYLOAD_ALLOWED, + TOKEN_USAGE_ALLOWED, } from '../../lib/exporters/field-allowlist.mjs'; // ---- path-validator: CWE-22 mitigation ------------------------------------- @@ -241,6 +242,40 @@ test('field-allowlist: Object.freeze on allowlists (drift-pin)', () => { assert.equal(Object.isFrozen(POST_BASH_STATS_ALLOWED), true, 'POST_BASH_STATS_ALLOWED must be frozen — runtime mutation prevention'); assert.equal(Object.isFrozen(EVENT_EMIT_PAYLOAD_ALLOWED), true); + assert.equal(Object.isFrozen(TOKEN_USAGE_ALLOWED), true, + 'TOKEN_USAGE_ALLOWED must be frozen — runtime mutation prevention'); +}); + +// ---- token-usage allowlist (SKAL-2, CWE-212) ------------------------------- + +test('field-allowlist: token-usage INCLUDES numeric/label fields, EXCLUDES session_id/transcript_path/cwd (two-sided)', () => { + const record = { + ts: '2026-06-26T12:00:00.000Z', + session_id: 'uuid-secret', + transcript_path: '/Users/ktg/.claude/projects/x/sesn.jsonl', + cwd: '/Users/ktg/secret/project', + scope: 'main-context', + model: 'claude-opus-4-8', + tokens_input: 12345, + tokens_output: 6789, + tokens_cache_creation: 400, + tokens_cache_read: 5000, + cost_usd: 0.42, + is_estimate: false, + price_table_version: '2026-06-26', + }; + const out = applyFieldAllowlist(record, 'token-usage'); + // INCLUDED (numeric + low-cardinality labels) + for (const k of ['tokens_input', 'tokens_output', 'tokens_cache_creation', + 'tokens_cache_read', 'cost_usd', 'is_estimate', 'price_table_version', 'scope', 'model']) { + assert.equal(k in out, true, `${k} MUST be allowlisted`); + } + assert.equal(out.tokens_input, 12345); + assert.equal(out._schema_id, 'token-usage'); + // EXCLUDED (CWE-212 boundary) + assert.equal('session_id' in out, false, 'session_id MUST be stripped (CWE-212)'); + assert.equal('transcript_path' in out, false, 'transcript_path MUST be stripped (CWE-212)'); + assert.equal('cwd' in out, false, 'cwd MUST be stripped (CWE-212)'); }); test('field-allowlist: null/undefined record handled safely', () => { From 62ebc28e3f45af4945929c33b07f3ffd22f1a223 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:38:16 +0200 Subject: [PATCH 07/44] feat(stats): aggregate token/cost totals in cache-analyzer --- lib/stats/cache-analyzer.mjs | 24 ++++++++++ tests/lib/cache-analyzer.test.mjs | 78 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 tests/lib/cache-analyzer.test.mjs diff --git a/lib/stats/cache-analyzer.mjs b/lib/stats/cache-analyzer.mjs index 609ce72..2e5c7f4 100644 --- a/lib/stats/cache-analyzer.mjs +++ b/lib/stats/cache-analyzer.mjs @@ -43,6 +43,16 @@ export function summarize(lines) { unique_event_names: [], oldest_event_iso: null, newest_event_iso: null, + // SKAL-2 token/cost aggregation (additive; zero when no token records). + // With upsert semantics (one record per session_id), summing across lines + // = correct cross-session aggregate. cost_usd is summed only when finite; + // a null (refuse-to-estimate) record still counts in sessions_with_tokens. + total_tokens_input: 0, + total_tokens_output: 0, + total_tokens_cache_creation: 0, + total_tokens_cache_read: 0, + total_cost_usd: 0, + sessions_with_tokens: 0, }; const durations = []; @@ -71,6 +81,20 @@ export function summarize(lines) { if (newestMs === null || t > newestMs) newestMs = t; } } + + // SKAL-2: aggregate token/cost from token-bearing records (token-usage + // schema). Detect by presence of any numeric token field. + const tokenKeys = ['tokens_input', 'tokens_output', 'tokens_cache_creation', 'tokens_cache_read']; + const hasTokens = tokenKeys.some(k => typeof obj[k] === 'number' && Number.isFinite(obj[k])); + if (hasTokens) { + summary.sessions_with_tokens++; + if (Number.isFinite(obj.tokens_input)) summary.total_tokens_input += obj.tokens_input; + if (Number.isFinite(obj.tokens_output)) summary.total_tokens_output += obj.tokens_output; + if (Number.isFinite(obj.tokens_cache_creation)) summary.total_tokens_cache_creation += obj.tokens_cache_creation; + if (Number.isFinite(obj.tokens_cache_read)) summary.total_tokens_cache_read += obj.tokens_cache_read; + // cost_usd may be null (refuse-to-estimate) — sum finite values only. + if (Number.isFinite(obj.cost_usd)) summary.total_cost_usd += obj.cost_usd; + } } if (durations.length > 0) { diff --git a/tests/lib/cache-analyzer.test.mjs b/tests/lib/cache-analyzer.test.mjs new file mode 100644 index 0000000..17b4292 --- /dev/null +++ b/tests/lib/cache-analyzer.test.mjs @@ -0,0 +1,78 @@ +// tests/lib/cache-analyzer.test.mjs +// SKAL-2 Step 3 — token/cost aggregation in cache-analyzer's summarize(). +// Also the first test coverage for cache-analyzer (previously zero-coverage): +// includes a regression guard that the pre-existing duration/event-name fields +// still compute correctly on mixed input. +// +// Hermetic: no LLM, network, time, or randomness. Idiom: +// tests/lib/token-usage.test.mjs (pure-module node:test style). + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { summarize } from '../../lib/stats/cache-analyzer.mjs'; + +const jsonl = (objs) => objs.map(o => JSON.stringify(o)); + +// ---- token/cost aggregation ------------------------------------------------ + +test('summarize — two token-bearing records → summed totals + cost', () => { + const s = summarize(jsonl([ + { ts: '2026-06-26T12:00:00Z', session_id: 'S1', scope: 'main-context', tokens_input: 1000, tokens_output: 200, tokens_cache_creation: 400, tokens_cache_read: 5000, cost_usd: 0.03675 }, + { ts: '2026-06-26T13:00:00Z', session_id: 'S2', scope: 'main-context', tokens_input: 2000, tokens_output: 300, tokens_cache_creation: 600, tokens_cache_read: 1000, cost_usd: 0.05 }, + ])); + assert.equal(s.total_tokens_input, 3000); + assert.equal(s.total_tokens_output, 500); + assert.equal(s.total_tokens_cache_creation, 1000); + assert.equal(s.total_tokens_cache_read, 6000); + assert.equal(s.sessions_with_tokens, 2); + assert.ok(Math.abs(s.total_cost_usd - 0.08675) < 1e-9, `cost ${s.total_cost_usd}`); +}); + +test('summarize — cost_usd:null counts the session but is excluded from total_cost_usd', () => { + const s = summarize(jsonl([ + { session_id: 'S1', tokens_input: 100, cost_usd: 0.01 }, + { session_id: 'S2', tokens_input: 200, cost_usd: null, is_estimate: true }, + ])); + assert.equal(s.sessions_with_tokens, 2, 'null-cost session must still be counted'); + assert.equal(s.total_tokens_input, 300); + assert.ok(Math.abs(s.total_cost_usd - 0.01) < 1e-9, `cost ${s.total_cost_usd}`); +}); + +test('summarize — no token records → token totals stay zero (additive default)', () => { + const s = summarize(jsonl([ + { event: 'main-merge-gate', duration_ms: 100 }, + ])); + assert.equal(s.total_tokens_input, 0); + assert.equal(s.total_cost_usd, 0); + assert.equal(s.sessions_with_tokens, 0); +}); + +// ---- regression guard: pre-existing fields still computed ------------------ + +test('summarize — existing duration/event-name fields still correct on mixed input', () => { + const s = summarize(jsonl([ + { event: 'main-merge-gate', duration_ms: 100, ts: '2026-06-26T10:00:00Z' }, + { event: 'main-merge-approved', duration_ms: 200, ts: '2026-06-26T11:00:00Z' }, + { session_id: 'S1', tokens_input: 1000, cost_usd: 0.04, ts: '2026-06-26T12:00:00Z' }, + ])); + // Pre-existing behavior (regression guard) + assert.equal(s.total_events, 3); + assert.equal(s.events_with_duration, 2); + assert.equal(s.wall_time_ms_max, 200); + assert.deepEqual(s.unique_event_names, ['main-merge-approved', 'main-merge-gate']); + // New behavior coexists + assert.equal(s.total_tokens_input, 1000); + assert.equal(s.sessions_with_tokens, 1); + assert.ok(Math.abs(s.total_cost_usd - 0.04) < 1e-9); +}); + +test('summarize — malformed/empty lines tolerated alongside token records', () => { + const s = summarize([ + '', + 'not json', + JSON.stringify({ session_id: 'S1', tokens_input: 50, cost_usd: 0.001 }), + ]); + assert.equal(s.total_events, 1); + assert.equal(s.sessions_with_tokens, 1); + assert.equal(s.total_tokens_input, 50); +}); From 46d51f8088e3b7ad1c7b4ad25a4fe76957bd8620 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:42:58 +0200 Subject: [PATCH 08/44] feat(observability): opt-in token capture in Stop hook (VOYAGE_TOKEN_METER) --- hooks/scripts/otel-export.mjs | 29 ++++ lib/stats/token-usage.mjs | 69 +++++++- .../hooks/otel-export-token-capture.test.mjs | 147 ++++++++++++++++++ 3 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 tests/hooks/otel-export-token-capture.test.mjs diff --git a/hooks/scripts/otel-export.mjs b/hooks/scripts/otel-export.mjs index d36f46a..d45a8d4 100644 --- a/hooks/scripts/otel-export.mjs +++ b/hooks/scripts/otel-export.mjs @@ -17,6 +17,7 @@ // - All stderr prefixed with [voyage]. // - EXDEV mitigation: tmp file in same dir as target (do NOT use atomicWriteJson). +import { stdin } from 'node:process'; import { readFileSync, existsSync, writeFileSync, renameSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { transformToPrometheus } from '../../lib/exporters/textfile-format.mjs'; @@ -24,6 +25,13 @@ import { transformToOtlpJson } from '../../lib/exporters/otlp-format.mjs'; import { validateTextfilePath } from '../../lib/exporters/path-validator.mjs'; import { validateOtlpEndpoint } from '../../lib/exporters/endpoint-validator.mjs'; import { applyFieldAllowlist } from '../../lib/exporters/field-allowlist.mjs'; +import { captureTokenUsage } from '../../lib/stats/token-usage.mjs'; + +async function readStdin() { + let data = ''; + for await (const chunk of stdin) data += chunk; + return data; +} const VALID_MODES = new Set(['off', 'textfile', 'otlp']); const TEXTFILE_NAME = 'voyage.prom'; @@ -38,6 +46,7 @@ const STATS_FILES = [ { file: 'trekexecute-stats.jsonl', schema: 'trekexecute' }, { file: 'trekreview-stats.jsonl', schema: 'trekreview' }, { file: 'trekcontinue-stats.jsonl', schema: 'trekcontinue' }, + { file: 'token-usage-stats.jsonl', schema: 'token-usage' }, ]; function loadAndAllowlist(dataDir) { @@ -134,6 +143,26 @@ async function exportOtlp(records, env) { (async () => { try { const env = process.env; + + // SKAL-2: opt-in main-context token/cost capture (default off → zero added + // latency). Rides this existing Stop hook rather than a new hook script — + // avoids a second-Stop-hook race and keeps the README hook-count pin intact. + // Runs BEFORE the export-mode gate so capture is independent of export. + // Fail-open: any error is swallowed; capture must never block Stop. + if (env.VOYAGE_TOKEN_METER) { + try { + const raw = await readStdin(); + if (raw.trim()) { + const payload = JSON.parse(raw); + captureTokenUsage({ + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + dataDir: env.CLAUDE_PLUGIN_DATA, + }); + } + } catch { /* fail-open: never block Stop, never throw */ } + } + const mode = (env.VOYAGE_EXPORT_MODE || 'off').toLowerCase(); if (mode === 'off') return; diff --git a/lib/stats/token-usage.mjs b/lib/stats/token-usage.mjs index 72036ab..229da4f 100644 --- a/lib/stats/token-usage.mjs +++ b/lib/stats/token-usage.mjs @@ -15,11 +15,15 @@ // main-transcript sum UNDER-counts total Voyage cost. Every record is stamped // scope:'main-context' so no reader mistakes cost_usd for the session total. // -// All functions in this module are PURE (no I/O). The impure capture shell -// (captureTokenUsage) is added in Step 4. +// The four core functions (parseTranscriptUsage, deriveCost, buildRecord, +// upsertSessionRecord) are PURE (no I/O). captureTokenUsage is the impure +// shell that wires them to the filesystem (read transcript → upsert jsonl). // // Zero npm dependencies. Node stdlib only. +import { readFileSync, existsSync, writeFileSync, renameSync, statSync } from 'node:fs'; +import { join, dirname } from 'node:path'; + // Per-Mtok USD prices, resolved 2026-06-26 via the claude-api skill reference: // base input/output from the model table; cache rates from the prompt-caching // doc multipliers (cache_read 0.1x, write_5m 1.25x, write_1h 2.0x of input). @@ -159,3 +163,64 @@ export function upsertSessionRecord(existingText, record) { if (!replaced) out.push(JSON.stringify(record)); return out.join('\n') + '\n'; } + +/** + * Last main-chain (non-sidechain) assistant model in the transcript. Used to + * pick the price-table key. Returns null when no model is found (→ deriveCost + * refuses to estimate). Pure. + */ +export function lastMainChainModel(text) { + const lines = (text || '').split('\n'); + let model = null; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === '') continue; + let obj; + try { obj = JSON.parse(trimmed); } + catch { continue; } + if (!obj || obj.type !== 'assistant' || obj.isSidechain === true) continue; + const m = obj.message && obj.message.model; + if (typeof m === 'string' && m) model = m; + } + return model; +} + +/** + * Impure capture shell (Step 4). Reads the transcript, derives main-context + * token totals + cost, and UPSERTS a one-line-per-session record into + * {dataDir}/token-usage-stats.jsonl via an atomic temp+rename write + * (EXDEV mitigation: tmp lives in the same dir as the target). + * + * Returns the written record, or null when skipped (no path / no dataDir / + * transcript not a readable regular file). Throws propagate to the caller — + * the Stop hook wraps this in try/catch so capture stays fail-open. + * + * @param {{transcriptPath:string, sessionId:string, dataDir:string, + * now?:string}} args + */ +export function captureTokenUsage({ transcriptPath, sessionId, dataDir, now }) { + if (!transcriptPath || !dataDir) return null; + let st; + try { st = statSync(transcriptPath); } + catch { return null; } + if (!st.isFile()) return null; + + const text = readFileSync(transcriptPath, 'utf-8'); + const totals = parseTranscriptUsage(text); + const model = lastMainChainModel(text); + const record = buildRecord({ + sessionId, + model, + totals, + now: now || new Date().toISOString(), + }); + + const outPath = join(dataDir, 'token-usage-stats.jsonl'); + const existing = existsSync(outPath) ? readFileSync(outPath, 'utf-8') : ''; + const updated = upsertSessionRecord(existing, record); + + const tmpPath = join(dirname(outPath), '.token-usage-stats.jsonl.tmp'); + writeFileSync(tmpPath, updated); + renameSync(tmpPath, outPath); + return record; +} diff --git a/tests/hooks/otel-export-token-capture.test.mjs b/tests/hooks/otel-export-token-capture.test.mjs new file mode 100644 index 0000000..3fdb1da --- /dev/null +++ b/tests/hooks/otel-export-token-capture.test.mjs @@ -0,0 +1,147 @@ +// tests/hooks/otel-export-token-capture.test.mjs +// SKAL-2 Step 4 — opt-in main-context token/cost capture folded into the Stop +// hook (otel-export.mjs), gated by VOYAGE_TOKEN_METER. +// +// Fail-open contract: any error → exit 0, no token file, Stop never blocked. +// Pattern: tests/hooks/otel-export.test.mjs (setupDataDir + runHookWithEnv). + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { runHookWithEnv } from '../helpers/hook-helper.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const HOOK_PATH = join(HERE, '..', '..', 'hooks', 'scripts', 'otel-export.mjs'); +const FIXTURE = join(HERE, '..', 'fixtures', 'token-usage', 'sample-transcript.jsonl'); +const TOKEN_FILE = 'token-usage-stats.jsonl'; + +const mkDataDir = () => mkdtempSync(join(tmpdir(), 'voyage-token-capture-')); +const stdinFor = (extra = {}) => JSON.stringify({ transcript_path: FIXTURE, session_id: 'sess-1', ...extra }); +const readToken = (dir) => readFileSync(join(dir, TOKEN_FILE), 'utf-8').trim().split('\n').filter(Boolean).map(JSON.parse); + +test('VOYAGE_TOKEN_METER=1 → writes token-usage-stats.jsonl with expected totals + scope:main-context', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), true, 'token file must be written'); + const recs = readToken(dir); + assert.equal(recs.length, 1); + const rec = recs[0]; + // Fixture totals: dedup-by-requestId + sidechain-exclusion (verified in Step 1). + assert.equal(rec.tokens_input, 3000); + assert.equal(rec.tokens_output, 500); + assert.equal(rec.tokens_cache_creation, 1000); + assert.equal(rec.tokens_cache_read, 6000); + assert.equal(rec.scope, 'main-context'); + assert.equal(rec.model, 'claude-opus-4-8'); + assert.ok(Math.abs(rec.cost_usd - 0.03675) < 1e-9, `cost ${rec.cost_usd}`); + assert.equal(rec.session_id, 'sess-1'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('UPSERT — running twice for the same session_id keeps ONE line (not two)', async () => { + const dir = mkDataDir(); + try { + const env = { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }; + await runHookWithEnv(HOOK_PATH, stdinFor(), env); + const r2 = await runHookWithEnv(HOOK_PATH, stdinFor(), env); + assert.equal(r2.code, 0); + const recs = readToken(dir); + assert.equal(recs.length, 1, 'upsert must NOT append a second line for the same session'); + assert.equal(recs[0].tokens_input, 3000); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('UPSERT — distinct session_ids append (one line per session)', async () => { + const dir = mkDataDir(); + try { + const env = { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }; + await runHookWithEnv(HOOK_PATH, stdinFor({ session_id: 'A' }), env); + await runHookWithEnv(HOOK_PATH, stdinFor({ session_id: 'B' }), env); + const recs = readToken(dir); + assert.equal(recs.length, 2); + assert.deepEqual(recs.map(r => r.session_id).sort(), ['A', 'B']); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('OPT-IN — VOYAGE_TOKEN_METER unset → NO token file written', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '', // explicit empty mimics unset + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false, 'no token file without opt-in'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('FAIL-OPEN — malformed stdin → exit 0, no token file, no throw', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, 'not valid json {{{', { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('FAIL-OPEN — missing transcript file → exit 0, no token file', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv( + HOOK_PATH, + JSON.stringify({ transcript_path: '/no/such/transcript.jsonl', session_id: 'x' }), + { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }, + ); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('capture coexists with export — VOYAGE_TOKEN_METER=1 + textfile mode writes BOTH', async () => { + const dir = mkDataDir(); + try { + // Seed a stats file so the export path has something to write. + writeFileSync(join(dir, 'trekplan-stats.jsonl'), + JSON.stringify({ ts: '2026-06-26T08:00:00.000Z', slug: 't', mode: 'default', codebase_files: 10, profile: 'premium' }) + '\n'); + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'textfile', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), true, 'token file written'); + assert.equal(existsSync(join(dir, 'voyage.prom')), true, 'export still completed'); + // The exported textfile should now also carry the token metric (STATS_FILES wiring). + const prom = readFileSync(join(dir, 'voyage.prom'), 'utf-8'); + assert.match(prom, /voyage_token_usage_tokens_input/); + assert.ok(!prom.includes('sess-1'), 'session_id must not leak into export (CWE-212)'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 7e78d076f993f521da5fff6fd2f42c525c8c1708 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:47:24 +0200 Subject: [PATCH 09/44] docs(observability): document token-usage schema + main-context v1 scope --- docs/observability.md | 42 +++++++++++++++++++++++++++++++++ tests/fixtures/jsonl-schemas.md | 5 +++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/observability.md b/docs/observability.md index cc0fe36..3c64102 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -60,6 +60,7 @@ operator-private data (paths, prompts, brief content). | `VOYAGE_TEXTFILE_DIR` | `${CLAUDE_PLUGIN_DATA}` | Directory for `voyage.prom` (textfile mode) | | `VOYAGE_OTEL_ENDPOINT` | _(none)_ | HTTPS URL for OTLP/HTTP POST | | `VOYAGE_OTEL_ALLOW_PRIVATE` | _(unset)_ | Set to `1` to allow loopback / RFC1918 endpoints | +| `VOYAGE_TOKEN_METER` | _(unset)_ | Set to a truthy value to capture per-session token/cost into `token-usage-stats.jsonl` on Stop (default off → zero added latency). See **Token/cost metering** below. | ## Docker Compose quickstart @@ -88,6 +89,47 @@ the allowlist explicitly. This is intentional: `${CLAUDE_PLUGIN_DATA}` is trusted local storage; OTel endpoints are operator-controlled and may be external. +## Token/cost metering + +> **SKAL-2.** Opt-in capture of per-session token usage and a cache-aware USD +> cost estimate, folded into the existing Stop hook (`hooks/scripts/otel-export.mjs`). +> No new hook is added — capture rides the same Stop event, so there is no +> second-Stop-hook ordering race. + +**Activation.** Set `VOYAGE_TOKEN_METER` to any truthy value. When unset (the +default) the capture path is skipped entirely — zero added Stop latency. When +set, the hook reads the Claude Code transcript (`transcript_path` from the Stop +payload), sums token usage, derives cost, and **upserts** one record per session +into `${CLAUDE_PLUGIN_DATA}/token-usage-stats.jsonl`. Capture is fail-open: any +error (malformed payload, unreadable transcript) is swallowed and never blocks +Stop. Once captured, the record is exported like any other stats file when +`VOYAGE_EXPORT_MODE` is `textfile` or `otlp`. + +**Schema (`token-usage`).** Flat numeric record: +`ts`, `session_id`, `scope`, `model`, `tokens_input`, `tokens_output`, +`tokens_cache_creation`, `tokens_cache_read`, `cost_usd`, `is_estimate`, +`price_table_version`. The field allowlist (`lib/exporters/field-allowlist.mjs`, +`TOKEN_USAGE_ALLOWED`) admits only the numeric + low-cardinality-label fields and +**strips `session_id` at export** (CWE-212). The exporter auto-promotes each +numeric field to a metric: `voyage_token_usage_tokens_input` (Prometheus) / +`voyage.token-usage.tokens_input` (OTLP). + +**Cost contract (honesty).** `cost_usd` is computed from a dated, in-source +`PRICE_TABLE` (per-MTok USD), cache-aware: +`input + output + cache_creation×(5m write rate) + cache_read×(read rate)`. +Each record carries `price_table_version` (the date the prices were resolved) +and `is_estimate`. When the transcript's model is **not** in the price table, the +meter **refuses to guess**: `cost_usd` is `null` and `is_estimate` is `true`. +Prices are volatile — re-resolve them against the `claude-api` reference and bump +`PRICE_TABLE_VERSION` when they change. + +**v1 limitation — MAIN-CONTEXT only.** The meter reads the main-session +transcript, which contains only `isSidechain:false` records. Sub-agent (swarm) +turns live in separate `agent-*.jsonl` sibling files and are **not** counted, so +`cost_usd` is a lower bound on total Voyage cost, not the session total. Every +record is stamped `scope:'main-context'` to make this explicit. Per-subagent +attribution is a documented v2 follow-on (it was a Non-Goal for v1). + ## Security The exporter is hardened against three CWE classes: diff --git a/tests/fixtures/jsonl-schemas.md b/tests/fixtures/jsonl-schemas.md index 0275466..1a6ab5f 100644 --- a/tests/fixtures/jsonl-schemas.md +++ b/tests/fixtures/jsonl-schemas.md @@ -27,6 +27,7 @@ | trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) | | trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | commands/trekreview.md (orchestrator-emit Phase 8) | trekreview.md:255 | profile, phase_models, profile_source | none | | trekcontinue-stats | ts, project, next_session_label, status | commands/trekcontinue.md (orchestrator-emit Phase 5) | trekcontinue.md:289 | profile, profile_source | none | +| token-usage | ts, session_id, scope, model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, cost_usd, is_estimate, price_table_version | lib/stats/token-usage.mjs `captureTokenUsage` (via hooks/scripts/otel-export.mjs Stop hook, opt-in `VOYAGE_TOKEN_METER`) | token-usage.mjs:buildRecord | n/a (SKAL-2 schema) | session_id (stripped at export) | ## Field-allowlist input for Step 11 @@ -46,7 +47,9 @@ critic_verdict, guardian_verdict, outcome, plan_type, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step, verdict, reviewed_files_count, duration_ms, status, next_session_label, event, known_event, success, scope, -profile, profile_source, parallel_agents, external_research_enabled +profile, profile_source, parallel_agents, external_research_enabled, +model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, +cost_usd, is_estimate, price_table_version ``` **EXPORT_DENYLIST** (PII or high-cardinality, never export): From 68c9bef38fe3a6b1fd37cc438764b679e9e2f990 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 17:14:03 +0200 Subject: [PATCH 10/44] test(observability): close 2 MAJOR test gaps from SKAL-2 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 cache-analyzer regression guard (SC5): pin percentile (wall_time_ms_p50/p90) and time-range (oldest/newest_event_iso) — 2 of 3 'unchanged' categories were previously un-asserted on the mixed-input fixture. F2 lastMainChainModel: add direct coverage — last-wins across 2 distinct main-chain models, sidechain exclusion (even when the sidechain is the last record), and model-absent → null propagating to deriveCost refuse-to-estimate. 804 -> 807 tests (805 pass / 0 fail / 2 skipped). No test pins the test count. --- tests/lib/cache-analyzer.test.mjs | 6 +++++ tests/lib/token-usage.test.mjs | 37 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/tests/lib/cache-analyzer.test.mjs b/tests/lib/cache-analyzer.test.mjs index 17b4292..474f2c7 100644 --- a/tests/lib/cache-analyzer.test.mjs +++ b/tests/lib/cache-analyzer.test.mjs @@ -58,8 +58,14 @@ test('summarize — existing duration/event-name fields still correct on mixed i // Pre-existing behavior (regression guard) assert.equal(s.total_events, 3); assert.equal(s.events_with_duration, 2); + // Percentiles: durations [100,200] → floor(2·0.5)=floor(2·0.9)=idx 1 → both 200. + assert.equal(s.wall_time_ms_p50, 200); + assert.equal(s.wall_time_ms_p90, 200); assert.equal(s.wall_time_ms_max, 200); assert.deepEqual(s.unique_event_names, ['main-merge-approved', 'main-merge-gate']); + // Time range spans all ts-bearing lines (incl. the token-only record at 12:00). + assert.equal(s.oldest_event_iso, '2026-06-26T10:00:00.000Z'); + assert.equal(s.newest_event_iso, '2026-06-26T12:00:00.000Z'); // New behavior coexists assert.equal(s.total_tokens_input, 1000); assert.equal(s.sessions_with_tokens, 1); diff --git a/tests/lib/token-usage.test.mjs b/tests/lib/token-usage.test.mjs index 36e7f8d..bccdfe7 100644 --- a/tests/lib/token-usage.test.mjs +++ b/tests/lib/token-usage.test.mjs @@ -16,6 +16,7 @@ import { deriveCost, buildRecord, upsertSessionRecord, + lastMainChainModel, PRICE_TABLE, PRICE_TABLE_VERSION, } from '../../lib/stats/token-usage.mjs'; @@ -140,3 +141,39 @@ test('upsertSessionRecord — empty file → single record, trailing newline', ( assert.ok(out.endsWith('\n')); assert.deepEqual(out.trim().split('\n').map(JSON.parse).length, 1); }); + +// ---- lastMainChainModel ---------------------------------------------------- + +test('lastMainChainModel — last main-chain model wins; sidechain excluded even when last', () => { + // Two distinct main-chain models (opus then sonnet) → sonnet (last) wins. + // A sidechain record sits AFTER sonnet: if exclusion broke it would win; + // if last-wins broke, opus (first) would win. Correct answer pins both. + const text = [ + JSON.stringify({ type: 'assistant', isSidechain: false, message: { model: 'claude-opus-4-8', usage: {} } }), + JSON.stringify({ type: 'assistant', isSidechain: false, message: { model: 'claude-sonnet-4-6', usage: {} } }), + JSON.stringify({ type: 'assistant', isSidechain: true, message: { model: 'claude-sidechain-ZZZ', usage: {} } }), + JSON.stringify({ type: 'user', message: { content: 'ignored' } }), + ].join('\n'); + assert.equal(lastMainChainModel(text), 'claude-sonnet-4-6'); +}); + +test('lastMainChainModel — no main-chain model → null → deriveCost refuses to estimate', () => { + // Only a sidechain model, a user line, and a malformed line: no main-chain model. + const text = [ + JSON.stringify({ type: 'assistant', isSidechain: true, message: { model: 'claude-opus-4-8', usage: {} } }), + JSON.stringify({ type: 'user', message: { content: 'hi' } }), + 'not json', + ].join('\n'); + const model = lastMainChainModel(text); + assert.equal(model, null); + // null model must propagate to refuse-to-estimate (honesty contract). + const { cost_usd, is_estimate } = deriveCost({ tokens_input: 1000 }, model); + assert.equal(cost_usd, null); + assert.equal(is_estimate, true); +}); + +test('lastMainChainModel — empty / nullish input → null', () => { + assert.equal(lastMainChainModel(''), null); + assert.equal(lastMainChainModel(null), null); + assert.equal(lastMainChainModel(undefined), null); +}); From be183e4617607b8d0d582c12512e393f939e6dd2 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 17:48:22 +0200 Subject: [PATCH 11/44] =?UTF-8?q?chore(voyage):=20release=20v5.7.0=20?= =?UTF-8?q?=E2=80=94=20opt-in=20token/cost=20metering=20(SKAL-2)=20+=20eva?= =?UTF-8?q?l=20foundation=20(SKAL-1=C2=B74a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive, no breaking change. Bundles the unreleased work since v5.6.1: SKAL-2 token/cost metering (parser+cache-aware cost, CWE-212 export boundary, cache-analyzer aggregation, opt-in VOYAGE_TOKEN_METER Stop-hook capture) and SKAL-1·4a eval foundation (gold corpus, review-coordinator contract, BRIEF_* gate coverage). Version sync across 5 refs + CHANGELOG; canonical node --test 807. --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 20 ++++++++++++++++++++ README.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index bc9c396..eb6c095 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "voyage", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", - "version": "5.6.1", + "version": "5.7.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index e5f539b..1ae729e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## v5.7.0 — 2026-06-26 — opt-in token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a) + +Additive — no breaking change. Two unreleased work-streams land together. + +### Opt-in token/cost metering (SKAL-2) — the headline + +- Pure token-usage parser + cache-aware USD cost (`lib/stats/token-usage.mjs`): parses MAIN-CONTEXT usage from the transcript, dedups by requestId (last-wins, streaming-placeholder mitigation), excludes sidechain records, and REFUSES to estimate (`cost_usd:null, is_estimate:true`) for models absent from the frozen PRICE_TABLE. +- CWE-212 export boundary: token-usage schema allowlist in the OTLP exporter; `session_id`/`transcript_path`/`cwd` stripped at export, asserted both ways. +- Cross-session aggregation in `cache-analyzer` (total tokens + cost). +- Capture folded into the EXISTING `otel-export.mjs` Stop hook, gated behind the `VOYAGE_TOKEN_METER` env var (default off), fail-open. v1 scope = main-context only; sub-agent turns are a documented v2 follow-on. + +### Eval foundation (SKAL-1·4a) + +- `gold.json` golden corpus + loader/validator; review-coordinator contract reference impl + deterministic test; two-sided gate coverage for the `BRIEF_*` BLOCKERs; eval-corpus frozen-failure home. + +### Release hygiene + +- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.7.0`, guarded by `doc-consistency.test.mjs`. +- Canonical `node --test`: **807** (805 pass / 0 fail / 2 skipped). + ## v5.6.1 — 2026-06-24 — leaner always-loaded agent listing (reference/dormant agent descriptions trimmed) Additive — no breaking change, **no runtime behavior change**. Trims the always-loaded token cost of the agent listing that Claude Code injects into every session. diff --git a/README.md b/README.md index f95ce45..a4a823d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # trekplan — Brief, Research, Plan, Execute, Review, Continue -![Version](https://img.shields.io/badge/version-5.6.1-blue) +![Version](https://img.shields.io/badge/version-5.7.0-blue) ![License](https://img.shields.io/badge/license-MIT-green) ![Platform](https://img.shields.io/badge/platform-Claude%20Code-purple) diff --git a/package-lock.json b/package-lock.json index 2c62c2a..ac531f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voyage", - "version": "5.6.1", + "version": "5.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voyage", - "version": "5.6.1", + "version": "5.7.0", "license": "MIT", "engines": { "node": ">=18" diff --git a/package.json b/package.json index bca638c..2f3a494 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voyage", - "version": "5.6.1", + "version": "5.7.0", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any heading/paragraph/list-item, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", "type": "module", "engines": { From 6bd50a42dd0c5d4a6772ac70db007c68555d91ac Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 20:15:34 +0200 Subject: [PATCH 12/44] docs(voyage): track agent-description token-trim brief (M4 input) Cross-session coordination brief from the config-audit machine-tuning session, dropped into voyage per operator instruction. brief_version 2.2, framing: refine. Locked as the next session's task (Alternative A). Tracking it makes the next session's /trekplan --brief input deterministic and durable on the private remote. --- docs/agent-description-token-trim-brief.md | 153 +++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 docs/agent-description-token-trim-brief.md diff --git a/docs/agent-description-token-trim-brief.md b/docs/agent-description-token-trim-brief.md new file mode 100644 index 0000000..2dfb3ab --- /dev/null +++ b/docs/agent-description-token-trim-brief.md @@ -0,0 +1,153 @@ +--- +type: trekbrief +brief_version: "2.2" +created: 2026-06-26 +task: "Relocate blocks out of voyage agent description frontmatter to cut always-loaded tokens" +slug: agent-description-token-trim +project_dir: .claude/projects/2026-06-26-agent-description-token-trim/ +research_topics: 0 +research_status: skipped +auto_research: false +interview_turns: 0 +source: manual +framing: refine +phase_signals: + - phase: research + effort: low + - phase: plan + effort: standard + - phase: execute + effort: standard + - phase: review + effort: standard +--- + +# Task: Trim voyage agent `description:` frontmatter — relocate `` blocks into the agent body + +> **Cross-session coordination brief.** Authored 2026-06-26 by the **config-audit machine-tuning session** (a parallel session that audits this machine's Claude Code always-loaded token footprint). It is dropped here per operator instruction so the **voyage session** owns the implementation — config-audit does **not** edit the voyage repo. The source measurement lives in config-audit's local worklist (`config-audit/docs/machine-tuning-worklist.local.md`, item **M4**, session m). This brief is the contract; `/trekplan` can consume it directly. + +## TL;DR + +voyage's 24 agent `description:` fields cost **~4,418 always-loaded tokens every turn** (injected into the system prompt of every session that has voyage enabled). **17** of those agents carry **two `` blocks each (34 total, ~3,147 tok)** in their frontmatter `description`. Those example blocks exist to drive *autonomous* agent-selection — but voyage agents are launched **by explicit name** from the orchestrator commands, so the examples are cost without function. Move them into each agent's body (preserve, don't delete). Expected saving: **~2,500–3,100 always-loaded tokens.** framing: **refine** — this continues the token-trim line started in M1a (v5.6.1). + +## Intent + +The machine's #1 tuning lever is *always-loaded* tokens — context injected on every turn, paid on every request. config-audit's Fase-1 measurement (2026-06-26) found the machine's global layer is already lean (CLAUDE.md trimmed, rules/agents empty, output style off, MCP deferred), and the **single largest remaining tunable source is voyage's agent listing (~4,418 tok)**. The bulk of that is `` blocks in the spawnable agents' `description:` frontmatter. These blocks follow the documented agent-authoring pattern whose purpose is to help the *main loop autonomously decide* when to delegate to an agent. voyage doesn't rely on that path: `/trekplan`, `/trekresearch`, and `/trekreview` launch their agents **deterministically by name** ("Launch the **architecture-mapper** agent", explicit per-codebase-size launch tables, explicit `voyage:review-coordinator` references). So the examples are paying a per-turn token tax for an auto-selection behavior voyage never uses. + +## Goal + +Each of the **17** example-bearing voyage agents has its frontmatter `description:` reduced to its **triggering lead sentence(s)** (the "Use this agent to/when…" summary — what any residual autonomous selection actually needs), with the `` blocks **relocated verbatim into the agent's markdown body** under a clearly-marked section (e.g. `## When to use — examples`). No agent is deleted, no behavior changes, no example content is lost. The injected agent-listing cost drops by ~2,500–3,100 tokens, landing on every machine that reloads voyage after release. + +## Non-Goals + +- **Do NOT delete the examples** — relocate them into the body. They remain useful documentation and `/trekbrief`/onboarding reference. +- **Do NOT change any agent's system prompt, body logic, tools, model, or `name`** (`name` is the agent's identity). +- **Do NOT touch the 7 zero-example agents** (`planning-orchestrator`, `research-orchestrator`, `review-orchestrator`, `synthesis-agent`, `review-coordinator`, `code-correctness-reviewer`, `brief-conformance-reviewer`) — already lean (M1a handled the reference/dormant ones; the orchestrators keep their pinned "reference document, not a spawnable capability" phrasing). +- **Do NOT modify the orchestrator commands** (`/trekplan` etc.) or change which agents exist. +- This is **not** a behavioral or capability change — purely a token/packaging optimization. + +## Constraints + +- Frontmatter must remain valid YAML. The retained `description:` must keep a meaningful **triggering lead sentence**, because a few agents (notably the `*-researcher` agents) *can* legitimately be selected autonomously when a user asks a research question directly — keep their first 1–2 sentences descriptive enough to still trigger. +- **The test suite must stay green.** `node --test 'tests/**/*.test.mjs'` is the gate. voyage has inventory/frontmatter-pin tests — M1a's note warned that "synthesis-schema + inventory tests may pin description content." If any test asserts on `` presence/count or description length, update that test's expectation **deliberately** (it is asserting the old structure) and document why. +- Follow voyage's release discipline: version-sync (plugin.json + package.json + README badge/What's-new + CHANGELOG), annotated tag, and catalog `ref` bump with `check-versions` green. This is a perf/token change → a **patch** release (e.g. v5.6.2) or fold into the work already in progress on the branch. +- The saving only reaches a machine after the operator reloads voyage (`/plugin marketplace update` + update + `/exit`) — note this in the CHANGELOG so the delta isn't expected instantly. + +## Preferences + +- Relocate the two examples per agent into a single body section with a stable heading so they're easy to find and so any future inventory test can pin the body instead of the frontmatter. +- Keep the diff mechanical and uniform across the 17 agents (same heading, same ordering) for reviewability. +- If a lead sentence is currently entangled with the first ``, lightly rewrite it into a clean 1–2 sentence trigger — minimal, not a rewrite. + +## Non-Functional Requirements + +- **Zero new dependencies.** +- **Always-loaded `description` budget:** total frontmatter `description` chars across all 24 agents drops from **~17,672** to **≤ ~6,000** (~2,500–3,100 tok saved). +- No regression in agent auto-selection for the agents that are legitimately autonomous (the researchers) — spot-check that a direct research-style prompt still routes sensibly. + +## Success Criteria + +- **Full suite green:** `node --test 'tests/**/*.test.mjs'` exits 0 (same pass count, or deliberately updated pins with rationale). +- **No `` in any frontmatter `description`:** the frontmatter-only scan below prints `0`: + ```bash + python3 - <<'PY' + import re,glob + n=0 + for f in glob.glob('agents/*.md'): + t=open(f).read(); m=re.match(r'^---\n(.*?)\n---', t, re.S) + fm=m.group(1) if m else '' + dm=re.search(r'description:\s*(.*?)(?=\n[a-zA-Z_][a-zA-Z_-]*:\s|\Z)', fm, re.S) + n+=(dm.group(1).count('') if dm else 0) + print(n) # must be 0 + PY + ``` +- **Examples preserved in bodies:** `grep -l '' agents/*.md` still lists the 17 agents (the blocks moved, not vanished). +- **Agent count unchanged:** 24 agents, each with non-empty `name` + `description`. Plugin validates (plugin-validator / inventory tests green). +- **Token budget met:** total frontmatter `description` chars ≤ ~6,000 (measure with the per-agent script in the worklist / the snippet above adapted to sum lengths). +- **Behavior unchanged:** `/trekplan` (and trekresearch/trekreview) still launch the same agents by name — diff touches only `agents/*.md`, no command files. + +## Research Plan + +No external research needed — the codebase and this brief contain sufficient context for planning. (research_topics = 0.) + +## Open Questions / Assumptions + +- **[ASSUMPTION]** The 17 example-bearing agents are invoked **by name** from orchestrator commands — verified by the config-audit session against `voyage/commands/*.md` (explicit "Launch the X agent" prose + per-size launch tables + explicit `voyage:` references). The `` auto-trigger blocks are therefore non-load-bearing for voyage's actual invocation path. +- **[Q]** Do any voyage tests assert on `` presence/count or `description` length in `agents/*.md`? **Check first** (`grep -rn 'example\|description' tests/`), so a pin is updated intentionally, not discovered as a red test. +- **[ASSUMPTION]** The `*-researcher` agents (community/contrarian/docs/security) and `gemini-bridge` may also be autonomously selected outside the trek pipeline → keep their lead sentences trigger-worthy (don't reduce to a bare label). + +## Prior Attempts + +- **M1a — voyage v5.6.1 (2026-06-24):** trimmed the 4 non-spawnable reference/dormant agents' (`planning/research/review-orchestrator` + `synthesis-agent`) verbose `description:` fields to one-liners; orchestrators kept their pinned phrase, synthesis kept its DORMANT flag. Suite stayed green (756). **M4 extends that same always-loaded-token-trim to the 17 spawnable agents' `` blocks** — the larger remaining chunk M1a deliberately left. + +## Reference data (config-audit Fase-1 measurement, 2026-06-26) + +24 agents, **17,672 `description` chars (~4,418 tok)**, **34 `` blocks**, ~12,591 chars (~3,147 tok) inside `…`. The 17 example-bearing agents (descending `description` size): + +| agent | desc chars | examples | +|---|---:|---:| +| community-researcher | 1280 | 2 | +| contrarian-researcher | 1270 | 2 | +| gemini-bridge | 1226 | 2 | +| security-researcher | 1207 | 2 | +| docs-researcher | 1138 | 2 | +| session-decomposer | 946 | 2 | +| convention-scanner | 939 | 2 | +| brief-reviewer | 870 | 2 | +| research-scout | 847 | 2 | +| test-strategist | 823 | 2 | +| dependency-tracer | 818 | 2 | +| task-finder | 817 | 2 | +| git-historian | 795 | 2 | +| risk-assessor | 794 | 2 | +| architecture-mapper | 789 | 2 | +| scope-guardian | 750 | 2 | +| plan-critic | 692 | 2 | + +Zero-example (leave as-is): review-coordinator (345), code-correctness-reviewer (292), brief-conformance-reviewer (266), synthesis-agent (228), planning-orchestrator (184), research-orchestrator (179), review-orchestrator (177). + +## Metadata + +- **Created:** 2026-06-26 +- **Interview turns:** 0 +- **Auto-research opted in:** no +- **Source:** manual (cross-session coordination brief from the config-audit machine-tuning session) + +--- + +## How to continue + +```bash +# (optional) confirm no test pins in descriptions first: +grep -rn 'example' tests/ | grep -i descr + +# plan + execute via the voyage pipeline: +/trekplan --project .claude/projects/2026-06-26-agent-description-token-trim/ +/trekexecute --project .claude/projects/2026-06-26-agent-description-token-trim/ + +# or implement directly (it's a uniform, mechanical 17-file edit) and release as a patch (v5.6.2): +# - move each agent's 2 blocks from frontmatter description into a body +# "## When to use — examples" section +# - node --test 'tests/**/*.test.mjs' (green) +# - version-sync + CHANGELOG + tag + catalog ref-bump (check-versions green) +``` From 6dea478de2b2c39974b0cfac04f26a7d1a3ff6ec Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 20:59:32 +0200 Subject: [PATCH 13/44] =?UTF-8?q?chore(voyage):=20make=20STATE.md=20LOCAL-?= =?UTF-8?q?ONLY=20=E2=80=94=20open/=20is=20a=20public=20mirror?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit origin is open/voyage (an OFFENTLIG/public mirror per ~/.claude/CLAUDE.md, which names open/ as the public example). STATE.md was wrongly tracked since 5bf574a (23 commits) under a mistaken 'PRIVAT Forgejo' premise. Its content has now been scrubbed from all history via git filter-repo; this gitignores it going forward and corrects the false tracked/private comments. STATE continuity stays local. --- .gitignore | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 822742e..b92beb2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,10 @@ blob-report/ # Local configuration / session files *.local.* -# STATE.md — current state-of-play. TRACKED continuity per ~/.claude/CLAUDE.md -# (overrides the polyrepo gitignore convention); pushed to private Forgejo, never public. +# STATE.md — current state-of-play. LOCAL-ONLY per ~/.claude/CLAUDE.md: +# origin is open/ (an OFFENTLIG/public mirror) → STATE must NEVER be pushed there. +# Kept local for continuity only. History scrubbed 2026-06-26 (was wrongly tracked 23 commits). +STATE.md # Local planning docs (briefs, design notes, observations) — never committed. # Existing tracked files in docs/ predate this rule; new planning docs stay local. @@ -30,7 +32,7 @@ docs/ultracontinue-design-notes.md # Ultraplan project directories — briefs, research, plans, progress all local. .claude/projects/ -# --- session/local state (gitignored) — STATE.md er nå tracked, se ~/.claude/CLAUDE.md --- +# --- session/local state (gitignored) — STATE.md is LOCAL-ONLY (open/ = public), se ~/.claude/CLAUDE.md --- REMEMBER.md ROADMAP.md TODO.md From 4bda2621eb2ad1a8c56fca14bfbac3e0f57a2d94 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 10:11:53 +0200 Subject: [PATCH 14/44] test(agents): pin examples-in-body invariant (RED before M4 relocation) --- tests/lib/agent-frontmatter.test.mjs | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/lib/agent-frontmatter.test.mjs b/tests/lib/agent-frontmatter.test.mjs index 9132f40..5340f9e 100644 --- a/tests/lib/agent-frontmatter.test.mjs +++ b/tests/lib/agent-frontmatter.test.mjs @@ -127,3 +127,39 @@ test('non-orchestrator agents do NOT include the Agent tool (no recursive swarmi ); } }); + +// M4 (v5.7.1): examples-relocation invariant. The 34 blocks belong in +// agent BODIES, not in the always-loaded `description:` frontmatter (voyage +// launches its agents by name, so the auto-selection examples are cost without +// function there). This pins the migration: examples cannot regress into +// frontmatter, and cannot be silently lost during the move. +function bodyOf(text) { + // Slice the file content AFTER the closing frontmatter `---`. + // Do NOT split on /^---$/m — a horizontal rule in the body would mis-split. + const m = text.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/); + return m ? text.slice(m[0].length) : text; +} + +test('no agents/*.md frontmatter contains an block (M4: examples live in the body)', () => { + for (const f of agentFiles) { + const fm = extractFrontmatter(read(`agents/${f}`)); + assert.ok(fm !== null, `agents/${f}: missing frontmatter`); + assert.equal( + (fm.match(//g) || []).length, + 0, + `agents/${f}: blocks must live in the body, not the always-loaded description: frontmatter (M4)`, + ); + } +}); + +test('agent bodies retain at least 34 blocks (M4: relocation moves, never deletes)', () => { + let total = 0; + for (const f of agentFiles) { + total += (bodyOf(read(`agents/${f}`)).match(//g) || []).length; + } + assert.ok( + total >= 34, + `expected >= 34 blocks across agent bodies (17 agents x 2), got ${total} ` + + `— examples may have been deleted instead of relocated (M4)`, + ); +}); From 816bf2a5fc45bcbca69ad53d7e39c43189da0b0c Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 10:14:35 +0200 Subject: [PATCH 15/44] refactor(agents): relocate example blocks to body (researchers + gemini-bridge) --- agents/community-researcher.md | 42 +++++++++++++++++---------------- agents/contrarian-researcher.md | 42 +++++++++++++++++---------------- agents/docs-researcher.md | 42 +++++++++++++++++---------------- agents/gemini-bridge.md | 40 ++++++++++++++++--------------- agents/security-researcher.md | 42 +++++++++++++++++---------------- 5 files changed, 109 insertions(+), 99 deletions(-) diff --git a/agents/community-researcher.md b/agents/community-researcher.md index 6317d86..89eb8b6 100644 --- a/agents/community-researcher.md +++ b/agents/community-researcher.md @@ -4,26 +4,6 @@ description: | Use this agent when the research task requires practical, real-world experience rather than official documentation — community sentiment, production war stories, known gotchas, and what developers actually encounter when using a technology. - - - Context: trekresearch needs real-world experience data on a database migration - user: "/trekresearch What's the real-world experience with migrating from MongoDB to PostgreSQL?" - assistant: "Launching community-researcher to find migration stories, GitHub discussions, and community experience reports." - - Official docs won't cover migration regrets or production war stories. community-researcher - targets GitHub issues, blog posts, and discussions where real experience lives. - - - - - Context: trekresearch is building a technology comparison - user: "/trekresearch Research community sentiment around adopting SvelteKit vs Next.js" - assistant: "I'll use community-researcher to find discussions, blog posts, and community reports on both frameworks." - - Framework comparisons live in community discourse, not official docs. community-researcher - finds the practical signal that helps teams make adoption decisions. - - model: opus color: green tools: ["WebSearch", "WebFetch", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research"] @@ -133,3 +113,25 @@ End with a summary table: Do not pick a side — report the split. - **Flag if a "problem" has since been fixed.** Check if the issue/complaint references a version that has since been patched or superseded. + +## When to use — examples + + +Context: trekresearch needs real-world experience data on a database migration +user: "/trekresearch What's the real-world experience with migrating from MongoDB to PostgreSQL?" +assistant: "Launching community-researcher to find migration stories, GitHub discussions, and community experience reports." + +Official docs won't cover migration regrets or production war stories. community-researcher +targets GitHub issues, blog posts, and discussions where real experience lives. + + + + +Context: trekresearch is building a technology comparison +user: "/trekresearch Research community sentiment around adopting SvelteKit vs Next.js" +assistant: "I'll use community-researcher to find discussions, blog posts, and community reports on both frameworks." + +Framework comparisons live in community discourse, not official docs. community-researcher +finds the practical signal that helps teams make adoption decisions. + + diff --git a/agents/contrarian-researcher.md b/agents/contrarian-researcher.md index 5411ec2..a1027b1 100644 --- a/agents/contrarian-researcher.md +++ b/agents/contrarian-researcher.md @@ -4,26 +4,6 @@ description: | Use this agent when the research task has an emerging conclusion that needs adversarial stress-testing — find counter-evidence, overlooked alternatives, and reasons the leading answer might be wrong. - - - Context: trekresearch has found evidence favoring a technology and needs the other side - user: "/trekresearch We're leaning toward adopting Kafka for our event streaming needs" - assistant: "Launching contrarian-researcher to find the strongest arguments against Kafka and what alternatives might serve better." - - The research equivalent of plan-critic. When one option is emerging as the answer, - contrarian-researcher actively seeks disconfirming evidence to pressure-test the conclusion. - - - - - Context: trekresearch is comparing options and needs the downsides of the leading candidate - user: "/trekresearch Compare Redis vs Memcached — initial research favors Redis" - assistant: "I'll use contrarian-researcher to find the strongest case against Redis and scenarios where Memcached wins." - - Contrarian-researcher finds the downsides of the leading option — not to be negative, - but to ensure the final recommendation is genuinely considered. - - model: opus effort: high color: red @@ -152,3 +132,25 @@ Followed by a **Verdict** section: apply to a read-heavy workload. Assess relevance before reporting. - **Check recency.** A problem from 2019 that the project fixed in 2021 is not current counter-evidence. Flag whether issues are current or historical. + +## When to use — examples + + +Context: trekresearch has found evidence favoring a technology and needs the other side +user: "/trekresearch We're leaning toward adopting Kafka for our event streaming needs" +assistant: "Launching contrarian-researcher to find the strongest arguments against Kafka and what alternatives might serve better." + +The research equivalent of plan-critic. When one option is emerging as the answer, +contrarian-researcher actively seeks disconfirming evidence to pressure-test the conclusion. + + + + +Context: trekresearch is comparing options and needs the downsides of the leading candidate +user: "/trekresearch Compare Redis vs Memcached — initial research favors Redis" +assistant: "I'll use contrarian-researcher to find the strongest case against Redis and scenarios where Memcached wins." + +Contrarian-researcher finds the downsides of the leading option — not to be negative, +but to ensure the final recommendation is genuinely considered. + + diff --git a/agents/docs-researcher.md b/agents/docs-researcher.md index 364a3e8..6a99790 100644 --- a/agents/docs-researcher.md +++ b/agents/docs-researcher.md @@ -3,26 +3,6 @@ name: docs-researcher description: | Use this agent when the research task requires authoritative information from official documentation, RFCs, vendor specifications, or Microsoft/Azure documentation. - - - Context: trekresearch needs to ground an OAuth2 implementation in official specs - user: "/trekresearch Research OAuth2 PKCE flow for our SPA" - assistant: "Launching docs-researcher to find the official RFC and vendor documentation for OAuth2 PKCE." - - docs-researcher targets authoritative sources — RFCs, specs, official vendor docs — - not community opinions. This is the right agent for protocol and standards questions. - - - - - Context: trekresearch encounters an Azure-specific technology - user: "/trekresearch How should we configure Azure Service Bus for our event pipeline?" - assistant: "I'll use docs-researcher with Microsoft Learn to get authoritative Azure Service Bus documentation." - - Microsoft/Azure technologies have dedicated MCP tools (microsoft_docs_search, - microsoft_docs_fetch) that docs-researcher uses for higher-quality results. - - model: opus color: blue tools: ["WebSearch", "WebFetch", "Read", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research", "mcp__microsoft-learn__microsoft_docs_search", "mcp__microsoft-learn__microsoft_docs_fetch"] @@ -119,3 +99,25 @@ End with a summary table: - **Flag conflicts between official sources.** When vendor docs and the spec disagree, report both. - **Stay focused.** Research only what the research question asks. Do not explore tangentially. - **Official sources only.** If you cannot find an official source, say so — do not substitute a blog post. + +## When to use — examples + + +Context: trekresearch needs to ground an OAuth2 implementation in official specs +user: "/trekresearch Research OAuth2 PKCE flow for our SPA" +assistant: "Launching docs-researcher to find the official RFC and vendor documentation for OAuth2 PKCE." + +docs-researcher targets authoritative sources — RFCs, specs, official vendor docs — +not community opinions. This is the right agent for protocol and standards questions. + + + + +Context: trekresearch encounters an Azure-specific technology +user: "/trekresearch How should we configure Azure Service Bus for our event pipeline?" +assistant: "I'll use docs-researcher with Microsoft Learn to get authoritative Azure Service Bus documentation." + +Microsoft/Azure technologies have dedicated MCP tools (microsoft_docs_search, +microsoft_docs_fetch) that docs-researcher uses for higher-quality results. + + diff --git a/agents/gemini-bridge.md b/agents/gemini-bridge.md index a15de55..1a3c964 100644 --- a/agents/gemini-bridge.md +++ b/agents/gemini-bridge.md @@ -5,25 +5,6 @@ description: | needed on a technology choice, architectural question, or complex research topic. Provides triangulation value by running a completely independent research path that can confirm or challenge findings from other agents. - - - Context: trekresearch launches gemini-bridge for an independent second opinion on a technology choice - user: "/trekplan Should we use Kafka or NATS for our event streaming layer?" - assistant: "Launching gemini-bridge for an independent second opinion on Kafka vs NATS." - - Technology choice with significant architectural implications triggers gemini-bridge - to provide an independent research path alongside local exploration agents. - - - - - Context: user wants deep research via Gemini on a complex architectural question - user: "Get me a Gemini deep research on event sourcing patterns for distributed systems" - assistant: "I'll use the gemini-bridge agent to run a deep research on event sourcing patterns." - - Direct request for Gemini research on a complex architectural question triggers the agent. - - model: opus color: magenta tools: ["mcp__gemini-mcp__gemini_deep_research", "mcp__gemini-mcp__gemini_get_research_status", "mcp__gemini-mcp__gemini_get_research_result", "mcp__gemini-mcp__gemini_research_followup"] @@ -147,3 +128,24 @@ and other external agents:* - **Graceful degradation at every step.** Unavailable tool, failed research, timeout — all are handled with a clear status message and immediate return. Never leave the pipeline hanging. + +## When to use — examples + + +Context: trekresearch launches gemini-bridge for an independent second opinion on a technology choice +user: "/trekplan Should we use Kafka or NATS for our event streaming layer?" +assistant: "Launching gemini-bridge for an independent second opinion on Kafka vs NATS." + +Technology choice with significant architectural implications triggers gemini-bridge +to provide an independent research path alongside local exploration agents. + + + + +Context: user wants deep research via Gemini on a complex architectural question +user: "Get me a Gemini deep research on event sourcing patterns for distributed systems" +assistant: "I'll use the gemini-bridge agent to run a deep research on event sourcing patterns." + +Direct request for Gemini research on a complex architectural question triggers the agent. + + diff --git a/agents/security-researcher.md b/agents/security-researcher.md index d960c0d..a4516ca 100644 --- a/agents/security-researcher.md +++ b/agents/security-researcher.md @@ -3,26 +3,6 @@ name: security-researcher description: | Use this agent when the research task requires security investigation of a technology, dependency, or library — CVEs, audit history, supply chain risks, and OWASP relevance. - - - Context: trekresearch is evaluating whether a dependency is safe to adopt - user: "/trekresearch Research whether we should trust the `node-fetch` library" - assistant: "Launching security-researcher to check CVE history, supply chain risk, and audit reports for node-fetch." - - Before adopting a dependency, security-researcher checks the attack surface: known - vulnerabilities, maintainer health, and whether past issues were handled responsibly. - - - - - Context: trekresearch is assessing the security posture of a technology choice - user: "/trekresearch Evaluate the security implications of using JWT for session management" - assistant: "I'll use security-researcher to check known JWT vulnerabilities, OWASP guidance, and community security reports." - - Technology choices have security tradeoffs. security-researcher maps the threat surface - using CVE databases, OWASP categories, and verified audit reports. - - model: opus color: red tools: ["WebSearch", "WebFetch", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research"] @@ -140,3 +120,25 @@ End with an overall security summary table: risks from incomplete information. - **Severity matters.** A CVSS 9.8 is not equivalent to a CVSS 3.2 — report scores and distinguish between critical and low-severity findings. + +## When to use — examples + + +Context: trekresearch is evaluating whether a dependency is safe to adopt +user: "/trekresearch Research whether we should trust the `node-fetch` library" +assistant: "Launching security-researcher to check CVE history, supply chain risk, and audit reports for node-fetch." + +Before adopting a dependency, security-researcher checks the attack surface: known +vulnerabilities, maintainer health, and whether past issues were handled responsibly. + + + + +Context: trekresearch is assessing the security posture of a technology choice +user: "/trekresearch Evaluate the security implications of using JWT for session management" +assistant: "I'll use security-researcher to check known JWT vulnerabilities, OWASP guidance, and community security reports." + +Technology choices have security tradeoffs. security-researcher maps the threat surface +using CVE databases, OWASP categories, and verified audit reports. + + From 27a7573b0684f7335a831636ab9fd01a74b4ed68 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 10:16:48 +0200 Subject: [PATCH 16/44] refactor(agents): relocate example blocks to body (retrieval agents) --- agents/architecture-mapper.md | 38 ++++++++++++++++++----------------- agents/convention-scanner.md | 38 ++++++++++++++++++----------------- agents/dependency-tracer.md | 38 ++++++++++++++++++----------------- agents/git-historian.md | 38 ++++++++++++++++++----------------- agents/task-finder.md | 38 ++++++++++++++++++----------------- agents/test-strategist.md | 38 ++++++++++++++++++----------------- 6 files changed, 120 insertions(+), 108 deletions(-) diff --git a/agents/architecture-mapper.md b/agents/architecture-mapper.md index 448f7e1..9a5e6ad 100644 --- a/agents/architecture-mapper.md +++ b/agents/architecture-mapper.md @@ -3,24 +3,6 @@ name: architecture-mapper description: | Use this agent when you need deep architecture analysis of a codebase — structure, tech stack, patterns, anti-patterns, and key abstractions. - - - Context: Voyage exploration phase needs architecture overview - user: "/trekplan Add authentication to the API" - assistant: "Launching architecture-mapper to analyze codebase structure and patterns." - - Phase 5 of trekplan triggers this agent for every codebase size. - - - - - Context: User wants to understand an unfamiliar codebase - user: "Map out the architecture of this project" - assistant: "I'll use the architecture-mapper agent to analyze the codebase structure." - - Direct architecture analysis request triggers the agent. - - model: opus effort: medium color: cyan @@ -104,3 +86,23 @@ Structure your report with clear sections matching the 7 areas above. Include: - A brief "Architecture Summary" paragraph at the top (3-4 sentences) Do NOT include raw file listings — synthesize and organize the information. + +## When to use — examples + + +Context: Voyage exploration phase needs architecture overview +user: "/trekplan Add authentication to the API" +assistant: "Launching architecture-mapper to analyze codebase structure and patterns." + +Phase 5 of trekplan triggers this agent for every codebase size. + + + + +Context: User wants to understand an unfamiliar codebase +user: "Map out the architecture of this project" +assistant: "I'll use the architecture-mapper agent to analyze the codebase structure." + +Direct architecture analysis request triggers the agent. + + diff --git a/agents/convention-scanner.md b/agents/convention-scanner.md index 686715f..3f6f868 100644 --- a/agents/convention-scanner.md +++ b/agents/convention-scanner.md @@ -5,24 +5,6 @@ description: | Produces a structured conventions report covering naming, directory layout, import style, error handling, test patterns, git commit style, and documentation patterns. Uses concrete examples from the codebase. - - - Context: Voyage exploration phase for a medium+ codebase - user: "/trekplan Add authentication to the API" - assistant: "Launching convention-scanner to discover coding patterns." - - Phase 5 of trekplan triggers this agent for medium+ codebases (50+ files). - - - - - Context: User wants to understand a project's conventions before contributing - user: "What are the coding conventions in this project?" - assistant: "I'll use the convention-scanner agent to analyze the codebase." - - Direct convention discovery request triggers the agent. - - model: opus color: yellow tools: ["Read", "Glob", "Grep", "Bash"] @@ -159,3 +141,23 @@ Based on existing conventions, new code should: than scanning everything. - **Stay focused.** This is about conventions — not architecture, dependencies, or risks. Those are handled by other agents. + +## When to use — examples + + +Context: Voyage exploration phase for a medium+ codebase +user: "/trekplan Add authentication to the API" +assistant: "Launching convention-scanner to discover coding patterns." + +Phase 5 of trekplan triggers this agent for medium+ codebases (50+ files). + + + + +Context: User wants to understand a project's conventions before contributing +user: "What are the coding conventions in this project?" +assistant: "I'll use the convention-scanner agent to analyze the codebase." + +Direct convention discovery request triggers the agent. + + diff --git a/agents/dependency-tracer.md b/agents/dependency-tracer.md index ed0f90e..25c9d57 100644 --- a/agents/dependency-tracer.md +++ b/agents/dependency-tracer.md @@ -3,24 +3,6 @@ name: dependency-tracer description: | Use this agent when you need to trace import chains, map data flow, or understand how modules connect and what side effects they produce. - - - Context: Voyage needs to understand module relationships for a task - user: "/trekplan Refactor the payment processing pipeline" - assistant: "Launching dependency-tracer to map module connections and data flow." - - Phase 5 of trekplan triggers this agent to trace dependencies relevant to the task. - - - - - Context: User needs to understand impact of changing a module - user: "What would break if I change the User model?" - assistant: "I'll use the dependency-tracer agent to trace all dependents of the User model." - - Impact analysis request triggers the agent. - - model: opus effort: medium color: blue @@ -93,3 +75,23 @@ Structure as: 6. **Risk Flags** — circular deps, tight coupling, hidden side effects Include file paths and line numbers for every finding. + +## When to use — examples + + +Context: Voyage needs to understand module relationships for a task +user: "/trekplan Refactor the payment processing pipeline" +assistant: "Launching dependency-tracer to map module connections and data flow." + +Phase 5 of trekplan triggers this agent to trace dependencies relevant to the task. + + + + +Context: User needs to understand impact of changing a module +user: "What would break if I change the User model?" +assistant: "I'll use the dependency-tracer agent to trace all dependents of the User model." + +Impact analysis request triggers the agent. + + diff --git a/agents/git-historian.md b/agents/git-historian.md index a28230e..2c90a51 100644 --- a/agents/git-historian.md +++ b/agents/git-historian.md @@ -3,24 +3,6 @@ name: git-historian description: | Use this agent to analyze git history for planning context — recent changes, code ownership, hot files, and active branches relevant to the task. - - - Context: Voyage exploration phase needs git context - user: "/trekplan Refactor the database layer" - assistant: "Launching git-historian to check recent changes and ownership of DB code." - - Phase 2 of trekplan triggers this agent for every codebase size. - - - - - Context: User wants to understand change history before modifying code - user: "Who has been changing the auth module recently?" - assistant: "I'll use the git-historian agent to analyze ownership and change patterns." - - Git history analysis request triggers the agent. - - model: opus effort: medium color: yellow @@ -122,3 +104,23 @@ Run `git status --short` to check for: are risks the planner needs to know about. - **Use relative time.** "2 days ago" is more useful than a raw timestamp. - **Never expose email addresses.** Use author names only. + +## When to use — examples + + +Context: Voyage exploration phase needs git context +user: "/trekplan Refactor the database layer" +assistant: "Launching git-historian to check recent changes and ownership of DB code." + +Phase 2 of trekplan triggers this agent for every codebase size. + + + + +Context: User wants to understand change history before modifying code +user: "Who has been changing the auth module recently?" +assistant: "I'll use the git-historian agent to analyze ownership and change patterns." + +Git history analysis request triggers the agent. + + diff --git a/agents/task-finder.md b/agents/task-finder.md index 81ef3a3..fee90ea 100644 --- a/agents/task-finder.md +++ b/agents/task-finder.md @@ -4,24 +4,6 @@ description: | Use this agent to find all files, functions, types, and interfaces directly related to the planning task. Replaces generic Explore agents with targeted, structured code discovery. - - - Context: Voyage exploration phase needs task-relevant code - user: "/trekplan Add authentication to the API" - assistant: "Launching task-finder to locate auth-related code, endpoints, and models." - - Phase 2 of trekplan triggers this agent for every codebase size. - - - - - Context: User wants to find code related to a specific feature - user: "Find all code related to payment processing" - assistant: "I'll use the task-finder agent to locate payment-related code." - - Direct code discovery request triggers the agent. - - model: opus effort: medium color: green @@ -146,3 +128,23 @@ Structure your report using three tiers: - **Stay focused on the task.** Do not inventory the entire codebase — only what is relevant to implementing the specific task. - **Never read file contents that look like secrets or credentials.** + +## When to use — examples + + +Context: Voyage exploration phase needs task-relevant code +user: "/trekplan Add authentication to the API" +assistant: "Launching task-finder to locate auth-related code, endpoints, and models." + +Phase 2 of trekplan triggers this agent for every codebase size. + + + + +Context: User wants to find code related to a specific feature +user: "Find all code related to payment processing" +assistant: "I'll use the task-finder agent to locate payment-related code." + +Direct code discovery request triggers the agent. + + diff --git a/agents/test-strategist.md b/agents/test-strategist.md index b95e0cb..756b7f1 100644 --- a/agents/test-strategist.md +++ b/agents/test-strategist.md @@ -3,24 +3,6 @@ name: test-strategist description: | Use this agent when you need to design a test strategy for an implementation task — discovers existing patterns, maps coverage gaps, and recommends what tests to write. - - - Context: Voyage exploration phase for medium+ codebase - user: "/trekplan Add rate limiting to the API" - assistant: "Launching test-strategist to analyze existing test patterns and design test coverage." - - Phase 5 of trekplan triggers this agent for medium and large codebases. - - - - - Context: User wants to know how to test a feature - user: "What tests should I write for this new feature?" - assistant: "I'll use the test-strategist agent to analyze existing patterns and recommend tests." - - Test planning request triggers the agent. - - model: opus color: green tools: ["Read", "Glob", "Grep", "Bash"] @@ -95,3 +77,23 @@ For each test, provide: 5. **Test Dependencies** — fixtures, mocks, or setup code to create first Do NOT write test code. Describe what each test should verify and which patterns to follow. + +## When to use — examples + + +Context: Voyage exploration phase for medium+ codebase +user: "/trekplan Add rate limiting to the API" +assistant: "Launching test-strategist to analyze existing test patterns and design test coverage." + +Phase 5 of trekplan triggers this agent for medium and large codebases. + + + + +Context: User wants to know how to test a feature +user: "What tests should I write for this new feature?" +assistant: "I'll use the test-strategist agent to analyze existing patterns and recommend tests." + +Test planning request triggers the agent. + + From b148b6f8b8054811460039098a3b37dbcfc0baf8 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 10:19:24 +0200 Subject: [PATCH 17/44] refactor(agents): relocate example blocks to body (reviewer/planning agents) --- agents/brief-reviewer.md | 38 +++++++++++++++++++----------------- agents/plan-critic.md | 38 +++++++++++++++++++----------------- agents/research-scout.md | 38 +++++++++++++++++++----------------- agents/risk-assessor.md | 38 +++++++++++++++++++----------------- agents/scope-guardian.md | 38 +++++++++++++++++++----------------- agents/session-decomposer.md | 38 +++++++++++++++++++----------------- 6 files changed, 120 insertions(+), 108 deletions(-) diff --git a/agents/brief-reviewer.md b/agents/brief-reviewer.md index 1eead8d..e129fda 100644 --- a/agents/brief-reviewer.md +++ b/agents/brief-reviewer.md @@ -5,24 +5,6 @@ description: | checks completeness, consistency, testability, scope clarity, and research-plan validity. Catches problems early to avoid wasting tokens on exploration with a flawed brief. - - - Context: Voyage runs brief review before exploration - user: "/trekplan --project .claude/projects/2026-04-18-notifications" - assistant: "Reviewing brief quality before launching exploration agents." - - Orchestrator Phase 1b triggers this agent after the brief is available. - - - - - Context: User wants to validate a brief before planning - user: "Review this brief for completeness" - assistant: "I'll use the brief-reviewer agent to check brief quality." - - Brief review request triggers the agent. - - model: opus color: magenta tools: ["Read", "Glob", "Grep"] @@ -302,3 +284,23 @@ information that would strengthen the brief. List only if actionable.} files or technologies exist, but deep code analysis is not your job. - **Research-plan checks are load-bearing.** A brief with `research_status: pending` and missing research files is a scope hazard — flag it as a major risk. + +## When to use — examples + + +Context: Voyage runs brief review before exploration +user: "/trekplan --project .claude/projects/2026-04-18-notifications" +assistant: "Reviewing brief quality before launching exploration agents." + +Orchestrator Phase 1b triggers this agent after the brief is available. + + + + +Context: User wants to validate a brief before planning +user: "Review this brief for completeness" +assistant: "I'll use the brief-reviewer agent to check brief quality." + +Brief review request triggers the agent. + + diff --git a/agents/plan-critic.md b/agents/plan-critic.md index 5c08407..b2aa6ed 100644 --- a/agents/plan-critic.md +++ b/agents/plan-critic.md @@ -3,24 +3,6 @@ name: plan-critic description: | Use this agent when an implementation plan needs adversarial review — it finds problems, never praises. - - - Context: Voyage adversarial review phase - user: "/trekplan Implement WebSocket real-time updates" - assistant: "Launching plan-critic to stress-test the implementation plan." - - Phase 9 of trekplan triggers this agent to review the generated plan. - - - - - Context: User wants a plan reviewed before execution - user: "Review this plan and find problems" - assistant: "I'll use the plan-critic agent to perform adversarial review." - - Plan review request triggers the agent. - - model: opus effort: high color: red @@ -297,3 +279,23 @@ short, stable id for the finding class (used for exact-match dedup). Emit Be specific. Reference exact plan sections, step numbers, and file paths. Never use "generally" or "usually" — cite the specific problem in this specific plan. + +## When to use — examples + + +Context: Voyage adversarial review phase +user: "/trekplan Implement WebSocket real-time updates" +assistant: "Launching plan-critic to stress-test the implementation plan." + +Phase 9 of trekplan triggers this agent to review the generated plan. + + + + +Context: User wants a plan reviewed before execution +user: "Review this plan and find problems" +assistant: "I'll use the plan-critic agent to perform adversarial review." + +Plan review request triggers the agent. + + diff --git a/agents/research-scout.md b/agents/research-scout.md index 09efc9a..9f3e8e8 100644 --- a/agents/research-scout.md +++ b/agents/research-scout.md @@ -3,24 +3,6 @@ name: research-scout description: | Use this agent when the implementation task involves unfamiliar technologies, external APIs, or libraries where official documentation and known issues should be checked. - - - Context: Voyage detects external technology in the task - user: "/trekplan Integrate Stripe payment processing" - assistant: "Launching research-scout to find Stripe documentation and best practices." - - Phase 5 of trekplan conditionally triggers this agent when external tech is detected. - - - - - Context: User needs research before implementation - user: "Research the best approach for WebSocket scaling" - assistant: "I'll use the research-scout agent to find documentation and best practices." - - Research request for external technology triggers the agent. - - model: opus color: blue tools: ["WebSearch", "WebFetch", "Read"] @@ -118,3 +100,23 @@ End with a summary table: - **Date everything.** Documentation ages — the reader needs to judge freshness. - **Flag conflicts.** If official docs and community advice disagree, report both. - **Stay focused.** Research only what the task needs. Do not explore tangentially. + +## When to use — examples + + +Context: Voyage detects external technology in the task +user: "/trekplan Integrate Stripe payment processing" +assistant: "Launching research-scout to find Stripe documentation and best practices." + +Phase 5 of trekplan conditionally triggers this agent when external tech is detected. + + + + +Context: User needs research before implementation +user: "Research the best approach for WebSocket scaling" +assistant: "I'll use the research-scout agent to find documentation and best practices." + +Research request for external technology triggers the agent. + + diff --git a/agents/risk-assessor.md b/agents/risk-assessor.md index 16a2531..7fd14a2 100644 --- a/agents/risk-assessor.md +++ b/agents/risk-assessor.md @@ -3,24 +3,6 @@ name: risk-assessor description: | Use this agent when you need to identify risks, edge cases, failure modes, and technical debt that could affect an implementation task. - - - Context: Voyage exploration phase identifies potential risks - user: "/trekplan Migrate database from PostgreSQL to MongoDB" - assistant: "Launching risk-assessor to identify failure modes and edge cases for this migration." - - Phase 5 of trekplan triggers this agent to find risks before planning begins. - - - - - Context: User wants to understand risks before a change - user: "What could go wrong with this refactor?" - assistant: "I'll use the risk-assessor agent to map risks and failure modes." - - Risk analysis request triggers the agent. - - model: opus effort: high color: yellow @@ -106,3 +88,23 @@ Produce a prioritized risk list: **Low** = minor concerns worth noting Follow with a narrative section expanding on each Critical and High risk. + +## When to use — examples + + +Context: Voyage exploration phase identifies potential risks +user: "/trekplan Migrate database from PostgreSQL to MongoDB" +assistant: "Launching risk-assessor to identify failure modes and edge cases for this migration." + +Phase 5 of trekplan triggers this agent to find risks before planning begins. + + + + +Context: User wants to understand risks before a change +user: "What could go wrong with this refactor?" +assistant: "I'll use the risk-assessor agent to map risks and failure modes." + +Risk analysis request triggers the agent. + + diff --git a/agents/scope-guardian.md b/agents/scope-guardian.md index bd3f425..c36fd3a 100644 --- a/agents/scope-guardian.md +++ b/agents/scope-guardian.md @@ -3,24 +3,6 @@ name: scope-guardian description: | Use this agent when you need to verify that an implementation plan matches its requirements — catches scope creep and scope gaps. - - - Context: Voyage adversarial review phase checks scope alignment - user: "/trekplan Add caching to the API layer" - assistant: "Launching scope-guardian to verify plan matches requirements." - - Phase 9 of trekplan triggers this agent alongside plan-critic. - - - - - Context: User wants to verify plan doesn't do too much or too little - user: "Does this plan match what I asked for?" - assistant: "I'll use the scope-guardian agent to check scope alignment." - - Scope verification request triggers the agent. - - model: opus color: magenta tools: ["Read", "Glob", "Grep"] @@ -144,3 +126,23 @@ One object per scope-creep / gap / dependency finding. `file`/`line` point at th plan step (or the brief requirement) the finding concerns; `rule_key` is a short, stable id for the finding class (used for exact-match dedup against plan-critic). Emit `"findings": []` when the plan is fully aligned. + +## When to use — examples + + +Context: Voyage adversarial review phase checks scope alignment +user: "/trekplan Add caching to the API layer" +assistant: "Launching scope-guardian to verify plan matches requirements." + +Phase 9 of trekplan triggers this agent alongside plan-critic. + + + + +Context: User wants to verify plan doesn't do too much or too little +user: "Does this plan match what I asked for?" +assistant: "I'll use the scope-guardian agent to check scope alignment." + +Scope verification request triggers the agent. + + diff --git a/agents/session-decomposer.md b/agents/session-decomposer.md index fb82457..96526ab 100644 --- a/agents/session-decomposer.md +++ b/agents/session-decomposer.md @@ -4,24 +4,6 @@ description: | Use this agent to decompose an trekplan into self-contained headless sessions. Reads a plan file, analyzes step dependencies, groups steps into sessions, identifies parallelism, and generates session specs + dependency graph + launch script. - - - Context: User wants to run a plan across multiple headless sessions - user: "/trekplan --decompose .claude/plans/trekplan-2026-04-06-auth-refactor.md" - assistant: "Launching session-decomposer to split the plan into headless sessions." - - The --decompose flag triggers this agent to analyze and split the plan. - - - - - Context: User has a large plan and wants parallel execution - user: "Split this plan into sessions I can run in parallel" - assistant: "I'll use the session-decomposer to identify parallel session groups." - - Plan decomposition request for parallel headless execution. - - model: opus color: green tools: ["Read", "Glob", "Grep", "Write"] @@ -310,3 +292,23 @@ After all sessions complete, run: wrong sequentiality only costs time. - **Verify file existence.** Use Glob to confirm that files referenced in the plan actually exist before assigning them to sessions. + +## When to use — examples + + +Context: User wants to run a plan across multiple headless sessions +user: "/trekplan --decompose .claude/plans/trekplan-2026-04-06-auth-refactor.md" +assistant: "Launching session-decomposer to split the plan into headless sessions." + +The --decompose flag triggers this agent to analyze and split the plan. + + + + +Context: User has a large plan and wants parallel execution +user: "Split this plan into sessions I can run in parallel" +assistant: "I'll use the session-decomposer to identify parallel session groups." + +Plan decomposition request for parallel headless execution. + + From 2b8ca9a044a2f72cfe5f1d80abb000ddf50c902b Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 10:23:29 +0200 Subject: [PATCH 18/44] chore(voyage): release v5.7.1 - relocate agent example blocks (always-loaded token trim) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 12 ++++++++++++ README.md | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index eb6c095..095328f 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "voyage", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", - "version": "5.7.0", + "version": "5.7.1", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ae729e..6aac024 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## v5.7.1 — 2026-06-29 — relocate agent `` blocks to body (always-loaded token trim) + +Performance/packaging change — no behavior change, no breaking change. (M4) + +### Always-loaded token trim + +- The 17 example-bearing agents each carried two `` blocks (34 total) inside their `description:` frontmatter. voyage launches its agents **by name** from the orchestrator commands, so those auto-selection examples were paying a per-turn token tax for a path the pipeline never uses. They are now relocated **verbatim** into each agent's body under a `## When to use — examples` section — preserved, not deleted. +- Frontmatter `description` chars across all 24 agents: **17,672 → 4,945** — roughly **3,180 fewer always-loaded tokens per turn** for every session with voyage enabled. The 7 zero-example agents are untouched; no system prompt, tools, model, or `name` changed. +- New invariant test in `tests/lib/agent-frontmatter.test.mjs`: no `` in any frontmatter `description`, and ≥ 34 `` retained across agent bodies (relocation moves, never deletes). Suite: **807 pass / 0 fail / 2 skip**. +- **The saving only reaches a machine after `/plugin marketplace update` + reload** — the delta is not instant on the machine that ships the release. +- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.7.1`, guarded by `doc-consistency.test.mjs`. + ## v5.7.0 — 2026-06-26 — opt-in token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a) Additive — no breaking change. Two unreleased work-streams land together. diff --git a/README.md b/README.md index a4a823d..d920b6d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # trekplan — Brief, Research, Plan, Execute, Review, Continue -![Version](https://img.shields.io/badge/version-5.7.0-blue) +![Version](https://img.shields.io/badge/version-5.7.1-blue) ![License](https://img.shields.io/badge/license-MIT-green) ![Platform](https://img.shields.io/badge/platform-Claude%20Code-purple) @@ -10,7 +10,7 @@ A [Claude Code](https://docs.anthropic.com/en/docs/claude-code) plugin for deep implementation planning, multi-source research, autonomous execution, independent post-hoc review, and zero-friction multi-session resumption. Six commands, one pipeline: -> **What's new — v5.6.1: leaner always-loaded agent listing.** The four reference/dormant agents (the three `*-orchestrator` reference docs + the dormant `synthesis-agent`) now carry one-line `description:` frontmatter — their full rationale already lives in each file's body — trimming ~700 tokens from the agent listing injected into every session, with no behavior change. **v5.6.0: `/trekexecute` loop hardening.** Execution-loop termination is now machine-verifiable (completion gate, Hard Rule 18) and recovery is bounded by an explicit `TREKEXECUTE_MAX_RECOVERY_ITERATIONS` budget (default 25, Hard Rule 20). The prior coordinated release **v5.5.0** added brief **framing** enforcement (`brief_version 2.2`) + a `/trekreview` reviewer-schema contract. Additive — no breaking changes. **Full version history → [CHANGELOG.md](CHANGELOG.md).** +> **What's new — v5.7.1: leaner always-loaded agent listing (round 2).** The 17 example-bearing agents now keep their `` blocks in the body (under `## When to use — examples`) instead of the `description:` frontmatter — voyage launches its agents by name, so the auto-selection examples were cost without function there. This trims ~3,180 always-loaded tokens (frontmatter `description` chars 17,672 → 4,945) from the agent listing injected into every session, with no behavior change. The saving lands after `/plugin marketplace update` + reload. **v5.7.0:** opt-in per-session token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a). **v5.6.1:** one-line `description:` for the four reference/dormant agents (~700 tok). **v5.5.0:** brief **framing** enforcement (`brief_version 2.2`) + a `/trekreview` reviewer-schema contract. Additive — no breaking changes. **Full version history → [CHANGELOG.md](CHANGELOG.md).** | Command | What it does | |---------|-------------| diff --git a/package-lock.json b/package-lock.json index ac531f7..9efc757 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voyage", - "version": "5.7.0", + "version": "5.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voyage", - "version": "5.7.0", + "version": "5.7.1", "license": "MIT", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 2f3a494..6c4c740 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voyage", - "version": "5.7.0", + "version": "5.7.1", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any heading/paragraph/list-item, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", "type": "module", "engines": { From 15d172521dd96820e348ce8909c18c77c6da1f04 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 14:49:32 +0200 Subject: [PATCH 19/44] docs(claude-md): trim CLAUDE.md to invariants (always-loaded token trim, S53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relocate the verbose rationale/history from six always-loaded block-quote notes (L5/L7/L9/L11/L56/L58) to the docs that already own it, leaving a terse invariant + pointer in each. No invariant fact deleted — moved. L5 synthesis-PoC Δ≈0 caveat → kept one clause; detail in T1-synthesis-poc-results.md L7 v3.0.0 architect note → CHANGELOG.md [3.0.0] L9 Trinity Tier 2/3 detail → HANDOVER-CONTRACTS.md §Handover 1 (added the producer-context para) L11 brief-framing 3-layer gate → HANDOVER-CONTRACTS.md §Handover 1 (already owned) L56 sonnet-downgrade rationale → voyage-vs-cc-balance-analysis.md §10 (D3) L58 per-agent effort table → profiles.md §Model & effort axes Measured (config-audit manifest scanner): voyage project CLAUDE.md 2261 → 1759 always-loaded tok (-502, -22%). Tables (Commands/Agents) left byte-exact; doc-consistency pins (parallel wall-clock, 21 spawnable, 3 orchestrator reference docs, synthesis-agent dormant) preserved. Docs-only: no version bump, no catalog ref. Suite 809 (807/0/2) green; doc-consistency 87/0. Also tracks the task brief (provenance). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RZHLfnJXGx55G4euxPuxim --- CLAUDE.md | 12 +++++----- docs/HANDOVER-CONTRACTS.md | 2 ++ docs/claudemd-token-trim-brief.md | 40 +++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 docs/claudemd-token-trim-brief.md diff --git a/CLAUDE.md b/CLAUDE.md index 9c1c22e..9ebdfd1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,13 +2,13 @@ Voyage — a contract-driven Claude Code pipeline: brief, research, plan, execute, review, continue. Deep implementation planning and research with specialized agent swarms, external research, adversarial review, session decomposition, disciplined execution, and headless support. -**Design principle: Context Engineering** — build the right context by orchestrating specialized agents. Each step in the pipeline (brief → research → plan → execute) produces a structured artifact that the next step consumes. (The claim that fanning out to a swarm *relieves* the main context is asserted-by-design, not measured end-to-end: the one delegation actually measured — the Phase-7 synthesis PoC — found Δ main-context ≈ 0, see `docs/T1-synthesis-poc-results.md`. The parallel wall-clock + structured artifact handoffs are the load-bearing benefit; main-context relief is not yet demonstrated.) +**Design principle: Context Engineering** — build the right context by orchestrating specialized agents. Each step in the pipeline (brief → research → plan → execute) produces a structured artifact that the next step consumes. The load-bearing benefit is the parallel wall-clock + structured artifact handoffs; main-context relief is asserted-by-design, not measured (Δ ≈ 0 in the one PoC — see `docs/T1-synthesis-poc-results.md`). -> **v3.0.0 — architect step extracted from this plugin.** The plan command still auto-discovers `architecture/overview.md` if present, so any compatible producer (architect plugin no longer publicly distributed; the architecture/overview.md slot remains available for any compatible producer) plugs into the same slot. See [CHANGELOG.md](CHANGELOG.md) for migration history. +> **Architecture slot.** The plan command auto-discovers `architecture/overview.md` if present — any compatible producer plugs in (the architect plugin is no longer publicly distributed). Migration history (v3.0.0 extraction) → [CHANGELOG.md](CHANGELOG.md). -> **Trinity context (2026-05-13, informational).** Voyage is Tier 1 (per-task) of a three-tier architecture in active design under the author's private marketplace: Tier 2 `app-creator` (per-app — "what does the app need, what's the next brief?") produces briefs Voyage consumes; Tier 3 `app-factory` (per-portfolio — "which app needs me now?") aggregates state across multiple app-creator instances. Both are pre-implementation and will ship to Forgejo when ready. **Asymmetry is a hard invariant:** Voyage stays unaware of Tier 2/3. Handover 1 (brief format) is the only integration point — any compatible producer can feed Voyage, app-creator is not privileged. Brief-schema changes are therefore breaking changes for downstream consumers, formalized as a public contract in v5.5.0 — see `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT). +> **Trinity context (informational).** Voyage is Tier 1 (per-task) of a three-tier architecture. **Asymmetry is a hard invariant:** Voyage stays unaware of Tier 2/3; Handover 1 (brief format) is the only integration point, no producer is privileged, and brief-schema changes are breaking for downstream consumers (formalized as a public contract in v5.5.0). Tier 2/3 producer detail + the public contract → `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT). -> **Cross-cutting invariant: brief framing must match operator intent (2026-05-15).** Etablert etter residiv. Briefen er pipelinens source of truth; operatørens intent lever i hodet + i memory-filer (`feedback_*`, `project_*`); pipelinen tvinger ikke alignment. Høyere reasoning-kraft polerer feil premiss istedenfor å utfordre det. **Tre lag av forsvar (input-siden), implementert i S6 som `brief_version 2.2`-gate (v5.5), alle BLOCKER ved brudd for briefer som deklarerer ≥ 2.2:** (1) eksplisitt `framing: preserve|refine|replace|new-direction` i brief-frontmatter, `AskUserQuestion`-validert i `/trekbrief` Phase 2.5 før brief-prosa skrives (ikke-skippbar, også i `--quick`); enum-feil → `BRIEF_INVALID_FRAMING` (alle versjoner), fravær ved ≥ 2.2 → `BRIEF_MISSING_FRAMING`; (2) memory-alignment som dimensjon 6 i `brief-reviewer` — sammenlikner brief-prosa + `framing` mot relevante memory-filer, rapporterer kun eksplisitte motsigelser (degraderer til score 5 N/A uten memory-kontekst), wired til Phase 4e-gate (`memory_alignment.score ≥ 4`); (3) obligatorisk `## TL;DR`-seksjon (≤ 5 linjer, soft-cap → `BRIEF_TLDR_TOO_LONG`) øverst i `brief.md`. Eksisterende 2.0/2.1-briefer forblir gyldige (forward+backward-compat, speiler `phase_signals ≥ 2.1`-presedensen); `trekreview`-briefer er unntatt. **Schema-akse bumpet (2.1→2.2); plugin-versjon-badge + CHANGELOG bumpes ved den koordinerte releasen (S10).** Kontrakt-evolusjon dokumentert i `docs/HANDOVER-CONTRACTS.md` §Handover 1. Den gamle manuelle stopgap-sjekken er dermed retired for ≥ 2.2-briefer. +> **Cross-cutting invariant: brief framing must match operator intent.** The brief is the pipeline's source of truth; operator intent lives in memory files — the pipeline must not polish a wrong premise. Enforced as the `brief_version 2.2` gate (v5.5), all BLOCKER for briefs declaring ≥ 2.2: (1) explicit `framing: preserve|refine|replace|new-direction` frontmatter, `AskUserQuestion`-validated in `/trekbrief` Phase 2.5; (2) memory-alignment as `brief-reviewer` dimension 6; (3) mandatory `## TL;DR`. Existing 2.0/2.1 briefs stay valid; `trekreview` briefs are exempt. Full implementation + contract evolution → `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT). ## Commands @@ -53,9 +53,9 @@ Full flag reference for each command (modes, `--gates`, `--profile`, breaking ch | contrarian-researcher | opus | Counter-evidence, overlooked alternatives | | gemini-bridge | opus | Gemini Deep Research second opinion (conditional) | -> **Inventory (S33 reconcile).** 24 agent files = **21 spawnable** (one of which, `synthesis-agent`, ships **dormant** — Δ≈0, wired to nothing, see `docs/T1-synthesis-poc-results.md`) **+ 3 orchestrator reference docs** (`planning-/research-/review-orchestrator` document the inline `/trek*` workflow; they are reference documentation, not spawnable capabilities). All 24 stay `model: opus` (operator pin `40d8742`); the glue/mechanical/retrieval/dormant roles (V09 gemini-bridge, V16 session-decomposer, V11 retrieval agents, V08 researchers, V35 synthesis) were reconsidered for a sonnet downgrade and **kept opus** — document-only, no frontmatter change (the decision record lives in `docs/voyage-vs-cc-balance-analysis.md` §10). +> **Inventory (S33 reconcile).** 24 agent files = **21 spawnable** (one, `synthesis-agent`, ships **dormant** — Δ≈0, wired to nothing) **+ 3 orchestrator reference docs** (`planning-/research-/review-orchestrator` document the inline `/trek*` workflow, not spawnable capabilities). All 24 stay `model: opus` (operator pin `40d8742`); the glue/mechanical/retrieval/dormant roles were reconsidered for a sonnet downgrade and **kept opus** — decision record: `docs/voyage-vs-cc-balance-analysis.md` §10. -> **Model & effort (CC 2.1.154+).** `opus` resolves to **Opus 4.8** (default reasoning effort `high`); `sonnet` to Sonnet 4.6. Select agents carry native `effort:` — retrieval agents (`task-finder`, `git-historian`, `dependency-tracer`, `architecture-mapper`) at `medium`, adversarial-reasoning agents (`plan-critic`, `risk-assessor`, `contrarian-researcher`, `review-coordinator`) at `high`. This native per-spawn **reasoning** effort is a different axis from brief `phase_signals.effort` (orchestration shape — which agents/passes run). See `docs/profiles.md` §Model & effort axes. +> **Model & effort.** `opus` = Opus 4.8 (default reasoning effort `high`); `sonnet` = Sonnet 4.6. Select agents carry native per-spawn `effort:` (retrieval → `medium`, adversarial-reasoning → `high`) — a different axis from brief `phase_signals.effort` (orchestration shape: which agents/passes run). Per-agent table + axes → `docs/profiles.md` §Model & effort axes. ## Reference docs (read on demand) diff --git a/docs/HANDOVER-CONTRACTS.md b/docs/HANDOVER-CONTRACTS.md index 89d01c9..92cde1b 100644 --- a/docs/HANDOVER-CONTRACTS.md +++ b/docs/HANDOVER-CONTRACTS.md @@ -45,6 +45,8 @@ Every validator exposes a CLI: `node lib/validators/.mjs --json ` re **Stability tier: PUBLIC CONTRACT.** Handover 1 is the *only* public integration boundary of the pipeline: `brief.md` is what an upstream producer hands to Voyage, and Voyage consumes it without any knowledge of who produced it. This asymmetry is a hard invariant (see `CLAUDE.md` §Trinity context) — the interactive `/trekbrief` interview is just one producer; a `manual` brief, or an external per-app / per-portfolio producer, is equally valid as long as the artifact conforms to the schema below. No producer is privileged. The consequence: changing this schema — renaming or removing a field, narrowing an enum, or promoting an optional field to required — is a **breaking change for every downstream consumer** and MUST follow the [breaking-change protocol](#breaking-change-protocol) (version bump + N-1 compatibility window). Additive *optional* fields are non-breaking by design: the validator tolerates unknown frontmatter keys silently (forward-compat), so a newer brief still validates against an older consumer. The v5.5.0 contract formalization established `brief_version` **2.1** as the public-contract baseline and, in the same coordinated release, **evolved it to `2.2`** under the breaking-change protocol — adding the required `framing` field + `## TL;DR` section gated at ≥ 2.2 (existing 2.0/2.1 briefs stay valid). See the Versioning paragraph below for the 2.2 details. +**Trinity producer context (informational).** In the author's private three-tier design, Voyage is **Tier 1** (per-task). Its upstream producers are **Tier 2 `app-creator`** (per-app — "what does the app need, what's the next brief?") and **Tier 3 `app-factory`** (per-portfolio — "which app needs me now?"), both pre-implementation and bound for Forgejo when ready. None is privileged: any schema-conforming producer is equally valid, and Voyage stays unaware of Tier 2/3 — this Handover is the only coupling. + **Producer:** `/trekbrief` Phase 4g (after `brief-reviewer` stop-gate passes or iteration cap is hit). **Consumer:** `/trekresearch` Phase 1 (mode parse + brief validation). diff --git a/docs/claudemd-token-trim-brief.md b/docs/claudemd-token-trim-brief.md new file mode 100644 index 0000000..b37c776 --- /dev/null +++ b/docs/claudemd-token-trim-brief.md @@ -0,0 +1,40 @@ +# Brief — trim voyage CLAUDE.md to invariants (always-loaded token reduction) + +**Author:** config-audit machine-tuning loop (KTG). **Date:** 2026-06-29. +**Why config-audit didn't do this directly:** voyage is the operator's plugin — config-audit only writes briefs here, never code. Run this in a voyage session. + +## Goal + +`CLAUDE.md` is injected **every turn** while working in the voyage repo (the whole per-repo always-loaded delta). Measured with config-audit `manifest`: **2,261 always-loaded tokens**. The tables (commands, agents) are invariant; the cost sits in six design-history / context **block-quote notes** whose detail already lives in referenced docs. Moving them out (leaving a terse invariant + pointer) should cut **~900–1,200 tok (~40–50 %)** with zero capability loss. + +## Hard constraints (do NOT break these) + +- `tests/lib/doc-consistency.test.mjs` reads CLAUDE.md. Keep the **Commands table** and the **Agents table including the `Model` column** and any **count** the test cross-checks. After editing, `node --test 'tests/**/*.test.mjs'` and `bash verify.sh` must stay green. +- Keep every **invariant fact** — just relocate the verbose *rationale/history* to the doc that already owns it, and leave a one-line pointer. Don't delete facts; move them. +- Docs-only change → no version bump / no catalog ref unless the doc-consistency test demands a badge sync (it shouldn't for a CLAUDE.md prose trim). + +## Trim targets (line numbers as of commit at brief time; match by content) + +| Block | ~chars / ~tok | Action | Destination (already referenced) | +|-------|---------------|--------|----------------------------------| +| L5 design-principle parenthetical (synthesis-PoC Δ≈0 caveat) | 620 / ~155 | **Keep the honesty caveat in one short clause** ("main-context relief is asserted-by-design, not measured — see doc"), move the PoC explanation out | `docs/T1-synthesis-poc-results.md` | +| L7 v3.0.0 architect-extraction note | 376 / ~94 | Replace with a one-liner: the plan command auto-discovers `architecture/overview.md` if present (migration history → CHANGELOG) | `CHANGELOG.md` | +| L9 Trinity context (Tier 1/2/3) | 859 / ~215 | Keep the **asymmetry invariant** terse ("Voyage stays unaware of Tier 2/3; Handover 1 = the only integration point; brief-schema changes are breaking"), move the Tier 2/3 description out | `docs/HANDOVER-CONTRACTS.md` §Handover 1 | +| **L11 brief-framing 3-layer gate** | **1,585 / ~396** | Biggest. Keep one invariant line ("brief framing must match operator intent — enforced as the `brief_version 2.2` gate: `framing` frontmatter + memory-alignment dim 6 + `## TL;DR`; all BLOCKER for ≥2.2"), move the implementation detail out | `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT) | +| L56 S33 agent-inventory reconcile | 740 / ~185 | Keep the terse fact ("24 agent files = 21 spawnable + 3 orchestrator reference docs; `synthesis-agent` dormant; all `opus`"), move the sonnet-downgrade rationale out | `docs/voyage-vs-cc-balance-analysis.md` §10 | +| L58 Model & effort axes | 588 / ~147 | Keep one line ("`opus`=Opus 4.8 / `high`; select agents carry native `effort:`; distinct from brief `phase_signals.effort`"), move the per-agent effort table + axes explanation out | `docs/profiles.md` §Model & effort axes | + +**Net:** ~2,261 → ~1,300 tok. (Verify the after-figure with `node /path/to/config-audit/scanners/manifest.mjs ` → the `project` `claude-md` source.) + +## Procedure + +1. For each block above, confirm the destination doc already contains the detail (it's referenced, so it should) — if not, move the prose there first. +2. Replace each block-quote with its terse invariant + pointer. +3. Leave Commands/Agents tables byte-exact (counts + model column). +4. `node --test 'tests/**/*.test.mjs'` + `bash verify.sh` green; confirm `doc-consistency` passes. +5. Re-measure; commit `docs(claude-md): trim CLAUDE.md to invariants` to voyage's Forgejo. Δ lands on the machine after operator `/exit` + reload. + +## Notes + +- This mirrors the same trim already done directly in config-audit (−662 tok), linkedin-studio (−2,266), ms-ai-architect (−346), okr (−15), portfolio-optimiser (−93) — same principle: CLAUDE.md carries invariants for working on the plugin; history/rationale lives in CHANGELOG + docs/. +- The voyage honesty culture (no overclaiming) is *preserved*, not trimmed: the synthesis-PoC Δ≈0 caveat stays as a one-liner; only the long-form explanation moves to its doc. From da418e653d678e892064a4fadc7634a8f71c8c69 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 06:37:28 +0200 Subject: [PATCH 20/44] fix(verify): exclude cc-upgrade decision-matrix from SC1 (legitimate CC refs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SC1 (zero `ultra` refs) was a pre-existing false-positive on docs/cc-upgrade-2.1.181-decision-matrix.md, which cites real CC features: `ultracode` (a CC keyword, 2.1.160) and the `ultra-cc-architect` plugin name. Rewording would make the doc factually wrong, so the file is excluded via exclude_path() — same pattern as CHANGELOG/MIGRATION. Tooling-only (verify.sh); no version bump. verify.sh 7/0, suite 809 (807/0/2). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_013J12WFAbngQNMTJMoybD7N --- verify.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/verify.sh b/verify.sh index af8acbb..4a91fbe 100755 --- a/verify.sh +++ b/verify.sh @@ -26,6 +26,9 @@ fail() { echo "[FAIL] SC$1 — $2"; FAIL=$((FAIL + 1)); exit 1; } # Tracked-file exclusions (paths preserved verbatim from old name). # - CHANGELOG/TRADEMARKS/MIGRATION legitimately reference the old name. +# - cc-upgrade-2.1.181-decision-matrix.md cites CC features: `ultracode` +# (a real CC keyword, 2.1.160) and the `ultra-cc-architect` plugin name — +# both legitimate, not rebrand leftovers. # - architecture-discovery.mjs + project-discovery.mjs are Q8 exceptions # pointing at the upstream architect producer slot. # - verify.sh self-references the forbidden patterns to detect them. @@ -33,6 +36,7 @@ fail() { echo "[FAIL] SC$1 — $2"; FAIL=$((FAIL + 1)); exit 1; } exclude_path() { case "$1" in *CHANGELOG.md|*TRADEMARKS.md|*MIGRATION.md) return 0 ;; + *cc-upgrade-2.1.181-decision-matrix.md) return 0 ;; *lib/validators/architecture-discovery.mjs) return 0 ;; *lib/parsers/project-discovery.mjs) return 0 ;; *verify.sh) return 0 ;; From 440594f1b286c557dc6e5990da13b9fa3224254b Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:00:33 +0200 Subject: [PATCH 21/44] =?UTF-8?q?feat(eval):=20SKAL-1=C2=B74b=20offline=20?= =?UTF-8?q?gold-scored=20output=20eval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scores committed agent-run fixtures against the golden corpus at (file, rule_key) granularity, building on the deterministic coordinator contract (4a). Offline: committed reviewer payloads, no live agent spawn, no LLM, no network (the LLM-in-the-loop grading is the separate 4c tier). - lib/review/gold-scorer.mjs: scoreFindings (precision/recall/f1 at (file,rule_key) granularity, line+severity ignored) + scoreVerdict; pure, with documented vacuous-set conventions. - tests/fixtures/bakeoff-rich/runs/run-perfect.json: committed run that reproduces all 5 seeded gold findings through runContract. - tests/lib/gold-eval.test.mjs: the scoring RUN (precision/recall/f1 = 1.0, verdict == expected_verdict BLOCK, nothing suppressed/skipped). - lib/util/test-census.mjs: third census category (goldEval) — a scoring run is neither behavior coverage nor a doc-pin; honest-count invariant now 3-way. - docs/eval-corpus/README.md: 4b moved from Future hardening to implemented. Suite 809 -> 822 (820/0/2). gold-scorer covers TP+FP+FN+degenerate paths. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01BJQYC5vpkJWxndS55vQQZ6 --- docs/eval-corpus/README.md | 42 +++++- lib/review/gold-scorer.mjs | Bin 0 -> 3359 bytes lib/util/test-census.mjs | 29 ++-- .../bakeoff-rich/runs/run-perfect.json | 17 +++ tests/lib/gold-eval.test.mjs | 49 +++++++ tests/lib/gold-scorer.test.mjs | 126 ++++++++++++++++++ tests/lib/test-census.test.mjs | 11 +- 7 files changed, 255 insertions(+), 19 deletions(-) create mode 100644 lib/review/gold-scorer.mjs create mode 100644 tests/fixtures/bakeoff-rich/runs/run-perfect.json create mode 100644 tests/lib/gold-eval.test.mjs create mode 100644 tests/lib/gold-scorer.test.mjs diff --git a/docs/eval-corpus/README.md b/docs/eval-corpus/README.md index 8f9d675..59e0632 100644 --- a/docs/eval-corpus/README.md +++ b/docs/eval-corpus/README.md @@ -7,9 +7,11 @@ review/coordinator agent **misfires** on a real task, the case is distilled into a machine-readable record and added here, so the failure can never silently regress. -This tier is the **staging ground** for the offline gold-scored output eval -(SKAL-1·4b). It is deliberately **not wired into CI** yet — these are committed -fixtures + a schema, not a scoring run. +This corpus is the gold that the **offline gold-scored output eval (SKAL-1·4b)** +scores against — that eval is now implemented and wired into `node --test` +(see [§Gold-scored output eval](#gold-scored-output-eval-skal-14b) below). The +corpus records here are committed fixtures + a schema; the scoring run is the +separate piece that consumes them. ## Seed example @@ -65,10 +67,38 @@ A corpus file is one JSON object: 3. Add (or extend) a loader test in the `tests/lib/gold-corpus.test.mjs` shape so the record's shape + catalogue membership are pinned under `node --test`. +## Gold-scored output eval (SKAL-1·4b) + +The offline scoring run that grades a recorded agent run against this corpus. +**Offline** = committed reviewer payloads, no live agent spawn, no LLM, no +network (the LLM-in-the-loop grading is the separate 4c tier). + +- **Committed runs** live under `tests/fixtures/bakeoff-rich/runs/`. Each file is + the JSON **reviewer payloads** (one object per reviewer, `{ reviewer, findings }`) + that a recorded run produced. `run-perfect.json` is the regression guard: fed + through the coordinator contract it must reproduce every seeded gold finding. +- **The contract** `lib/review/coordinator-contract.mjs::runContract(payloads)` + turns those payloads into the deterministic coordinator output (4a). +- **The scorer** `lib/review/gold-scorer.mjs`: + - `scoreFindings(runFindings, goldFindings)` matches at **`(file, rule_key)` + granularity** (line + severity ignored) → `{ tp, fp, fn, precision, recall, + f1, matched, missed, spurious }`. Vacuous-set conventions (empty run → + recall 0; empty gold → precision 0; f1 collapses to 0) are documented in the + module header. + - `scoreVerdict(runVerdict, goldVerdict)` → exact verdict match. +- **The scoring run** `tests/lib/gold-eval.test.mjs` asserts `run-perfect` + reproduces gold at precision/recall/f1 = 1.0 and `verdict === expected_verdict`. +- **Third test-census category.** `lib/util/test-census.mjs` now reports a + `goldEval` bucket (matched by `GOLD_EVAL_FILE_RE`) separately from `behavior` + and `docPins` — a scoring run is neither behavior coverage nor a prose pin, so + the honest-count invariant is now a 3-way sum. + ## Future hardening (not in this tier) -- SKAL-1·4b: an offline gold-scored output eval that scores recorded agent runs - against these records at `(file, rule_key)` granularity, with committed runs - and a third test-census category. - A prose↔JSON cross-assertion (parse the fixture README table, diff against `gold.json`) to mechanically bound the two-source-of-truth drift. +- Degraded-run fixtures (a run that misses or invents findings) to exercise the + scorer's discriminating path end-to-end; today that path is covered by + `tests/lib/gold-scorer.test.mjs` with inline synthetic findings. +- SKAL-1·4c: the LLM-in-the-loop eval that grades live agent runs (needs a + filesystem + model judgement, deliberately excluded from this deterministic tier). diff --git a/lib/review/gold-scorer.mjs b/lib/review/gold-scorer.mjs new file mode 100644 index 0000000000000000000000000000000000000000..638611f7b29b8c9ce6470f37d1ffb96d79565724 GIT binary patch literal 3359 zcma)8+iu)85bg7R#ncFDt-Y(Yi#$1T(g1bQBF+VMnis=JXo=coERm{6+1WY^^dtH& zeeb9AOL~SEt(-VO4cL|_&Y3eaXNFuXs8*MYPF^c{vsjwCoZG^5vdfzfcCuK|+4Ef>0R49lGmc9Xbekl%utpDbrMgdEJ-Y-cJr>+yOt`Y zOQo0iz7$GZhkXnyHDPnx+i1l=*|k~2bk08JylHVbBP%79rmzdY8f+H7o;>^U#Yx_j zQxcA{O4V{kU0=(0S8_dNtYs&3UyDw;b&lW>jL;>(t|c##=zuDb6xj(U>ovW6^@du} zImAwGC8-gvSP{^U7>!qaYkPP?kafK?^Dgfrhn(v5N}Bo!r4 zS~%0q3!_~p3ddEZ?4g(*0TJXHQzBL5`c7LIG>vjjmJAdWB0Dfz*G80xNIT{S1UV&}j$s^K+ zep$SlF&rLYG2{PQx*OA7`DdpZR33{ccWD|etxZXJD~jIqc5bCZkouZ)Z4htq7%?kR}1~u196rI16s$j&-wfj{| z$H&KXFon?*I;0HzDp6ZWt+C1>dh0dHn1F;<0w8h$uYgO)$QP$Pfa(MqFe~I0od(-K zWM>Av>JY)!wP-C`67~lknq@ztZ@T0c&JNIHhLj`g#M%p!^UXgYezFL%mh+Gq=)Gw~ zx26gpEaFk_AQ3-Ys1Wd;PZ;OrZ>Q)wRG$@jgm56dR=5>14Em{M5wfSqip7)w8&8Vy zKNx=ZQ%c}Le~<-d@QbGBR9n|^LD04N1W*_@XD6@en0CL}eKeU=y@n~O$L;o9uCr=N zw-Z7txD)98gWD=+ewzokFvH&}Py9CTA5Atso}c?h=y@kaDJ~HTve59z5TS?R8N`qo zKR#r+XfP4*&qB`%VuY{ObPgn^`CUGz;!@Tb?C)^mNMKT9z+~Vwqg~paCMstTHe83X z@MS*yPyql&aAz7Y9J;qw*5&k59L!uLrpcL;7$HzfrDP3lL5%iFwXK)+3ZWH3Lz;1J zRhXha>d!*~FATjf^+PnF78aGz@`m_3f*>!UEPSgtl4L|rpVIk7FeOqIVx#ZABe!lP z@A5_R&9EB^zI*3us@a_#35>|*qAatm1UwU>4fybH6oU;(-rJkd9-Z+42d$0kB zl-lyX=PbH&5HZ*f+zSoenoCyl`Nbz3Q}rkGvPAf@+_0Zink{nMVP+R7#N|3n<8;(w z_y0m=q5~}V)!jolrQTzO99X(VckE#fxE7<>&&3&4<7YjrhmFI49Mip* zsHGfbqcY5VveK8aq_HWSy~n;2SlJ^xFRh?&&i3a!$0fm5h%fAo89}$4UvQhm?nxsg zE%TU(-P3aSBrZwn+2Au@37fT3dYaNj$YX??=D35cd_RN{)J{EZ?uAIlkFbHw=WYit z$uIZow~0)2J&iT}sLu!ak0{4y$BJfD+MJcREt(jy}r!Vtu8TiB0XFgQGb!|Zz~c)MpW?wK-;?Q${= z`iR5n7Hqi_@vCQB4BUYIWdh5%H%AqvZo+w*1hg5?wv&&zAcoE197E-BYad-B33ms+ z+zc1^G(89ngb^mocX0=|)iB?}X!hgmZ2-9ME30Sxc^EyH#?(@1$na~O8qK)j;$!I3 S_!8t8Gq&C1YcTT4x%dw?$z(hL literal 0 HcmV?d00001 diff --git a/lib/util/test-census.mjs b/lib/util/test-census.mjs index 1281e38..2992ce4 100644 --- a/lib/util/test-census.mjs +++ b/lib/util/test-census.mjs @@ -1,10 +1,16 @@ // lib/util/test-census.mjs -// Census of the test suite: split top-level test() declarations into -// "behavior" tests vs "doc-consistency pins" (string/existence assertions that -// pin documentation against a source-of-truth). Makes the cited test count -// honest — a prose-pin is not the same coverage as a behavior test, so a single +// Census of the test suite: split top-level test() declarations into three +// honest categories: +// - "behavior" — ordinary behavior coverage (the default bucket). +// - "docPins" — doc-consistency pins (string/existence assertions that pin +// documentation against a source-of-truth). +// - "goldEval" — the offline gold-scored output eval scoring RUN (SKAL-1·4b): +// neither behavior nor prose-pin, but a run that scores a +// committed agent-run fixture against the golden corpus. +// Makes the cited test count honest — a prose-pin is not the same coverage as a +// behavior test, and a scoring run is a third thing again, so a single // conflated total oversells behavior coverage. -// Devil's-advocate audit §Top changes #8 (S19). +// Devil's-advocate audit §Top changes #8 (S19); third bucket added in SKAL-1·4b. // // Metric: top-level `test(` declarations (static, deterministic, in-process). // This is distinct from node:test's runtime total, which additionally counts @@ -18,6 +24,11 @@ import { join } from 'node:path'; // is bucketed correctly without editing this module. export const PIN_FILE_RE = /(doc-consistency|prose-pins)\.test\.mjs$/; +// Files whose tests are the offline gold-scored output eval scoring run +// (SKAL-1·4b). Kept as a regex (not a single filename) so a future split-out +// is bucketed correctly without editing this module. +export const GOLD_EVAL_FILE_RE = /gold-eval\.test\.mjs$/; + function walk(dir) { const out = []; for (const e of readdirSync(dir, { withFileTypes: true })) { @@ -32,18 +43,20 @@ function countTests(file) { return (readFileSync(file, 'utf-8').match(/^\s*test\(/gm) || []).length; } -// Returns { behavior, docPins, total, byFile } for all *.test.mjs under -// testsRoot. behavior + docPins === total by construction. +// Returns { behavior, docPins, goldEval, total, byFile } for all *.test.mjs +// under testsRoot. behavior + docPins + goldEval === total by construction. export function censusTests(testsRoot) { const files = walk(testsRoot).sort(); const byFile = {}; let behavior = 0; let docPins = 0; + let goldEval = 0; for (const f of files) { const n = countTests(f); byFile[f] = n; if (PIN_FILE_RE.test(f)) docPins += n; + else if (GOLD_EVAL_FILE_RE.test(f)) goldEval += n; else behavior += n; } - return { behavior, docPins, total: behavior + docPins, byFile }; + return { behavior, docPins, goldEval, total: behavior + docPins + goldEval, byFile }; } diff --git a/tests/fixtures/bakeoff-rich/runs/run-perfect.json b/tests/fixtures/bakeoff-rich/runs/run-perfect.json new file mode 100644 index 0000000..1c44cde --- /dev/null +++ b/tests/fixtures/bakeoff-rich/runs/run-perfect.json @@ -0,0 +1,17 @@ +[ + { + "reviewer": "brief-conformance-reviewer", + "findings": [ + { "severity": "BLOCKER", "rule_key": "UNIMPLEMENTED_CRITERION", "file": "lib/handlers/login.mjs", "line": 17 }, + { "severity": "MAJOR", "rule_key": "PLAN_EXECUTE_DRIFT", "file": "lib/handlers/login.mjs", "line": 13 } + ] + }, + { + "reviewer": "code-correctness-reviewer", + "findings": [ + { "severity": "BLOCKER", "rule_key": "SECURITY_INJECTION", "file": "lib/auth/jwt.mjs", "line": 19 }, + { "severity": "MAJOR", "rule_key": "MISSING_TEST", "file": "lib/auth/refresh.mjs", "line": 0 }, + { "severity": "MINOR", "rule_key": "MISSING_ERROR_HANDLING", "file": "lib/auth/refresh.mjs", "line": 10 } + ] + } +] diff --git a/tests/lib/gold-eval.test.mjs b/tests/lib/gold-eval.test.mjs new file mode 100644 index 0000000..a34c27f --- /dev/null +++ b/tests/lib/gold-eval.test.mjs @@ -0,0 +1,49 @@ +// tests/lib/gold-eval.test.mjs +// SKAL-1·4b — the offline gold-scored output eval (the scoring RUN). +// +// This is the third test-census category (see lib/util/test-census.mjs): +// neither a behavior unit test nor a doc-consistency pin, but a SCORING RUN — +// it feeds a committed agent-run fixture through the deterministic coordinator +// contract (4a) and scores the result against the golden corpus at +// (file, rule_key) granularity. Offline: committed reviewer payloads, no live +// agent spawn, no LLM, no network. The all-agree foundation; the +// LLM-in-the-loop eval is the separate 4c tier. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { runContract } from '../../lib/review/coordinator-contract.mjs'; +import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '..', '..'); +const gold = JSON.parse(readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/gold.json'), 'utf-8')); +const runPerfect = JSON.parse( + readFileSync(join(ROOT, 'tests/fixtures/bakeoff-rich/runs/run-perfect.json'), 'utf-8'), +); + +test('gold-eval — run-perfect reproduces gold at (file,rule_key): precision/recall/f1 = 1.0', () => { + const result = runContract(runPerfect); + const score = scoreFindings(result.findings, gold.findings); + assert.equal(score.precision, 1, `precision ${score.precision}; spurious=${JSON.stringify(score.spurious)}`); + assert.equal(score.recall, 1, `recall ${score.recall}; missed=${JSON.stringify(score.missed)}`); + assert.equal(score.f1, 1); + assert.equal(score.tp, gold.findings.length); +}); + +test('gold-eval — run-perfect coordinator verdict matches gold expected_verdict (BLOCK)', () => { + const result = runContract(runPerfect); + assert.equal(result.verdict, gold.expected_verdict); + assert.ok(scoreVerdict(result.verdict, gold.expected_verdict)); +}); + +test('gold-eval — run-perfect drops nothing (no suppressed, no schema-skipped payloads)', () => { + // A clean run is the regression guard: any future contract change that + // silently suppresses or skips one of the 5 seeded findings breaks here. + const result = runContract(runPerfect); + assert.equal(result.skipped.length, 0, `skipped payloads: ${JSON.stringify(result.skipped)}`); + assert.equal(result.suppressed.length, 0, `suppressed findings: ${result.suppressed.length}`); + assert.equal(result.findings.length, gold.findings.length); +}); diff --git a/tests/lib/gold-scorer.test.mjs b/tests/lib/gold-scorer.test.mjs new file mode 100644 index 0000000..a9eae63 --- /dev/null +++ b/tests/lib/gold-scorer.test.mjs @@ -0,0 +1,126 @@ +// tests/lib/gold-scorer.test.mjs +// SKAL-1·4b — behavior test for the gold scorer. +// +// scoreFindings compares a recorded agent-run's findings against a golden +// corpus record at (file, rule_key) granularity (line + severity deliberately +// ignored). This pins the precision/recall/f1 math AND the discriminating +// paths (false negatives + false positives), not merely the all-match case — +// a scorer that always returned 1.0 would pass an all-match-only test. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { scoreFindings, scoreVerdict } from '../../lib/review/gold-scorer.mjs'; + +// A minimal gold set: two distinct (file, rule_key) pairs. +const GOLD = [ + { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, + { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, +]; + +test('scoreFindings — perfect match scores precision/recall/f1 = 1', () => { + // Same pairs, different line/severity (must be ignored at (file,rule_key) granularity). + const run = [ + { file: 'a.mjs', line: 99, rule_key: 'SECURITY_INJECTION', severity: 'MINOR' }, + { file: 'b.mjs', line: 7, rule_key: 'MISSING_TEST', severity: 'BLOCKER' }, + ]; + const s = scoreFindings(run, GOLD); + assert.equal(s.tp, 2); + assert.equal(s.fp, 0); + assert.equal(s.fn, 0); + assert.equal(s.precision, 1); + assert.equal(s.recall, 1); + assert.equal(s.f1, 1); +}); + +test('scoreFindings — a missed gold pair is a false negative (recall < 1)', () => { + const run = [{ file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }]; + const s = scoreFindings(run, GOLD); + assert.equal(s.tp, 1); + assert.equal(s.fp, 0); + assert.equal(s.fn, 1); + assert.equal(s.precision, 1); + assert.equal(s.recall, 0.5); + assert.deepEqual(s.missed, ['b.mjs MISSING_TEST']); +}); + +test('scoreFindings — a spurious pair is a false positive (precision < 1)', () => { + const run = [ + { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, + { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, + { file: 'c.mjs', line: 3, rule_key: 'PLACEHOLDER_IN_CODE', severity: 'MAJOR' }, + ]; + const s = scoreFindings(run, GOLD); + assert.equal(s.tp, 2); + assert.equal(s.fp, 1); + assert.equal(s.fn, 0); + assert.equal(s.recall, 1); + assert.equal(s.precision, 2 / 3); + assert.deepEqual(s.spurious, ['c.mjs PLACEHOLDER_IN_CODE']); +}); + +test('scoreFindings — mixed FN + FP', () => { + const run = [ + { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, // match + { file: 'c.mjs', line: 3, rule_key: 'PLACEHOLDER_IN_CODE', severity: 'MAJOR' }, // spurious + ]; + const s = scoreFindings(run, GOLD); + assert.equal(s.tp, 1); + assert.equal(s.fp, 1); + assert.equal(s.fn, 1); + assert.equal(s.precision, 0.5); + assert.equal(s.recall, 0.5); + assert.equal(s.f1, 0.5); +}); + +test('scoreFindings — duplicate (file,rule_key) pairs collapse (set semantics)', () => { + const run = [ + { file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, + { file: 'a.mjs', line: 22, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }, // same pair + { file: 'b.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, + ]; + const s = scoreFindings(run, GOLD); + assert.equal(s.tp, 2); + assert.equal(s.fp, 0); + assert.equal(s.fn, 0); +}); + +test('scoreFindings — empty run: recall 0 (nothing found), precision vacuously 1, f1 0', () => { + const s = scoreFindings([], GOLD); + assert.equal(s.tp, 0); + assert.equal(s.fp, 0); + assert.equal(s.fn, 2); + assert.equal(s.recall, 0); + assert.equal(s.precision, 1); + assert.equal(s.f1, 0); +}); + +test('scoreFindings — empty gold: precision 0 (all spurious), recall vacuously 1, f1 0', () => { + const run = [{ file: 'a.mjs', line: 10, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' }]; + const s = scoreFindings(run, []); + assert.equal(s.tp, 0); + assert.equal(s.fp, 1); + assert.equal(s.fn, 0); + assert.equal(s.precision, 0); + assert.equal(s.recall, 1); + assert.equal(s.f1, 0); +}); + +test('scoreFindings — both empty: precision/recall/f1 = 1 (matched nothing perfectly)', () => { + const s = scoreFindings([], []); + assert.equal(s.precision, 1); + assert.equal(s.recall, 1); + assert.equal(s.f1, 1); +}); + +test('scoreFindings — tolerates null/undefined finding arrays', () => { + const s = scoreFindings(null, undefined); + assert.equal(s.tp, 0); + assert.equal(s.fp, 0); + assert.equal(s.fn, 0); +}); + +test('scoreVerdict — exact verdict match is true, mismatch is false', () => { + assert.equal(scoreVerdict('BLOCK', 'BLOCK'), true); + assert.equal(scoreVerdict('WARN', 'BLOCK'), false); + assert.equal(scoreVerdict('ALLOW', 'ALLOW'), true); +}); diff --git a/tests/lib/test-census.test.mjs b/tests/lib/test-census.test.mjs index 92fb0d9..81328a4 100644 --- a/tests/lib/test-census.test.mjs +++ b/tests/lib/test-census.test.mjs @@ -15,20 +15,21 @@ import { censusTests } from '../../lib/util/test-census.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); const TESTS_ROOT = join(HERE, '..'); -test('suite census splits behavior tests from doc-consistency pins (S19)', (t) => { +test('suite census splits behavior / doc-pins / gold-eval (S19 + SKAL-1·4b)', (t) => { const c = censusTests(TESTS_ROOT); - // Honest-count invariant: the two buckets must account for every top-level - // test() declaration — no silent drift between behavior and pin counts. - assert.equal(c.behavior + c.docPins, c.total, + // Honest-count invariant: the three buckets must account for every top-level + // test() declaration — no silent drift between behavior, pin, and eval counts. + assert.equal(c.behavior + c.docPins + c.goldEval, c.total, 'census buckets must sum to the total declaration count'); assert.ok(c.docPins > 0, 'doc-consistency pin bucket must be non-empty (regex/glob sanity)'); assert.ok(c.behavior > 0, 'behavior bucket must be non-empty (regex/glob sanity)'); + assert.ok(c.goldEval > 0, 'gold-eval scoring-run bucket must be non-empty (SKAL-1·4b present)'); // Report the split so the cited count is honest (audit §Top changes #8). // Metric = top-level test() declarations; node:test's runtime total counts // subtests too and is therefore ≥ this number. t.diagnostic( `behavior=${c.behavior} doc-consistency-pins=${c.docPins} ` + - `total=${c.total} (top-level test() declarations)`, + `gold-eval=${c.goldEval} total=${c.total} (top-level test() declarations)`, ); }); From cb5dba9542f96b37cc92bed231d06d60d58c6a80 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:00:40 +0200 Subject: [PATCH 22/44] docs(brief): add opt-in /deep-research engine brief to backlog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the operator-supplied brief for an --engine {swarm|deep-research} choice on /trekresearch's external phase (swarm stays default; deep-research delegates the external phase to the built-in workflow with auto-fallback). Backlog item only — not implemented. Lives beside the work (docs/), tracked. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01BJQYC5vpkJWxndS55vQQZ6 --- docs/deep-research-engine-brief.md | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/deep-research-engine-brief.md diff --git a/docs/deep-research-engine-brief.md b/docs/deep-research-engine-brief.md new file mode 100644 index 0000000..b7d1c78 --- /dev/null +++ b/docs/deep-research-engine-brief.md @@ -0,0 +1,48 @@ +--- +# Avstem frontmatter mot lib/validators/brief-validator.mjs før /trekplan +brief_version: "2.2" +status: draft +brief_quality: complete +research_topics: 3 +--- + +# Brief — Opt-in `/deep-research`-motor for `/trekresearch` + +## Intent +`/trekresearch` eier i dag hele den eksterne research-fasen selv (Tavily / MS-Learn / Gemini-sverm). Det er portabelt, men du vedlikeholder fan-out, kryssjekk og syntese selv. Anthropic sin innebygde `/deep-research` vedlikeholder fan-out, adversariell påstandsverifisering, sitatfiltrering og websøk for deg. Lar vi operatøren *velge* `/deep-research` for den eksterne fasen, får brukeren en vedlikeholdsfri "turbo" når den er tilgjengelig — uten at Voyage mister sin egen lokal-analyse, triangulering eller H2-kontrakt. + +## Goal +`/trekresearch` får et `--engine {swarm|deep-research}`-valg på den eksterne fasen. `swarm` er default (dagens oppførsel). `deep-research` delegerer den eksterne fasen til den innebygde workflowen og adapterer resultatet inn i research-brief-skjemaet. Lokal analyse, triangulering og H2-output (`research/NN-*.md`) er uendret uansett motor. + +## Non-Goals +- Bygge om `/trekresearch` på et eget dynamic-workflow-substrat (vurdert og forkastet, Δ≈0). +- Multi-provider (Perplexity / Google) — hører til en senere egen `workflow`-motor, ikke denne. +- Endre den lokale svermen, handover-kontraktene eller validator-skjemaene. +- Gjøre `/deep-research` til default. + +## Constraints / NFR +- `/deep-research` krever Claude Code v2.1.154+ og må være på (Pro: via `/config`). Må feature-detekteres med auto-fallback til `swarm` — aldri hard feil. +- Output må passere `research-validator.mjs` (confidence ∈ [0,1], dimensions ≥ 1, påkrevde body-seksjoner). +- Ingen nye `lib/`-avhengigheter; overflate-endring i `commands/` + `agents/`. +- Doc-consistency: oppdater command-tabellen i `CLAUDE.md` + `README.md`, kjør `npm test`. + +## Success Criteria +1. `/trekresearch --external --engine swarm ""` gir identisk oppførsel som før (regresjon grønn). +2. `/trekresearch --external --engine deep-research ""` produserer en `research/NN-*.md` der `research-validator.mjs --json` rapporterer `valid: true`. +3. Med dynamic workflows avskrudd faller `--engine deep-research` automatisk tilbake til `swarm` og logger valget — ingen exception. +4. `npm test` (inkl. doc-consistency) er grønn. + +## Research Plan +1. **Programmatisk trigging av `/deep-research`** — Kan en plugin-kommando starte den innebygde workflowen og fange artefakten, eller må `commands/trekresearch.md` instruere Claude til å kjøre den? Scope: external · Konfidens: høy · Kost: lav. + `/trekresearch --external "Kan en Claude Code plugin-kommando programmatisk starte /deep-research og hente rapporten?"` +2. **Output-adapter** — Hvordan mappe `/deep-research` sin siterte rapport til research-brief-skjemaet (dimensjoner, confidence, sitater)? Scope: both · Konfidens: høy · Kost: lav. + `/trekresearch --project --local "Hva krever research-validator.mjs av research/NN-*.md?"` +3. **Feature-deteksjon + fallback** — Hvordan oppdage om workflows er på, og hvor i fasen fallback-grenen bør sitte? Scope: local · Konfidens: middels · Kost: lav. + +## Open Questions / Assumptions +- Antar at `/deep-research`-rapporten kan reduseres til ≥ 1 dimensjon med per-påstand-sitater uten å bryte trianguleringen. Verifiseres i topic 2. +- Uavklart om delegering skjer via instruksjon (trygt) eller programmatisk API (raskere) — topic 1 avgjør. + +## Prior Attempts +- Dvalende synthesis-agent bygget, målt (Δ≈0) og forkastet → samme lærdom gjelder substrat-bytte: mål før du adopterer. +- Workflow-substratet er allerede dokumentert som vurdert-og-valgt-bort i `docs/architecture.md` §Primitives per step. From 0c634b663696eec120dc7d40dee76d909d9cb3c7 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:02:39 +0200 Subject: [PATCH 23/44] =?UTF-8?q?chore(voyage):=20release=20v5.8.0=20-=20S?= =?UTF-8?q?KAL-1=C2=B74b=20offline=20gold-scored=20output=20eval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version sync 5.7.1 -> 5.8.0 across plugin.json, package.json, package-lock.json, README badge, and CHANGELOG top entry (guarded by doc-consistency.test.mjs). Suite 822 (820/0/2). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01BJQYC5vpkJWxndS55vQQZ6 --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 15 +++++++++++++++ README.md | 4 ++-- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 095328f..0476d8b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "voyage", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", - "version": "5.7.1", + "version": "5.8.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aac024..5827ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## v5.8.0 — 2026-06-30 — offline gold-scored output eval (SKAL-1·4b) + +Additive — no behavior change, no breaking change. Internal eval infrastructure only (`lib/` + `tests/` + docs); no command, agent, profile, or Handover contract touched. + +### Gold-scored output eval (SKAL-1·4b) + +- The review-coordinator self-eval gains its **scoring run**, building on the deterministic coordinator contract + golden corpus shipped as the 4a foundation in v5.7.0. +- `lib/review/gold-scorer.mjs`: `scoreFindings(runFindings, goldFindings)` matches at **`(file, rule_key)` granularity** (line + severity deliberately ignored) → `{ tp, fp, fn, precision, recall, f1, matched, missed, spurious }`; `scoreVerdict` checks exact verdict match. Pure — no I/O, no LLM, no network. Vacuous-set conventions (empty run → recall 0; empty gold → precision 0; f1 collapses to 0) are documented in the module header. +- `tests/fixtures/bakeoff-rich/runs/run-perfect.json`: the first **committed run** — reviewer payloads that, fed through `runContract`, reproduce all 5 seeded gold findings. The regression guard: any future contract change that silently suppresses or skips a seeded finding breaks the eval. +- `tests/lib/gold-eval.test.mjs`: the **scoring run**, wired into `node --test`. Asserts `run-perfect` scores precision/recall/f1 = 1.0 and `verdict == expected_verdict` (BLOCK). Offline: committed payloads, no live agent spawn (the LLM-in-the-loop grading is the separate 4c tier). +- **Third test-census category.** `lib/util/test-census.mjs` now reports a `goldEval` bucket (matched by `GOLD_EVAL_FILE_RE`) separately from `behavior` and `docPins` — a scoring run is neither behavior coverage nor a prose pin, so the honest-count invariant is a 3-way sum. +- `docs/eval-corpus/README.md`: SKAL-1·4b moved from "Future hardening" to an implemented section documenting the scorer, run format, and census category. +- Suite: **822 (820 pass / 0 fail / 2 skip)**, up from 809 (+10 scorer behavior tests, +3 eval scoring-run tests). The scorer test covers the discriminating paths (false negatives + false positives) and the degenerate empty-run / empty-gold cases, not merely the all-match case. +- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.8.0`, guarded by `doc-consistency.test.mjs`. + ## v5.7.1 — 2026-06-29 — relocate agent `` blocks to body (always-loaded token trim) Performance/packaging change — no behavior change, no breaking change. (M4) diff --git a/README.md b/README.md index d920b6d..144bc26 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # trekplan — Brief, Research, Plan, Execute, Review, Continue -![Version](https://img.shields.io/badge/version-5.7.1-blue) +![Version](https://img.shields.io/badge/version-5.8.0-blue) ![License](https://img.shields.io/badge/license-MIT-green) ![Platform](https://img.shields.io/badge/platform-Claude%20Code-purple) @@ -10,7 +10,7 @@ A [Claude Code](https://docs.anthropic.com/en/docs/claude-code) plugin for deep implementation planning, multi-source research, autonomous execution, independent post-hoc review, and zero-friction multi-session resumption. Six commands, one pipeline: -> **What's new — v5.7.1: leaner always-loaded agent listing (round 2).** The 17 example-bearing agents now keep their `` blocks in the body (under `## When to use — examples`) instead of the `description:` frontmatter — voyage launches its agents by name, so the auto-selection examples were cost without function there. This trims ~3,180 always-loaded tokens (frontmatter `description` chars 17,672 → 4,945) from the agent listing injected into every session, with no behavior change. The saving lands after `/plugin marketplace update` + reload. **v5.7.0:** opt-in per-session token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a). **v5.6.1:** one-line `description:` for the four reference/dormant agents (~700 tok). **v5.5.0:** brief **framing** enforcement (`brief_version 2.2`) + a `/trekreview` reviewer-schema contract. Additive — no breaking changes. **Full version history → [CHANGELOG.md](CHANGELOG.md).** +> **What's new — v5.8.0: offline gold-scored output eval (SKAL-1·4b).** The review-coordinator self-eval gains a scoring run: `lib/review/gold-scorer.mjs` grades a committed agent-run fixture against the golden corpus at `(file, rule_key)` granularity (precision/recall/f1 + verdict match), and the suite census gains a third category (`goldEval`) so a scoring run is counted apart from behavior coverage and doc-pins. Offline + deterministic — committed reviewer payloads, no live agent spawn (the LLM-in-the-loop tier is the separate 4c). Internal eval infrastructure; no command/agent/Handover change. **v5.7.1:** leaner always-loaded agent listing — `` blocks relocated to agent bodies (~3,180 tok/turn, no behavior change). **v5.7.0:** opt-in per-session token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a). **v5.6.1:** one-line `description:` for the four reference/dormant agents (~700 tok). **v5.5.0:** brief **framing** enforcement (`brief_version 2.2`) + a `/trekreview` reviewer-schema contract. Additive — no breaking changes. **Full version history → [CHANGELOG.md](CHANGELOG.md).** | Command | What it does | |---------|-------------| diff --git a/package-lock.json b/package-lock.json index 9efc757..4da4c04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voyage", - "version": "5.7.1", + "version": "5.8.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voyage", - "version": "5.7.1", + "version": "5.8.0", "license": "MIT", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 6c4c740..5720b87 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voyage", - "version": "5.7.1", + "version": "5.8.0", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any heading/paragraph/list-item, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", "type": "module", "engines": { From 926b768543cc3b8c28674f0ea8612cea22be4c57 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:48:28 +0200 Subject: [PATCH 24/44] fix(validators): brief-validator CLI no-flag invocation bailed to Usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The documented `brief-validator.mjs ` invocation (no flags) always bailed to Usage/exit 2. Root cause: when --min-version is absent, minIdx is -1, so the skip index minIdx+1 was 0 — excluding argv index 0, exactly where the file positional sits in the no-flag case. Any leading flag (--soft, --json) pushed the file to index >=1 and masked the bug, so the function-level tests never caught it. Guard the skip index to -1 when --min-version is absent. Add two CLI regression tests (execFileSync, matching the next-session-prompt-validator pattern): no-flag invocation reaches validation, and --min-version still skips its value token to find the file. Suite 822 -> 824 (822/0/2). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01WH1krHamUehZh6JqVqs85t --- lib/validators/brief-validator.mjs | 6 +++- tests/validators/brief-validator.test.mjs | 39 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/lib/validators/brief-validator.mjs b/lib/validators/brief-validator.mjs index f93d2e2..7f25d90 100644 --- a/lib/validators/brief-validator.mjs +++ b/lib/validators/brief-validator.mjs @@ -252,7 +252,11 @@ if (import.meta.url === `file://${process.argv[1]}`) { const minIdx = args.indexOf('--min-version'); const minBriefVersion = minIdx >= 0 ? args[minIdx + 1] : undefined; // filePath is the first positional, skipping the --min-version value token. - const filePath = args.find((a, i) => !a.startsWith('--') && i !== minIdx + 1); + // Guard: when --min-version is absent (minIdx === -1) the skip index must be -1, + // not 0 — otherwise the no-flag invocation `brief-validator.mjs ` drops + // the file (which sits at index 0) and bails to Usage. + const skipIdx = minIdx >= 0 ? minIdx + 1 : -1; + const filePath = args.find((a, i) => !a.startsWith('--') && i !== skipIdx); if (!filePath) { process.stderr.write('Usage: brief-validator.mjs [--soft] [--min-version ] \n'); process.exit(2); diff --git a/tests/validators/brief-validator.test.mjs b/tests/validators/brief-validator.test.mjs index a24d9d4..3a05928 100644 --- a/tests/validators/brief-validator.test.mjs +++ b/tests/validators/brief-validator.test.mjs @@ -1,5 +1,9 @@ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; const GOOD_BRIEF = `--- @@ -393,3 +397,38 @@ test('validateBrief — S18 min-version: trekreview brief is exempt (no framing const r = validateBriefContent(REVIEW_AS_BRIEF, { minBriefVersion: '2.2' }); assert.ok(!r.warnings.find(w => w.code === 'BRIEF_VERSION_BELOW_MINIMUM')); }); + +// S56 — CLI arg-parsing regression. The no-flag invocation `brief-validator.mjs ` +// used to bail to Usage/exit 2 because the --min-version skip index was 0 when the flag +// was absent, dropping the file positional (which sits at argv index 0). +test('CLI — no-flag invocation reaches validation, does not bail to Usage (S56 regression)', () => { + const dir = mkdtempSync(join(tmpdir(), 'brief-cli-')); + try { + const file = join(dir, 'brief.md'); + writeFileSync(file, GOOD_BRIEF); + // execFileSync throws on non-zero exit; pre-fix this bailed to Usage (exit 2). + const out = execFileSync(process.execPath, [ + 'lib/validators/brief-validator.mjs', + file, + ], { encoding: 'utf-8' }); + assert.match(out, /PASS/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('CLI — --min-version still locates the file positional after the value token (S56)', () => { + const dir = mkdtempSync(join(tmpdir(), 'brief-cli-')); + try { + const file = join(dir, 'brief.md'); + writeFileSync(file, GOOD_BRIEF); + const out = execFileSync(process.execPath, [ + 'lib/validators/brief-validator.mjs', + '--min-version', '2.0', + file, + ], { encoding: 'utf-8' }); + assert.match(out, /PASS/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 60e9e7ae5c1034b56d91b3d96c7e81b29100e4eb Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:48:28 +0200 Subject: [PATCH 25/44] docs(brief): reconcile deep-research-engine brief to validator-green (2.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-delivered draft (S55) carried only brief_version/status/brief_quality/ research_topics and an explicit "reconcile frontmatter against brief-validator before /trekplan" note. Add the required fields (type, task, slug, research_status, phase_signals_partial) plus the brief_version 2.2 gates: framing: refine (operator-confirmed — additive opt-in engine, swarm stays default, no contract change) and a 5-line ## TL;DR. brief-validator passes strict + --min-version 2.2. status stays draft pending operator review before the /trekplan run. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01WH1krHamUehZh6JqVqs85t --- docs/deep-research-engine-brief.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/deep-research-engine-brief.md b/docs/deep-research-engine-brief.md index b7d1c78..ac22280 100644 --- a/docs/deep-research-engine-brief.md +++ b/docs/deep-research-engine-brief.md @@ -1,13 +1,25 @@ --- -# Avstem frontmatter mot lib/validators/brief-validator.mjs før /trekplan +type: trekbrief brief_version: "2.2" +task: "Opt-in /deep-research-motor for /trekresearch ekstern-fase" +slug: deep-research-engine +framing: refine status: draft brief_quality: complete research_topics: 3 +research_status: pending +phase_signals_partial: true --- # Brief — Opt-in `/deep-research`-motor for `/trekresearch` +## TL;DR +- `--engine {swarm|deep-research}` på `/trekresearch` ekstern-fase; `swarm` default (uendret oppførsel). +- `deep-research` delegerer ekstern research til Claude Codes innebygde workflow og adapterer inn i research-brief-skjemaet. +- Lokal analyse, triangulering og H2-output (`research/NN-*.md`) er motor-uavhengig. +- Feature-detekteres med auto-fallback til `swarm` — aldri hard feil. +- Surface-only: `commands/` + `agents/`, ingen nye `lib/`-avhengigheter. + ## Intent `/trekresearch` eier i dag hele den eksterne research-fasen selv (Tavily / MS-Learn / Gemini-sverm). Det er portabelt, men du vedlikeholder fan-out, kryssjekk og syntese selv. Anthropic sin innebygde `/deep-research` vedlikeholder fan-out, adversariell påstandsverifisering, sitatfiltrering og websøk for deg. Lar vi operatøren *velge* `/deep-research` for den eksterne fasen, får brukeren en vedlikeholdsfri "turbo" når den er tilgjengelig — uten at Voyage mister sin egen lokal-analyse, triangulering eller H2-kontrakt. From 9d8e04395988f7c6a4d07ffef8a536cac889a1c2 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 10:39:03 +0200 Subject: [PATCH 26/44] docs(research): resolve deep-research-engine topic-1 (/deep-research trigging) Operator reviewed the brief (S57) and approved as-is, then chose research-first at the /trekplan research gate (option A: investigate only the genuinely-external topic 1; fold local topics 2/3 into /trekplan exploration). Topic 1 finding (validator-green; claude-code-guide + CC 2.1.196 binary + a real local /deep-research run, all cited): /deep-research is a built-in *dynamic workflow* (not a skill), outside the Skill-tool allowlist. Trigging is prose-instruction only; the report lands inline in context with no on-disk artifact (only the .js script is written). => the engine must be instruction-based + in-context transform, surface-only. Brief's v2.1.154+ / Pro-via-/config constraints verified correct; SC3 ("dynamic workflows off -> fallback to swarm") is correct as written (an earlier review note that called it a conflation rested on a wrong premise and is retracted). - docs/deep-research-engine-research.md: new, research-validator green strict. - docs/deep-research-engine-brief.md: status draft->ready (operator-approved), research_status pending->complete (option-A decision recorded), Research Plan traceability note. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_019ddpUq3MNQFoV5h3qYrVYj --- docs/deep-research-engine-brief.md | 6 +- docs/deep-research-engine-research.md | 210 ++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 docs/deep-research-engine-research.md diff --git a/docs/deep-research-engine-brief.md b/docs/deep-research-engine-brief.md index ac22280..bd779eb 100644 --- a/docs/deep-research-engine-brief.md +++ b/docs/deep-research-engine-brief.md @@ -4,10 +4,10 @@ brief_version: "2.2" task: "Opt-in /deep-research-motor for /trekresearch ekstern-fase" slug: deep-research-engine framing: refine -status: draft +status: ready brief_quality: complete research_topics: 3 -research_status: pending +research_status: complete phase_signals_partial: true --- @@ -51,6 +51,8 @@ phase_signals_partial: true `/trekresearch --project --local "Hva krever research-validator.mjs av research/NN-*.md?"` 3. **Feature-deteksjon + fallback** — Hvordan oppdage om workflows er på, og hvor i fasen fallback-grenen bør sitte? Scope: local · Konfidens: middels · Kost: lav. +> **Research-status (2026-06-30, operatør-beslutning «option A»):** Topic 1 (eneste ekte eksterne ukjente) er undersøkt → `docs/deep-research-engine-research.md` (validator-grønn). Funn: `/deep-research` er en innebygd **dynamic workflow** (ikke skill) → trigging KUN via prosa-instruksjon, output **inline i kontekst** (ingen on-disk-artefakt) → motoren må være instruksjons-basert + in-context transform. **SC3 er korrekt som skrevet** (`/deep-research` ER en dynamic workflow). Topic 2 (validator-skjema: `type/created/question` + `## Executive Summary`/`## Dimensions`, `dimensions ≥ 1`) og topic 3 (fallback-plassering) er lokale kode-spørsmål reklassifisert til `/trekplan`-utforskning. `research_status: complete` reflekterer denne beslutningen. + ## Open Questions / Assumptions - Antar at `/deep-research`-rapporten kan reduseres til ≥ 1 dimensjon med per-påstand-sitater uten å bryte trianguleringen. Verifiseres i topic 2. - Uavklart om delegering skjer via instruksjon (trygt) eller programmatisk API (raskere) — topic 1 avgjør. diff --git a/docs/deep-research-engine-research.md b/docs/deep-research-engine-research.md new file mode 100644 index 0000000..d65a13a --- /dev/null +++ b/docs/deep-research-engine-research.md @@ -0,0 +1,210 @@ +--- +type: trekresearch-brief +created: 2026-06-30 +question: "Can a Claude Code plugin command programmatically trigger the built-in /deep-research workflow and capture its report, or must it instruct Claude to run it?" +confidence: 0.85 +dimensions: 4 +mcp_servers_used: [] +local_agents_used: [claude-code-guide] +external_agents_used: [] +slug: deep-research-engine +feeds_brief: docs/deep-research-engine-brief.md +research_topic: 1 +--- + +# Research — Programmatic trigging of `/deep-research` from a plugin command + +> Targeted single-topic research for the `deep-research-engine` brief (Topic 1 of +> the brief's Research Plan). Topics 2 & 3 are local Voyage-code questions folded +> into `/trekplan` exploration; only Topic 1 was a genuine external unknown. +> Method: `claude-code-guide` agent (Anthropic docs + CHANGELOG, cited) + +> direct inspection of this machine (CC 2.1.196 binary, a real local `/deep-research` +> run). Not run through the `/trekresearch` swarm — see brief reconcile note. + +## Research Question + +Can a Claude Code **plugin slash-command** (markdown under `commands/`) +**programmatically** start the built-in `/deep-research` workflow and **capture its +report artifact** for adaptation into the research-brief schema — or must the command +instead **instruct** Claude (in prose) to run `/deep-research` and then transform the +in-context result? + +## Executive Summary + +Programmatic trigger + file-based capture is **not feasible**: `/deep-research` is a +built-in **dynamic workflow** (not a skill), deliberately outside the Skill-tool +allowlist, and it returns its report **inline into conversation context with no +documented on-disk report artifact**. The **only reliable path is instruction-based +delegation** — the command's prose tells Claude to run `/deep-research `, then +transforms the in-context report in the same turn. Confidence **high** (Anthropic docs + +CHANGELOG + local run), with one residual gap: there is no positive "is it enabled?" +probe, so feature-detection must lean on the documented *disable* switches + version +floor + a `swarm` default. + +## Dimensions + +### 1. Origin + gating -- Confidence: high + +**External findings:** +- `/deep-research` is a **built-in dynamic workflow**, not a command and not a bundled + skill. `commands.md` marks the `/deep-research ` row as **[Workflow]**; + `workflows.md`: "Claude Code includes `/deep-research` as a built-in workflow." + [VERIFIED — code.claude.com/docs/en/commands.md, code.claude.com/docs/en/workflows.md] +- Gating: available on **all paid plans** (pro/max/team/enterprise) + API/Bedrock/Vertex/ + Foundry. **On Pro it must be turned on** in the *Dynamic workflows* row of `/config`. + Also requires the **WebSearch tool** to be available. + [VERIFIED — workflows.md, commands.md bundled-workflows row] +- Version floor: dynamic workflows were **introduced in CC 2.1.154**; "Dynamic workflows + require Claude Code v2.1.154 or later." The brief's `v2.1.154+` + `(Pro: via /config)` + constraints are **both correct**. The exact version that first shipped the *named* + `/deep-research` workflow is **[NOT DOCUMENTED]** (only a 2.1.196 bugfix mentions it by + name) — treat 2.1.154 as the substrate floor, not a proven introduction point. + [VERIFIED — workflows.md + CHANGELOG 2.1.154] + +**Local findings:** +- This machine runs **CC 2.1.196** (`claude --version`) — substrate floor satisfied. +- The exact skill-description string lives **compiled into the binary** + (`/Users/ktg/.local/share/claude/versions/2.1.196`, Mach-O 235 MB); there is **no + `SKILL.md`** for it anywhere under `~/.claude` (system-wide `find`/`grep` — only hits + are this brief + unrelated harness notes). Confirms "Anthropic-bundled, not user skill." + +### 2. Invocation mechanism (programmatic vs instruction) -- Confidence: high + +**External findings:** +- **Not** via the `Skill` tool. The Skill-tool built-in allowlist is closed: only + `/init`, `/review`, `/security-review` are reachable; "Other built-in commands such as + `/compact` are not." `/deep-research` is a Workflow and is not on that list. + [VERIFIED — code.claude.com/docs/en/skills.md] +- **Instruction-based delegation is the documented mechanism.** Workflows launch when the + user types the command, or **when Claude is asked in natural language** ("use a + workflow" / "run a workflow") or via the `ultracode` keyword. A plugin command whose + markdown instructs Claude to run `/deep-research ` is therefore the supported path. + [VERIFIED — workflows.md "Have Claude write a workflow"] +- **Approval gate caveat:** launching a workflow triggers a per-run approval prompt — + *every run* in default/acceptEdits; *first launch only* in auto; **never in `claude -p` + / Agent SDK / bypass-permissions** ("the run starts immediately"). Voyage's headless + surface (`claude -p`) thus delegates without an interactive gate; interactive sessions + hit a prompt. [VERIFIED — workflows.md "Behavior and limits"] +- No documented blanket "commands/skills cannot nest" prohibition beyond the Skill-tool + allowlist + workflow runtime limits (no mid-run user input; 16 concurrent agents; + 1000 agents/run). [VERIFIED — workflows.md; NOT DOCUMENTED for a general nesting ban] + +### 3. Output capture -- Confidence: high + +**External findings:** +- "When the run finishes, **the report lands in your session**"; "Claude's context holds + only the final answer." The report is **in-context**, not a file. + [VERIFIED — workflows.md] +- What *is* written to disk is the orchestration **script**, not the report: "Every run + writes its script to a file under your session's directory in `~/.claude/projects/`." + [VERIFIED — workflows.md "How a workflow runs"] + +**Local findings:** +- A real `/deep-research` run on this machine left exactly one file — + `~/.claude/projects//workflows/scripts/deep-research-wf_.js` — and **no + `.md` report** beside it. [VERIFIED — local filesystem inspection by claude-code-guide] +- **Consequence:** a calling command cannot `grep` a results file off disk (none is + documented to exist). It can only consume the report **as it sits in conversation + context in the same turn** — Claude reads its own prior output and transforms it into + research-brief schema. [INFERENCE — file-based capture not feasible; in-context + transform is the only avenue] + +### 4. Feature detection + fallback -- Confidence: medium + +**External findings:** +- **No positive enumeration / "is-enabled" API or flag is documented.** `claude --help` + exposes `--disable-slash-commands` but no "list skills/workflows" or "is-feature-on" + flag. [VERIFIED — local `claude --help`; NOT DOCUMENTED for any positive probe] +- The documented signals are the **off-switches**, read defensively: `/config` Dynamic- + workflows off; `disableWorkflows: true` / `disableBundledSkills: true` in settings.json; + `CLAUDE_CODE_DISABLE_WORKFLOWS=1` / `CLAUDE_CODE_DISABLE_BUNDLED_SKILLS=1`. + [VERIFIED — workflows.md "Turn workflows off"; CHANGELOG 2.1.x] +- **The gap:** the Pro `/config` *on*-state is the very thing you most need to detect, and + only the *disable* keys are documented; the persisted key/value for the Pro enable-state + is **[NOT DOCUMENTED]**, so it cannot be reliably grepped. A failed/disabled + `/deep-research` is also **not documented** to raise a signal a sibling command can + catch. [VERIFIED gap] + +**Local findings:** +- On this machine both disable keys are unset/absent — workflows are not disabled. + +## External Knowledge + +### Best Practice +The "Have Claude write a workflow" + "Behavior and limits" sections of `workflows.md` +establish that workflows are operator/Claude-launched, run isolated, and return one +in-context report. The Skill-tool allowlist (`skills.md`) is the authoritative statement +that only three built-ins are tool-invocable. + +### Known Issues +Per-run approval prompts outside `-p`/SDK/bypass mean an interactive `/trekresearch +--engine deep-research` will pause for operator approval on each launch — acceptable, but +worth documenting in the command UX. The absence of a positive availability probe is the +single biggest design constraint (see Dimension 4). + +## Synthesis + +Three cross-cutting insights that only emerge from combining the docs with Voyage's brief: + +1. **The brief's SC3 is correct as written — an earlier review note was wrong.** Because + `/deep-research` *is itself* a dynamic workflow, "med dynamic workflows avskrudd faller + `--engine deep-research` tilbake til swarm" is the right feature-detection axis. A + prior brief-review remark that SC3 "conflated `/deep-research` (skill) with dynamic + workflows" was based on a wrong premise (that `/deep-research` was a skill) and is + retracted. **No SC3 brief edit is needed.** + +2. **The engine must be instruction-based + in-context, never file-based.** Topic 1's + open question ("instruction (safe) vs. programmatic API (faster) — topic 1 decides") + resolves decisively to **instruction-based**: there is no programmatic API and no + on-disk report. The `deep-research` engine path in `commands/trekresearch.md` must + (a) instruct Claude to run `/deep-research `, then (b) transform the in-context + report into the `research/NN-*.md` schema in the same turn. This is surface-only + (`commands/` prose), matching the brief's "ingen nye `lib/`-avhengigheter." + +3. **SC3's "ingen exception" cannot rest on runtime detection — pin it to a default.** + Since no positive availability probe exists, robust fallback = version-floor check + (≥ 2.1.154) + disable-key heuristic (`disableWorkflows` / env) + **`--engine` default + of `swarm`** (explicit opt-in). The fallback is "graceful degradation by design," + not "catch an exception at runtime." This refines, but does not contradict, SC3. + +## Open Questions + +- **Adapter fidelity (brief Topic 2, partly answered locally):** `research-validator.mjs` + requires `type: trekresearch-brief` + `created` + `question`; validates `confidence ∈ + [0,1]` and `dimensions ≥ 1` if present; body must carry `## Executive Summary` + + `## Dimensions`. So the `/deep-research` report **can** be reduced to ≥ 1 dimension with + per-claim citations and a confidence number — the brief's assumption holds. The + remaining open part (how cleanly the in-context report maps to per-dimension + local/external splits) is a `/trekplan` exploration concern, not an external unknown. +- **Exact intro version of the *named* `/deep-research` workflow** — not documented; the + 2.1.154 dynamic-workflows floor is the safe pin. + +## Recommendation + +Build the `deep-research` engine as **instruction-based delegation with in-context +adaptation**, not a programmatic trigger: +1. `--engine deep-research` makes `commands/trekresearch.md`'s external phase instruct + Claude to run `/deep-research ` and transform the returned in-context report into + `research/NN-*.md` (validator-conformant: `## Executive Summary` + `## Dimensions`, + `confidence`, `dimensions ≥ 1`). +2. Feature-detect by **graceful degradation**: version floor + disable-key heuristic + + `--engine` default `swarm`. Do not depend on a positive availability probe (none + exists). Log the chosen engine. This satisfies SC3 without a runtime exception. +3. Keep `swarm` the default (brief's Non-Goal: "ikke default-bytte"). Document the + per-run approval prompt for interactive (non-`-p`) sessions. + +Confidence in the recommendation: **high** for the mechanism (instruction-based + +in-context), **medium** for the exact fallback-detection ergonomics (the one documented +gap). This is sufficient to green-light `/trekplan` with Topic 1 resolved. + +## Sources + +| # | Source | Type | Quality | Used in | +|---|--------|------|---------|---------| +| 1 | code.claude.com/docs/en/workflows.md | official | high | Dim 1,2,3,4 + Synthesis | +| 2 | code.claude.com/docs/en/commands.md | official | high | Dim 1 (Workflow classification, WebSearch req) | +| 3 | code.claude.com/docs/en/skills.md | official | high | Dim 2 (Skill-tool allowlist) | +| 4 | github.com/anthropics/claude-code CHANGELOG (2.1.154, 2.1.x, 2.1.196) | official | high | Dim 1,4 (version floor, disable keys) | +| 5 | Local: CC 2.1.196 binary inspection (`find`/`grep`, `claude --version`) | codebase | high | Dim 1 (bundled, no SKILL.md) | +| 6 | Local: real `/deep-research` run — only `.js` script written, no `.md` report | codebase | high | Dim 3 (no on-disk artifact) | +| 7 | Local: `lib/validators/research-validator.mjs` schema | codebase | high | Open Questions (adapter feasibility / Topic 2) | From 581489a5133812c0ad1bd46f1e8a5aefad04c934 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:38:09 +0200 Subject: [PATCH 27/44] test(trekresearch): pin deep-research adapter output contract --- tests/commands/trekresearch-engine.test.mjs | 35 +++++++++++++ .../research-deep-research-adapted.md | 49 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/commands/trekresearch-engine.test.mjs create mode 100644 tests/fixtures/research-deep-research-adapted.md diff --git a/tests/commands/trekresearch-engine.test.mjs b/tests/commands/trekresearch-engine.test.mjs new file mode 100644 index 0000000..fbcea25 --- /dev/null +++ b/tests/commands/trekresearch-engine.test.mjs @@ -0,0 +1,35 @@ +// tests/commands/trekresearch-engine.test.mjs +// Step 1 (deep-research-engine): pin the contract the `--engine deep-research` +// adapter must hit. The adapted in-context `/deep-research` report, reduced into +// the research-brief schema, must pass research-validator under the strict +// default; and a brief missing a required section must fail. This is the one +// genuinely automatable slice of SC2 (schema, not provenance). + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateResearchContent } from '../../lib/validators/research-validator.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(HERE, '..', '..'); +const FIXTURE = join(ROOT, 'tests', 'fixtures', 'research-deep-research-adapted.md'); + +test('deep-research adapter output contract — valid brief passes, missing section fails', () => { + const text = readFileSync(FIXTURE, 'utf-8'); + + // (a) positive: the adapter's target output passes the validator (default = strict). + const okResult = validateResearchContent(text); + assert.equal(okResult.valid, true, JSON.stringify(okResult.errors)); + + // (b) negative: stripping a required section makes it fail with RESEARCH_MISSING_SECTION, + // giving the contract teeth (a fixture that always passes proves nothing). + const mutated = text.replace('## Dimensions', '## Removed'); + const badResult = validateResearchContent(mutated); + assert.equal(badResult.valid, false); + assert.ok( + badResult.errors.find(e => e.code === 'RESEARCH_MISSING_SECTION'), + 'expected RESEARCH_MISSING_SECTION; got ' + JSON.stringify(badResult.errors), + ); +}); diff --git a/tests/fixtures/research-deep-research-adapted.md b/tests/fixtures/research-deep-research-adapted.md new file mode 100644 index 0000000..c1189cb --- /dev/null +++ b/tests/fixtures/research-deep-research-adapted.md @@ -0,0 +1,49 @@ +--- +type: trekresearch-brief +created: 2026-06-30 +question: "Should /trekresearch delegate its external phase to the built-in /deep-research workflow?" +confidence: 0.8 +dimensions: 2 +mcp_servers_used: [] +local_agents_used: [] +external_agents_used: + - deep-research +--- + +# Deep-research engine adapter output + +> Fixture: a `/deep-research` in-context report reduced into the research-brief +> schema by the `--engine deep-research` adapter (Step 4). Models the target the +> adapter must hit; not real engine output. + +## Executive Summary + +Delegating the external phase to the built-in `/deep-research` workflow is a +viable opt-in engine that supplies fan-out and cited claim-verification for free. +Confidence is medium-high on the mechanism but lower on availability, because the +workflow exposes no positive "is-enabled" probe. The load-bearing caveat is +provenance: structural validity does not certify that the cited URLs are real, so +a swarm fallback plus a human URL spot-check stay mandatory. + +## Dimensions + +### Engine mechanism -- Confidence: high + +**External findings:** +- `/deep-research` is a built-in dynamic workflow reachable only by prose instruction, with no programmatic API (https://code.claude.com/docs/workflows). +- Its report lands in-context with no on-disk artifact, so the adapter must transform what is already in the turn (https://code.claude.com/docs/commands). + +### Fallback ergonomics -- Confidence: high + +**External findings:** +- There is no positive availability probe; only `disableWorkflows` / `CLAUDE_CODE_DISABLE_WORKFLOWS` off-switches and a 2.1.154 version floor are documented (https://code.claude.com/docs/skills). +- Disabled-workflow behavior under `claude -p` is undocumented, so the post-hoc presence check must be robust to every failure manifestation (https://github.com/anthropics/claude-code/issues/52272). + +## Sources + +| # | Source | Type | Quality | Used in | +|---|--------|------|---------|---------| +| 1 | https://code.claude.com/docs/workflows | official | high | Engine mechanism | +| 2 | https://code.claude.com/docs/commands | official | high | Engine mechanism | +| 3 | https://code.claude.com/docs/skills | official | high | Fallback ergonomics | +| 4 | https://github.com/anthropics/claude-code/issues/52272 | community | medium | Fallback ergonomics | From a6bed277d081dd9aae8b4be5d2f6f7662f19663f Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:41:19 +0200 Subject: [PATCH 28/44] feat(trekresearch): parse --engine {swarm|deep-research} flag --- commands/trekresearch.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/commands/trekresearch.md b/commands/trekresearch.md index 08bc064..b616067 100644 --- a/commands/trekresearch.md +++ b/commands/trekresearch.md @@ -1,7 +1,7 @@ --- name: trekresearch description: Deep research combining local codebase analysis with external knowledge, producing structured research briefs with triangulation and confidence ratings -argument-hint: "[--project ] [--quick | --local | --external | --fg] " +argument-hint: "[--project ] [--quick | --local | --external | --fg] [--engine swarm|deep-research] " model: opus allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion, WebSearch, WebFetch, mcp__tavily__tavily_search, mcp__tavily__tavily_research --- @@ -80,6 +80,16 @@ Supported flags: enforcement only fires at `≥ 2.2`. Absent → no version check. See `docs/HANDOVER-CONTRACTS.md` §Handover 1 for the pre-2.2 enforcement hole. +8. `--engine ` — opt-in external-research engine. Accepts `--engine ` + where `` is `swarm` or `deep-research`. **Default: `swarm`** (unchanged + behavior). `swarm` runs Voyage's own external-research agent swarm; + `deep-research` delegates the external phase to Claude Code's built-in + `/deep-research` dynamic workflow and adapts its report into the research-brief + schema (requires Claude Code 2.1.154+ and dynamic workflows enabled; falls back + to `swarm` and notes the fallback if unavailable — never hard-fails). Orthogonal + to `--profile`/`phase_signals`; only affects the external phase. Set + **engine = {swarm|deep-research}** (the *requested* engine). + Flags can be combined: - `--local` — local-only research - `--external --quick` — external-only, lightweight @@ -87,7 +97,7 @@ Flags can be combined: - `--quick` alone implies both local and external (lightweight) Defaults: **scope = both**, **execution = foreground** (only mode as of -v2.4.0), **project_dir = none**. +v2.4.0), **project_dir = none**, **engine = swarm**. After stripping flags, the remaining text is the **research question**. @@ -108,6 +118,7 @@ Modes: --external Only external research agents (skip codebase analysis) --fg No-op alias (foreground is the only mode as of v2.4.0) --project Write brief into an trekbrief project folder (auto-indexed) + --engine Opt-in external-research engine: swarm (default) | deep-research Flags can be combined: --local, --external --quick, --project --external @@ -118,6 +129,7 @@ Examples: /trekresearch --external What are the security implications of using Redis for sessions? /trekresearch --fg --local What patterns does this codebase use for database access? /trekresearch --project .claude/projects/2026-04-18-jwt-auth --external What JWT library is best for Node.js? + /trekresearch --project --external --engine deep-research ``` Do not continue past this step if no question was provided. @@ -126,6 +138,7 @@ Report the detected mode: ``` Mode: {default | quick}, Scope: {both | local | external}, Execution: foreground Project: {project_dir or "-"} +Engine (requested): {swarm | deep-research} Question: {research question} ``` @@ -418,6 +431,7 @@ Record format (one JSON line): "question": "{research question (first 100 chars)}", "mode": "{default|quick}", "scope": "{both|local|external}", + "engine": "{effective engine: swarm|deep-research}", "slug": "{brief slug}", "project_dir": "{project_dir or null}", "brief_path": "{brief_destination}", @@ -455,8 +469,8 @@ VOYAGE_PROFILE=balanced /trekresearch ``` Stats records emit `profile`, `phase_models`, `parallel_agents`, -`external_research_enabled`, and `profile_source` so operators can audit -which profile drove which session. +`external_research_enabled`, `profile_source`, and `engine` so operators can +audit which profile and engine drove which session. ## Composition rule (v5.1) From 0e657de023ef1bfccda47820b457e5ea95b53d74 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:43:20 +0200 Subject: [PATCH 29/44] feat(trekresearch): add deep-research engine-selection fork --- commands/trekresearch.md | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/commands/trekresearch.md b/commands/trekresearch.md index b616067..4a6d220 100644 --- a/commands/trekresearch.md +++ b/commands/trekresearch.md @@ -305,6 +305,63 @@ For each local agent, prompt with the research question, NOT a task description: - convention-scanner: "Discover coding conventions relevant to evaluating {question}. What patterns would a solution need to follow?" +### Engine selection (scope = both or external) + +`--engine` affects ONLY the external portion of research. The local agents +(`### Local agents` above) and Phases 6–7 (triangulation, synthesis, brief +writing) are **engine-agnostic** — they run identically regardless of engine. + +`--engine` is **moot** (treated as `swarm`) whenever the external phase does not +run at all: `--local`, `--quick`, `effort == 'low'`, or a profile with +`external_research_enabled == false` (the `economy`/`balanced` auto-disable — see +Profile below). The profile's on/off switch wins. Initialize +`effective_engine = {requested engine}`. + +**engine = swarm (default):** run the `### External agents` + `### Bridge agent` +blocks below unchanged. This is byte-for-byte the current path, so `--engine swarm` +changes nothing (SC1). Keep the native-swarm anchors intact ("in parallel", +"single message", `model: "opus"`). + +**engine = deep-research:** + +1. **Coarse pre-gate (best-effort, NOT a trust signal).** `Bash: claude --version`; + parse the leading `X.Y.Z` (e.g. from `2.1.196 (Claude Code)`) and compare + numerically against `2.1.154` — split each on `.` and compare major, then minor, + then patch as integers (do NOT string-compare; lexical comparison mis-orders + multi-digit patch numbers). If the version is `< 2.1.154`, OR if + `disableWorkflows: true` / `CLAUDE_CODE_DISABLE_WORKFLOWS=1` is set, skip to the + fallback (step 4). **If `claude` is not on PATH inside the Bash tool (possible + under `claude -p`) or the version cannot be parsed, treat the pre-gate as + *indeterminate* and proceed to step 2 — do NOT hard-fail.** There is no positive + availability probe (research Dim 4), so a passing pre-gate does not guarantee the + workflow runs; the post-hoc check (step 3) is the authoritative guard. + +2. **Run.** Instruct Claude (in prose, this turn) to run + `/deep-research ` and request per-claim citations. Note: + interactive default/acceptEdits triggers a per-run approval prompt; `claude -p` / + SDK / bypass runs immediately. + +3. **Post-hoc presence + provenance check (the real guard).** Verify a real, cited + `/deep-research` report actually landed in context — substantive findings with + citations, not an empty/denied/errored turn and not bare error text. This check + must be **robust to all failure manifestations** (workflow disabled, approval + denied, runtime error, empty output), because the disabled-headless behavior is + undocumented: no recognizable cited report in context → fall back, regardless of + how the failure surfaces. + +4. **On no real report (fallback):** set `effective_engine = swarm`, run the swarm + blocks below, and **log the fallback at this decision point** — print + `Engine: deep-research → swarm (fallback: )` and carry the reason into the + Phase-8 Present summary and the brief's `## Executive Summary`. **NEVER fabricate + or synthesize a substitute report** — a structurally-valid-but-invented brief + passes the structure-only validator and silently poisons `/trekplan`; that is the + worst outcome of this feature. + +5. **On a real report:** keep `effective_engine = deep-research`, log + `Engine: deep-research (active)`, and carry the report into Phase 6 triangulation + as the external-findings input (adapted in Phase 7 — see the Deep-research engine + adapter below). + ### External agents (scope = both or external) Launch the new research-specialized agents: @@ -397,6 +454,7 @@ Present a summary to the user: **Mode:** {default | quick}, Scope: {both | local | external} **Brief:** {brief_destination} **Project:** {project_dir or "-"} +**Engine (effective):** {swarm | deep-research}{, with fallback reason if it fell back} **Confidence:** {overall confidence 0.0-1.0} **Dimensions:** {N} researched **Agents:** {N} local + {N} external + {gemini: used | unavailable | skipped} From 4ec979747b1019657924477e3417638f1085f294 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:44:37 +0200 Subject: [PATCH 30/44] feat(trekresearch): add deep-research in-context adapter + self-check --- commands/trekresearch.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/commands/trekresearch.md b/commands/trekresearch.md index 4a6d220..2fa5798 100644 --- a/commands/trekresearch.md +++ b/commands/trekresearch.md @@ -443,6 +443,45 @@ Write the brief to the `brief_destination` computed in Phase 1: Create the parent directory if it does not exist. +### Deep-research engine adapter (engine = deep-research only) + +**Only when `effective_engine == deep-research`.** The swarm path skips this +entirely — its findings already flow through Phases 6–7 unchanged (SC1). + +Transform the in-context `/deep-research` report INTO +`@${CLAUDE_PLUGIN_ROOT}/templates/research-brief-template.md` — do NOT paste the +raw report. Specifically: + +- Reduce the report to ≥ 1 `### {Dimension} -- Confidence: {high|medium|low}` + entry, each carrying **External findings** bullets with per-claim source URLs. + Local findings still come from the local agents (Phase 4) and are merged in per + dimension as usual. +- Emit a numeric `confidence ∈ [0,1]` in frontmatter and a 3-sentence + `## Executive Summary` (answer, confidence, key caveat). +- Populate `## Sources` from the report's citations. +- **If the report lacks per-claim URLs, lower the confidence and note the gap in + `## Open Questions` — do NOT fabricate URLs.** Provenance you cannot cite is not + provenance. +- If the report is large, bound the transform to the top dimensions to avoid + context truncation. + +### Output self-check (engine = deep-research only) + +**Only when `effective_engine == deep-research`.** After writing to +`brief_destination`, run the output validator and repair-or-fall-back. This mirrors +the trekplan Phase-8 write→validate→repair self-check; the swarm path does NOT run +it, so swarm behavior is unchanged (SC1): + +```bash +node ${CLAUDE_PLUGIN_ROOT}/lib/validators/research-validator.mjs --json "{brief_destination}" +``` + +On `valid: false`, repair the brief to satisfy the reported errors and re-run the +validator. If it cannot be made valid (e.g. the report was too thin to yield even +one dimension), set `effective_engine = swarm`, fall back to the swarm engine for +this run (and log the fallback per the Engine selection step), rather than emit an +invalid brief. + ## Phase 8 — Present and track Present a summary to the user: From 76818b2459a78a8e29ea9169372b2e3baab1224a Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:46:39 +0200 Subject: [PATCH 31/44] docs(trekresearch): document --engine in command-modes, CLAUDE, README --- CLAUDE.md | 2 +- README.md | 1 + docs/command-modes.md | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9ebdfd1..b746629 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ Voyage — a contract-driven Claude Code pipeline: brief, research, plan, execut | Command | Description | Model | |---------|-------------|-------| | `/trekbrief` | Brief — interactive interview produces a task brief with explicit research plan; optionally orchestrates the pipeline | opus | -| `/trekresearch` | Research — deep local + external research, produces structured research brief | opus | +| `/trekresearch` | Research — deep local + external research, produces structured research brief. Opt-in `--engine {swarm\|deep-research}` delegates the external phase to Claude Code's built-in `/deep-research` workflow (swarm default) | opus | | `/trekplan` | Plan — brief-reviewer, explore, plan, review. Requires `--brief` or `--project`. Auto-discovers `architecture/overview.md` if present | opus | | `/trekexecute` | Execute — disciplined plan/session-spec executor with failure recovery | opus | | `/trekreview` | Review — independent post-hoc review of delivered code against the brief. Produces `review.md` with severity-tagged findings (Handover 6) | opus | diff --git a/README.md b/README.md index 144bc26..c90b119 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ Output: | **External** | `/trekresearch --external ` | Only external research agents (skip codebase analysis) | | **Foreground** | `/trekresearch --fg ` | No-op alias (foreground is default since v2.4.0) | | **Profile** | `/trekresearch --profile ` | (v4.1.0) Pin model profile for the research phase. See [Profile system](#profile-system-v410). | +| **Engine** | `/trekresearch --external --engine deep-research ` | Delegate the external phase to Claude Code's built-in `/deep-research` workflow; falls back to `swarm` if unavailable. Default `swarm`. | Flags combine: `--project --external`. diff --git a/docs/command-modes.md b/docs/command-modes.md index e68cf0a..5390150 100644 --- a/docs/command-modes.md +++ b/docs/command-modes.md @@ -26,6 +26,7 @@ Always interactive. Phase 3 is a section-driven completeness loop (no hard cap o | `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. | | `--min-brief-version ` | (S18) Warn — never block — if an attached `--project` brief declares a version below `` (e.g. `2.2`), i.e. sidesteps framing enforcement | | `--profile ` | (v4.1.0) Model profile for the research phase. | +| `--engine {swarm\|deep-research}` | (deep-research-engine) Opt-in external-research engine; `deep-research` delegates the external phase to Claude Code's built-in `/deep-research` workflow (CC 2.1.154+), falls back to `swarm`. Default `swarm`. | Flags combine: `--project --local`, `--external --quick`. From 937482067ddb5e4598a64666a2c15c080bd80450 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:49:02 +0200 Subject: [PATCH 32/44] test(trekresearch): pin --engine doc-consistency across surfaces --- tests/lib/doc-consistency.test.mjs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs index f8c03c9..cfd00c9 100644 --- a/tests/lib/doc-consistency.test.mjs +++ b/tests/lib/doc-consistency.test.mjs @@ -1065,6 +1065,32 @@ test('S18: --min-brief-version is documented at the trekplan + trekresearch boun } }); +test('deep-research-engine: --engine is documented + consistent across surfaces', () => { + // Cross-doc pin mirroring S18 (:1057). The opt-in external-research engine flag + // must be discoverable wherever /trekresearch flags live: the command itself + // plus the three reference surfaces. + for (const f of ['commands/trekresearch.md', 'docs/command-modes.md', 'CLAUDE.md', 'README.md']) { + assert.ok( + read(f).includes('--engine'), + `${f} must document the --engine flag (deep-research-engine)`, + ); + } + // README documents it specifically as an **Engine** mode-table row. + assert.ok( + /\*\*Engine\*\*/.test(read('README.md')), + 'README.md must document --engine as an **Engine** mode row', + ); + // The command prose must name both engine values and the swarm fallback, so the + // opt-in + graceful-degradation contract is pinned — not merely the flag string. + const research = read('commands/trekresearch.md'); + assert.ok(/\bswarm\b/.test(research), 'trekresearch.md must name the swarm engine value'); + assert.ok(/\bdeep-research\b/.test(research), 'trekresearch.md must name the deep-research engine value'); + assert.ok( + /fall back|falls back/.test(research), + 'trekresearch.md must document the swarm fallback (graceful degradation)', + ); +}); + test('S18: HANDOVER-CONTRACTS documents the pre-2.2 zero-framing-enforcement hole', () => { // The framing defense is producer-elective: a brief declaring ≤ 2.1 sidesteps // it entirely. Handover 1 (PUBLIC CONTRACT) must disclose this and name the remedy. From 357e17b176d9d019c882ff51386e2da5893c9cfc Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 16:54:08 +0200 Subject: [PATCH 33/44] feat(validators): add fable to BASE_ALLOWED_MODELS with accept/reject coverage --- lib/validators/profile-validator.mjs | 4 +- tests/validators/profile-validator.test.mjs | 72 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/lib/validators/profile-validator.mjs b/lib/validators/profile-validator.mjs index 37909e8..b5fb3a0 100644 --- a/lib/validators/profile-validator.mjs +++ b/lib/validators/profile-validator.mjs @@ -21,7 +21,7 @@ // PROFILE_READ_ERROR — file unreadable or parse-error // PROFILE_NOT_FOUND — file does not exist // -// Allowed model values: ['sonnet', 'opus']. Haiku is allowed only when +// Allowed model values: ['sonnet', 'opus', 'fable']. Haiku is allowed only when // VOYAGE_ALLOW_HAIKU=1 (per global CLAUDE.md modellvalg-prinsipp: Haiku skal // ikke brukes som default; eksplisitt opt-in for spesielle bruksmønstre). @@ -42,7 +42,7 @@ export const PROFILE_REQUIRED_PHASES = Object.freeze([ 'brief', 'research', 'plan', 'execute', 'review', 'continue', ]); -export const BASE_ALLOWED_MODELS = Object.freeze(['sonnet', 'opus']); +export const BASE_ALLOWED_MODELS = Object.freeze(['sonnet', 'opus', 'fable']); function getAllowedModels(env = process.env) { if (env.VOYAGE_ALLOW_HAIKU === '1') { diff --git a/tests/validators/profile-validator.test.mjs b/tests/validators/profile-validator.test.mjs index 37c3806..7cd3fec 100644 --- a/tests/validators/profile-validator.test.mjs +++ b/tests/validators/profile-validator.test.mjs @@ -12,6 +12,7 @@ import { validateProfileContent, PROFILE_REQUIRED_FIELDS, PROFILE_REQUIRED_PHASES, + BASE_ALLOWED_MODELS, } from '../../lib/validators/profile-validator.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -93,6 +94,77 @@ brief_reviewer_iter_cap: 1 `expected valid with VOYAGE_ALLOW_HAIKU=1, got: ${JSON.stringify(allowed.errors)}`); }); +// Fable tier (v5.9): fable accepted under DEFAULT env (no opt-in flag), +// unknown models still rejected — the allowlist gate must demonstrably fire. + +test('fable accepted in phase_models under default env (no env flag)', () => { + const fableProfile = `--- +profile_version: "1.0" +name: fable-inline +phase_models: + - phase: brief + model: fable + - phase: research + model: fable + - phase: plan + model: fable + - phase: execute + model: fable + - phase: review + model: fable + - phase: continue + model: fable +parallel_agents_min: 6 +parallel_agents_max: 8 +external_research_enabled: true +brief_reviewer_iter_cap: 3 +--- +`; + const r = validateProfileContent(fableProfile, { env: { /* default: no flags */ } }); + assert.equal(r.valid, true, + `expected fable accepted under default env, got: ${JSON.stringify(r.errors)}`); + assert.equal(r.errors.length, 0); +}); + +test('unknown model gpt5 rejected with PROFILE_INVALID_MODEL under default env', () => { + const gpt5Profile = `--- +profile_version: "1.0" +name: gpt5-inline +phase_models: + - phase: brief + model: gpt5 + - phase: research + model: sonnet + - phase: plan + model: opus + - phase: execute + model: sonnet + - phase: review + model: opus + - phase: continue + model: sonnet +parallel_agents_min: 2 +parallel_agents_max: 4 +external_research_enabled: false +brief_reviewer_iter_cap: 1 +--- +`; + const r = validateProfileContent(gpt5Profile, { env: { /* default: no flags */ } }); + assert.equal(r.valid, false); + const found = r.errors.find(e => e.code === 'PROFILE_INVALID_MODEL' && /gpt5/.test(e.message)); + assert.ok(found, `expected PROFILE_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`); +}); + +// BASE_ALLOWED_MODELS allowlist drift-pin (mirrors the PROFILE_REQUIRED_FIELDS pin) + +test('BASE_ALLOWED_MODELS export drift-pin', () => { + assert.deepEqual( + [...BASE_ALLOWED_MODELS], + ['sonnet', 'opus', 'fable'], + 'BASE_ALLOWED_MODELS contract drift — pin contract', + ); +}); + // Required fields presence test('PROFILE_MISSING_FIELD when name absent', () => { From 8b7a849a762b6f182d6088d316dd657201d1f7a4 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 16:56:26 +0200 Subject: [PATCH 34/44] test(validators): cover fable accept + unknown-model reject in both gate layers --- tests/fixtures/brief-effort-fable.md | 45 +++++++++++++++++++++++ tests/lib/phase-signal-resolver.test.mjs | 6 ++- tests/validators/brief-validator.test.mjs | 21 ++++++++++- 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/brief-effort-fable.md diff --git a/tests/fixtures/brief-effort-fable.md b/tests/fixtures/brief-effort-fable.md new file mode 100644 index 0000000..b1d50fb --- /dev/null +++ b/tests/fixtures/brief-effort-fable.md @@ -0,0 +1,45 @@ +--- +type: trekbrief +brief_version: "2.1" +created: 2026-07-02 +task: "Fixture: high-effort all phases on fable (v5.9 allowlist test)" +slug: brief-effort-fable +project_dir: .claude/projects/2026-07-02-brief-effort-fable/ +research_topics: 0 +research_status: complete +auto_research: false +interview_turns: 4 +source: fixture +phase_signals: + - phase: research + effort: high + model: fable + - phase: plan + effort: high + model: fable + - phase: execute + effort: high + model: fable + - phase: review + effort: high + model: fable +--- + +# Task: High-effort fable fixture + +## Intent + +Test fixture for the v5.9 fable model tier — all 4 phases at the +high effort tier with explicit fable model overrides. Mirrors +brief-effort-high.md with `model: opus` replaced by `model: fable`. + +## Goal + +Resolver returns `{effort: 'high', model: 'fable'}` for each of the 4 +PHASE_SIGNAL_PHASES. + +## Success Criteria + +- Validator passes with no BRIEF_INVALID_MODEL. +- resolvePhaseSignal(fm, phase).effort === 'high' for all 4 phases. +- resolvePhaseSignal(fm, phase).model === 'fable' for all 4 phases. diff --git a/tests/lib/phase-signal-resolver.test.mjs b/tests/lib/phase-signal-resolver.test.mjs index bfb1872..8799b2e 100644 --- a/tests/lib/phase-signal-resolver.test.mjs +++ b/tests/lib/phase-signal-resolver.test.mjs @@ -50,7 +50,7 @@ test('resolvePhaseSignal — defensive: null/non-object input returns null', () test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in-depth gate)', () => { // MAJOR fix (S4): line that copies `model` must gate against the same - // allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus']), + // allowlist brief-validator uses (BASE_ALLOWED_MODELS = ['sonnet','opus','fable']), // mirroring how effort is gated against EFFORT_LEVELS. A brief that slipped // validation (hand-edited, validation skipped) must not hand a junk model // string to a command that then spawns an agent with `model: `. @@ -70,15 +70,17 @@ test('resolvePhaseSignal — drops model not in BASE_ALLOWED_MODELS (defense-in- assert.ok(!('model' in review), 'model key absent for haiku'); }); -test('resolvePhaseSignal — keeps valid models (sonnet, opus) after gating', () => { +test('resolvePhaseSignal — keeps valid models (sonnet, opus, fable) after gating', () => { const fm = { phase_signals: [ { phase: 'research', effort: 'low', model: 'sonnet' }, { phase: 'execute', effort: 'high', model: 'opus' }, + { phase: 'review', effort: 'high', model: 'fable' }, ], }; assert.equal(resolvePhaseSignal(fm, 'research').model, 'sonnet'); assert.equal(resolvePhaseSignal(fm, 'execute').model, 'opus'); + assert.equal(resolvePhaseSignal(fm, 'review').model, 'fable'); }); test('resolvePhaseSignalFromFile + CLI shim — writes JSON to stdout, exit 0', () => { diff --git a/tests/validators/brief-validator.test.mjs b/tests/validators/brief-validator.test.mjs index 3a05928..314435f 100644 --- a/tests/validators/brief-validator.test.mjs +++ b/tests/validators/brief-validator.test.mjs @@ -1,7 +1,7 @@ import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { execFileSync } from 'node:child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { validateBriefContent } from '../../lib/validators/brief-validator.mjs'; @@ -255,6 +255,25 @@ test('validateBrief — v5.1.1: UNQUOTED brief_version 2.1 WITH phase_signals is assert.ok(!r.errors.find(e => e.code === 'BRIEF_V51_MISSING_SIGNALS')); }); +// --- v5.9 — fable model tier (BASE_ALLOWED_MODELS widened to three values) --- + +test('validateBrief — v5.9: fable phase_signals fixture accepted (no BRIEF_INVALID_MODEL)', () => { + const t = readFileSync(new URL('../fixtures/brief-effort-fable.md', import.meta.url), 'utf-8'); + const r = validateBriefContent(t, { strict: true }); + assert.equal(r.valid, true, JSON.stringify(r.errors)); + assert.ok(!r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL')); +}); + +test('validateBrief — v5.9: unknown model gpt5 in phase_signals rejected with BRIEF_INVALID_MODEL', () => { + const t = GOOD_BRIEF + .replace('brief_version: "2.0"', 'brief_version: "2.1"') + .replace('source: interview\n', `source: interview\n${SIGNALS_BLOCK.replace('model: opus', 'model: gpt5')}`); + const r = validateBriefContent(t, { strict: true }); + assert.equal(r.valid, false); + assert.ok(r.errors.find(e => e.code === 'BRIEF_INVALID_MODEL'), + `expected BRIEF_INVALID_MODEL for gpt5, got: ${JSON.stringify(r.errors)}`); +}); + // --- v5.5 — framing enforcement + obligatory TL;DR (gated at brief_version ≥ 2.2) --- // Operator decision (S6, option A1): framing + TL;DR are hard BLOCKERs for briefs // declaring brief_version ≥ 2.2; existing 2.0/2.1 briefs stay valid (forward-compat, From 84fbee231312e1807095caa98b0483ffa8975916 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 16:59:26 +0200 Subject: [PATCH 35/44] feat(profiles): add built-in fable profile (all six phases on fable) --- lib/parsers/manifest-yaml.mjs | 2 +- lib/profiles/fable.yaml | 21 ++++++++++++++++ lib/profiles/phase-signal-resolver.mjs | 5 ++++ lib/profiles/resolver.mjs | 27 ++++++++++++++++----- tests/validators/profile-validator.test.mjs | 6 ++--- 5 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 lib/profiles/fable.yaml diff --git a/lib/parsers/manifest-yaml.mjs b/lib/parsers/manifest-yaml.mjs index c5a3903..93d386d 100644 --- a/lib/parsers/manifest-yaml.mjs +++ b/lib/parsers/manifest-yaml.mjs @@ -32,7 +32,7 @@ const OPTIONAL_KEYS = [ const OPTIONAL_BOOLEAN_KEYS = new Set(OPTIONAL_KEYS); // Optional string-typed manifest keys (v4.1 Step 3 — additive forward-compat). -// `profile_used`: name of the model profile (economy|balanced|premium|) the +// `profile_used`: name of the model profile (economy|balanced|premium|fable|) the // step was executed under. Absence is fine (v4.0 manifests have no // profile concept); presence MUST be a string. // Unlike OPTIONAL_BOOLEAN_KEYS, absence is NOT defaulted — the field is simply diff --git a/lib/profiles/fable.yaml b/lib/profiles/fable.yaml new file mode 100644 index 0000000..8d69b18 --- /dev/null +++ b/lib/profiles/fable.yaml @@ -0,0 +1,21 @@ +--- +profile_version: "1.0" +name: fable +phase_models: + - phase: brief + model: fable + - phase: research + model: fable + - phase: plan + model: fable + - phase: execute + model: fable + - phase: review + model: fable + - phase: continue + model: fable +parallel_agents_min: 6 +parallel_agents_max: 8 +external_research_enabled: true +brief_reviewer_iter_cap: 3 +--- diff --git a/lib/profiles/phase-signal-resolver.mjs b/lib/profiles/phase-signal-resolver.mjs index 965e560..0292c71 100644 --- a/lib/profiles/phase-signal-resolver.mjs +++ b/lib/profiles/phase-signal-resolver.mjs @@ -67,6 +67,11 @@ export function resolvePhaseSignalFromFile(briefPath, phase) { } // CLI shim — mirrors lib/validators/brief-validator.mjs:168 pattern. +// Footgun guard (v5.9): this shim's `model` output is brief-signal-only — it +// never consults the profile layer. For command wiring, the composed resolver +// CLI (`resolver.mjs --resolve-phase-model`, brief > profile > default) is the +// single resolution source for {effort, model}. Do not re-wire commands/*.md +// Bash blocks back to this shim. if (import.meta.url === `file://${process.argv[1]}`) { const args = process.argv.slice(2); const getArg = (name) => { diff --git a/lib/profiles/resolver.mjs b/lib/profiles/resolver.mjs index 3d17bc1..b2ec44b 100644 --- a/lib/profiles/resolver.mjs +++ b/lib/profiles/resolver.mjs @@ -45,7 +45,7 @@ import { resolvePhaseSignal } from './phase-signal-resolver.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const BUILTIN_PROFILES_DIR = __dirname; // lib/profiles/ -const BUILTIN_NAMES = new Set(['economy', 'balanced', 'premium']); +const BUILTIN_NAMES = new Set(['economy', 'balanced', 'premium', 'fable']); /** * Resolve the path to a profile file. @@ -221,7 +221,13 @@ export function validateProfileFile(path, opts = {}) { * @param {string|null} briefPath Absolute or repo-relative path to brief.md, or null * @param {string[]|object} argv Full process.argv array OR parsed flags object * @param {object} [env] Environment-variable record (defaults to process.env) - * @returns {{model: string, source: 'brief-signal'|'flag'|'env'|'default'}} + * @returns {{effort?: string, model: string, source: 'brief-signal'|'flag'|'env'|'default'}} + * + * `effort` (v5.9 ADDITIVE) is the brief signal's effort passed through when + * present — so commands consume ONE coherent {effort, model, source} result + * instead of two split CLI calls. Absent when the brief carries no valid + * effort signal for the phase (commands default to 'standard' per the + * composition rule). * * Error handling contract: * - Never throws. Any failure (ENOENT on briefPath, malformed YAML, missing @@ -234,7 +240,10 @@ export function validateProfileFile(path, opts = {}) { * directly; commands must inject {resolved model} at Agent-tool spawn sites. */ export function resolvePhaseModel(phase, briefPath, argv, env = process.env) { - // Step 1: brief-signal lookup + // Step 1: brief-signal lookup. `effort` is captured independently of `model` + // so a signal like {effort: high} (no model) still passes effort through + // while the model falls to the profile layer. + let effort; if (typeof briefPath === 'string' && briefPath.length > 0 && existsSync(briefPath)) { let fm = null; try { @@ -246,8 +255,11 @@ export function resolvePhaseModel(phase, briefPath, argv, env = process.env) { } if (fm) { const signal = resolvePhaseSignal(fm, phase); + if (signal && typeof signal.effort === 'string') effort = signal.effort; if (signal && typeof signal.model === 'string' && signal.model.length > 0) { - return { model: signal.model, source: 'brief-signal' }; + return effort !== undefined + ? { effort, model: signal.model, source: 'brief-signal' } + : { model: signal.model, source: 'brief-signal' }; } } } @@ -278,7 +290,9 @@ export function resolvePhaseModel(phase, briefPath, argv, env = process.env) { } } const model = phaseModels[phase] || 'opus'; - return { model, source: profile_source }; + return effort !== undefined + ? { effort, model, source: profile_source } + : { model, source: profile_source }; } // CLI shim — invoked by commands/trek*.md via Bash. @@ -300,7 +314,8 @@ if (import.meta.url === `file://${process.argv[1]}`) { if (args.includes('--json')) { process.stdout.write(JSON.stringify(r) + '\n'); } else { - process.stdout.write(`model=${r.model} source=${r.source}\n`); + const effort = 'effort' in r ? ` effort=${r.effort}` : ''; + process.stdout.write(`model=${r.model} source=${r.source}${effort}\n`); } process.exit(0); } diff --git a/tests/validators/profile-validator.test.mjs b/tests/validators/profile-validator.test.mjs index 7cd3fec..f86d8a0 100644 --- a/tests/validators/profile-validator.test.mjs +++ b/tests/validators/profile-validator.test.mjs @@ -1,5 +1,5 @@ // tests/validators/profile-validator.test.mjs -// SC #1, #2, #3: profile-validator validates lib/profiles/{economy,balanced,premium}.yaml +// SC #1, #2, #3: profile-validator validates lib/profiles/{economy,balanced,premium,fable}.yaml // (innebygde profiler) plus rejects invalid models and invalid enum types. import { test } from 'node:test'; @@ -18,9 +18,9 @@ import { const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(__dirname, '..', '..'); -// SC #1: alle 3 innebygde profiler grønne +// SC #1: alle 4 innebygde profiler grønne -for (const profileName of ['economy', 'balanced', 'premium']) { +for (const profileName of ['economy', 'balanced', 'premium', 'fable']) { test(`SC #1: lib/profiles/${profileName}.yaml validates clean`, () => { const r = validateProfile(join(REPO_ROOT, 'lib', 'profiles', `${profileName}.yaml`)); assert.equal(r.valid, true, From 5c37b95dfb7810637d02046fa51c009c05b0eeeb Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:00:43 +0200 Subject: [PATCH 36/44] test(profiles): pin fable profile resolution end-to-end --- tests/lib/profile-application.test.mjs | 14 +++++++++++++ tests/lib/profile-resolver.test.mjs | 27 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/lib/profile-application.test.mjs b/tests/lib/profile-application.test.mjs index 6a36513..5f334a5 100644 --- a/tests/lib/profile-application.test.mjs +++ b/tests/lib/profile-application.test.mjs @@ -50,6 +50,20 @@ test('SC #5: loadProfile("premium") returns all-opus', () => { } }); +test('SC #5: loadProfile("fable") returns all-fable (BUILTIN_NAMES canary)', () => { + // Throws PROFILE_NOT_FOUND if 'fable' regresses out of BUILTIN_NAMES — + // the silent-fallback-to-premium failure mode this pin exists to catch. + const p = loadProfile('fable'); + assert.equal(p.name, 'fable'); + for (const phase of ['brief', 'research', 'plan', 'execute', 'review', 'continue']) { + assert.equal(p.phase_models[phase], 'fable', `fable ${phase} should be fable`); + } + assert.equal(p.parallel_agents_min, 6); + assert.equal(p.parallel_agents_max, 8); + assert.equal(p.external_research_enabled, true); + assert.equal(p.brief_reviewer_iter_cap, 3); +}); + test('SC #5: loadProfile throws PROFILE_NOT_FOUND for unknown profile', () => { try { loadProfile('does-not-exist-xyz'); diff --git a/tests/lib/profile-resolver.test.mjs b/tests/lib/profile-resolver.test.mjs index 4eef940..f87fb42 100644 --- a/tests/lib/profile-resolver.test.mjs +++ b/tests/lib/profile-resolver.test.mjs @@ -60,3 +60,30 @@ test('resolvePhaseModel — Case 6 (defensive): null briefPath falls through to assert.equal(r.model, 'opus', 'premium.plan default = opus'); assert.equal(r.source, 'default'); }); + +// v5.9 — fable profile composition (brief SC 4) + effort passthrough coherence + +test('resolvePhaseModel — --profile fable: all six phases resolve model fable (no brief signal)', () => { + for (const phase of ['brief', 'research', 'plan', 'execute', 'review', 'continue']) { + const r = resolvePhaseModel(phase, null, ['--profile', 'fable'], {}); + assert.equal(r.model, 'fable', `fable.${phase} should be fable; got ${JSON.stringify(r)}`); + assert.equal(r.source, 'flag'); + } +}); + +test('resolvePhaseModel — brief signal (opus) beats --profile fable (composition precedence pin)', () => { + // brief-effort-high.md pins execute to model: opus. Brief must beat the fable profile. + const r = resolvePhaseModel('execute', FIXTURE('brief-effort-high.md'), ['--profile', 'fable'], {}); + assert.equal(r.model, 'opus', `brief signal should beat fable profile; got ${JSON.stringify(r)}`); + assert.equal(r.source, 'brief-signal'); +}); + +test('resolvePhaseModel — composed output carries {effort, model} atomically (v5.9 passthrough)', () => { + // brief-effort-high.md: execute → {effort: high, model: opus}. The composed + // resolver must return BOTH fields from one call — the split-CLI design this + // passthrough replaced could drift effort and model apart. + const r = resolvePhaseModel('execute', FIXTURE('brief-effort-high.md'), [], {}); + assert.equal(r.effort, 'high', `effort must pass through; got ${JSON.stringify(r)}`); + assert.equal(r.model, 'opus'); + assert.equal(r.source, 'brief-signal'); +}); From dcc71d9577f87d298c69064c60cd7759818ba2a1 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:02:56 +0200 Subject: [PATCH 37/44] feat(trekbrief): add fable tier option to Phase 3.5 loop --- commands/trekbrief.md | 14 ++++++++++++-- templates/trekbrief-template.md | 5 +++-- tests/commands/trekbrief.test.mjs | 18 ++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/commands/trekbrief.md b/commands/trekbrief.md index c9f02d8..f16d81f 100644 --- a/commands/trekbrief.md +++ b/commands/trekbrief.md @@ -367,13 +367,14 @@ in the question body so the operator sees why it was picked. ### The loop — 4 tier-coupled AskUserQuestion calls Loop over `[research, plan, execute, review]` in order. For each phase, -issue one `AskUserQuestion` with 3 options: +issue one `AskUserQuestion` with 4 options: | Option | Maps to phase_signals entry | |--------|----------------------------| | **Low effort** | `{phase: , effort: low, model: sonnet}` | | **Standard (default)** | `{phase: , effort: standard}` *(model omitted — composition falls through to profile)* | | **High effort** | `{phase: , effort: high, model: opus}` | +| **Fable (max quality)** | `{phase: , effort: high, model: fable}` | The proposed tier per phase (from the default-derivation heuristic) MUST be labelled `(default)` in the option list so the operator can one-click @@ -384,6 +385,15 @@ The mapping table is canonical: - `low → {effort: low, model: sonnet}` (force sonnet for the low-cost path) - `standard → {effort: standard}` (model omitted; composition rule resolves via profile) - `high → {effort: high, model: opus}` (force opus for the high-confidence path) +- `fable → {effort: high, model: fable}` (force Fable 5 for the max-quality path) + +The fable tier reuses `effort: high` semantics — full swarm, contrarian + +gemini always-on; `EFFORT_LEVELS` is unchanged (Voyage effort is orchestration +shape, not model reasoning effort). Model reasoning effort is inherited from +the session: Fable 5's default effort is `high`, NOT xhigh. To run xhigh, the +operator sets it at session level via `/effort xhigh`, the `effortLevel` +setting, or `CLAUDE_CODE_EFFORT_LEVEL` — switching model resets effort to the +model default, so it does not follow the model. ### Force-stop handling @@ -891,7 +901,7 @@ Never let stats failures block the workflow. ## Profile (v4.1) Accepts `--profile ` where `` is one of `economy`, `balanced`, -`premium`, or a custom profile under `voyage-profiles/`. Default: `premium`. +`premium`, `fable`, or a custom profile under `voyage-profiles/`. Default: `premium`. Resolution order (per `lib/profiles/resolver.mjs`): 1. `--profile` flag (source: `flag`) diff --git a/templates/trekbrief-template.md b/templates/trekbrief-template.md index 242e954..c8ec99e 100644 --- a/templates/trekbrief-template.md +++ b/templates/trekbrief-template.md @@ -17,8 +17,9 @@ source: {interview | manual} # plan polishing a wrong premise after a rejected iteration). framing: {preserve | refine | replace | new-direction} # v5.1 — per-phase effort + model signal (Phase 3.5). -# `effort` ∈ {low, standard, high}. Omit `model:` for `standard` so composition -# falls through to profile resolver. Force-stop alternative is the commented +# `effort` ∈ {low, standard, high}; `model` ∈ {sonnet, opus, fable} (v5.9). +# Omit `model:` for `standard` so composition falls through to profile +# resolver. Force-stop alternative is the commented # `phase_signals_partial: true` below (mutually exclusive with `phase_signals`). phase_signals: - phase: research diff --git a/tests/commands/trekbrief.test.mjs b/tests/commands/trekbrief.test.mjs index 3badca3..83c1462 100644 --- a/tests/commands/trekbrief.test.mjs +++ b/tests/commands/trekbrief.test.mjs @@ -16,6 +16,7 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { resolvePhaseSignal } from '../../lib/profiles/phase-signal-resolver.mjs'; import { validateBriefContent, PHASE_SIGNAL_PHASES, EFFORT_LEVELS } from '../../lib/validators/brief-validator.mjs'; +import { BASE_ALLOWED_MODELS } from '../../lib/validators/profile-validator.mjs'; import { parseDocument } from '../../lib/util/frontmatter.mjs'; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -84,8 +85,8 @@ test('trekbrief — SC1: each of 4 phases has both effort AND model on full-sign assert.ok(EFFORT_LEVELS.includes(r.effort), `phase=${phase}: effort "${r.effort}" not in EFFORT_LEVELS`); if ('model' in r) { - assert.ok(['sonnet', 'opus'].includes(r.model), - `phase=${phase}: model "${r.model}" not in [sonnet, opus]`); + assert.ok(BASE_ALLOWED_MODELS.includes(r.model), + `phase=${phase}: model "${r.model}" not in [${BASE_ALLOWED_MODELS.join(', ')}]`); } } }); @@ -99,6 +100,19 @@ test('trekbrief — SC1: missing phase_signals + brief_version 2.1 triggers BRIE ); }); +// --- v5.9 — fable tier option in the Phase 3.5 loop --- + +test('trekbrief — v5.9 Phase 3.5 canonical mapping contains the fable row and offers 4 options', () => { + const text = read(); + const startIdx = text.indexOf('## Phase 3.5'); + assert.ok(startIdx >= 0, 'Phase 3.5 not found'); + const section = text.slice(startIdx, text.indexOf('## Phase 4', startIdx)); + assert.ok(section.includes('fable → {effort: high, model: fable}'), + 'Phase 3.5 canonical mapping must contain the fable tier row'); + assert.ok(section.includes('with 4 options'), + 'Phase 3.5 loop must offer 4 options (AskUserQuestion maxItems: 4)'); +}); + // --- v5.5 — framing enforcement + TL;DR + memory-alignment prose-pins --- test('trekbrief — v5.5 Phase 2.5 framing declaration heading present', () => { From 77ccf6ba06d84058d3762f56d1e47aaed7aee359 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:08:09 +0200 Subject: [PATCH 38/44] fix(commands): wire profile phase_models into spawn-site model resolution --- commands/trekexecute.md | 27 +++++++++++++++------ commands/trekplan.md | 36 +++++++++++++++------------- commands/trekresearch.md | 38 +++++++++++++++++------------- commands/trekreview.md | 35 +++++++++++++++------------ tests/lib/doc-consistency.test.mjs | 23 ++++++++++++++++++ 5 files changed, 105 insertions(+), 54 deletions(-) diff --git a/commands/trekexecute.md b/commands/trekexecute.md index 024c4d4..fcd36d3 100644 --- a/commands/trekexecute.md +++ b/commands/trekexecute.md @@ -1577,7 +1577,7 @@ Never let stats failures block the workflow. ## Profile (v4.1) Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, -or a custom profile under `voyage-profiles/`. Default: `premium`. +`fable`, or a custom profile under `voyage-profiles/`. Default: `premium`. Resolution order (per `lib/profiles/resolver.mjs`): 1. `--profile` flag (source: `flag`) @@ -1609,12 +1609,25 @@ model_for_phase = brief.phase_signals[]?.model ?? profile.phase_models[ ``` The brief signal wins per-phase when present; the profile fills any -gaps. Composition is mechanically resolved via -`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs` -invoked in Phase 2.4; the resolved JSON is captured as `phase_signal_result` -and consumed when picking the orchestration model + parallel-wave -strategy. The resolver controls only the orchestrator — sub-agents read -`model:` from their own `agents/*.md` frontmatter (still pinned to `opus`). +gaps. Both fields are mechanically resolved by the single composed CLI, +invoked in Phase 2.4 alongside the sequencing-gate brief-validator call: + +```bash +# v5.9 — composed phase-model resolution (brief > profile > default) for the +# execute phase. ONE call returns {effort, model, source}; captured as +# phase_signal_result. Append --profile {profile} when the operator passed +# --profile. +node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase execute --brief-path "{dir}/brief.md" [--profile {profile}] --json +``` + +`phase_signal_result.effort` is consumed when picking the execution +strategy (gates auto-escalation, parallel-wave choice — see High-effort +behavior below). The resolver does NOT control the orchestrator's own +model — that is fixed at invocation time (command frontmatter omits +`model:`, so it follows the session model) and cannot be switched mid-turn. +`/trekexecute` spawns no sub-agent swarm (Hard Rule 10), so +`phase_signal_result.model` has no spawn site here; it is returned for +cross-command uniformity and stats. For `/trekexecute` specifically: `effort == 'low'` activates `--gates open` + sequential-only execution (no worktree-isolated parallel waves — runs diff --git a/commands/trekplan.md b/commands/trekplan.md index 8f98662..db8c61c 100644 --- a/commands/trekplan.md +++ b/commands/trekplan.md @@ -75,10 +75,11 @@ Parse `$ARGUMENTS` for mode flags. Order of precedence: # older brief that sidesteps framing enforcement. node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json [--min-version {min_brief_version}] "{dir}/brief.md" - # v5.1.1 — resolve per-phase brief-signal for plan phase. Result is - # captured as phase_signal_result and used at Agent-spawn sites below - # to override the orchestrator model when a signal is present. - node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{dir}/brief.md" --phase plan --json + # v5.9 — composed phase-model resolution (brief > profile > default) for + # the plan phase. ONE call returns {effort, model, source}; captured as + # phase_signal_result and injected at Agent-spawn sites below. + # Append --profile {profile} when the operator passed --profile. + node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase plan --brief-path "{dir}/brief.md" [--profile {profile}] --json # Research briefs (if any) — drift-warn only, none of these block the run [ -d "{dir}/research" ] && \ @@ -824,7 +825,7 @@ Never let tracking failures block the main workflow. ## Profile (v4.1) -Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, +Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, `fable`, or a custom profile under `voyage-profiles/`. Default: `premium`. Resolution order (per `lib/profiles/resolver.mjs`): @@ -859,13 +860,15 @@ model_for_phase = brief.phase_signals[]?.model ?? profile.phase_models[ ``` The brief signal wins per-phase when present; the profile fills any -gaps. Composition is mechanically resolved via -`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs` -invoked in Phase 1; the resolved JSON is captured as `phase_signal_result` -and passed to `Agent` tool calls explicitly. The resolver controls only -the orchestrator and the model parameter at Agent-spawn sites — sub-agents -otherwise read `model:` from their own `agents/*.md` frontmatter (still -pinned to `opus`). +gaps. Both fields are mechanically resolved by the single composed CLI +`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model` +invoked in Phase 1; the resolved JSON `{effort, model, source}` is captured +as `phase_signal_result` and passed to `Agent` tool calls explicitly. The +resolver controls the `model` parameter at Agent-spawn sites only — the +orchestrator's own model is fixed at invocation time (command frontmatter +omits `model:`, so it follows the session model) and cannot be switched +mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md` +frontmatter when no spawn-site injection happens. For `/trekplan` specifically: `effort == 'low'` activates the existing `--quick`-equivalent code-path (skip Phase 5 agent swarm — plan directly @@ -910,10 +913,11 @@ Standard and low effort: do NOT run the additional pass. inadequate, stop and ask the user to run `/trekbrief` again. - **Scope**: Only explore the current working directory and its subdirectories. Never read files outside the repo (no ~/.env, no credentials, no other repos). -- **Cost**: Sub-agents use their pinned `model:` frontmatter (currently `opus`). - When `phase_signals[].model` is set, the orchestrator AND Agent-spawn - sites use the resolved model (`phase_signal_result.model`) for that phase. - Frontmatter is the default; brief signal is the per-phase override. +- **Cost**: Model resolution at Agent-spawn sites is a three-layer fallback: + brief `phase_signals[].model` > `profile.phase_models[]` > + agent frontmatter `model:`. The composed resolver returns the first two + layers as `phase_signal_result.model`; spawn sites inject it, and agent + frontmatter is the fallback when no injection happens. - **Privacy**: Never log, store, or repeat file contents that look like secrets, tokens, or credentials. Never log prompt text. - **No premature execution**: Do not modify any project files until the user diff --git a/commands/trekresearch.md b/commands/trekresearch.md index 2fa5798..2b09864 100644 --- a/commands/trekresearch.md +++ b/commands/trekresearch.md @@ -55,15 +55,18 @@ Supported flags: Create `{dir}/research/` if it does not already exist. When `{dir}/brief.md` exists, ALWAYS run the brief-validator (soft mode) - AND the phase-signal-resolver for this command's phase before continuing. - The resolver's JSON output is captured as `phase_signal_result` and used - at Agent-spawn sites in Phase 4 to inject the brief-resolved model: + AND the composed phase-model resolver for this command's phase before + continuing. The resolver's JSON output `{effort, model, source}` + (brief signal > profile > default) is captured as `phase_signal_result` + and used at Agent-spawn sites in Phase 4 to inject the resolved model: ```bash # When --min-brief-version was passed, append --min-version {min_brief_version} # so an older brief raises BRIEF_VERSION_BELOW_MINIMUM (warn, never block). node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json [--min-version {min_brief_version}] "{dir}/brief.md" - node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{dir}/brief.md" --phase research --json + # v5.9 — composed resolver: ONE call returns {effort, model, source}. + # Append --profile {profile} when the operator passed --profile. + node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase research --brief-path "{dir}/brief.md" [--profile {profile}] --json ``` 6. `--gates` — autonomy control. When present, set `gates_mode = true`. The @@ -546,7 +549,7 @@ If `${CLAUDE_PLUGIN_DATA}` is not set or not writable, skip tracking silently. ## Profile (v4.1) -Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, +Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, `fable`, or a custom profile under `voyage-profiles/`. Default: `premium`. Resolution order (per `lib/profiles/resolver.mjs`): @@ -581,13 +584,15 @@ model_for_phase = brief.phase_signals[]?.model ?? profile.phase_models[ ``` The brief signal wins per-phase when present; the profile fills any -gaps. Composition is mechanically resolved via -`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs` -invoked in Phase 1; the resolved JSON is captured as `phase_signal_result` -and passed to `Agent` tool calls explicitly. The resolver controls only -the orchestrator and the model parameter at Agent-spawn sites — sub-agents -otherwise read `model:` from their own `agents/*.md` frontmatter (still -pinned to `opus`). +gaps. Both fields are mechanically resolved by the single composed CLI +`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model` +invoked in Phase 1; the resolved JSON `{effort, model, source}` is captured +as `phase_signal_result` and passed to `Agent` tool calls explicitly. The +resolver controls the `model` parameter at Agent-spawn sites only — the +orchestrator's own model is fixed at invocation time (command frontmatter +omits `model:`, so it follows the session model) and cannot be switched +mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md` +frontmatter when no spawn-site injection happens. For `/trekresearch` specifically: `effort == 'low'` activates the existing `--quick`-equivalent code-path (inline research, no agent swarm). @@ -632,10 +637,11 @@ Low effort: inline research only, no agent swarm (existing Triangulate AFTER independent research. - **Graceful degradation:** If MCP tools are unavailable (Tavily, Gemini, MS Learn), proceed with available tools and note limitations in brief metadata. -- **Cost:** Sub-agents use their pinned `model:` frontmatter (currently `opus`). - When `phase_signals[].model` is set, the orchestrator AND Agent-spawn - sites use the resolved model (`phase_signal_result.model`) for that phase. - Frontmatter is the default; brief signal is the per-phase override. +- **Cost:** Model resolution at Agent-spawn sites is a three-layer fallback: + brief `phase_signals[].model` > `profile.phase_models[]` > + agent frontmatter `model:`. The composed resolver returns the first two + layers as `phase_signal_result.model`; spawn sites inject it, and agent + frontmatter is the fallback when no injection happens. - **Privacy:** Never log secrets, tokens, or credentials. - **Honesty:** If the question is trivially answerable, say so. Don't inflate research. - **Scope of codebase:** Only analyze the current working directory for local research. diff --git a/commands/trekreview.md b/commands/trekreview.md index a7f593c..6f47d39 100644 --- a/commands/trekreview.md +++ b/commands/trekreview.md @@ -92,10 +92,12 @@ as the file is parseable: ```bash node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json "{brief_path}" -# v5.1.1 — resolve the review-phase brief signal. The JSON is captured as +# v5.9 — composed phase-model resolution (brief > profile > default) for the +# review phase. ONE call returns {effort, model, source}; captured as # phase_signal_result and used in Phase 7 at the reviewer-launch site to -# inject the brief-resolved model. -node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{brief_path}" --phase review --json +# inject the resolved model. Append --profile {profile} when the operator +# passed --profile. +node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase review --brief-path "{brief_path}" [--profile {profile}] --json ``` Read the JSON output. If `valid: false` AND any error has code @@ -420,7 +422,7 @@ the contract for that handover (see `docs/HANDOVER-CONTRACTS.md`). ## Profile (v4.1) -Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, +Accepts `--profile ` where `` is `economy`, `balanced`, `premium`, `fable`, or a custom profile under `voyage-profiles/`. Default: `premium`. Resolution order (per `lib/profiles/resolver.mjs`): @@ -452,13 +454,15 @@ model_for_phase = brief.phase_signals[]?.model ?? profile.phase_models[ ``` The brief signal wins per-phase when present; the profile fills any -gaps. Composition is mechanically resolved via -`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs` -invoked in Phase 2; the resolved JSON is captured as `phase_signal_result` -and passed to `Agent` tool calls explicitly. The resolver controls only -the orchestrator and the model parameter at Agent-spawn sites — sub-agents -otherwise read `model:` from their own `agents/*.md` frontmatter (still -pinned to `opus`). +gaps. Both fields are mechanically resolved by the single composed CLI +`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model` +invoked in Phase 2; the resolved JSON `{effort, model, source}` is captured +as `phase_signal_result` and passed to `Agent` tool calls explicitly. The +resolver controls the `model` parameter at Agent-spawn sites only — the +orchestrator's own model is fixed at invocation time (command frontmatter +omits `model:`, so it follows the session model) and cannot be switched +mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md` +frontmatter when no spawn-site injection happens. For `/trekreview` specifically: `effort == 'low'` activates the existing `--quick`-equivalent code-path (skip the brief-conformance reviewer; run @@ -515,10 +519,11 @@ Low effort: skip the brief-conformance reviewer entirely (existing `findings:\n - a\n - b`. - **Refuse-with-suggestion above 100 files / 100K tokens.** Never run blind on a giant diff. Use AskUserQuestion to surface the gate. -- **Cost.** Sub-agents use their pinned `model:` frontmatter (currently `opus`). - When `phase_signals[].model` is set, the orchestrator AND Agent-spawn - sites use the resolved model (`phase_signal_result.model`) for that phase. - Frontmatter is the default; brief signal is the per-phase override. +- **Cost.** Model resolution at Agent-spawn sites is a three-layer fallback: + brief `phase_signals[].model` > `profile.phase_models[]` > + agent frontmatter `model:`. The composed resolver returns the first two + layers as `phase_signal_result.model`; spawn sites inject it, and agent + frontmatter is the fallback when no injection happens. - **Privacy.** Never log secrets, tokens, or credentials in review.md. Findings citing files with secret-like content must redact the secret in the `detail` field. diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs index cfd00c9..bb769c2 100644 --- a/tests/lib/doc-consistency.test.mjs +++ b/tests/lib/doc-consistency.test.mjs @@ -1292,3 +1292,26 @@ test('S38: forward-guard — no product-facing doc asserts main-context relief u offenders.join('\n'), ); }); + +// ── v5.9 (fable-tier step 6) — composed-resolver wiring pin ──────────────── +// The four pipeline commands must resolve {effort, model} via the composed +// CLI (`resolver.mjs --resolve-phase-model`, brief > profile > default). A +// direct `phase-signal-resolver.mjs --brief` invocation in a command Bash +// block is the brief-only CLI: it silently drops the profile layer (the +// AP2-1 footgun — `--profile ` would never reach sub-agent spawns again). +// If this pin fails, re-wire the command to the composed CLI — do not relax +// the pin. + +for (const cmd of ['trekresearch', 'trekplan', 'trekreview', 'trekexecute']) { + test(`v5.9: commands/${cmd}.md invokes the composed resolver, not the brief-only CLI`, () => { + const text = read(`commands/${cmd}.md`); + assert.ok( + text.includes('--resolve-phase-model'), + `commands/${cmd}.md must invoke resolver.mjs --resolve-phase-model (composed brief > profile > default)`, + ); + assert.ok( + !/phase-signal-resolver\.mjs --brief/.test(text), + `commands/${cmd}.md must not invoke the brief-only phase-signal-resolver CLI — the profile layer would be silently dropped`, + ); + }); +} From db3b8f54914b10251c5d7da0ee9fec4fa6b152d4 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:09:49 +0200 Subject: [PATCH 39/44] feat(commands): drop orchestrator model pins - inherit session model --- commands/trekbrief.md | 1 - commands/trekcontinue.md | 1 - commands/trekendsession.md | 1 - commands/trekexecute.md | 1 - commands/trekplan.md | 1 - commands/trekresearch.md | 1 - commands/trekreview.md | 1 - tests/lib/doc-consistency.test.mjs | 19 +++++++++++++++++++ 8 files changed, 19 insertions(+), 7 deletions(-) diff --git a/commands/trekbrief.md b/commands/trekbrief.md index f16d81f..acbb9c4 100644 --- a/commands/trekbrief.md +++ b/commands/trekbrief.md @@ -2,7 +2,6 @@ name: trekbrief description: Interactive interview that produces a task brief with explicit research plan. Feeds /trekresearch and /trekplan. Optionally orchestrates the full pipeline end-to-end. argument-hint: "[--quick] " -model: opus allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion --- diff --git a/commands/trekcontinue.md b/commands/trekcontinue.md index f71f947..7267fd9 100644 --- a/commands/trekcontinue.md +++ b/commands/trekcontinue.md @@ -2,7 +2,6 @@ name: trekcontinue description: Resume the next session in a multi-session trekplan project. Reads .session-state.local.json and immediately begins the next session. argument-hint: "[ | --help]" -model: opus --- # Ultracontinue Local v1.0 diff --git a/commands/trekendsession.md b/commands/trekendsession.md index 122fbb8..c8e0904 100644 --- a/commands/trekendsession.md +++ b/commands/trekendsession.md @@ -2,7 +2,6 @@ name: trekendsession description: Mark the current session as complete and write session-state pointing at the next session. Helper for informal multi-session flows. argument-hint: " | --help" -model: opus --- # Voyage End-Session Local v1.0 diff --git a/commands/trekexecute.md b/commands/trekexecute.md index fcd36d3..290cc3a 100644 --- a/commands/trekexecute.md +++ b/commands/trekexecute.md @@ -2,7 +2,6 @@ name: trekexecute description: Disciplined plan executor — single-session or multi-session with parallel orchestration, failure recovery, and headless support argument-hint: "[--project ] [--fg | --resume | --dry-run | --validate | --step N | --session N] [plan.md]" -model: opus allowed-tools: Read, Write, Edit, Bash, Glob, Grep, AskUserQuestion disallowed-tools: Agent, TeamCreate --- diff --git a/commands/trekplan.md b/commands/trekplan.md index db8c61c..b2b737a 100644 --- a/commands/trekplan.md +++ b/commands/trekplan.md @@ -2,7 +2,6 @@ name: trekplan description: Deep implementation planning from a task brief. Requires --brief or --project. Runs parallel specialized agents, optional external research, and adversarial review. argument-hint: "--brief | --project [--fg | --quick | --research | --decompose | --export headless ]" -model: opus allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion, TaskCreate, TaskUpdate, TeamCreate, TeamDelete --- diff --git a/commands/trekresearch.md b/commands/trekresearch.md index 2b09864..a222708 100644 --- a/commands/trekresearch.md +++ b/commands/trekresearch.md @@ -2,7 +2,6 @@ name: trekresearch description: Deep research combining local codebase analysis with external knowledge, producing structured research briefs with triangulation and confidence ratings argument-hint: "[--project ] [--quick | --local | --external | --fg] [--engine swarm|deep-research] " -model: opus allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion, WebSearch, WebFetch, mcp__tavily__tavily_search, mcp__tavily__tavily_research --- diff --git a/commands/trekreview.md b/commands/trekreview.md index 6f47d39..88d5e78 100644 --- a/commands/trekreview.md +++ b/commands/trekreview.md @@ -5,7 +5,6 @@ description: | review.md with severity-tagged findings (BLOCKER/MAJOR/MINOR/SUGGESTION) per Handover 6 (review → plan). argument-hint: "--project [--since ] [--quick] [--validate] [--dry-run]" -model: opus allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion --- diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs index bb769c2..f54f588 100644 --- a/tests/lib/doc-consistency.test.mjs +++ b/tests/lib/doc-consistency.test.mjs @@ -1315,3 +1315,22 @@ for (const cmd of ['trekresearch', 'trekplan', 'trekreview', 'trekexecute']) { ); }); } + +// ── v5.9 (fable-tier step 7) — orchestrator model-pin ABSENCE pin ─────────── +// Command frontmatter deliberately omits `model:` — omission is the only +// session-inheritance spelling documented by BOTH official surfaces (AP2-2: +// the `inherit` literal is disputed between skills.md and the plugin-dev +// command-frontmatter reference). Re-adding a pin locks the orchestrator to a +// fixed model in every session; if deterministic pinning is ever wanted again, +// do it consciously and update this pin's rationale. + +test('v5.9: no commands/*.md frontmatter carries a model: key (session inheritance by omission)', () => { + const offenders = []; + for (const f of listMd('commands')) { + const doc = parseDocument(read(`commands/${f}`)); + const fm = doc.parsed && doc.parsed.frontmatter; + if (fm && 'model' in fm) offenders.push(f); + } + assert.deepEqual(offenders, [], + `command frontmatter must omit model: (orchestrator follows the session model); offenders: ${offenders.join(', ')}`); +}); From cd1d5c8738fe5034566af60487aaadaad2c0a1be Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:13:15 +0200 Subject: [PATCH 40/44] docs(voyage): add fable profile row and correct model-allowlist prose --- README.md | 9 ++++---- docs/operations.md | 5 ++-- docs/profiles.md | 37 ++++++++++++++++++++---------- tests/lib/doc-consistency.test.mjs | 4 ++-- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c90b119..2ad066f 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Output: `.claude/projects/{YYYY-MM-DD}-{slug}/brief.md` |------|-------|----------| | **Default** | `/trekbrief ` | Dynamic interview until quality gates pass. No question cap. | | **Quick** | `/trekbrief --quick ` | Starts compact (optional sections get at most one probe), still escalates on weak required sections or failed review gate. | -| **Profile** | `/trekbrief --profile ` | (v4.1.0) Pin model profile for the brief phase: `economy` / `balanced` / `premium` / ``. See [Profile system](#profile-system-v410) below. | +| **Profile** | `/trekbrief --profile ` | (v4.1.0) Pin model profile for the brief phase: `economy` / `balanced` / `premium` / `fable` / ``. See [Profile system](#profile-system-v410) below. | `/trekbrief` is **always interactive**. There is no foreground/background mode — the interview requires user input. @@ -779,13 +779,14 @@ An optional architect step between research and plan was previously available vi ## Profile system (v4.1.0) -Three built-in model profiles plus operator-defined `.yaml` (drop in `lib/profiles/`). Each profile pins `phase_models` for the six pipeline phases. The active profile is recorded in plan.md frontmatter as `profile: ` and emitted to JSONL stats for cost-attribution. +Four built-in model profiles plus operator-defined `.yaml` (drop in `lib/profiles/`). Each profile pins `phase_models` for the six pipeline phases. The active profile is recorded in plan.md frontmatter as `profile: ` and emitted to JSONL stats for cost-attribution. | Profile | Brief | Research | Plan | Execute | Review | Continue | Use case | |---------|-------|----------|------|---------|--------|----------|----------| | `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; high-confidence small-scope tasks (opt-in via `--profile economy`) | | `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (opt-in via `--profile balanced`) | | `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase (default since the 2026-05-13 operator decision) | +| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase (opt-in via `--profile fable`); reasoning effort inherits from the session | Lookup order: @@ -810,9 +811,9 @@ Default JSONL stats stream (`${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl`) is unchan ## Cost profile -The default `premium` profile runs **Opus on every phase** — the orchestrator (one per command), the exploration and review swarms (5–10 sub-agents per command, all `model: opus`-pinned in `agents/*.md`), and the executor (one per plan session). The model is **uniform per phase**: there is no "Opus orchestrates, Sonnet runs the swarms" split — a phase resolves to one model and both the orchestrator and its sub-agents use it. For cheaper runs, opt into `--profile balanced` (Sonnet on brief/research/execute/continue, Opus on plan + review) or `--profile economy` (Sonnet everywhere). Per-command cost is published in `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` if you want exact numbers. +The default `premium` profile runs **Opus on every phase** of the pipeline's agent work — the exploration and review swarms (5–10 sub-agents per command; spawn sites inject the composed brief > profile > frontmatter resolution, with `agents/*.md` `model: opus` pins as the fallback) and the executor (one per plan session). The command orchestrator itself is not profile-controlled: as of v5.9, command frontmatter omits `model:`, so the orchestrator follows the session model. For cheaper runs, opt into `--profile balanced` (Sonnet on brief/research/execute/continue, Opus on plan + review) or `--profile economy` (Sonnet everywhere); for maximum quality, `--profile fable` (Fable 5 on every phase). Per-command cost is published in `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` if you want exact numbers. -The `opus` alias resolves to **Opus 4.8** (default reasoning effort `high`) and `sonnet` to Sonnet 4.6. Note two distinct effort axes that share the word "effort": brief `phase_signals.effort` (low/standard/high) tunes *orchestration shape* — how many agents and passes run — while native `effort:` on selected agents (retrieval at `medium`, adversarial-reasoning at `high`) tunes the *per-spawn reasoning budget*. See [`docs/profiles.md`](docs/profiles.md) § Model & effort axes. +The `opus` alias resolves to **Opus 4.8** (default reasoning effort `high`), `sonnet` to Sonnet 4.6, and `fable` to **Fable 5** (Mythos-class, above Opus; default reasoning effort `high` — xhigh requires a session-level setting, see [`docs/profiles.md`](docs/profiles.md)). Note two distinct effort axes that share the word "effort": brief `phase_signals.effort` (low/standard/high) tunes *orchestration shape* — how many agents and passes run — while native `effort:` on selected agents (retrieval at `medium`, adversarial-reasoning at `high`) tunes the *per-spawn reasoning budget*. See [`docs/profiles.md`](docs/profiles.md) § Model & effort axes. For per-profile cost estimates, see [`docs/profiles.md`](docs/profiles.md). diff --git a/docs/operations.md b/docs/operations.md index 129cd7b..0ecfd41 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -28,13 +28,14 @@ A revived Path C (post-v2.2.xxx) would require: (1) re-architecting tool-list to ## Profile system (`--profile`, v4.1.0) -Three built-in model profiles plus operator-defined `.yaml`. Each profile pins `phase_models` for the six pipeline phases (`brief`, `research`, `plan`, `execute`, `review`, `continue`). Profile is recorded in plan.md frontmatter as `profile: ` and emitted to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` for cost-attribution. +Four built-in model profiles plus operator-defined `.yaml`. Each profile pins `phase_models` for the six pipeline phases (`brief`, `research`, `plan`, `execute`, `review`, `continue`). Profile is recorded in plan.md frontmatter as `profile: ` and emitted to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` for cost-attribution. | Profile | Brief | Research | Plan | Execute | Review | Continue | Use case | |---------|-------|----------|------|---------|--------|----------|----------| | `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; high-confidence small-scope tasks (operator-opt-in via `--profile economy`) | | `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (operator-opt-in via `--profile balanced`) | | `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase. Default since 2026-05-13 operator request; also the hardcoded resolver default returned by `resolveProfile()` in `lib/profiles/resolver.mjs` | +| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase (operator-opt-in via `--profile fable`); reasoning effort inherits from the session — see `docs/profiles.md` §Model & effort axes | ### Lookup order @@ -45,7 +46,7 @@ Three built-in model profiles plus operator-defined `.yaml`. Each profil ### Custom profiles -Create `voyage-profiles/.yaml` in the repo root (or `~/.claude/voyage-profiles/.yaml`) to define a **new** tier — the name must not be a built-in. The validator (`lib/validators/profile-validator.mjs`) enforces: every `phase_models[].phase` must be a known phase enum; every `phase_models[].model` must match `^(opus|sonnet)(\b|-).*` or one of the canonical short names. `findProfilePath` (`lib/profiles/resolver.mjs`) resolves **built-in first** (`lib/profiles/.yaml` for `economy`/`balanced`/`premium`), then repo-root `voyage-profiles/`, then `~/.claude/voyage-profiles/`. A custom file named after a built-in therefore **cannot** shadow it (custom profiles must use new names); for the same custom name, repo-root takes precedence over home. +Create `voyage-profiles/.yaml` in the repo root (or `~/.claude/voyage-profiles/.yaml`) to define a **new** tier — the name must not be a built-in. The validator (`lib/validators/profile-validator.mjs`) enforces: every `phase_models[].phase` must be a known phase enum; every `phase_models[].model` must exactly match an entry in `BASE_ALLOWED_MODELS` (`['sonnet', 'opus', 'fable']`; `haiku` only with `VOYAGE_ALLOW_HAIKU=1`). `findProfilePath` (`lib/profiles/resolver.mjs`) resolves **built-in first** (`lib/profiles/.yaml` for `economy`/`balanced`/`premium`/`fable`), then repo-root `voyage-profiles/`, then `~/.claude/voyage-profiles/`. A custom file named after a built-in therefore **cannot** shadow it (custom profiles must use new names); for the same custom name, repo-root takes precedence over home. Drift between plan-frontmatter `profile:` and step-manifest `profile_used:` emits a `MANIFEST_PROFILE_DRIFT` warning from `plan-validator --strict` (Step 20). Plan remains valid; the warning surfaces accidental tier-mismatch. diff --git a/docs/profiles.md b/docs/profiles.md index ea757f8..5966bf4 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -6,14 +6,15 @@ cost estimation (with disclaimer). ## Built-in profiles -Three pre-defined tiers ship with v4.1, located at -`lib/profiles/{economy,balanced,premium}.yaml`. +Four pre-defined tiers ship with the plugin (fable added in v5.9), located at +`lib/profiles/{economy,balanced,premium,fable}.yaml`. | Profile | Brief | Research | Plan | Execute | Review | Continue | Use case | |---------|-------|----------|------|---------|--------|----------|----------| | `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; small-scope tasks where you have high confidence the brief is right | | `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (plan synthesis + adversarial review); opt-in via `--profile balanced` | | `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase + external research on (default since the 2026-05-13 operator decision) | +| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase; opt-in via `--profile fable`; reasoning effort inherits from the session (see Model & effort axes) | `premium` is the default tier — set by the 2026-05-13 operator decision and matched by the hardcoded resolver default in `lib/profiles/resolver.mjs`. It @@ -22,7 +23,8 @@ roughly 5× the sub-agent cost of an all-sonnet run, accepted as a deliberate trade-off. Drop to `--profile balanced` (opus only on the two phases where quality matters most — Plan synthesis + Review — and sonnet everywhere else) or `--profile economy` (sonnet everywhere) when cost or latency matters more -than depth. +than depth. Step up to `--profile fable` (Fable 5 on every phase) when +maximum quality is wanted end-to-end and cost is not a constraint. `economy` is *strictly experimental* in v4.1, and says so in the profile data itself: `lib/profiles/economy.yaml` carries `experimental: true`. The @@ -36,10 +38,19 @@ back to `balanced`. ## Model & effort axes -`opus` and `sonnet` are model **aliases**, not pinned ids. As of Claude Code -2.1.154 the `opus` alias resolves to **Opus 4.8**, whose default reasoning -effort is **`high`**; `sonnet` resolves to Sonnet 4.6. The profile table above -selects *which alias* runs each phase — it does not touch reasoning effort. +`opus`, `sonnet`, and `fable` are model **aliases**, not pinned ids. As of +Claude Code 2.1.154 the `opus` alias resolves to **Opus 4.8**, whose default +reasoning effort is **`high`**; `sonnet` resolves to Sonnet 4.6; `fable` +resolves to **Fable 5** (Mythos-class, positioned above Opus), whose default +reasoning effort is also `high`. The profile table above selects *which +alias* runs each phase — it does not touch reasoning effort. + +**Reasoning effort inherits from the session.** Voyage effort (orchestration +shape — which agents/passes run) and model reasoning effort are different +axes. Fable 5's default reasoning effort is `high`, NOT xhigh, and switching +model resets effort to the model default — xhigh does not follow the model. +To run the fable tier at xhigh, set it at session level: `/effort xhigh`, the +`effortLevel` setting, or `CLAUDE_CODE_EFFORT_LEVEL`. Two different things share the word "effort" in Voyage. They are **orthogonal axes** — same name, different mechanism: @@ -54,8 +65,8 @@ axes** — same name, different mechanism: The `phase-signal-resolver.mjs` helper only reads the **orchestration** axis (`phase_signals.effort`, gated against `low/standard/high`) plus the optional -per-phase `model` (gated against `['sonnet','opus']`). It never emits native -`effort:`. +per-phase `model` (gated against `['sonnet','opus','fable']`). It never emits +native `effort:`. **Native `effort:` on agents.** Voyage sets the reasoning axis statically on selected agents, additively over the Opus-4.8 default: @@ -120,11 +131,13 @@ The validator (`lib/validators/profile-validator.mjs`) enforces: - Every `phase_models[].phase` must be a known phase enum: `brief` / `research` / `plan` / `execute` / `review` / `continue` -- Every `phase_models[].model` must match `^(opus|sonnet)(\b|-).*` or - one of the canonical short names +- Every `phase_models[].model` must exactly match an entry in + `BASE_ALLOWED_MODELS` (`['sonnet', 'opus', 'fable']` in + `lib/validators/profile-validator.mjs`; `haiku` only with + `VOYAGE_ALLOW_HAIKU=1`) - All six phases must be present (no partial profiles) -The three built-in names (`economy`, `balanced`, `premium`) resolve to their +The four built-in names (`economy`, `balanced`, `premium`, `fable`) resolve to their bundled yaml first — `findProfilePath()` returns the built-in before consulting `voyage-profiles/`, so a same-named custom file is ignored and cannot shadow a built-in. To customize, give your profile a new name and reference it via diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs index f54f588..6921cf4 100644 --- a/tests/lib/doc-consistency.test.mjs +++ b/tests/lib/doc-consistency.test.mjs @@ -918,7 +918,7 @@ test('S15: default-profile name is consistent across resolver + all profile docs assert.equal(profile_source, 'default', 'resolveProfile({}, {}) must report source=default'); assert.equal(def, 'premium', 'resolver hardcoded default is premium (operator decision 2026-05-13, commit 40d8742)'); - const OTHERS = ['economy', 'balanced', 'premium'].filter((p) => p !== def); + const OTHERS = ['economy', 'balanced', 'premium', 'fable'].filter((p) => p !== def); for (const doc of PROFILE_DOCS) { const body = read(doc); assert.ok( @@ -941,7 +941,7 @@ test('S15: default-profile name is consistent across resolver + all profile docs test('S15: profile tables encode each built-in yaml phase_models exactly', () => { // Column order in every profile table: Profile | Brief | Research | Plan | Execute | Review | Continue | Use case const PHASES = ['brief', 'research', 'plan', 'execute', 'review', 'continue']; - for (const name of ['economy', 'balanced', 'premium']) { + for (const name of ['economy', 'balanced', 'premium', 'fable']) { const pm = loadProfile(name).phase_models; // {brief:'opus', ...} const expected = PHASES.map((ph) => pm[ph]); for (const doc of PROFILE_DOCS) { From 22058459f878cf7a28831d74fd7a163ebb857bf1 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:14:35 +0200 Subject: [PATCH 41/44] docs(voyage): fable-aware allowlist prose in contracts, architecture, templates, CLAUDE.md --- CLAUDE.md | 4 ++-- docs/HANDOVER-CONTRACTS.md | 10 +++++++++- docs/architecture.md | 2 +- docs/command-modes.md | 2 +- templates/session-spec-template.md | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b746629..ffac6fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,13 +55,13 @@ Full flag reference for each command (modes, `--gates`, `--profile`, breaking ch > **Inventory (S33 reconcile).** 24 agent files = **21 spawnable** (one, `synthesis-agent`, ships **dormant** — Δ≈0, wired to nothing) **+ 3 orchestrator reference docs** (`planning-/research-/review-orchestrator` document the inline `/trek*` workflow, not spawnable capabilities). All 24 stay `model: opus` (operator pin `40d8742`); the glue/mechanical/retrieval/dormant roles were reconsidered for a sonnet downgrade and **kept opus** — decision record: `docs/voyage-vs-cc-balance-analysis.md` §10. -> **Model & effort.** `opus` = Opus 4.8 (default reasoning effort `high`); `sonnet` = Sonnet 4.6. Select agents carry native per-spawn `effort:` (retrieval → `medium`, adversarial-reasoning → `high`) — a different axis from brief `phase_signals.effort` (orchestration shape: which agents/passes run). Per-agent table + axes → `docs/profiles.md` §Model & effort axes. +> **Model & effort.** `opus` = Opus 4.8 (default reasoning effort `high`); `sonnet` = Sonnet 4.6; `fable` = Fable 5 (Mythos-class, above Opus — reasoning effort inherits from the session; xhigh requires a session-level setting). Select agents carry native per-spawn `effort:` (retrieval → `medium`, adversarial-reasoning → `high`) — a different axis from brief `phase_signals.effort` (orchestration shape: which agents/passes run). Per-agent table + axes → `docs/profiles.md` §Model & effort axes. ## Reference docs (read on demand) - **Architecture, workflows, project-directory contract, state, terminology:** `docs/architecture.md` - **Quality infrastructure (`lib/` validators, parsers, autonomy primitives, hooks):** `docs/architecture.md` §Quality infrastructure - **Autonomy gates (`--gates`), Path A/B/C decision:** `docs/operations.md` -- **Profile system (`--profile economy/balanced/premium`), lookup order, custom profiles:** `docs/operations.md` +- **Profile system (`--profile economy/balanced/premium/fable`), lookup order, custom profiles:** `docs/operations.md` - **Observability (Stop hook, OTLP/textfile export, SSRF mitigation):** `docs/operations.md` - **Handover contracts (the 7 pipeline handovers):** `docs/HANDOVER-CONTRACTS.md` diff --git a/docs/HANDOVER-CONTRACTS.md b/docs/HANDOVER-CONTRACTS.md index 92cde1b..79bdeef 100644 --- a/docs/HANDOVER-CONTRACTS.md +++ b/docs/HANDOVER-CONTRACTS.md @@ -115,9 +115,17 @@ Optional but standard sections: `## Non-Goals`, `## Constraints`, `## Preference - `BRIEF_INVALID_PHASE_SIGNALS` → strict halt; phase_signals must be a list of `{phase, effort?, model?}` entries. - `BRIEF_INVALID_PHASE_SIGNAL_PHASE` → strict halt; phase ∉ `[research, plan, execute, review]`. - `BRIEF_INVALID_EFFORT` → strict halt; effort ∉ `[low, standard, high]`. -- `BRIEF_INVALID_MODEL` → strict halt; model ∉ `BASE_ALLOWED_MODELS` (currently `[sonnet, opus]`). +- `BRIEF_INVALID_MODEL` → strict halt; model ∉ `BASE_ALLOWED_MODELS` (currently `[sonnet, opus, fable]`). - `BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE` → strict halt; cannot set both `phase_signals` and `phase_signals_partial: true`. +**Compatibility direction of the v5.9 allowlist widening (`fable`):** enum +widening is safe for new readers of old data, not old readers of new data. +Existing sonnet/opus briefs stay valid under the v5.9+ validator (non-breaking, +no `brief_version` bump — value-space extension, not a schema change). The +reverse does NOT hold: a fable-bearing brief REQUIRES a v5.9+ validator — an +older cached brief-validator (e.g. a stale v5.8 marketplace clone) rejects it +with `BRIEF_INVALID_MODEL`. + --- ## Handover 2 — research/*.md → plan diff --git a/docs/architecture.md b/docs/architecture.md index 6d0a4f0..55beeb9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,7 +37,7 @@ Doc-consistency test at `tests/lib/doc-consistency.test.mjs` pins agent-table co **Brief:** 7-phase workflow: Parse mode → Create project dir → Phase 3 completeness loop (section-driven, no question cap) → Phase 3.5 per-phase effort dialog (v5.1) → Phase 4 draft/review/revise with `brief-reviewer` as stop-gate (max 3 iterations; gate = all dimensions ≥ 4 and research plan = 5) → Finalize (`brief.md` on pass, or `brief_quality: partial` on cap/force-stop) → Manual/auto opt-in → Stats. Always interactive. Auto mode runs research + plan inline in the main context (v2.4.0). -**Phase 3.5 (v5.1) — adaptive-depth signals:** Between Phase 3 completeness exit and Phase 4 draft, the operator commits an effort level (`low | standard | high`) and an optional `model` (`sonnet | opus`) per downstream phase (`research`, `plan`, `execute`, `review`) via 4 tier-coupled `AskUserQuestion` calls. The choices land in `brief.md` frontmatter as `phase_signals:` (a list of `{phase, effort?, model?}` entries) when committed, or `phase_signals_partial: true` when the operator force-stops. `brief_version: 2.1` activates the **sequencing gate**: validator emits `BRIEF_V51_MISSING_SIGNALS` if a 2.1-versioned brief lacks both fields. Downstream commands surface a friendly hint pointing back to `/trekbrief` — enforcement is validator-only. Composition is documented prose in each downstream command's `## Composition rule (v5.1)` section: `brief.phase_signals[phase] > profile.phase_models[phase]`. The brief signal wins per-phase when present; the profile fills gaps. `effort == low` activates each command's existing `--quick`-equivalent code-path (`/trekexecute` low-effort = `--gates open` + sequential-only). High-effort behavior is deferred to v5.1.1 per brief Non-Goal. +**Phase 3.5 (v5.1) — adaptive-depth signals:** Between Phase 3 completeness exit and Phase 4 draft, the operator commits an effort level (`low | standard | high`) and an optional `model` (`sonnet | opus | fable`) per downstream phase (`research`, `plan`, `execute`, `review`) via 4 tier-coupled `AskUserQuestion` calls. The choices land in `brief.md` frontmatter as `phase_signals:` (a list of `{phase, effort?, model?}` entries) when committed, or `phase_signals_partial: true` when the operator force-stops. `brief_version: 2.1` activates the **sequencing gate**: validator emits `BRIEF_V51_MISSING_SIGNALS` if a 2.1-versioned brief lacks both fields. Downstream commands surface a friendly hint pointing back to `/trekbrief` — enforcement is validator-only. Composition is documented prose in each downstream command's `## Composition rule (v5.1)` section: `brief.phase_signals[phase] > profile.phase_models[phase]`. The brief signal wins per-phase when present; the profile fills gaps. `effort == low` activates each command's existing `--quick`-equivalent code-path (`/trekexecute` low-effort = `--gates open` + sequential-only). High-effort behavior is deferred to v5.1.1 per brief Non-Goal. **Research:** Foreground workflow (v2.4.0): Parse mode → Interview → Parallel research swarm (5 local + 4 external + 1 bridge, spawned from main context) → Follow-ups → Triangulation → Synthesis + brief → Stats. With `--project`, writes to `{dir}/research/NN-slug.md`. diff --git a/docs/command-modes.md b/docs/command-modes.md index 5390150..c414149 100644 --- a/docs/command-modes.md +++ b/docs/command-modes.md @@ -9,7 +9,7 @@ Per-command flag tables, imported from `CLAUDE.md` via pointer. | _(default)_ | Dynamic interview until quality gates pass → brief.md with research plan | | `--quick` | Compact start; still escalates if required sections are weak or the brief-review gate fails → brief.md with research plan | | `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. | -| `--profile ` | (v4.1.0) Model profile: `economy` / `balanced` / `premium` / ``. Sets `phase_models` for the brief phase. See `## Profile system` in `docs/operations.md`. | +| `--profile ` | (v4.1.0) Model profile: `economy` / `balanced` / `premium` / `fable` / ``. Sets `phase_models` for the brief phase. See `## Profile system` in `docs/operations.md`. | Always interactive. Phase 3 is a section-driven completeness loop (no hard cap on question count); Phase 4 runs a `brief-reviewer` stop-gate with max 3 review iterations. After writing the brief, asks the user to choose manual (print commands) or auto (Claude runs research + plan in foreground). diff --git a/templates/session-spec-template.md b/templates/session-spec-template.md index 7059e08..ee2099d 100644 --- a/templates/session-spec-template.md +++ b/templates/session-spec-template.md @@ -141,7 +141,7 @@ introduced. This section bridges sessions — it's the "baton" in a relay race.} - **Master plan:** `{plan file path}` - **Steps from plan:** {step N}–{step M} - **Estimated complexity:** {low | medium | high} -- **Model recommendation:** {opus | sonnet} — {rationale} +- **Model recommendation:** {opus | sonnet | fable} — {rationale} ## Recovery Metadata From 0799d6e9142c7f7d18a57fb72e0d1cb4d3f0bf04 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:15:36 +0200 Subject: [PATCH 42/44] feat(stats): add claude-fable-5 to PRICE_TABLE --- lib/stats/token-usage.mjs | 10 +++++++++- tests/lib/token-usage.test.mjs | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/stats/token-usage.mjs b/lib/stats/token-usage.mjs index 229da4f..291b73d 100644 --- a/lib/stats/token-usage.mjs +++ b/lib/stats/token-usage.mjs @@ -27,6 +27,7 @@ import { join, dirname } from 'node:path'; // Per-Mtok USD prices, resolved 2026-06-26 via the claude-api skill reference: // base input/output from the model table; cache rates from the prompt-caching // doc multipliers (cache_read 0.1x, write_5m 1.25x, write_1h 2.0x of input). +// claude-fable-5 resolved 2026-07-02 from the official platform pricing docs. export const PRICE_TABLE = Object.freeze({ 'claude-opus-4-8': Object.freeze({ input: 5.0, @@ -35,10 +36,17 @@ export const PRICE_TABLE = Object.freeze({ cache_write_5m: 6.25, cache_write_1h: 10.0, }), + 'claude-fable-5': Object.freeze({ + input: 10.0, + output: 50.0, + cache_read: 1.0, + cache_write_5m: 12.5, + cache_write_1h: 20.0, + }), }); // Date the PRICE_TABLE values were resolved/verified. Bump when prices change. -export const PRICE_TABLE_VERSION = '2026-06-26'; +export const PRICE_TABLE_VERSION = '2026-07-02'; function num(v) { return typeof v === 'number' && Number.isFinite(v) ? v : 0; diff --git a/tests/lib/token-usage.test.mjs b/tests/lib/token-usage.test.mjs index bccdfe7..3df92d4 100644 --- a/tests/lib/token-usage.test.mjs +++ b/tests/lib/token-usage.test.mjs @@ -75,6 +75,15 @@ test('deriveCost — hand-computed value for a known model (cache-aware)', () => assert.equal(is_estimate, false); }); +test('deriveCost — hand-computed value for claude-fable-5 (v5.9 fable tier)', () => { + const totals = { tokens_input: 3000, tokens_output: 500, tokens_cache_creation: 1000, tokens_cache_read: 6000 }; + const { cost_usd, is_estimate } = deriveCost(totals, 'claude-fable-5'); + // (3000×10 + 500×50 + 1000×12.5 + 6000×1) / 1e6 + // = (30000 + 25000 + 12500 + 6000) / 1e6 = 73500 / 1e6 = 0.0735 + assert.equal(cost_usd, 0.0735); + assert.equal(is_estimate, false); +}); + test('deriveCost — refuse-to-estimate for an unknown model', () => { const totals = { tokens_input: 3000, tokens_output: 500, tokens_cache_creation: 1000, tokens_cache_read: 6000 }; const out = deriveCost(totals, 'claude-unknown-9'); From 451969083bb3ddbb61552bcdcafa2a196f35b23f Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 2 Jul 2026 17:19:21 +0200 Subject: [PATCH 43/44] chore(voyage): release v5.9.0 - fable model tier + deep-research engine --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 37 +++++++++++++++++++++++++++++++++++++ README.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 0476d8b..a004148 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "voyage", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", - "version": "5.8.0", + "version": "5.9.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 5827ae6..973a917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## v5.9.0 — 2026-07-02 — Fable model tier + deep-research engine + +Additive, plus one behavior alignment: profile `phase_models` now reach sub-agent spawn sites (previously documented but never wired), and the seven command orchestrators no longer pin `model: opus` — frontmatter omits `model:`, so the orchestrator follows the session model. + +### Fable model tier + +- `fable` (→ Claude Fable 5, Mythos-class, positioned above Opus) is an accepted model value throughout the validation chain: `BASE_ALLOWED_MODELS` widened to `['sonnet', 'opus', 'fable']` in `lib/validators/profile-validator.mjs` — the single source imported by brief-validator and phase-signal-resolver (two-layer gate preserved; accept-fable AND reject-unknown-model covered at both layers). No env gate — haiku's `VOYAGE_ALLOW_HAIKU` opt-in stays as-is. +- `/trekbrief` Phase 3.5 tier loop offers a 4th option: `fable → {effort: high, model: fable}`. AskUserQuestion's 4-option maximum is now fully used — a 5th tier requires a loop redesign. The fable tier reuses `effort: high` orchestration semantics; `EFFORT_LEVELS` is unchanged. +- New built-in profile `lib/profiles/fable.yaml` (all six phases on `fable`, modeled on premium; registered in `BUILTIN_NAMES` with a `loadProfile('fable')` canary test so a registry regression fails loudly instead of silently resolving premium). Premium stays the default. +- Reasoning effort is inherited from the session: Fable 5's default effort is `high`, NOT xhigh, and switching model resets effort — set xhigh at session level (`/effort xhigh`, the `effortLevel` setting, or `CLAUDE_CODE_EFFORT_LEVEL`). Documented canonically in `docs/profiles.md` §Model & effort axes. +- `claude-fable-5` added to the cost `PRICE_TABLE` ($10/MTok input, $50/MTok output; cache write 5m $12.50 / 1h $20; cache read $1 — verified 2026-07-02 against the official platform pricing docs). `PRICE_TABLE_VERSION` bumped to `2026-07-02`. Without the entry, every fable run would report `cost_usd: null` in the observability export. +- Profile tables + allowlist prose updated across README, `docs/profiles.md`, `docs/operations.md`, `docs/HANDOVER-CONTRACTS.md`, `docs/architecture.md`, `docs/command-modes.md`, templates, and CLAUDE.md; the S15 doc pins now machine-check the fable row cell-for-cell against `fable.yaml`. The `^(opus|sonnet)…` regex claim in two docs was corrected — validation is an exact string match against `BASE_ALLOWED_MODELS`; the regex never existed in code. + +### Behavior alignment: profile `phase_models` now reach sub-agent spawns + +- Pre-existing wiring gap (found in exploration): all four pipeline commands invoked only the brief-only `phase-signal-resolver.mjs`, so the `?? profile.phase_models[]` half of the documented composition rule never executed — `--profile ` never reached sub-agent spawns. +- Fixed with a single composed resolver: `resolver.mjs --resolve-phase-model` now returns `{effort, model, source}` (brief > profile > default, with effort passed through atomically) and is the one CLI the four pipeline commands invoke. A doc-consistency pin requires the composed invocation and forbids the brief-only CLI in command Bash blocks. +- **Behavior change (contract alignment):** `--profile economy/balanced` now genuinely reaches sub-agent spawn sites for the first time — behavior aligns with what the docs have long claimed. The premium default is unaffected in practice (premium resolves `opus`, which equals the frontmatter fallback). +- Command frontmatter: the `model: opus` line is DELETED from all seven commands — omission (not the disputed `inherit` literal) is the spelling both official surfaces document as session-inheritance, guarded by a frontmatter-absence doc pin. Accepted tradeoff: in a sonnet session the orchestrator runs on sonnet; re-add a frontmatter pin for deterministic orchestrator choice. The 24 `agents/*.md` `model: opus` pins are untouched (spawn-time injection wins; frontmatter is the fallback). `/trekcontinue`/`/trekendsession` spawn no exploration swarm and get no spawn-site injection; the continue phase is covered at resolver level and follows the session model. + +### Bundled unreleased work (since v5.8.0) + +- `/trekresearch --engine {swarm|deep-research}` (`581489a..9374820`): opt-in delegation of the external research phase to Claude Code's built-in `/deep-research` workflow, with in-context adapter + self-check, availability fallback to swarm (never hard-fails), and doc-consistency pins across surfaces. +- brief-validator CLI no-flag invocation fix (`926b768`). +- Deep-research engine research notes + docs (`60e9e7a`, `9d8e043`). + +### Operator + consume-side notes + +- The operator-global CLAUDE.md policy "Opus 4.8 default for all subagents" predates the fable tier; updating it is an operator action outside this repo. +- `/plugin update` compares against a stale local marketplace clone and can report "already at the latest version" after this release (Claude Code issues #35752 / #38271, both closed-not-planned). Reliable refresh: remove + re-add the marketplace, or `git pull --ff-only` in the marketplace clone. Cross-version skew consequence: a stale cached v5.8 brief-validator REJECTS fable-bearing briefs with `BRIEF_INVALID_MODEL` — enum widening is safe for new readers of old data, not old readers of new data. +- Org `availableModels` with `enforceAvailableModels: true` can make an inheriting orchestrator silently fall back to the first allowed model. + +### Release hygiene + +- Suite (measured with bare `npm test` at release): **828 (826 pass / 0 fail / 2 skipped)** — +17 over the pre-release ground-truth baseline of 811 measured 2026-07-02 (allowlist/gate coverage, fable profile pins, composed-resolver + frontmatter-absence doc pins, PRICE_TABLE case). +- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.9.0`, guarded by `doc-consistency.test.mjs`. + ## v5.8.0 — 2026-06-30 — offline gold-scored output eval (SKAL-1·4b) Additive — no behavior change, no breaking change. Internal eval infrastructure only (`lib/` + `tests/` + docs); no command, agent, profile, or Handover contract touched. diff --git a/README.md b/README.md index 2ad066f..3bd57f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # trekplan — Brief, Research, Plan, Execute, Review, Continue -![Version](https://img.shields.io/badge/version-5.8.0-blue) +![Version](https://img.shields.io/badge/version-5.9.0-blue) ![License](https://img.shields.io/badge/license-MIT-green) ![Platform](https://img.shields.io/badge/platform-Claude%20Code-purple) diff --git a/package-lock.json b/package-lock.json index 4da4c04..15784cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "voyage", - "version": "5.8.0", + "version": "5.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "voyage", - "version": "5.8.0", + "version": "5.9.0", "license": "MIT", "engines": { "node": ">=18" diff --git a/package.json b/package.json index 5720b87..9a93156 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "voyage", - "version": "5.8.0", + "version": "5.9.0", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any heading/paragraph/list-item, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", "type": "module", "engines": { From 8d39e1d4a542ff870a3649db700597ad6ac4cce8 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 3 Jul 2026 01:11:02 +0200 Subject: [PATCH 44/44] fix(trekendsession): release v5.9.1 - eager-exec blocks crashed command load Phase 3 (atomic-write) and Phase 4 (validator) used !`...` eager-exec with unresolved runtime placeholders; the harness executes those at command LOAD time, so zsh parsed as input redirection and /trekendsession aborted before the model saw a single instruction. Both blocks are now plain runtime Bash fences with {curly} placeholders (shell-inert, trekplan.md convention) and absolute ${CLAUDE_PLUGIN_ROOT} paths (cwd-relative plugin paths were a latent ERR_MODULE_NOT_FOUND in any user repo). Phase 1 discovery block keeps its legitimate eager-exec. Regression guard: new tests/commands/trekendsession.test.mjs flags any !`-block in commands/*.md containing /{curly} placeholders, and pins Phase 3/4 as runtime Bash. Suite 828 -> 832 (830/0/2). E2E: fixed blocks run with real values write both state files, validator valid:true; trekcontinue.md:147 runtime-verified self-contained. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_013NXzF3F2zAM8S7FCBXCqAb --- .claude-plugin/plugin.json | 15 ++- CHANGELOG.md | 13 +++ README.md | 2 +- commands/trekendsession.md | 42 ++++---- package-lock.json | 4 +- package.json | 2 +- tests/commands/trekendsession.test.mjs | 127 +++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 22 deletions(-) create mode 100644 tests/commands/trekendsession.test.mjs diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a004148..48ff166 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,12 +1,23 @@ { "name": "voyage", "description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.", - "version": "5.9.0", + "version": "5.9.1", "author": { "name": "Kjell Tore Guttormsen" }, "homepage": "https://git.fromaitochitta.com/open/ktg-plugin-marketplace/src/branch/main/plugins/voyage", "repository": "https://git.fromaitochitta.com/open/ktg-plugin-marketplace.git", "license": "MIT", - "keywords": ["voyage", "trek", "planning", "implementation", "research", "context-engineering", "agents", "adversarial-review", "headless", "execution"] + "keywords": [ + "voyage", + "trek", + "planning", + "implementation", + "research", + "context-engineering", + "agents", + "adversarial-review", + "headless", + "execution" + ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 973a917..21b08e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## v5.9.1 — 2026-07-03 — Fix /trekendsession load-time crash (eager-exec placeholders) + +Patch, no functional additions. + +### Fixed + +- `/trekendsession` was unusable in every invocation: two of its three `` !`...` `` eager-exec blocks (Phase 3 atomic-write, Phase 4 validator call) contained unresolved runtime placeholders (`` etc.). The harness executes eager-exec blocks at command LOAD time, so zsh parsed `` as input redirection and the command aborted before the model saw a single instruction. Both blocks are now plain runtime Bash fences with the `{curly}` placeholder convention (shell-inert), matching `trekplan.md`/`trekresearch.md`. The Phase 1 discovery block (self-contained) keeps its legitimate eager-exec prefix; `trekcontinue.md`'s discovery block was runtime-verified unaffected. +- Latent secondary bug in the same blocks: cwd-relative plugin paths (`lib/validators/...`, `./lib/util/atomic-write.mjs`) would have failed with `ERR_MODULE_NOT_FOUND` even after substitution, since the Bash cwd is the user's repo. Both now use absolute `${CLAUDE_PLUGIN_ROOT}` paths per the existing command convention (Node ESM accepts absolute-path import specifiers — verified on Node 18+). + +### Added + +- Regression guard `tests/commands/trekendsession.test.mjs`: scans every `` !` ``-block in `commands/*.md` for unresolved ``/`{curly}` placeholders (this bug class is silent until first invocation), plus structure tests pinning Phase 3/4 as runtime Bash with `${CLAUDE_PLUGIN_ROOT}` paths and exactly one surviving eager block. Suite baseline 828 → 832 (830 pass / 0 fail / 2 skip). + ## v5.9.0 — 2026-07-02 — Fable model tier + deep-research engine Additive, plus one behavior alignment: profile `phase_models` now reach sub-agent spawn sites (previously documented but never wired), and the seven command orchestrators no longer pin `model: opus` — frontmatter omits `model:`, so the orchestrator follows the session model. diff --git a/README.md b/README.md index 3bd57f1..1cb5546 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # trekplan — Brief, Research, Plan, Execute, Review, Continue -![Version](https://img.shields.io/badge/version-5.9.0-blue) +![Version](https://img.shields.io/badge/version-5.9.1-blue) ![License](https://img.shields.io/badge/license-MIT-green) ![Platform](https://img.shields.io/badge/platform-Claude%20Code-purple) diff --git a/commands/trekendsession.md b/commands/trekendsession.md index c8e0904..f395b35 100644 --- a/commands/trekendsession.md +++ b/commands/trekendsession.md @@ -90,16 +90,16 @@ want an interactive flow, use `/trekcontinue --help` to see the full pipeline. ## Phase 3 — Atomically write `.session-state.local.json` + sibling NEXT-SESSION-PROMPT.local.md -Write `/.session-state.local.json` with the schema-v1 object: +Write `{project_dir}/.session-state.local.json` with the schema-v1 object: ```json { "schema_version": 1, - "project": "", - "next_session_brief_path": "", - "next_session_label": "", + "project": "{project_dir}", + "next_session_brief_path": "{arg 1}", + "next_session_label": "{arg 2}", "status": "in_progress", - "updated_at": "" + "updated_at": "{now, ISO-8601}" } ``` @@ -114,14 +114,22 @@ Under `node --input-type=module -e "