From eb5a7324a4b04c56d2fcd5e8d721fe3a88843078 Mon Sep 17 00:00:00 2001
From: Kjell Tore Guttormsen <hello@fromaitochitta.com>
Date: Fri, 19 Jun 2026 06:24:27 +0200
Subject: [PATCH] =?UTF-8?q?fix(voyage):=20S17=20=E2=80=94=20downgrade=20NW?=
 =?UTF-8?q?2=20bake-off=20verdict=20+=20drop=20orchestrator=20Agent-grants?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Survivor #3 (bake-off data un-archived): per-run JSON a1..b3.json was never
committed (git diff-filter=A empty), so the medians/jaccard ladder cannot be
re-derived or audited. Commit-path would require fabricated data (forbidden) →
downgraded "POSITIVE" to "opt-in-defensible (single un-archived run)" in
docs/T2-bakeoff-results.md (verdict line + §5 header + new §Reproducibility
caveat) and the v5.5.0 CHANGELOG entry. New doc-consistency pin fails if the
doc re-asserts bare POSITIVE or drops the un-archived disclosure.

Survivor #5 (latent dispatch risk): planning/research/review orchestrators are
inline reference docs no command invokes, yet shipped tools:["Agent",...]. The
harness cannot spawn sub-agents with Agent, so the grant was pure latent risk.
Dropped "Agent" from all three frontmatters; flipped the agent-frontmatter
invariant (its canonical home) from "must include Agent" to "must NOT include
Agent". Removed the duplicate orchestrator pin from doc-consistency to avoid a
double guard (S19 anti-bloat).

Tests 690 → 691 (689 pass / 2 skip / 0 fail). claude plugin validate passes
(1 accepted CLAUDE.md-at-root warning).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
---
 CHANGELOG.md                         |  2 +-
 agents/planning-orchestrator.md      |  2 +-
 agents/research-orchestrator.md      |  2 +-
 agents/review-orchestrator.md        |  2 +-
 docs/T2-bakeoff-results.md           | 19 ++++++++++++++++---
 tests/lib/agent-frontmatter.test.mjs | 12 ++++++++----
 tests/lib/doc-consistency.test.mjs   | 20 ++++++++++++++++++++
 7 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 32475c6..883553b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,7 +38,7 @@ Existing `2.0`/`2.1` briefs stay valid (forward + backward compatible), mirrorin
 ### Narrow wins (W1 — NW1–NW3)
 
 - **NW1 — reviewer-output schema contract** (ungated, shipped). `lib/review/findings-schema.mjs` validates each `/trekreview` Phase 5 reviewer's JSON against a schema; on *schema* failure main re-asks for conforming JSON (bounded N=2), retiring the fragile "collect trailing JSON / re-ask on parse error" contract.
-- **NW2 — trekreview Workflow port as opt-in `--workflow` flag.** Phase 5–6 (`parallel([conformance, correctness])` → plain-JS dedup-by-`(file,line,rule_key)` → `agent(coordinator)`) ported to a Workflow script, reachable via `--workflow` (default stays prose, preserving the 2.1.154+ portability floor). Bake-off verdict **POSITIVE**: fidelity-equivalent `review.md`, zero classifier interference at 9-agent concurrency, acceptable cost (≈ +4.4% tokens).
+- **NW2 — trekreview Workflow port as opt-in `--workflow` flag.** Phase 5–6 (`parallel([conformance, correctness])` → plain-JS dedup-by-`(file,line,rule_key)` → `agent(coordinator)`) ported to a Workflow script, reachable via `--workflow` (default stays prose, preserving the 2.1.154+ portability floor). Bake-off verdict **opt-in-defensible** (POSITIVE on the measured axes — fidelity-equivalent `review.md`, zero classifier interference at 9-agent concurrency, ≈ +4.4% tokens — but on a single un-archived run: the raw per-run data was never committed, so the numbers cannot be re-derived; see `docs/T2-bakeoff-results.md` §Reproducibility caveat).
 - **NW3 — synthesis-agent shipped dormant (declined per measurement).** Built + measured deterministically: Δ main-context = **0%** at all realistic BASE — delegating only trekplan Phase 7 evicts nothing, since the Phase-5 foreground reads make the outputs resident first. Not wired; re-measurement is cheap via `scripts/synthesis-measure.mjs`. See `docs/T1-synthesis-poc-results.md`.
 
 ### CC 2.1.130→181 verification dispositions
diff --git a/agents/planning-orchestrator.md b/agents/planning-orchestrator.md
index 4332aa4..cfd576f 100644
--- a/agents/planning-orchestrator.md
+++ b/agents/planning-orchestrator.md
@@ -13,7 +13,7 @@ description: |
   session.
 model: opus
 color: cyan
-tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
+tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
 ---
 
 <!-- Phase mapping: orchestrator → command
diff --git a/agents/research-orchestrator.md b/agents/research-orchestrator.md
index 4ab1d4f..53e9ae2 100644
--- a/agents/research-orchestrator.md
+++ b/agents/research-orchestrator.md
@@ -12,7 +12,7 @@ description: |
   directly in the main session.
 model: opus
 color: cyan
-tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash"]
+tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash"]
 ---
 
 <!-- Phase mapping: orchestrator → command
diff --git a/agents/review-orchestrator.md b/agents/review-orchestrator.md
index fe16f58..6169be0 100644
--- a/agents/review-orchestrator.md
+++ b/agents/review-orchestrator.md
@@ -14,7 +14,7 @@ description: |
   session.
 model: opus
 color: red
-tools: ["Agent", "Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
+tools: ["Read", "Glob", "Grep", "Write", "Edit", "Bash", "TaskCreate", "TaskUpdate"]
 ---
 
 <!-- Phase mapping: orchestrator → command
diff --git a/docs/T2-bakeoff-results.md b/docs/T2-bakeoff-results.md
index 5d47c0f..2d904ed 100644
--- a/docs/T2-bakeoff-results.md
+++ b/docs/T2-bakeoff-results.md
@@ -2,8 +2,9 @@
 
 **Status (S10):** Part A (build + smoke) **and** part B (full ≥3-runs/arm
 measurement) **COMPLETE**. Operator GO 2026-06-18 (choice "a" — full ≥3× run).
-**T2 §5 verdict: POSITIVE** — the Workflow substrate is fidelity-neutral; proceed to
-the opt-in `--workflow` flag at S11. Full evidence in **§Full run (S10 part B)** at the
+**T2 §5 verdict: opt-in-defensible** (POSITIVE on the measured axes, on a single
+un-archived run — see **§Reproducibility caveat**) — the Workflow substrate is
+fidelity-neutral on what was measured; proceed to the opt-in `--workflow` flag at S11. Full evidence in **§Full run (S10 part B)** at the
 end of this doc. The smoke section below is retained for history.
 
 Resolves: `docs/W1-narrow-wins-plan.md §S10` (build + de-risk **and** full measurement).
@@ -160,7 +161,7 @@ Arm A median ~150s/run; Arm B ~230s ⇒ **+54%**. **Caveat:** not a controlled p
 
 Arm B runs in the background; intermediate findings visible in the workflow transcript + `/workflows`; returns structured `{verdict, findings}`. Phase 7 rendering stays shared/prose; the operator-gate (review.md write) is unaffected. Arm B **frees the main context** during the review (the 3 workflows ran while main did other work) — Arm A occupies it end-to-end.
 
-## VERDICT: **POSITIVE**
+## VERDICT: **opt-in-defensible** (POSITIVE on measured axes; single un-archived run)
 
 The Workflow substrate (Arm B) is **fidelity-equivalent** to the prose path (Arm A) on
 the operator-meaningful axes — verdict 1.0, file-set 1.0, issue-coverage 100%,
@@ -177,6 +178,18 @@ It is **not** a regression: the granularity ladder and within-vs-cross compariso
 the divergence as intrinsic LLM nondeterminism, equal in both arms. Arm B's wall-time is
 ~+54% but non-blocking.
 
+## Reproducibility caveat (Survivor #3, added S17)
+
+The per-run arm outputs (`a1.json`..`b3.json`) were **never committed** — `git log
+--diff-filter=A` finds no such file in any ref, and `scripts/bakeoff-fidelity.mjs`
+requires them as input. The medians and the jaccard ladder above therefore **cannot be
+regenerated, audited, or falsified** from the repo. This is why the verdict is labeled
+**opt-in-defensible** rather than a clean POSITIVE: it justifies shipping `--workflow`
+behind a flag (the measured axes did come out fidelity-neutral), but it rests on a
+**single un-archived run** and is not a reproducible result. To upgrade it, a future
+session must re-run the bake-off and commit the raw per-run JSON under
+`tests/fixtures/bakeoff-rich/runs/` with a test that re-derives these numbers.
+
 **→ S11:** proceed with the **opt-in `--workflow` flag** (prose stays default — preserves
 the 2.1.154+ portability floor; see §Open decisions in the narrow-wins plan). The bake-off
 supports making the Workflow path reachable; residuals are (a) the F4 `auto`/`bypass`
diff --git a/tests/lib/agent-frontmatter.test.mjs b/tests/lib/agent-frontmatter.test.mjs
index 1350d42..9132f40 100644
--- a/tests/lib/agent-frontmatter.test.mjs
+++ b/tests/lib/agent-frontmatter.test.mjs
@@ -90,7 +90,11 @@ test('every agents/*.md frontmatter name matches its filename', () => {
   }
 });
 
-test('orchestrator agents are model: opus and include the Agent tool', () => {
+test('orchestrator agents are model: opus and do NOT include the Agent tool (reference-only, never spawned)', () => {
+  // S17 / Survivor #5: these orchestrator .md files are inline reference docs.
+  // No command invokes them, and the harness does not expose the Agent tool to
+  // sub-agents, so an Agent grant here is purely a latent dispatch risk. They
+  // must NOT carry it — the real orchestration runs in the main session.
   for (const f of ORCHESTRATORS) {
     const path = `agents/${f}`;
     const fm = extractFrontmatter(read(path));
@@ -99,12 +103,12 @@ test('orchestrator agents are model: opus and include the Agent tool', () => {
     assert.equal(
       model,
       'opus',
-      `${path}: orchestrator must be model: opus (drives multi-agent swarm reasoning) — got "${model}"`,
+      `${path}: orchestrator must be model: opus (documents multi-agent swarm reasoning) — got "${model}"`,
     );
     const tools = getTopLevelValue(fm, 'tools');
     assert.ok(
-      tools && /\bAgent\b/.test(tools),
-      `${path}: orchestrator tools: must include "Agent" so it can spawn the swarm — got ${tools}`,
+      tools && !/\bAgent\b/.test(tools),
+      `${path}: orchestrator is reference-only — tools: must NOT include "Agent" (Survivor #5) — got ${tools}`,
     );
   }
 });
diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs
index 671e7b3..4f175ea 100644
--- a/tests/lib/doc-consistency.test.mjs
+++ b/tests/lib/doc-consistency.test.mjs
@@ -849,3 +849,23 @@ test('S15: README cost prose does not resurrect the false Sonnet-swarm claim', (
     );
   }
 });
+
+// NOTE (S17): the "reference-only orchestrators must not grant the Agent tool"
+// invariant (Survivor #5) is guarded in tests/lib/agent-frontmatter.test.mjs
+// (its canonical frontmatter home), not duplicated here.
+
+test('S17: NW2 bake-off verdict is not labeled unqualified POSITIVE (raw data un-archived)', () => {
+  // Survivor #3: the per-run JSON (a1..b3.json) was never committed
+  // (git log --diff-filter=A is empty), so the medians/jaccards cannot be
+  // re-derived, audited, or falsified. The verdict must be qualified as
+  // opt-in-defensible, and the un-reproducibility must be disclosed.
+  const doc = read('docs/T2-bakeoff-results.md');
+  assert.ok(
+    !/VERDICT:\s*\*\*POSITIVE\*\*/.test(doc),
+    'T2-bakeoff-results.md still labels the verdict bare POSITIVE — downgrade to opt-in-defensible (Survivor #3).',
+  );
+  assert.ok(
+    /un-archived|not committed|cannot be re-derived/i.test(doc),
+    'T2-bakeoff-results.md must disclose the raw per-run data is un-archived / un-reproducible.',
+  );
+});