feat(voyage): S12 — NW3 synthesis-agent built + measured → declined per measurement [skip-docs]

NW3 (CC-26 §6 PoC): delegate trekplan Phase 7 synthesis to a synthesis-agent,
adopt only if Δ main-context ≥30% with no quality loss. Operator chose the
deterministic-proof path (live ≥3-run bake-off was env-blocked: no API key;
installed plugin is a cache copy so a new agent is invisible to `claude -p`).

Decisive structural finding: trekplan Phase 5 runs the swarm FOREGROUND, so its
outputs are already resident in main before Phase 7. Delegating only Phase 7
evicts nothing → Δ_faithful = 0% (BASE-independent). The ≥30% saving needs an
out-of-scope Phase-5 redesign (swarm-writes-to-disk / nested orchestrator).
VERDICT: DECLINED per measurement.

- agents/synthesis-agent.md — dormant, schema-conformant deliverable (NOT wired)
- lib/plan/synthesis-digest-schema.mjs — digest output contract (+ tests)
- scripts/synthesis-measure.mjs — deterministic Δ-accounting core (+ tests)
- tests/fixtures/synthesis/ — 7 exploration outputs + representative digest
- docs/T1-synthesis-poc-results.md — measurement + verdict (reproducible)
- CLAUDE.md — agent table row (doc-consistency: 24 agents)

Tests 670 → 695 (693 pass / 2 skip / 0 fail). `claude plugin validate` clean
(only the pre-existing root-CLAUDE.md warning). commands/trekplan.md untouched.

[skip-docs] rationale: no user-facing feature ships (NW3 declined; agent dormant
and unwired). The substantive doc is docs/T1-synthesis-poc-results.md; the
README/CHANGELOG roll-up for NW1–NW3 is the S13 coordinated release per
docs/W1-narrow-wins-plan.md §S13.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
This commit is contained in:
Kjell Tore Guttormsen 2026-06-18 17:58:39 +02:00
commit 6b30483304
15 changed files with 1312 additions and 0 deletions

View file

@ -0,0 +1,101 @@
// tests/scripts/synthesis-measure.test.mjs
// NW3 (S12) — deterministic Δ main-context measurement core.
//
// The gate metric (T1 §2) is Δ main-context tokens for an equivalent-quality
// digest. This pins the pure accounting that turns fixture token counts into a
// verdict, under the two framings that decide NW3:
//
// - FAITHFUL (current flow): Phase 5 swarm runs FOREGROUND, so its outputs are
// already RESIDENT in main before Phase 7. Delegating only the synthesis read
// cannot evict them → main holds base+out+dig in BOTH arms → Δ ≈ 0.
// - DISK-POTENTIAL (upper bound): IF outputs were on disk (a separate Phase-5
// change, out of NW3 scope), the delegated arm holds base+dig only → Δ = out/(base+out+dig).
//
// POSITIVE adopt requires Δ ≥ 30% AND quality ≥ inline (T1 §5 thresholds).
import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import {
estimateTokens,
mainContextTokens,
deltaPct,
decideVerdict,
analyze,
} from '../../scripts/synthesis-measure.mjs';
test('estimateTokens: ~chars/4 heuristic, monotonic, non-negative', () => {
assert.equal(estimateTokens(''), 0);
assert.equal(estimateTokens('abcd'), 1);
assert.equal(estimateTokens('abcde'), 2); // ceil(5/4)
assert.ok(estimateTokens('a'.repeat(400)) === 100);
assert.ok(estimateTokens('x'.repeat(1000)) > estimateTokens('x'.repeat(500)));
});
test('mainContextTokens: inline arm holds base + out + dig', () => {
assert.equal(mainContextTokens({ base: 50000, out: 12000, dig: 1500, arm: 'inline' }), 63500);
});
test('mainContextTokens: delegated_faithful still holds out (Phase-5 resident) + dig', () => {
// The decisive structural fact: foreground swarm delivery already made `out`
// resident; delegating Phase 7 does not evict it.
assert.equal(
mainContextTokens({ base: 50000, out: 12000, dig: 1500, arm: 'delegated_faithful' }),
63500,
);
});
test('mainContextTokens: delegated_disk holds base + dig only (out lives in the sub-agent)', () => {
assert.equal(
mainContextTokens({ base: 50000, out: 12000, dig: 1500, arm: 'delegated_disk' }),
51500,
);
});
test('mainContextTokens: unknown arm throws (no silent default)', () => {
assert.throws(() => mainContextTokens({ base: 1, out: 1, dig: 1, arm: 'nope' }));
});
test('deltaPct: (A-B)/A; equal arms → 0; B smaller → positive fraction', () => {
assert.equal(deltaPct(100, 100), 0);
assert.equal(deltaPct(100, 75), 0.25);
assert.equal(deltaPct(0, 0), 0); // guard divide-by-zero
});
test('decideVerdict: ≥30% AND quality-ok → POSITIVE', () => {
assert.equal(decideVerdict(0.30, true), 'POSITIVE');
assert.equal(decideVerdict(0.45, true), 'POSITIVE');
});
test('decideVerdict: quality loss forces NEGATIVE even at a large Δ', () => {
assert.equal(decideVerdict(0.60, false), 'NEGATIVE');
});
test('decideVerdict: <15% → NEGATIVE; the [15%,30%) band → INCONCLUSIVE', () => {
assert.equal(decideVerdict(0.149, true), 'NEGATIVE');
assert.equal(decideVerdict(0.00, true), 'NEGATIVE');
assert.equal(decideVerdict(0.15, true), 'INCONCLUSIVE');
assert.equal(decideVerdict(0.2999, true), 'INCONCLUSIVE');
});
test('analyze: faithful arm is structurally Δ=0 → NEGATIVE regardless of sizes', () => {
const a = analyze({ baseTokens: 50000, outTokens: 12000, digTokens: 1500, qualityOK: true });
assert.equal(a.faithful.deltaPct, 0);
assert.equal(a.faithful.verdict, 'NEGATIVE');
assert.equal(a.faithful.armA, a.faithful.armB);
});
test('analyze: disk arm Δ = out/(base+out+dig)', () => {
const a = analyze({ baseTokens: 50000, outTokens: 12000, digTokens: 1500, qualityOK: true });
const expected = 12000 / (50000 + 12000 + 1500);
assert.ok(Math.abs(a.disk.deltaPct - expected) < 1e-9);
assert.equal(a.disk.armB, 51500);
});
test('analyze: disk verdict tracks BASE — large fixed baseline can sink the upper bound below 30%', () => {
// Small base → disk Δ clears 30%; large base → it does not. Demonstrates the
// upper bound is itself BASE-sensitive (sweep, do not assert one number).
const small = analyze({ baseTokens: 10000, outTokens: 12000, digTokens: 1500, qualityOK: true });
const large = analyze({ baseTokens: 200000, outTokens: 12000, digTokens: 1500, qualityOK: true });
assert.equal(small.disk.verdict, 'POSITIVE'); // 12000/23500 ≈ 0.51
assert.equal(large.disk.verdict, 'NEGATIVE'); // 12000/213500 ≈ 0.056
});