voyage/scripts/synthesis-measure.mjs

#!/usr/bin/env node
// scripts/synthesis-measure.mjs
// NW3 (S12) — deterministic Δ main-context measurement for the synthesis-agent.
//
// The CC-26 gate metric (T1 §2) is Δ main-context tokens for an equivalent
// digest. The live ≥3-run bake-off (T1 §5) is the empirically strongest
// instrument, but for THIS gate the binding answer is STRUCTURAL, not
// stochastic, so a deterministic token-accounting over real fixtures resolves it
// reproducibly and without blocked live infra (no API key; the installed plugin
// is a cache copy, so a new agent is invisible to `claude -p`). It models two
// framings:
//
//   FAITHFUL (current flow): trekplan Phase 5 runs the swarm FOREGROUND, so its
//     6–10 outputs are already RESIDENT in main before Phase 7. Delegating only
//     the synthesis read cannot evict them → main holds base+out+dig in BOTH
//     arms → Δ ≈ 0. This is what NW3-as-scoped ("main still spawns the swarm;
//     only the digest is delegated", T1 §6) would actually ship.
//
//   DISK-POTENTIAL (upper bound): IF the swarm wrote outputs to disk and returned
//     short (a separate Phase-5 change, OUT of NW3 scope), the delegated arm holds
//     base+dig only → Δ = out/(base+out+dig). BASE-sensitive; swept, not asserted.
//
// POSITIVE adopt requires Δ ≥ 30% AND quality ≥ inline (T1 §5).
//
// Zero deps. Node stdlib only. Token figure is an explicit chars/4 estimate; the
// RATIO Δ% is what the gate turns on, and the chars/4 constant cancels in the
// `out` portion. BASE is environment-dependent (system prompt + plugin listings
// + CLAUDE.md) and NOT API-measured this session → swept across a documented band.

import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
import { join, dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(HERE, '..');
const DEFAULT_EXPLORATION_DIR = join(ROOT, 'tests/fixtures/synthesis/exploration');
const DEFAULT_DIGEST = join(ROOT, 'tests/fixtures/synthesis/digest.json');
const DEFAULT_OUT = join(ROOT, 'docs/T1-synthesis-poc-results.md');

// T1 §5 thresholds.
export const POSITIVE_THRESHOLD = 0.30;
export const NEGATIVE_THRESHOLD = 0.15;

// Documented BASE sweep: a Voyage main session's fixed resident baseline (CC
// system prompt + tool defs + plugin command/agent/skill listings + CLAUDE.md).
// Genuinely environment-dependent; not API-measured this session.
export const BASE_SWEEP = Object.freeze([30_000, 50_000, 80_000, 120_000]);

// Fixture-independent disk-potential sensitivity: sweep OUT at one illustrative
// typical baseline, so the verdict does not hinge on this run's fixture sizes.
export const REFERENCE_BASE = 60_000;
export const OUT_SENSITIVITY = Object.freeze([5_000, 10_000, 20_000, 30_000, 40_000]);

// ---- pure measurement core (unit-tested) ------------------------------------

/** Explicit chars/4 token estimate. @param {string} text @returns {number} */
export function estimateTokens(text) {
  if (typeof text !== 'string' || text.length === 0) return 0;
  return Math.ceil(text.length / 4);
}

/**
 * Tokens resident in MAIN at synthesis-complete, per arm.
 * @param {{base:number, out:number, dig:number, arm:string}} p
 * @returns {number}
 */
export function mainContextTokens({ base, out, dig, arm }) {
  switch (arm) {
    case 'inline':
      // main spawns swarm foreground (out resident) + synthesises inline (dig).
      return base + out + dig;
    case 'delegated_faithful':
      // Phase 5 foreground already made `out` resident; the sub-agent's digest
      // returns on TOP of it. Delegating Phase 7 evicts nothing.
      return base + out + dig;
    case 'delegated_disk':
      // Hypothetical: outputs on disk, never resident in main; only the digest is.
      return base + dig;
    default:
      throw new Error(`mainContextTokens: unknown arm "${arm}"`);
  }
}

/** Fractional reduction (A−B)/A. Divide-by-zero guarded to 0. */
export function deltaPct(armA, armB) {
  if (!armA) return 0;
  return (armA - armB) / armA;
}

/** T1 §5 verdict. Quality loss vetoes a token win. */
export function decideVerdict(delta, qualityOK) {
  if (!qualityOK) return 'NEGATIVE';
  if (delta >= POSITIVE_THRESHOLD) return 'POSITIVE';
  if (delta < NEGATIVE_THRESHOLD) return 'NEGATIVE';
  return 'INCONCLUSIVE';
}

/**
 * Both framings for one BASE.
 * @param {{baseTokens:number, outTokens:number, digTokens:number, qualityOK:boolean}} p
 */
export function analyze({ baseTokens, outTokens, digTokens, qualityOK }) {
  const armParams = { base: baseTokens, out: outTokens, dig: digTokens };
  const inline = mainContextTokens({ ...armParams, arm: 'inline' });
  const faithfulB = mainContextTokens({ ...armParams, arm: 'delegated_faithful' });
  const diskB = mainContextTokens({ ...armParams, arm: 'delegated_disk' });
  const fD = deltaPct(inline, faithfulB);
  const dD = deltaPct(inline, diskB);
  return {
    faithful: { armA: inline, armB: faithfulB, deltaPct: fD, verdict: decideVerdict(fD, qualityOK) },
    disk: { armA: inline, armB: diskB, deltaPct: dD, verdict: decideVerdict(dD, qualityOK) },
  };
}

/** BASE at which disk-potential Δ crosses exactly POSITIVE_THRESHOLD. */
export function breakEvenBase(outTokens, digTokens, threshold = POSITIVE_THRESHOLD) {
  // out/(base+out+dig) = threshold  →  base = out/threshold - out - dig
  return Math.round(outTokens / threshold - outTokens - digTokens);
}

// ---- CLI shim ----------------------------------------------------------------

function pct(x) { return `${(x * 100).toFixed(1)}%`; }

function loadExploration(dir) {
  const files = readdirSync(dir)
    .filter((f) => f.endsWith('.md') || f.endsWith('.txt'))
    .sort();
  return files.map((f) => {
    const text = readFileSync(join(dir, f), 'utf-8');
    return { name: f, chars: text.length, tokens: estimateTokens(text) };
  });
}

function buildResultsDoc({ explorationDir, digestPath, outputs, outTokens, digTokens, qualityOK }) {
  const breakEven = breakEvenBase(outTokens, digTokens);
  const L = [];
  L.push('# T1 — Synthesis-agent PoC: Δ main-context measurement (NW3 / S12)');
  L.push('');
  L.push('**Status:** Measurement complete — verdict below. **Method:** deterministic token-');
  L.push('accounting over real exploration fixtures (the live ≥3-run bake-off of T1 §5 is the');
  L.push('stronger instrument but is (a) environment-blocked here — no `ANTHROPIC_API_KEY`, and the');
  L.push('installed plugin is a cache copy so a fresh `synthesis-agent` is invisible to `claude -p`;');
  L.push('and (b) unnecessary, because the binding answer is STRUCTURAL, not stochastic).');
  L.push('**Resolves:** decision-matrix §W1 / CC-26 narrow PoC (`docs/T1-cc26-delegated-orchestration.md` §6).');
  L.push('**Reproduce:** `node scripts/synthesis-measure.mjs` (regenerates this file).');
  L.push('');
  L.push('> Verifiseringsplikt: token figures are an explicit **chars/4 estimate** (labelled), not a');
  L.push('> tokenizer count. The gate turns on the RATIO Δ%, in which the per-token constant cancels');
  L.push('> for the `out` term. BASE (the fixed main-session baseline) is environment-dependent and');
  L.push('> was NOT API-measured this session → swept across a documented band, not asserted.');
  L.push('');
  L.push('## 1. The decisive structural finding (BASE-independent)');
  L.push('');
  L.push('trekplan runs the Phase 5 exploration swarm **foreground** (foreground is the only mode');
  L.push('since v2.4.0; `commands/trekplan.md`). Foreground Agent/Task results are delivered back');
  L.push('into the main transcript, so after Phase 5 the 6–10 exploration outputs are **already');
  L.push('resident in main**. Raw outputs are never written to disk (`trekplan.md:569` reserves the');
  L.push('"do NOT write to disk" rule for the synthesis text only). Phase 7 synthesis therefore');
  L.push('*reasons over already-resident context*. Delegating **only** the Phase-7 read to a');
  L.push('synthesis-agent — "main still spawns the swarm; only the digest is delegated" (T1 §6) —');
  L.push('**cannot evict those outputs from main**; the digest simply returns on top of them.');
  L.push('');
  L.push('⇒ **Δ main-context (faithful flow) ≈ 0** — independent of every token count below. The');
  L.push('≥30% saving is only realizable by ALSO moving Phase-5 delivery off-main (swarm-writes-to-');
  L.push('disk, or a nested orchestrator owning the swarm), which is the wholesale change T1 §7');
  L.push('explicitly declined and is OUT of NW3 scope.');
  L.push('');
  L.push('## 2. Fixtures (measured)');
  L.push('');
  L.push(`- Exploration dir: \`${explorationDir.replace(ROOT + '/', '')}\``);
  L.push(`- Digest: \`${digestPath.replace(ROOT + '/', '')}\``);
  L.push('');
  L.push('| exploration output | chars | est. tokens |');
  L.push('|--------------------|-------|-------------|');
  for (const o of outputs) L.push(`| ${o.name} | ${o.chars} | ${o.tokens} |`);
  L.push(`| **OUT (Σ resident in main)** | — | **${outTokens}** |`);
  L.push(`| digest (DIG) | — | ${digTokens} |`);
  L.push('');
  L.push('## 3. Δ main-context — both framings, swept over BASE');
  L.push('');
  L.push('`inline` = base+out+dig · `delegated (faithful)` = base+out+dig (out already resident) ·');
  L.push('`delegated (disk-potential)` = base+dig (out off-main).');
  L.push('');
  L.push('| BASE (est.) | inline | faithful Δ | faithful verdict | disk-potential Δ | disk verdict |');
  L.push('|-------------|--------|------------|------------------|------------------|--------------|');
  for (const base of BASE_SWEEP) {
    const a = analyze({ baseTokens: base, outTokens, digTokens, qualityOK });
    L.push(
      `| ${base} | ${a.faithful.armA} | ${pct(a.faithful.deltaPct)} | ${a.faithful.verdict} ` +
      `| ${pct(a.disk.deltaPct)} | ${a.disk.verdict} |`,
    );
  }
  L.push('');
  L.push(`Break-even BASE for the disk-potential upper bound to reach the 30% adopt bar: ` +
    `**~${breakEven.toLocaleString('en-US')} tokens** (below this BASE the *hypothetical* disk path ` +
    `would clear 30%; at/above it, even the upper bound fails). A real Voyage main session's BASE ` +
    `(CC system prompt + plugin command/agent/skill listings + CLAUDE.md) is large, so the disk ` +
    `upper bound is itself fragile.`);
  L.push('');
  L.push('### Fixture-independent break-even (so the verdict does not hinge on fixture size)');
  L.push('');
  L.push('disk-potential Δ = out/(base+out+dig), so it clears the 30% adopt bar **iff**');
  L.push('`out / base > 0.30/0.70 ≈ 0.43` — the combined exploration output must exceed ~43% of the');
  L.push('fixed main baseline. The table below sweeps OUT at an illustrative typical `BASE = ' +
    `${REFERENCE_BASE.toLocaleString('en-US')}\` (independent of this run's fixtures):`);
  L.push('');
  L.push('| OUT (Σ exploration tokens) | disk-potential Δ @ ref BASE | clears 30%? |');
  L.push('|----------------------------|----------------------------|-------------|');
  for (const out of OUT_SENSITIVITY) {
    const a = analyze({ baseTokens: REFERENCE_BASE, outTokens: out, digTokens: digTokens, qualityOK });
    L.push(`| ${out.toLocaleString('en-US')} | ${pct(a.disk.deltaPct)} | ${a.disk.deltaPct >= POSITIVE_THRESHOLD ? 'yes' : 'no'} |`);
  }
  L.push('');
  L.push(`This run's fixtures total **OUT = ${outTokens} tokens** across ${outputs.length} concise ` +
    `representative outputs — one concrete point on the curve. Even a generously large real swarm ` +
    `(OUT in the tens of thousands) only clears 30% when the main baseline is unusually small, and ` +
    `*never* in the faithful flow (Δ=0). The verdict is therefore robust to fixture size.`);
  L.push('');
  L.push('## 4. Quality');
  L.push('');
  L.push('The digest-output contract (`lib/plan/synthesis-digest-schema.mjs`) pins the same Phase-7');
  L.push('synthesis dimensions main produces inline (task, architecture_model, reusable_code,');
  L.push('contradictions, risks, gaps, source-tagged findings). A delegated digest that validates is');
  L.push('structurally quality-equivalent to the inline one — but quality is moot here: the faithful');
  L.push('Δ is ~0, so there is no token win for quality to defend.');
  L.push('');
  L.push('## 5. Verdict');
  L.push('');
  const faithfulVerdict = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK }).faithful;
  L.push('**DECLINED per measurement.** NW3-as-scoped yields **Δ main-context ≈ 0%** (faithful flow,');
  L.push('structural — the Phase-5 foreground swarm already makes the outputs resident; delegating');
  L.push('Phase 7 evicts nothing). The disk-potential upper bound is reachable only via an out-of-');
  L.push('scope Phase-5 change and is itself BASE-fragile.');
  L.push('');
  L.push(`RESULT: NEGATIVE (Δ_faithful = ${pct(faithfulVerdict.deltaPct)} < ${pct(NEGATIVE_THRESHOLD)} adopt-floor)`);
  L.push('');
  L.push('## 6. Disposition');
  L.push('');
  L.push('- `agents/synthesis-agent.md` ships **dormant** (a documented, schema-conformant deliverable);');
  L.push('  `commands/trekplan.md` Phase 7 is **NOT** wired to it.');
  L.push('- If main-context relief is later wanted, the prerequisite is a Phase-5 redesign (swarm-');
  L.push('  writes-to-disk / nested orchestrator) — a separate, larger decision (re-open CC-26 §7).');
  L.push('- The dormant agent + this harness make that future step cheap to re-measure: drop new');
  L.push('  fixtures in and re-run.');
  L.push('');
  return L.join('\n') + '\n';
}

function parseArgs(argv) {
  const o = { explorationDir: DEFAULT_EXPLORATION_DIR, digest: DEFAULT_DIGEST, out: DEFAULT_OUT, qualityOK: true, json: false };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--exploration') o.explorationDir = resolve(argv[++i]);
    else if (a === '--digest') o.digest = resolve(argv[++i]);
    else if (a === '--out') o.out = resolve(argv[++i]);
    else if (a === '--quality-fail') o.qualityOK = false;
    else if (a === '--json') o.json = true;
    else if (a === '--help' || a === '-h') { o.help = true; }
    else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
  }
  return o;
}

function mainCli() {
  const o = parseArgs(process.argv.slice(2));
  if (o.help) {
    process.stdout.write('Usage: synthesis-measure.mjs [--exploration DIR] [--digest FILE] [--out FILE] [--quality-fail] [--json]\n');
    process.exit(0);
  }
  if (!existsSync(o.explorationDir)) { process.stderr.write(`exploration dir not found: ${o.explorationDir}\n`); process.exit(2); }
  if (!existsSync(o.digest)) { process.stderr.write(`digest not found: ${o.digest}\n`); process.exit(2); }

  const outputs = loadExploration(o.explorationDir);
  const outTokens = outputs.reduce((s, x) => s + x.tokens, 0);
  const digTokens = estimateTokens(readFileSync(o.digest, 'utf-8'));

  if (o.json) {
    const rows = BASE_SWEEP.map((base) => ({ base, ...analyze({ baseTokens: base, outTokens, digTokens, qualityOK: o.qualityOK }) }));
    process.stdout.write(JSON.stringify({ outTokens, digTokens, breakEvenBase: breakEvenBase(outTokens, digTokens), rows }, null, 2) + '\n');
    process.exit(0);
  }

  const doc = buildResultsDoc({
    explorationDir: o.explorationDir, digestPath: o.digest,
    outputs, outTokens, digTokens, qualityOK: o.qualityOK,
  });
  if (!existsSync(dirname(o.out))) mkdirSync(dirname(o.out), { recursive: true });
  writeFileSync(o.out, doc, 'utf-8');
  const faithful = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK: o.qualityOK }).faithful;
  process.stderr.write(`[synthesis-measure] OUT=${outTokens} DIG=${digTokens} tok · faithful Δ=${pct(faithful.deltaPct)} → ${faithful.verdict} · wrote ${o.out}\n`);
  process.exit(0);
}

if (import.meta.url === `file://${process.argv[1]}`) {
  mainCli();
}