feat(voyage): S12 — NW3 synthesis-agent built + measured → declined per measurement [skip-docs]
NW3 (CC-26 §6 PoC): delegate trekplan Phase 7 synthesis to a synthesis-agent, adopt only if Δ main-context ≥30% with no quality loss. Operator chose the deterministic-proof path (live ≥3-run bake-off was env-blocked: no API key; installed plugin is a cache copy so a new agent is invisible to `claude -p`). Decisive structural finding: trekplan Phase 5 runs the swarm FOREGROUND, so its outputs are already resident in main before Phase 7. Delegating only Phase 7 evicts nothing → Δ_faithful = 0% (BASE-independent). The ≥30% saving needs an out-of-scope Phase-5 redesign (swarm-writes-to-disk / nested orchestrator). VERDICT: DECLINED per measurement. - agents/synthesis-agent.md — dormant, schema-conformant deliverable (NOT wired) - lib/plan/synthesis-digest-schema.mjs — digest output contract (+ tests) - scripts/synthesis-measure.mjs — deterministic Δ-accounting core (+ tests) - tests/fixtures/synthesis/ — 7 exploration outputs + representative digest - docs/T1-synthesis-poc-results.md — measurement + verdict (reproducible) - CLAUDE.md — agent table row (doc-consistency: 24 agents) Tests 670 → 695 (693 pass / 2 skip / 0 fail). `claude plugin validate` clean (only the pre-existing root-CLAUDE.md warning). commands/trekplan.md untouched. [skip-docs] rationale: no user-facing feature ships (NW3 declined; agent dormant and unwired). The substantive doc is docs/T1-synthesis-poc-results.md; the README/CHANGELOG roll-up for NW1–NW3 is the S13 coordinated release per docs/W1-narrow-wins-plan.md §S13. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
This commit is contained in:
parent
9fb536e2d8
commit
6b30483304
15 changed files with 1312 additions and 0 deletions
297
scripts/synthesis-measure.mjs
Normal file
297
scripts/synthesis-measure.mjs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/synthesis-measure.mjs
|
||||
// NW3 (S12) — deterministic Δ main-context measurement for the synthesis-agent.
|
||||
//
|
||||
// The CC-26 gate metric (T1 §2) is Δ main-context tokens for an equivalent
|
||||
// digest. The live ≥3-run bake-off (T1 §5) is the empirically strongest
|
||||
// instrument, but for THIS gate the binding answer is STRUCTURAL, not
|
||||
// stochastic, so a deterministic token-accounting over real fixtures resolves it
|
||||
// reproducibly and without blocked live infra (no API key; the installed plugin
|
||||
// is a cache copy, so a new agent is invisible to `claude -p`). It models two
|
||||
// framings:
|
||||
//
|
||||
// FAITHFUL (current flow): trekplan Phase 5 runs the swarm FOREGROUND, so its
|
||||
// 6–10 outputs are already RESIDENT in main before Phase 7. Delegating only
|
||||
// the synthesis read cannot evict them → main holds base+out+dig in BOTH
|
||||
// arms → Δ ≈ 0. This is what NW3-as-scoped ("main still spawns the swarm;
|
||||
// only the digest is delegated", T1 §6) would actually ship.
|
||||
//
|
||||
// DISK-POTENTIAL (upper bound): IF the swarm wrote outputs to disk and returned
|
||||
// short (a separate Phase-5 change, OUT of NW3 scope), the delegated arm holds
|
||||
// base+dig only → Δ = out/(base+out+dig). BASE-sensitive; swept, not asserted.
|
||||
//
|
||||
// POSITIVE adopt requires Δ ≥ 30% AND quality ≥ inline (T1 §5).
|
||||
//
|
||||
// Zero deps. Node stdlib only. Token figure is an explicit chars/4 estimate; the
|
||||
// RATIO Δ% is what the gate turns on, and the chars/4 constant cancels in the
|
||||
// `out` portion. BASE is environment-dependent (system prompt + plugin listings
|
||||
// + CLAUDE.md) and NOT API-measured this session → swept across a documented band.
|
||||
|
||||
import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(HERE, '..');
|
||||
const DEFAULT_EXPLORATION_DIR = join(ROOT, 'tests/fixtures/synthesis/exploration');
|
||||
const DEFAULT_DIGEST = join(ROOT, 'tests/fixtures/synthesis/digest.json');
|
||||
const DEFAULT_OUT = join(ROOT, 'docs/T1-synthesis-poc-results.md');
|
||||
|
||||
// T1 §5 thresholds.
|
||||
export const POSITIVE_THRESHOLD = 0.30;
|
||||
export const NEGATIVE_THRESHOLD = 0.15;
|
||||
|
||||
// Documented BASE sweep: a Voyage main session's fixed resident baseline (CC
|
||||
// system prompt + tool defs + plugin command/agent/skill listings + CLAUDE.md).
|
||||
// Genuinely environment-dependent; not API-measured this session.
|
||||
export const BASE_SWEEP = Object.freeze([30_000, 50_000, 80_000, 120_000]);
|
||||
|
||||
// Fixture-independent disk-potential sensitivity: sweep OUT at one illustrative
|
||||
// typical baseline, so the verdict does not hinge on this run's fixture sizes.
|
||||
export const REFERENCE_BASE = 60_000;
|
||||
export const OUT_SENSITIVITY = Object.freeze([5_000, 10_000, 20_000, 30_000, 40_000]);
|
||||
|
||||
// ---- pure measurement core (unit-tested) ------------------------------------
|
||||
|
||||
/** Explicit chars/4 token estimate. @param {string} text @returns {number} */
|
||||
export function estimateTokens(text) {
|
||||
if (typeof text !== 'string' || text.length === 0) return 0;
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens resident in MAIN at synthesis-complete, per arm.
|
||||
* @param {{base:number, out:number, dig:number, arm:string}} p
|
||||
* @returns {number}
|
||||
*/
|
||||
export function mainContextTokens({ base, out, dig, arm }) {
|
||||
switch (arm) {
|
||||
case 'inline':
|
||||
// main spawns swarm foreground (out resident) + synthesises inline (dig).
|
||||
return base + out + dig;
|
||||
case 'delegated_faithful':
|
||||
// Phase 5 foreground already made `out` resident; the sub-agent's digest
|
||||
// returns on TOP of it. Delegating Phase 7 evicts nothing.
|
||||
return base + out + dig;
|
||||
case 'delegated_disk':
|
||||
// Hypothetical: outputs on disk, never resident in main; only the digest is.
|
||||
return base + dig;
|
||||
default:
|
||||
throw new Error(`mainContextTokens: unknown arm "${arm}"`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fractional reduction (A−B)/A. Divide-by-zero guarded to 0. */
|
||||
export function deltaPct(armA, armB) {
|
||||
if (!armA) return 0;
|
||||
return (armA - armB) / armA;
|
||||
}
|
||||
|
||||
/** T1 §5 verdict. Quality loss vetoes a token win. */
|
||||
export function decideVerdict(delta, qualityOK) {
|
||||
if (!qualityOK) return 'NEGATIVE';
|
||||
if (delta >= POSITIVE_THRESHOLD) return 'POSITIVE';
|
||||
if (delta < NEGATIVE_THRESHOLD) return 'NEGATIVE';
|
||||
return 'INCONCLUSIVE';
|
||||
}
|
||||
|
||||
/**
|
||||
* Both framings for one BASE.
|
||||
* @param {{baseTokens:number, outTokens:number, digTokens:number, qualityOK:boolean}} p
|
||||
*/
|
||||
export function analyze({ baseTokens, outTokens, digTokens, qualityOK }) {
|
||||
const armParams = { base: baseTokens, out: outTokens, dig: digTokens };
|
||||
const inline = mainContextTokens({ ...armParams, arm: 'inline' });
|
||||
const faithfulB = mainContextTokens({ ...armParams, arm: 'delegated_faithful' });
|
||||
const diskB = mainContextTokens({ ...armParams, arm: 'delegated_disk' });
|
||||
const fD = deltaPct(inline, faithfulB);
|
||||
const dD = deltaPct(inline, diskB);
|
||||
return {
|
||||
faithful: { armA: inline, armB: faithfulB, deltaPct: fD, verdict: decideVerdict(fD, qualityOK) },
|
||||
disk: { armA: inline, armB: diskB, deltaPct: dD, verdict: decideVerdict(dD, qualityOK) },
|
||||
};
|
||||
}
|
||||
|
||||
/** BASE at which disk-potential Δ crosses exactly POSITIVE_THRESHOLD. */
|
||||
export function breakEvenBase(outTokens, digTokens, threshold = POSITIVE_THRESHOLD) {
|
||||
// out/(base+out+dig) = threshold → base = out/threshold - out - dig
|
||||
return Math.round(outTokens / threshold - outTokens - digTokens);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
function pct(x) { return `${(x * 100).toFixed(1)}%`; }
|
||||
|
||||
function loadExploration(dir) {
|
||||
const files = readdirSync(dir)
|
||||
.filter((f) => f.endsWith('.md') || f.endsWith('.txt'))
|
||||
.sort();
|
||||
return files.map((f) => {
|
||||
const text = readFileSync(join(dir, f), 'utf-8');
|
||||
return { name: f, chars: text.length, tokens: estimateTokens(text) };
|
||||
});
|
||||
}
|
||||
|
||||
function buildResultsDoc({ explorationDir, digestPath, outputs, outTokens, digTokens, qualityOK }) {
|
||||
const breakEven = breakEvenBase(outTokens, digTokens);
|
||||
const L = [];
|
||||
L.push('# T1 — Synthesis-agent PoC: Δ main-context measurement (NW3 / S12)');
|
||||
L.push('');
|
||||
L.push('**Status:** Measurement complete — verdict below. **Method:** deterministic token-');
|
||||
L.push('accounting over real exploration fixtures (the live ≥3-run bake-off of T1 §5 is the');
|
||||
L.push('stronger instrument but is (a) environment-blocked here — no `ANTHROPIC_API_KEY`, and the');
|
||||
L.push('installed plugin is a cache copy so a fresh `synthesis-agent` is invisible to `claude -p`;');
|
||||
L.push('and (b) unnecessary, because the binding answer is STRUCTURAL, not stochastic).');
|
||||
L.push('**Resolves:** decision-matrix §W1 / CC-26 narrow PoC (`docs/T1-cc26-delegated-orchestration.md` §6).');
|
||||
L.push('**Reproduce:** `node scripts/synthesis-measure.mjs` (regenerates this file).');
|
||||
L.push('');
|
||||
L.push('> Verifiseringsplikt: token figures are an explicit **chars/4 estimate** (labelled), not a');
|
||||
L.push('> tokenizer count. The gate turns on the RATIO Δ%, in which the per-token constant cancels');
|
||||
L.push('> for the `out` term. BASE (the fixed main-session baseline) is environment-dependent and');
|
||||
L.push('> was NOT API-measured this session → swept across a documented band, not asserted.');
|
||||
L.push('');
|
||||
L.push('## 1. The decisive structural finding (BASE-independent)');
|
||||
L.push('');
|
||||
L.push('trekplan runs the Phase 5 exploration swarm **foreground** (foreground is the only mode');
|
||||
L.push('since v2.4.0; `commands/trekplan.md`). Foreground Agent/Task results are delivered back');
|
||||
L.push('into the main transcript, so after Phase 5 the 6–10 exploration outputs are **already');
|
||||
L.push('resident in main**. Raw outputs are never written to disk (`trekplan.md:569` reserves the');
|
||||
L.push('"do NOT write to disk" rule for the synthesis text only). Phase 7 synthesis therefore');
|
||||
L.push('*reasons over already-resident context*. Delegating **only** the Phase-7 read to a');
|
||||
L.push('synthesis-agent — "main still spawns the swarm; only the digest is delegated" (T1 §6) —');
|
||||
L.push('**cannot evict those outputs from main**; the digest simply returns on top of them.');
|
||||
L.push('');
|
||||
L.push('⇒ **Δ main-context (faithful flow) ≈ 0** — independent of every token count below. The');
|
||||
L.push('≥30% saving is only realizable by ALSO moving Phase-5 delivery off-main (swarm-writes-to-');
|
||||
L.push('disk, or a nested orchestrator owning the swarm), which is the wholesale change T1 §7');
|
||||
L.push('explicitly declined and is OUT of NW3 scope.');
|
||||
L.push('');
|
||||
L.push('## 2. Fixtures (measured)');
|
||||
L.push('');
|
||||
L.push(`- Exploration dir: \`${explorationDir.replace(ROOT + '/', '')}\``);
|
||||
L.push(`- Digest: \`${digestPath.replace(ROOT + '/', '')}\``);
|
||||
L.push('');
|
||||
L.push('| exploration output | chars | est. tokens |');
|
||||
L.push('|--------------------|-------|-------------|');
|
||||
for (const o of outputs) L.push(`| ${o.name} | ${o.chars} | ${o.tokens} |`);
|
||||
L.push(`| **OUT (Σ resident in main)** | — | **${outTokens}** |`);
|
||||
L.push(`| digest (DIG) | — | ${digTokens} |`);
|
||||
L.push('');
|
||||
L.push('## 3. Δ main-context — both framings, swept over BASE');
|
||||
L.push('');
|
||||
L.push('`inline` = base+out+dig · `delegated (faithful)` = base+out+dig (out already resident) ·');
|
||||
L.push('`delegated (disk-potential)` = base+dig (out off-main).');
|
||||
L.push('');
|
||||
L.push('| BASE (est.) | inline | faithful Δ | faithful verdict | disk-potential Δ | disk verdict |');
|
||||
L.push('|-------------|--------|------------|------------------|------------------|--------------|');
|
||||
for (const base of BASE_SWEEP) {
|
||||
const a = analyze({ baseTokens: base, outTokens, digTokens, qualityOK });
|
||||
L.push(
|
||||
`| ${base} | ${a.faithful.armA} | ${pct(a.faithful.deltaPct)} | ${a.faithful.verdict} ` +
|
||||
`| ${pct(a.disk.deltaPct)} | ${a.disk.verdict} |`,
|
||||
);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`Break-even BASE for the disk-potential upper bound to reach the 30% adopt bar: ` +
|
||||
`**~${breakEven.toLocaleString('en-US')} tokens** (below this BASE the *hypothetical* disk path ` +
|
||||
`would clear 30%; at/above it, even the upper bound fails). A real Voyage main session's BASE ` +
|
||||
`(CC system prompt + plugin command/agent/skill listings + CLAUDE.md) is large, so the disk ` +
|
||||
`upper bound is itself fragile.`);
|
||||
L.push('');
|
||||
L.push('### Fixture-independent break-even (so the verdict does not hinge on fixture size)');
|
||||
L.push('');
|
||||
L.push('disk-potential Δ = out/(base+out+dig), so it clears the 30% adopt bar **iff**');
|
||||
L.push('`out / base > 0.30/0.70 ≈ 0.43` — the combined exploration output must exceed ~43% of the');
|
||||
L.push('fixed main baseline. The table below sweeps OUT at an illustrative typical `BASE = ' +
|
||||
`${REFERENCE_BASE.toLocaleString('en-US')}\` (independent of this run's fixtures):`);
|
||||
L.push('');
|
||||
L.push('| OUT (Σ exploration tokens) | disk-potential Δ @ ref BASE | clears 30%? |');
|
||||
L.push('|----------------------------|----------------------------|-------------|');
|
||||
for (const out of OUT_SENSITIVITY) {
|
||||
const a = analyze({ baseTokens: REFERENCE_BASE, outTokens: out, digTokens: digTokens, qualityOK });
|
||||
L.push(`| ${out.toLocaleString('en-US')} | ${pct(a.disk.deltaPct)} | ${a.disk.deltaPct >= POSITIVE_THRESHOLD ? 'yes' : 'no'} |`);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`This run's fixtures total **OUT = ${outTokens} tokens** across ${outputs.length} concise ` +
|
||||
`representative outputs — one concrete point on the curve. Even a generously large real swarm ` +
|
||||
`(OUT in the tens of thousands) only clears 30% when the main baseline is unusually small, and ` +
|
||||
`*never* in the faithful flow (Δ=0). The verdict is therefore robust to fixture size.`);
|
||||
L.push('');
|
||||
L.push('## 4. Quality');
|
||||
L.push('');
|
||||
L.push('The digest-output contract (`lib/plan/synthesis-digest-schema.mjs`) pins the same Phase-7');
|
||||
L.push('synthesis dimensions main produces inline (task, architecture_model, reusable_code,');
|
||||
L.push('contradictions, risks, gaps, source-tagged findings). A delegated digest that validates is');
|
||||
L.push('structurally quality-equivalent to the inline one — but quality is moot here: the faithful');
|
||||
L.push('Δ is ~0, so there is no token win for quality to defend.');
|
||||
L.push('');
|
||||
L.push('## 5. Verdict');
|
||||
L.push('');
|
||||
const faithfulVerdict = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK }).faithful;
|
||||
L.push('**DECLINED per measurement.** NW3-as-scoped yields **Δ main-context ≈ 0%** (faithful flow,');
|
||||
L.push('structural — the Phase-5 foreground swarm already makes the outputs resident; delegating');
|
||||
L.push('Phase 7 evicts nothing). The disk-potential upper bound is reachable only via an out-of-');
|
||||
L.push('scope Phase-5 change and is itself BASE-fragile.');
|
||||
L.push('');
|
||||
L.push(`RESULT: NEGATIVE (Δ_faithful = ${pct(faithfulVerdict.deltaPct)} < ${pct(NEGATIVE_THRESHOLD)} adopt-floor)`);
|
||||
L.push('');
|
||||
L.push('## 6. Disposition');
|
||||
L.push('');
|
||||
L.push('- `agents/synthesis-agent.md` ships **dormant** (a documented, schema-conformant deliverable);');
|
||||
L.push(' `commands/trekplan.md` Phase 7 is **NOT** wired to it.');
|
||||
L.push('- If main-context relief is later wanted, the prerequisite is a Phase-5 redesign (swarm-');
|
||||
L.push(' writes-to-disk / nested orchestrator) — a separate, larger decision (re-open CC-26 §7).');
|
||||
L.push('- The dormant agent + this harness make that future step cheap to re-measure: drop new');
|
||||
L.push(' fixtures in and re-run.');
|
||||
L.push('');
|
||||
return L.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { explorationDir: DEFAULT_EXPLORATION_DIR, digest: DEFAULT_DIGEST, out: DEFAULT_OUT, qualityOK: true, json: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--exploration') o.explorationDir = resolve(argv[++i]);
|
||||
else if (a === '--digest') o.digest = resolve(argv[++i]);
|
||||
else if (a === '--out') o.out = resolve(argv[++i]);
|
||||
else if (a === '--quality-fail') o.qualityOK = false;
|
||||
else if (a === '--json') o.json = true;
|
||||
else if (a === '--help' || a === '-h') { o.help = true; }
|
||||
else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
function mainCli() {
|
||||
const o = parseArgs(process.argv.slice(2));
|
||||
if (o.help) {
|
||||
process.stdout.write('Usage: synthesis-measure.mjs [--exploration DIR] [--digest FILE] [--out FILE] [--quality-fail] [--json]\n');
|
||||
process.exit(0);
|
||||
}
|
||||
if (!existsSync(o.explorationDir)) { process.stderr.write(`exploration dir not found: ${o.explorationDir}\n`); process.exit(2); }
|
||||
if (!existsSync(o.digest)) { process.stderr.write(`digest not found: ${o.digest}\n`); process.exit(2); }
|
||||
|
||||
const outputs = loadExploration(o.explorationDir);
|
||||
const outTokens = outputs.reduce((s, x) => s + x.tokens, 0);
|
||||
const digTokens = estimateTokens(readFileSync(o.digest, 'utf-8'));
|
||||
|
||||
if (o.json) {
|
||||
const rows = BASE_SWEEP.map((base) => ({ base, ...analyze({ baseTokens: base, outTokens, digTokens, qualityOK: o.qualityOK }) }));
|
||||
process.stdout.write(JSON.stringify({ outTokens, digTokens, breakEvenBase: breakEvenBase(outTokens, digTokens), rows }, null, 2) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const doc = buildResultsDoc({
|
||||
explorationDir: o.explorationDir, digestPath: o.digest,
|
||||
outputs, outTokens, digTokens, qualityOK: o.qualityOK,
|
||||
});
|
||||
if (!existsSync(dirname(o.out))) mkdirSync(dirname(o.out), { recursive: true });
|
||||
writeFileSync(o.out, doc, 'utf-8');
|
||||
const faithful = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK: o.qualityOK }).faithful;
|
||||
process.stderr.write(`[synthesis-measure] OUT=${outTokens} DIG=${digTokens} tok · faithful Δ=${pct(faithful.deltaPct)} → ${faithful.verdict} · wrote ${o.out}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
mainCli();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue