feat(scripts): add STORM measurement harness with pre-registered thresholds

scripts/storm-measure.mjs

@@ -0,0 +1,287 @@
+#!/usr/bin/env node
+// scripts/storm-measure.mjs
+// Step 11 — the STORM adoption gate: deterministic Δ accounting over
+// ${CLAUDE_PLUGIN_DATA}/trekresearch-stats.jsonl.
+//
+// What this decides: whether the bounded Phase 5 conversation loop buys enough
+// extra source/coverage breadth to justify flipping VOYAGE_STORM_ENABLED on by
+// default in lib/util/research-loop-cap.mjs. Nothing else. The thresholds live
+// in docs/storm-measurement.md and were committed BEFORE any measurement run —
+// that pre-registration is the whole point, so this script never invents them.
+//
+// What this does NOT measure: outline quality, answer correctness, or operator
+// satisfaction. It measures breadth (distinct sources, dimensions covered).
+// A breadth win is necessary for adoption, not sufficient on its own.
+//
+// Honesty properties, both load-bearing:
+//   - Runs with empty_turns > 0 are EXCLUDED from the gain and REPORTED. An
+//     empty turn means the loop spent budget and returned nothing; leaving those
+//     in decides adoption on a broken denominator.
+//   - A stats file carrying no `effort` field is a loud error. The silent
+//     failure this prevents is an empty treatment group reading as "no gain",
+//     which would decline the mechanism for a schema reason.
+//
+// Zero deps. Node stdlib only.
+
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+// Pre-registered in docs/storm-measurement.md. Do not tune to fit a result.
+export const ADOPT_THRESHOLD = 0.30;
+export const DECLINE_THRESHOLD = 0.15;
+
+const STATS_FILENAME = 'trekresearch-stats.jsonl';
+
+// ---- pure core (unit-tested) -------------------------------------------------
+
+/** @param {number[]} xs @returns {number|null} null for an empty list — 0 would read as a measurement. */
+export function median(xs) {
+  if (!Array.isArray(xs) || xs.length === 0) return null;
+  const s = [...xs].sort((a, b) => a - b);
+  const mid = s.length >> 1;
+  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+
+/**
+ * Parse a trekresearch-stats.jsonl body into effort-carrying records.
+ *
+ * Rows predating Step 9 have no `effort` field; they are dropped and counted as
+ * `legacy` rather than silently pooled into the control arm. A file where NO row
+ * carries `effort` throws — see the header note on silent failure.
+ *
+ * @param {string} text
+ * @returns {{records: object[], malformed: number, legacy: number}}
+ */
+export function parseStats(text) {
+  const lines = String(text ?? '').split('\n');
+  const records = [];
+  let malformed = 0;
+  let legacy = 0;
+  let parsed = 0;
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    let rec;
+    try {
+      rec = JSON.parse(line);
+    } catch {
+      malformed++;
+      continue;
+    }
+    parsed++;
+    if (typeof rec?.effort !== 'string' || rec.effort.length === 0) {
+      legacy++;
+      continue;
+    }
+    records.push(rec);
+  }
+
+  if (parsed === 0) {
+    throw new Error(`storm-measure: no records in stats file (${malformed} malformed line(s)).`);
+  }
+  if (records.length === 0) {
+    throw new Error(
+      `storm-measure: no records carry an \`effort\` field (${legacy} legacy row(s)). ` +
+      `The gate groups on \`effort\`; without it there is no treatment arm to measure. ` +
+      `Re-run the measurement set on a build that emits the Step 9 fields.`,
+    );
+  }
+  return { records, malformed, legacy };
+}
+
+/**
+ * Split off the runs that must not count toward a gain.
+ * @param {object[]} records
+ * @returns {{eligible: object[], excluded: number}}
+ */
+export function partitionEligible(records) {
+  const eligible = [];
+  let excluded = 0;
+  for (const r of records) {
+    const empty = Number(r.empty_turns ?? 0);
+    if (Number.isFinite(empty) && empty > 0) excluded++;
+    else eligible.push(r);
+  }
+  return { eligible, excluded };
+}
+
+function numbers(records, pick) {
+  return records.map(pick).filter((n) => Number.isFinite(n));
+}
+
+/** Relative gain (treatment − control)/control. null when control is absent or zero. */
+function relGain(control, treatment) {
+  if (control === null || treatment === null || control === 0) return null;
+  return (treatment - control) / control;
+}
+
+/**
+ * Full measurement over a parsed record set.
+ *
+ * Treatment arm = `effort: high` (the only effort at which the loop runs).
+ * Control arm   = every other effort.
+ *
+ * - sources:    between-arm median gain in `unique_sources`.
+ * - dimensions: within-run median gain of (dimensions − dimensions_baseline)
+ *               / dimensions_baseline across the treatment arm. It is a
+ *               within-run delta by construction, so it needs no control arm —
+ *               the control arm's is 0, the loop being inert there.
+ *
+ * @param {object[]} records
+ */
+export function measure(records) {
+  const { eligible, excluded } = partitionEligible(records);
+  const treatment = eligible.filter((r) => r.effort === 'high');
+  const control = eligible.filter((r) => r.effort !== 'high');
+
+  const srcControl = median(numbers(control, (r) => Number(r.unique_sources)));
+  const srcTreatment = median(numbers(treatment, (r) => Number(r.unique_sources)));
+  const sourcesGain = relGain(srcControl, srcTreatment);
+
+  const dimDeltas = treatment
+    .map((r) => ({ d: Number(r.dimensions), b: Number(r.dimensions_baseline) }))
+    .filter(({ d, b }) => Number.isFinite(d) && Number.isFinite(b) && b > 0)
+    .map(({ d, b }) => (d - b) / b);
+  const dimensionsGain = median(dimDeltas);
+
+  return {
+    control: { n: control.length, sources: srcControl },
+    treatment: { n: treatment.length, sources: srcTreatment },
+    sources: { control: srcControl, treatment: srcTreatment, gain: sourcesGain },
+    dimensions: { gain: dimensionsGain, n: dimDeltas.length },
+    excluded,
+    verdict: decideVerdict(sourcesGain, dimensionsGain),
+  };
+}
+
+/**
+ * Pre-registered mapping. Adopt requires BOTH metrics to clear the bar: a
+ * breadth win on one axis alone is not the effect the loop claims.
+ *
+ * @param {number|null} sourcesGain
+ * @param {number|null} dimensionsGain
+ * @returns {'adopt'|'decline'|'inconclusive'|'insufficient-data'}
+ */
+export function decideVerdict(sourcesGain, dimensionsGain) {
+  if (sourcesGain === null || sourcesGain === undefined) return 'insufficient-data';
+  if (dimensionsGain === null || dimensionsGain === undefined) return 'insufficient-data';
+  if (sourcesGain >= ADOPT_THRESHOLD && dimensionsGain >= ADOPT_THRESHOLD) return 'adopt';
+  if (sourcesGain < DECLINE_THRESHOLD && dimensionsGain < DECLINE_THRESHOLD) return 'decline';
+  return 'inconclusive';
+}
+
+/**
+ * SC activation check: did the most recent `effort: high` run actually discover
+ * dimensions beyond its interview baseline?
+ * @param {object[]} records
+ */
+export function activationCheck(records) {
+  const high = records.filter((r) => r.effort === 'high');
+  if (high.length === 0) {
+    return { ok: false, reason: 'no `effort: high` run found in stats', discovered_dimensions: null };
+  }
+  const last = high[high.length - 1];
+  const d = Number(last.dimensions);
+  const b = Number(last.dimensions_baseline);
+  if (!Number.isFinite(d) || !Number.isFinite(b)) {
+    return { ok: false, reason: 'latest high run lacks dimensions/dimensions_baseline', discovered_dimensions: null };
+  }
+  const discovered = d - b;
+  return {
+    ok: discovered >= 1,
+    ts: last.ts ?? null,
+    dimensions: d,
+    dimensions_baseline: b,
+    discovered_dimensions: discovered,
+    conv_turns: Number(last.conv_turns ?? 0),
+    empty_turns: Number(last.empty_turns ?? 0),
+  };
+}
+
+// ---- CLI shim ----------------------------------------------------------------
+
+function pct(x) {
+  return x === null ? 'n/a' : `${(x * 100).toFixed(1)}%`;
+}
+
+function defaultStatsPath(env = process.env) {
+  const dir = env.CLAUDE_PLUGIN_DATA;
+  return dir ? join(dir, STATS_FILENAME) : null;
+}
+
+function parseArgs(argv) {
+  const o = { stats: null, json: false, activation: false, help: false };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--stats') o.stats = argv[++i];
+    else if (a === '--json') o.json = true;
+    else if (a === '--activation-check') o.activation = true;
+    else if (a === '--help' || a === '-h') o.help = true;
+    else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
+  }
+  return o;
+}
+
+function mainCli() {
+  const o = parseArgs(process.argv.slice(2));
+  if (o.help) {
+    process.stdout.write(
+      'Usage: storm-measure.mjs [--stats FILE] [--activation-check] [--json]\n' +
+      '  Default --stats: ${CLAUDE_PLUGIN_DATA}/' + STATS_FILENAME + '\n' +
+      '  Thresholds (pre-registered, docs/storm-measurement.md): ' +
+      `adopt >= ${ADOPT_THRESHOLD * 100}%, decline < ${DECLINE_THRESHOLD * 100}%\n`,
+    );
+    process.exit(0);
+  }
+
+  const statsPath = o.stats || defaultStatsPath();
+  if (!statsPath) {
+    process.stderr.write('storm-measure: CLAUDE_PLUGIN_DATA is not set and no --stats FILE was given.\n');
+    process.exit(2);
+  }
+  if (!existsSync(statsPath)) {
+    process.stderr.write(`storm-measure: stats file not found: ${statsPath}\n`);
+    process.exit(2);
+  }
+
+  let parsed;
+  try {
+    parsed = parseStats(readFileSync(statsPath, 'utf-8'));
+  } catch (e) {
+    process.stderr.write(`${e.message}\n`);
+    process.exit(2);
+  }
+
+  if (o.activation) {
+    const res = activationCheck(parsed.records);
+    process.stdout.write(JSON.stringify(res, null, 2) + '\n');
+    process.exit(res.ok ? 0 : 1);
+  }
+
+  const m = measure(parsed.records);
+
+  if (o.json) {
+    process.stdout.write(JSON.stringify({ statsPath, ...m, malformed: parsed.malformed, legacy: parsed.legacy }, null, 2) + '\n');
+    process.exit(0);
+  }
+
+  const L = [];
+  L.push(`STORM adoption gate — ${statsPath}`);
+  L.push(`  control (effort != high):   n=${m.control.n}  median unique_sources=${m.control.sources ?? 'n/a'}`);
+  L.push(`  treatment (effort = high):  n=${m.treatment.n}  median unique_sources=${m.treatment.sources ?? 'n/a'}`);
+  L.push(`  excluded (empty_turns > 0): ${m.excluded}`);
+  if (parsed.legacy) L.push(`  legacy rows without \`effort\`: ${parsed.legacy}`);
+  if (parsed.malformed) L.push(`  malformed lines: ${parsed.malformed}`);
+  L.push('');
+  L.push(`  median gain, unique_sources:            ${pct(m.sources.gain)}`);
+  L.push(`  median gain, dimensions over baseline:  ${pct(m.dimensions.gain)}  (n=${m.dimensions.n})`);
+  L.push('');
+  L.push(`  thresholds: adopt >= ${pct(ADOPT_THRESHOLD)} on BOTH · decline < ${pct(DECLINE_THRESHOLD)} on BOTH`);
+  L.push(`  VERDICT: ${m.verdict}`);
+  process.stdout.write(L.join('\n') + '\n');
+  process.exit(0);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  mainCli();
+}