voyage/scripts/storm-measure.mjs

#!/usr/bin/env node
// scripts/storm-measure.mjs
// Step 11 — the STORM adoption gate: deterministic Δ accounting over
// ${CLAUDE_PLUGIN_DATA}/trekresearch-stats.jsonl.
//
// What this decides: whether the bounded Phase 5 conversation loop buys enough
// extra source/coverage breadth to justify flipping VOYAGE_STORM_ENABLED on by
// default in lib/util/research-loop-cap.mjs. Nothing else. The thresholds live
// in docs/storm-measurement.md and were committed BEFORE any measurement run —
// that pre-registration is the whole point, so this script never invents them.
//
// What this does NOT measure: outline quality, answer correctness, or operator
// satisfaction. It measures breadth (distinct sources, dimensions covered).
// A breadth win is necessary for adoption, not sufficient on its own.
//
// Honesty properties, both load-bearing:
//   - Runs with empty_turns > 0 are EXCLUDED from the gain and REPORTED. An
//     empty turn means the loop spent budget and returned nothing; leaving those
//     in decides adoption on a broken denominator.
//   - A stats file carrying no `effort` field is a loud error. The silent
//     failure this prevents is an empty treatment group reading as "no gain",
//     which would decline the mechanism for a schema reason.
//
// Zero deps. Node stdlib only.

import { readFileSync, existsSync } from 'node:fs';
import { join } from 'node:path';

// Pre-registered in docs/storm-measurement.md. Do not tune to fit a result.
export const ADOPT_THRESHOLD = 0.30;
export const DECLINE_THRESHOLD = 0.15;

const STATS_FILENAME = 'trekresearch-stats.jsonl';

// ---- pure core (unit-tested) -------------------------------------------------

/** @param {number[]} xs @returns {number|null} null for an empty list — 0 would read as a measurement. */
export function median(xs) {
  if (!Array.isArray(xs) || xs.length === 0) return null;
  const s = [...xs].sort((a, b) => a - b);
  const mid = s.length >> 1;
  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}

/**
 * Parse a trekresearch-stats.jsonl body into effort-carrying records.
 *
 * Rows predating Step 9 have no `effort` field; they are dropped and counted as
 * `legacy` rather than silently pooled into the control arm. A file where NO row
 * carries `effort` throws — see the header note on silent failure.
 *
 * @param {string} text
 * @returns {{records: object[], malformed: number, legacy: number}}
 */
export function parseStats(text) {
  const lines = String(text ?? '').split('\n');
  const records = [];
  let malformed = 0;
  let legacy = 0;
  let parsed = 0;

  for (const line of lines) {
    if (!line.trim()) continue;
    let rec;
    try {
      rec = JSON.parse(line);
    } catch {
      malformed++;
      continue;
    }
    parsed++;
    if (typeof rec?.effort !== 'string' || rec.effort.length === 0) {
      legacy++;
      continue;
    }
    records.push(rec);
  }

  if (parsed === 0) {
    throw new Error(`storm-measure: no records in stats file (${malformed} malformed line(s)).`);
  }
  if (records.length === 0) {
    throw new Error(
      `storm-measure: no records carry an \`effort\` field (${legacy} legacy row(s)). ` +
      `The gate groups on \`effort\`; without it there is no treatment arm to measure. ` +
      `Re-run the measurement set on a build that emits the Step 9 fields.`,
    );
  }
  return { records, malformed, legacy };
}

/**
 * Split off the runs that must not count toward a gain.
 *
 * A value that cannot be READ as a turn count is excluded, not treated as zero.
 * `Number('many')` is NaN, and testing `Number.isFinite(empty) && empty > 0`
 * sent NaN down the eligible branch — so a garbage field silently re-entered the
 * denominator, in the direction that flatters adoption: the run whose bookkeeping
 * broke is the run whose numbers deserve the least trust. Absent and null stay
 * eligible via `?? 0`, because a field that was never written is a genuine zero
 * on any run where the loop did not arm.
 *
 * @param {object[]} records
 * @returns {{eligible: object[], excluded: number}}
 */
export function partitionEligible(records) {
  const eligible = [];
  let excluded = 0;
  for (const r of records) {
    const empty = Number(r.empty_turns ?? 0);
    if (!Number.isFinite(empty) || empty > 0) excluded++;
    else eligible.push(r);
  }
  return { eligible, excluded };
}

function numbers(records, pick) {
  return records.map(pick).filter((n) => Number.isFinite(n));
}

/** Relative gain (treatment − control)/control. null when control is absent or zero. */
function relGain(control, treatment) {
  if (control === null || treatment === null || control === 0) return null;
  return (treatment - control) / control;
}

/**
 * Full measurement over a parsed record set.
 *
 * Treatment arm = `effort: high` (the only effort at which the loop runs).
 * Control arm   = every other effort.
 *
 * - sources:    between-arm median gain in `unique_sources`.
 * - dimensions: within-run median gain of (dimensions − dimensions_baseline)
 *               / dimensions_baseline across the treatment arm. It is a
 *               within-run delta by construction, so it needs no control arm —
 *               the control arm's is 0, the loop being inert there.
 *
 * @param {object[]} records
 */
export function measure(records) {
  const { eligible, excluded } = partitionEligible(records);
  const treatment = eligible.filter((r) => r.effort === 'high');
  const control = eligible.filter((r) => r.effort !== 'high');

  const srcControl = median(numbers(control, (r) => Number(r.unique_sources)));
  const srcTreatment = median(numbers(treatment, (r) => Number(r.unique_sources)));
  const sourcesGain = relGain(srcControl, srcTreatment);

  const dimDeltas = treatment
    .map((r) => ({ d: Number(r.dimensions), b: Number(r.dimensions_baseline) }))
    .filter(({ d, b }) => Number.isFinite(d) && Number.isFinite(b) && b > 0)
    .map(({ d, b }) => (d - b) / b);
  const dimensionsGain = median(dimDeltas);

  return {
    control: { n: control.length, sources: srcControl },
    treatment: { n: treatment.length, sources: srcTreatment },
    sources: { control: srcControl, treatment: srcTreatment, gain: sourcesGain },
    dimensions: { gain: dimensionsGain, n: dimDeltas.length },
    excluded,
    verdict: decideVerdict(sourcesGain, dimensionsGain),
  };
}

/**
 * Pre-registered mapping, verbatim from the brief: "median forbedring >= 30 %
 * på (a) eller (b) → adopt. < 15 % → decline." OR on both sides, adopt
 * evaluated first — so a strong win on one axis is an adopt even when the
 * other axis sits under the decline bar. A stricter AND rule may well be the
 * better decision procedure, but changing it here is changing the
 * pre-registration after the fact, which is the one thing the constraint
 * exists to prevent.
 *
 * @param {number|null} sourcesGain
 * @param {number|null} dimensionsGain
 * @returns {'adopt'|'decline'|'inconclusive'|'insufficient-data'}
 */
export function decideVerdict(sourcesGain, dimensionsGain) {
  if (sourcesGain === null || sourcesGain === undefined) return 'insufficient-data';
  if (dimensionsGain === null || dimensionsGain === undefined) return 'insufficient-data';
  if (sourcesGain >= ADOPT_THRESHOLD || dimensionsGain >= ADOPT_THRESHOLD) return 'adopt';
  if (sourcesGain < DECLINE_THRESHOLD || dimensionsGain < DECLINE_THRESHOLD) return 'decline';
  return 'inconclusive';
}

/**
 * SC activation check: did the most recent `effort: high` run actually discover
 * dimensions beyond its interview baseline?
 * @param {object[]} records
 */
export function activationCheck(records) {
  const high = records.filter((r) => r.effort === 'high');
  if (high.length === 0) {
    return { ok: false, reason: 'no `effort: high` run found in stats', discovered_dimensions: null };
  }
  const last = high[high.length - 1];
  const d = Number(last.dimensions);
  const b = Number(last.dimensions_baseline);
  if (!Number.isFinite(d) || !Number.isFinite(b)) {
    return { ok: false, reason: 'latest high run lacks dimensions/dimensions_baseline', discovered_dimensions: null };
  }
  const discovered = d - b;
  return {
    ok: discovered >= 1,
    ts: last.ts ?? null,
    dimensions: d,
    dimensions_baseline: b,
    discovered_dimensions: discovered,
    conv_turns: Number(last.conv_turns ?? 0),
    empty_turns: Number(last.empty_turns ?? 0),
  };
}

// ---- CLI shim ----------------------------------------------------------------

function pct(x) {
  return x === null ? 'n/a' : `${(x * 100).toFixed(1)}%`;
}

function defaultStatsPath(env = process.env) {
  const dir = env.CLAUDE_PLUGIN_DATA;
  return dir ? join(dir, STATS_FILENAME) : null;
}

function parseArgs(argv) {
  const o = { stats: null, json: false, activation: false, help: false };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--stats') o.stats = argv[++i];
    else if (a === '--json') o.json = true;
    else if (a === '--activation-check') o.activation = true;
    else if (a === '--help' || a === '-h') o.help = true;
    else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
  }
  return o;
}

function mainCli() {
  const o = parseArgs(process.argv.slice(2));
  if (o.help) {
    process.stdout.write(
      'Usage: storm-measure.mjs [--stats FILE] [--activation-check] [--json]\n' +
      '  Default --stats: ${CLAUDE_PLUGIN_DATA}/' + STATS_FILENAME + '\n' +
      '  Thresholds (pre-registered, docs/storm-measurement.md): ' +
      `adopt >= ${ADOPT_THRESHOLD * 100}%, decline < ${DECLINE_THRESHOLD * 100}%\n`,
    );
    process.exit(0);
  }

  const statsPath = o.stats || defaultStatsPath();
  if (!statsPath) {
    process.stderr.write('storm-measure: CLAUDE_PLUGIN_DATA is not set and no --stats FILE was given.\n');
    process.exit(2);
  }
  if (!existsSync(statsPath)) {
    process.stderr.write(`storm-measure: stats file not found: ${statsPath}\n`);
    process.exit(2);
  }

  let parsed;
  try {
    parsed = parseStats(readFileSync(statsPath, 'utf-8'));
  } catch (e) {
    process.stderr.write(`${e.message}\n`);
    process.exit(2);
  }

  if (o.activation) {
    const res = activationCheck(parsed.records);
    process.stdout.write(JSON.stringify(res, null, 2) + '\n');
    process.exit(res.ok ? 0 : 1);
  }

  const m = measure(parsed.records);

  if (o.json) {
    process.stdout.write(JSON.stringify({ statsPath, ...m, malformed: parsed.malformed, legacy: parsed.legacy }, null, 2) + '\n');
    process.exit(0);
  }

  const L = [];
  L.push(`STORM adoption gate — ${statsPath}`);
  L.push(`  control (effort != high):   n=${m.control.n}  median unique_sources=${m.control.sources ?? 'n/a'}`);
  L.push(`  treatment (effort = high):  n=${m.treatment.n}  median unique_sources=${m.treatment.sources ?? 'n/a'}`);
  L.push(`  excluded (empty_turns > 0): ${m.excluded}`);
  if (parsed.legacy) L.push(`  legacy rows without \`effort\`: ${parsed.legacy}`);
  if (parsed.malformed) L.push(`  malformed lines: ${parsed.malformed}`);
  L.push('');
  L.push(`  median gain, unique_sources:            ${pct(m.sources.gain)}`);
  L.push(`  median gain, dimensions over baseline:  ${pct(m.dimensions.gain)}  (n=${m.dimensions.n})`);
  L.push('');
  L.push(`  thresholds: adopt >= ${pct(ADOPT_THRESHOLD)} on EITHER · decline < ${pct(DECLINE_THRESHOLD)} on EITHER · adopt wins ties`);
  L.push(`  VERDICT: ${m.verdict}`);
  process.stdout.write(L.join('\n') + '\n');
  process.exit(0);
}

if (import.meta.url === `file://${process.argv[1]}`) {
  mainCli();
}