feat(scripts): add STORM measurement harness with pre-registered thresholds

tests/scripts/storm-measure.test.mjs

@@ -0,0 +1,199 @@
+// tests/scripts/storm-measure.test.mjs
+// Step 11 — the STORM adoption gate's deterministic accounting core.
+//
+// The gate decides ONE thing: does the bounded Phase 5 loop buy enough extra
+// source/coverage breadth to be worth flipping VOYAGE_STORM_ENABLED on by
+// default. Thresholds are pre-registered in docs/storm-measurement.md BEFORE
+// any measurement run, so this file pins the arithmetic that turns a
+// trekresearch-stats.jsonl into a verdict — not the verdict itself.
+//
+// Two properties carry the gate's honesty:
+//   - runs with empty_turns > 0 are EXCLUDED from the gain and COUNTED, so
+//     adoption is never decided on a broken denominator, and
+//   - a stats file with no `effort` field is a loud error, never a silently
+//     empty group that reads as "no gain".
+//
+// Pattern: tests/scripts/synthesis-measure.test.mjs (pure core, no fixtures on disk).
+
+import { test } from 'node:test';
+import { strict as assert } from 'node:assert';
+import {
+  median,
+  parseStats,
+  partitionEligible,
+  measure,
+  decideVerdict,
+  ADOPT_THRESHOLD,
+  DECLINE_THRESHOLD,
+} from '../../scripts/storm-measure.mjs';
+
+// ---------------------------------------------------------------------------
+// helpers — synthetic JSONL, one object per line, exactly as the orchestrator emits
+// ---------------------------------------------------------------------------
+function run({ effort, unique_sources, dimensions, dimensions_baseline, empty_turns = 0, conv_turns = 0 }) {
+  return JSON.stringify({
+    ts: '2026-08-12T00:00:00.000Z',
+    question: 'q',
+    mode: 'full',
+    scope: 'both',
+    engine: 'swarm',
+    effort,
+    unique_sources,
+    dimensions,
+    dimensions_baseline,
+    conv_turns,
+    empty_turns,
+  });
+}
+
+function jsonl(...lines) {
+  return lines.join('\n') + '\n';
+}
+
+// A control arm at 10 sources / 5 dimensions, and a treatment arm at 13
+// sources / 8 dimensions: +30.0% sources, +60.0% dimensions.
+const STANDARD = [
+  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
+  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
+  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
+];
+const HIGH = [
+  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
+  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
+  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
+];
+
+// ---------------------------------------------------------------------------
+// median
+// ---------------------------------------------------------------------------
+test('median: odd, even, single', () => {
+  assert.equal(median([3, 1, 2]), 2);
+  assert.equal(median([1, 2, 3, 4]), 2.5);
+  assert.equal(median([7]), 7);
+});
+
+test('median: empty list is null, never 0 — 0 would read as a real measurement', () => {
+  assert.equal(median([]), null);
+});
+
+test('median does not mutate its input', () => {
+  const xs = [3, 1, 2];
+  median(xs);
+  assert.deepEqual(xs, [3, 1, 2]);
+});
+
+// ---------------------------------------------------------------------------
+// parseStats — the loud-error requirement
+// ---------------------------------------------------------------------------
+test('parseStats: a file with no effort field throws, it does not yield empty groups', () => {
+  const noEffort = jsonl(
+    JSON.stringify({ ts: 'x', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
+    JSON.stringify({ ts: 'y', unique_sources: 11, dimensions: 5, dimensions_baseline: 5 }),
+  );
+  assert.throws(() => parseStats(noEffort), /effort/i);
+});
+
+test('parseStats: skips blank and malformed lines but keeps the good ones', () => {
+  const text = jsonl(STANDARD[0], '', 'not json', HIGH[0]);
+  const { records, malformed } = parseStats(text);
+  assert.equal(records.length, 2);
+  assert.equal(malformed, 1);
+});
+
+test('parseStats: an empty file throws rather than reporting a zero-gain verdict', () => {
+  assert.throws(() => parseStats(''), /no records/i);
+});
+
+// ---------------------------------------------------------------------------
+// exclusion of broken runs
+// ---------------------------------------------------------------------------
+test('partitionEligible: runs with empty_turns > 0 are excluded and counted', () => {
+  const { records } = parseStats(jsonl(
+    ...HIGH,
+    run({ effort: 'high', unique_sources: 99, dimensions: 8, dimensions_baseline: 5, empty_turns: 2 }),
+  ));
+  const { eligible, excluded } = partitionEligible(records);
+  assert.equal(eligible.length, 3);
+  assert.equal(excluded, 1);
+});
+
+test('partitionEligible: empty_turns === 0 is eligible; a missing field counts as 0', () => {
+  const { records } = parseStats(jsonl(
+    STANDARD[0],
+    JSON.stringify({ ts: 'z', effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
+  ));
+  const { eligible, excluded } = partitionEligible(records);
+  assert.equal(eligible.length, 2);
+  assert.equal(excluded, 0);
+});
+
+// ---------------------------------------------------------------------------
+// measure — the known-answer test
+// ---------------------------------------------------------------------------
+test('measure: median gains on synthetic runs give the known answer', () => {
+  const m = measure(parseStats(jsonl(...STANDARD, ...HIGH)).records);
+  assert.equal(m.control.n, 3);
+  assert.equal(m.treatment.n, 3);
+  assert.equal(m.sources.control, 10);
+  assert.equal(m.sources.treatment, 13);
+  assert.ok(Math.abs(m.sources.gain - 0.30) < 1e-9, `sources gain ${m.sources.gain}`);
+  // (8 - 5) / 5 = 0.60 within each treatment run.
+  assert.ok(Math.abs(m.dimensions.gain - 0.60) < 1e-9, `dimensions gain ${m.dimensions.gain}`);
+});
+
+test('measure: an excluded run cannot move the median', () => {
+  const withBroken = jsonl(
+    ...STANDARD,
+    ...HIGH,
+    run({ effort: 'high', unique_sources: 900, dimensions: 8, dimensions_baseline: 5, empty_turns: 1 }),
+  );
+  const m = measure(parseStats(withBroken).records);
+  assert.equal(m.excluded, 1);
+  assert.equal(m.sources.treatment, 13, 'the 900-source broken run must not reach the median');
+  assert.ok(Math.abs(m.sources.gain - 0.30) < 1e-9);
+});
+
+test('measure: reports null gain (not 0) when an arm has no eligible runs', () => {
+  const m = measure(parseStats(jsonl(...HIGH)).records);
+  assert.equal(m.control.n, 0);
+  assert.equal(m.sources.gain, null);
+  assert.equal(m.verdict, 'insufficient-data');
+});
+
+// ---------------------------------------------------------------------------
+// verdict mapping — both sides of both thresholds
+// ---------------------------------------------------------------------------
+test('decideVerdict: at and above the adopt threshold', () => {
+  assert.equal(decideVerdict(ADOPT_THRESHOLD, ADOPT_THRESHOLD), 'adopt');
+  assert.equal(decideVerdict(0.55, 0.44), 'adopt');
+});
+
+test('decideVerdict: just below the adopt threshold is inconclusive, not adopt', () => {
+  assert.equal(decideVerdict(ADOPT_THRESHOLD - 0.0001, 0.9), 'inconclusive');
+});
+
+test('decideVerdict: below the decline threshold on both metrics declines', () => {
+  assert.equal(decideVerdict(0.14, 0.05), 'decline');
+  assert.equal(decideVerdict(DECLINE_THRESHOLD - 0.0001, 0), 'decline');
+});
+
+test('decideVerdict: at the decline threshold is inconclusive, not decline', () => {
+  assert.equal(decideVerdict(DECLINE_THRESHOLD, DECLINE_THRESHOLD), 'inconclusive');
+});
+
+test('decideVerdict: adopt needs BOTH metrics — one strong metric does not carry a weak one', () => {
+  assert.equal(decideVerdict(0.90, 0.10), 'inconclusive');
+});
+
+test('decideVerdict: a null gain is insufficient data, never a decline', () => {
+  assert.equal(decideVerdict(null, 0.4), 'insufficient-data');
+  assert.equal(decideVerdict(0.4, null), 'insufficient-data');
+});
+
+// ---------------------------------------------------------------------------
+// pre-registration — the thresholds are the doc's, and the doc is committed first
+// ---------------------------------------------------------------------------
+test('thresholds are the pre-registered 30% / 15%', () => {
+  assert.equal(ADOPT_THRESHOLD, 0.30);
+  assert.equal(DECLINE_THRESHOLD, 0.15);
+});