voyage/tests/scripts/storm-measure.test.mjs

// tests/scripts/storm-measure.test.mjs
// Step 11 — the STORM adoption gate's deterministic accounting core.
//
// The gate decides ONE thing: does the bounded Phase 5 loop buy enough extra
// source/coverage breadth to be worth flipping VOYAGE_STORM_ENABLED on by
// default. Thresholds are pre-registered in docs/storm-measurement.md BEFORE
// any measurement run, so this file pins the arithmetic that turns a
// trekresearch-stats.jsonl into a verdict — not the verdict itself.
//
// Two properties carry the gate's honesty:
//   - runs with empty_turns > 0 are EXCLUDED from the gain and COUNTED, so
//     adoption is never decided on a broken denominator, and
//   - a stats file with no `effort` field is a loud error, never a silently
//     empty group that reads as "no gain".
//
// Pattern: tests/scripts/synthesis-measure.test.mjs (pure core, no fixtures on disk).

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync } from 'node:fs';
import {
  median,
  parseStats,
  partitionEligible,
  measure,
  decideVerdict,
  ADOPT_THRESHOLD,
  DECLINE_THRESHOLD,
} from '../../scripts/storm-measure.mjs';

// ---------------------------------------------------------------------------
// helpers — synthetic JSONL, one object per line, exactly as the orchestrator emits
// ---------------------------------------------------------------------------
function run({ effort, unique_sources, dimensions, dimensions_baseline, empty_turns = 0, conv_turns = 0 }) {
  return JSON.stringify({
    ts: '2026-08-12T00:00:00.000Z',
    question: 'q',
    mode: 'full',
    scope: 'both',
    engine: 'swarm',
    effort,
    unique_sources,
    dimensions,
    dimensions_baseline,
    conv_turns,
    empty_turns,
  });
}

function jsonl(...lines) {
  return lines.join('\n') + '\n';
}

// A control arm at 10 sources / 5 dimensions, and a treatment arm at 13
// sources / 8 dimensions: +30.0% sources, +60.0% dimensions.
const STANDARD = [
  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
  run({ effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
];
const HIGH = [
  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
  run({ effort: 'high', unique_sources: 13, dimensions: 8, dimensions_baseline: 5, conv_turns: 3 }),
];

// ---------------------------------------------------------------------------
// median
// ---------------------------------------------------------------------------
test('median: odd, even, single', () => {
  assert.equal(median([3, 1, 2]), 2);
  assert.equal(median([1, 2, 3, 4]), 2.5);
  assert.equal(median([7]), 7);
});

test('median: empty list is null, never 0 — 0 would read as a real measurement', () => {
  assert.equal(median([]), null);
});

test('median does not mutate its input', () => {
  const xs = [3, 1, 2];
  median(xs);
  assert.deepEqual(xs, [3, 1, 2]);
});

// ---------------------------------------------------------------------------
// parseStats — the loud-error requirement
// ---------------------------------------------------------------------------
test('parseStats: a file with no effort field throws, it does not yield empty groups', () => {
  const noEffort = jsonl(
    JSON.stringify({ ts: 'x', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
    JSON.stringify({ ts: 'y', unique_sources: 11, dimensions: 5, dimensions_baseline: 5 }),
  );
  assert.throws(() => parseStats(noEffort), /effort/i);
});

test('parseStats: skips blank and malformed lines but keeps the good ones', () => {
  const text = jsonl(STANDARD[0], '', 'not json', HIGH[0]);
  const { records, malformed } = parseStats(text);
  assert.equal(records.length, 2);
  assert.equal(malformed, 1);
});

test('parseStats: an empty file throws rather than reporting a zero-gain verdict', () => {
  assert.throws(() => parseStats(''), /no records/i);
});

// ---------------------------------------------------------------------------
// exclusion of broken runs
// ---------------------------------------------------------------------------
test('partitionEligible: runs with empty_turns > 0 are excluded and counted', () => {
  const { records } = parseStats(jsonl(
    ...HIGH,
    run({ effort: 'high', unique_sources: 99, dimensions: 8, dimensions_baseline: 5, empty_turns: 2 }),
  ));
  const { eligible, excluded } = partitionEligible(records);
  assert.equal(eligible.length, 3);
  assert.equal(excluded, 1);
});

test('partitionEligible: empty_turns === 0 is eligible; a missing field counts as 0', () => {
  const { records } = parseStats(jsonl(
    STANDARD[0],
    JSON.stringify({ ts: 'z', effort: 'standard', unique_sources: 10, dimensions: 5, dimensions_baseline: 5 }),
  ));
  const { eligible, excluded } = partitionEligible(records);
  assert.equal(eligible.length, 2);
  assert.equal(excluded, 0);
});

// A malformed empty_turns used to land in the ELIGIBLE arm: Number('many') is
// NaN, NaN fails the isFinite test, and the `else` branch pushed it in. The
// exclusion is one of the two properties carrying this gate's honesty, so a
// garbage value silently re-entering the denominator defeats it — and it does so
// in the direction that flatters adoption, since the run that broke is the run
// whose numbers are least trustworthy.
test('partitionEligible: a non-numeric empty_turns is EXCLUDED, never silently eligible', () => {
  const { records } = parseStats(jsonl(
    STANDARD[0],
    JSON.stringify({
      ts: 'm', effort: 'high', unique_sources: 999,
      dimensions: 8, dimensions_baseline: 5, empty_turns: 'many',
    }),
  ));
  const { eligible, excluded } = partitionEligible(records);
  assert.equal(excluded, 1, 'a value that cannot be read as a turn count is not evidence of zero empty turns');
  assert.equal(eligible.length, 1);
});

test('partitionEligible: an unparsable empty_turns cannot move the median either', () => {
  const m = measure(parseStats(jsonl(
    ...STANDARD,
    ...HIGH,
    JSON.stringify({
      ts: 'm', effort: 'high', unique_sources: 900,
      dimensions: 8, dimensions_baseline: 5, empty_turns: {},
    }),
  )).records);
  assert.equal(m.excluded, 1);
  assert.equal(m.sources.treatment, 13, 'the 900-source malformed run must not reach the median');
});

// ---------------------------------------------------------------------------
// measure — the known-answer test
// ---------------------------------------------------------------------------
test('measure: median gains on synthetic runs give the known answer', () => {
  const m = measure(parseStats(jsonl(...STANDARD, ...HIGH)).records);
  assert.equal(m.control.n, 3);
  assert.equal(m.treatment.n, 3);
  assert.equal(m.sources.control, 10);
  assert.equal(m.sources.treatment, 13);
  assert.ok(Math.abs(m.sources.gain - 0.30) < 1e-9, `sources gain ${m.sources.gain}`);
  // (8 - 5) / 5 = 0.60 within each treatment run.
  assert.ok(Math.abs(m.dimensions.gain - 0.60) < 1e-9, `dimensions gain ${m.dimensions.gain}`);
});

test('measure: an excluded run cannot move the median', () => {
  const withBroken = jsonl(
    ...STANDARD,
    ...HIGH,
    run({ effort: 'high', unique_sources: 900, dimensions: 8, dimensions_baseline: 5, empty_turns: 1 }),
  );
  const m = measure(parseStats(withBroken).records);
  assert.equal(m.excluded, 1);
  assert.equal(m.sources.treatment, 13, 'the 900-source broken run must not reach the median');
  assert.ok(Math.abs(m.sources.gain - 0.30) < 1e-9);
});

test('measure: reports null gain (not 0) when an arm has no eligible runs', () => {
  const m = measure(parseStats(jsonl(...HIGH)).records);
  assert.equal(m.control.n, 0);
  assert.equal(m.sources.gain, null);
  assert.equal(m.verdict, 'insufficient-data');
});

// ---------------------------------------------------------------------------
// verdict mapping — both sides of both thresholds
// ---------------------------------------------------------------------------
test('decideVerdict: at and above the adopt threshold', () => {
  assert.equal(decideVerdict(ADOPT_THRESHOLD, ADOPT_THRESHOLD), 'adopt');
  assert.equal(decideVerdict(0.55, 0.44), 'adopt');
});

test('decideVerdict: both metrics between the bars is inconclusive, not adopt', () => {
  assert.equal(decideVerdict(ADOPT_THRESHOLD - 0.0001, 0.2), 'inconclusive');
});

test('decideVerdict: below the decline threshold on both metrics declines', () => {
  assert.equal(decideVerdict(0.14, 0.05), 'decline');
  assert.equal(decideVerdict(DECLINE_THRESHOLD - 0.0001, 0), 'decline');
});

test('decideVerdict: at the decline threshold is inconclusive, not decline', () => {
  assert.equal(decideVerdict(DECLINE_THRESHOLD, DECLINE_THRESHOLD), 'inconclusive');
});

// The brief pre-registers "median forbedring >= 30 % på (a) eller (b) → adopt.
// < 15 % → decline." — OR on both sides, with adopt evaluated first.
test('decideVerdict: adopt needs EITHER metric — one strong metric carries a weak one', () => {
  assert.equal(decideVerdict(0.90, 0.10), 'adopt');
  assert.equal(decideVerdict(0.10, 0.90), 'adopt');
});

test('decideVerdict: either metric below the decline bar declines', () => {
  assert.equal(decideVerdict(0.02, 0.20), 'decline');
  assert.equal(decideVerdict(0.20, 0.02), 'decline');
});

test('decideVerdict: adopt outranks decline when one metric clears and the other is under the decline bar', () => {
  assert.equal(decideVerdict(0.90, 0.10), 'adopt');
});

test('decideVerdict: a null gain is insufficient data, never a decline', () => {
  assert.equal(decideVerdict(null, 0.4), 'insufficient-data');
  assert.equal(decideVerdict(0.4, null), 'insufficient-data');
});

// ---------------------------------------------------------------------------
// pre-registration — the thresholds are the doc's, and the doc is committed first
// ---------------------------------------------------------------------------
test('thresholds are the pre-registered 30% / 15%', () => {
  assert.equal(ADOPT_THRESHOLD, 0.30);
  assert.equal(DECLINE_THRESHOLD, 0.15);
});

// The human-readable summary is the only form of the rule most readers will
// ever see. It said "on BOTH" on both sides while decideVerdict evaluated OR —
// so the report described a stricter gate than the one that produced the verdict
// printed one line below it.
test('the printed threshold line states the OR rule that decideVerdict actually applies', () => {
  const src = readFileSync(new URL('../../scripts/storm-measure.mjs', import.meta.url), 'utf-8');
  const line = src.split('\n').find((l) => l.includes('thresholds: adopt'));
  assert.ok(line, 'the summary must still print its threshold rule');
  assert.doesNotMatch(line, /on BOTH/, 'the rule is OR on both sides — printing BOTH misstates the gate');
  assert.match(line, /EITHER/);
});