voyage/tests/scripts/yardstick.test.mjs

// tests/scripts/yardstick.test.mjs
// The yardstick prints Voyage's three numbers (use · trust · interrupts) with
// denominators read from their SOURCE, plus a fourth row that asks whether the
// end-state gate itself can be fooled by a signature-preserving mutant.
//
// Like the gate's own test, these tests pin the MECHANICS in both directions
// against throwaway fixtures - every number can be countable and can be not
// countable, the mutant row can read felled and not felled. They deliberately
// do NOT assert that the real data is red today: the yardstick is supposed to
// turn green, and a test that pins today's distance would fail on the day the
// work is done. Nothing here reads the real order queue or the real stats.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
  DEFAULT_CUTOFF,
  MARKER_SINCE,
  loadStats,
  readOrders,
  measureUse,
  measureTrust,
  measureInterrupts,
  emitterCallSites,
  buildM9Tree,
  measureMutants,
  measureAll,
  render,
  main,
} from '../../scripts/yardstick.mjs';

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');

// --- helpers ---------------------------------------------------------------

function fixture(files) {
  const dir = mkdtempSync(join(tmpdir(), 'yardstick-'));
  for (const [rel, body] of Object.entries(files)) {
    const p = join(dir, rel);
    mkdirSync(dirname(p), { recursive: true });
    writeFileSync(p, body);
  }
  return dir;
}

const cleanup = (...dirs) => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); };

const jsonl = (...rows) => `${rows.map((r) => (typeof r === 'string' ? r : JSON.stringify(r))).join('\n')}\n`;

// `marker` writes the order's solution-choice marker the way dispatch does: an
// HTML comment on its own line in the body, `<!-- loesningsvalg: <value>; grunn=... -->`.
function order({ id, date, workClass, slug, marker, extra = '' }) {
  const lines = ['---', 'from: .claude', 'to: some-repo', `order-id: ${id}`, `subject: order ${id}`, `date: ${date}`];
  if (workClass !== undefined) lines.push(`work-class: ${workClass}`);
  if (slug !== undefined) lines.push(`slug: ${slug}`);
  lines.push('---');
  if (marker !== undefined) lines.push(`<!-- loesningsvalg: ${marker}; grunn=fixture -->`);
  lines.push('', `# ORDER ${id}`, extra, '');
  return lines.join('\n');
}

const review = (ts, verdict, slug = 's') => ({ ts, slug, verdict });

// --- the stats loader --------------------------------------------------------

test('loadStats: an invalid JSON line is COUNTED, never silently dropped', () => {
  const dir = fixture({
    'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{not json', '', review('2026-09-02T00:00:00Z', 'BLOCK')),
  });
  try {
    const stats = loadStats(dir);
    const f = stats.files.find((x) => x.name === 'trekreview-stats.jsonl');
    assert.equal(f.lines, 3, 'blank lines are not lines; the invalid one is');
    assert.equal(f.records.length, 2);
    assert.deepEqual(f.invalid, [2], 'the invalid line is reported by its line number');
  } finally { cleanup(dir); }
});

test('loadStats: a missing data directory is reported, not read as zero', () => {
  const stats = loadStats(join(tmpdir(), 'yardstick-no-such-dir-8f3a'));
  assert.equal(stats.missing, true);
  assert.deepEqual(stats.files, []);
});

// --- 2 · TRUST ---------------------------------------------------------------

test('trust: the ALLOW streak is read in ts order, not file order', () => {
  // File order ends on ALLOW, ALLOW; chronologically the last review is a BLOCK.
  const dir = fixture({
    'trekreview-stats.jsonl': jsonl(
      review('2026-09-01T00:00:00Z', 'ALLOW'),
      review('2026-09-05T00:00:00Z', 'BLOCK'),
      review('2026-09-02T00:00:00Z', 'ALLOW'),
      review('2026-09-03T00:00:00Z', 'ALLOW'),
    ),
  });
  try {
    const row = measureTrust(loadStats(dir));
    assert.equal(row.countable, true);
    assert.equal(row.denominator, 4);
    assert.equal(row.value, 0, 'the chronologically last review is a BLOCK');
    assert.equal(row.facts.longestStreak, 3);
    assert.deepEqual(row.facts.verdicts, { ALLOW: 3, BLOCK: 1 });
    assert.equal(row.facts.last.verdict, 'BLOCK');
  } finally { cleanup(dir); }
});

test('trust: both spacings of the verdict field count the same (it is read as JSON, not grepped)', () => {
  const dir = fixture({
    'trekreview-stats.jsonl': jsonl(
      '{"ts": "2026-09-01T00:00:00Z", "slug": "a", "verdict": "BLOCK"}',
      '{"ts":"2026-09-02T00:00:00Z","slug":"b","verdict":"ALLOW"}',
    ),
  });
  try {
    const row = measureTrust(loadStats(dir));
    assert.deepEqual(row.facts.verdicts, { BLOCK: 1, ALLOW: 1 });
    assert.equal(row.value, 1);
  } finally { cleanup(dir); }
});

test('trust: an invalid line is left out of n and said out loud', () => {
  const dir = fixture({
    'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{"ts": oops'),
  });
  try {
    const row = measureTrust(loadStats(dir));
    assert.equal(row.denominator, 1);
    assert.equal(row.facts.invalid, 1);
    assert.match(row.detail, /1 invalid line/);
  } finally { cleanup(dir); }
});

test('trust: an empty or missing review file is NOT countable', () => {
  const empty = fixture({ 'trekreview-stats.jsonl': '' });
  const none = fixture({ 'trekplan-stats.jsonl': jsonl({ ts: '2026-09-01T00:00:00Z', slug: 'x' }) });
  try {
    for (const dir of [empty, none]) {
      const row = measureTrust(loadStats(dir));
      assert.equal(row.countable, false, dir);
      assert.equal(row.value, null);
      assert.ok(row.detail.length > 0, 'the row must say WHY it cannot be counted');
    }
  } finally { cleanup(empty, none); }
});

// --- 1 · USE -----------------------------------------------------------------

test('use: orders are read from orders/, archive/ and claimed/, and nothing else', () => {
  const coord = fixture({
    'repo-a/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
    'repo-a/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-01T10:00:00Z' }),
    'repo-b/orders/claimed/o3.md': order({ id: 'o3', date: '2026-09-22T10:00:00Z' }),
    'repo-b/orders/claimed/o3.claim': 'claimed',
    'repo-b/inbox/m1.md': order({ id: 'm1', date: '2026-09-22T10:00:00Z' }),
    'repo-b/orders/broken.md': 'no frontmatter at all\n',
  });
  try {
    const q = readOrders(coord);
    assert.equal(q.files, 4, 'three orders and one unreadable file, all .md under */orders/');
    assert.equal(q.orders.length, 3);
    assert.equal(q.unreadable.length, 1);
  } finally { cleanup(coord); }
});

test('use: a dot-mailbox is a mailbox - its orders count, and the detail names it', () => {
  // Measured 2026-09-22: a hand count over `coord/*/orders` came out lower than
  // the yardstick, because the shell's `*` skips dot-directories and the queue
  // does not. The detail names them, so the next re-measure can reconcile.
  const coord = fixture({
    'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
    '.pm/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-22T10:00:00Z' }),
  });
  try {
    const q = readOrders(coord);
    assert.equal(q.orders.length, 2);
    const row = measureUse({ queue: q, slugs: new Set(), cutoff: DEFAULT_CUTOFF });
    assert.match(row.detail, /2 order files in 2 mailboxes/);
    assert.match(row.detail, /dot-mailboxes .pm 1/);
  } finally { cleanup(coord); }
});

test('use: the marker is counted from 2026-09-28, the day pre-order-loesningsvalg.sh began enforcing it', () => {
  assert.equal(MARKER_SINCE, '2026-09-28');
});

test('use: no order carrying a loesningsvalg marker means NOT COUNTABLE, not 0 %', () => {
  const coord = fixture({
    'r/orders/o1.md': order({ id: 'o1', date: '2026-09-29T10:00:00Z' }),
    'r/orders/o2.md': order({ id: 'o2', date: '2026-09-30T10:00:00Z' }),
  });
  try {
    const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: DEFAULT_CUTOFF });
    assert.equal(row.countable, false);
    assert.equal(row.value, null);
    assert.equal(row.denominator, 0);
    assert.equal(row.facts.scanned, 2);
    assert.equal(row.facts.withMarker, 0);
    assert.equal(row.facts.notMeasured, 2, 'an unmarked order is not measured - never a no');
    assert.match(row.detail, /0 of 2 orders carry a loesningsvalg marker/);
    assert.match(row.detail, /2 not measured/);
  } finally { cleanup(coord); }
});

test('use: plan and prove are NEW work, ikke-nytt is not; unmarked and pre-2026-09-28 orders are not measured', () => {
  const coord = fixture({
    'r/orders/plan.md': order({ id: 'plan', date: '2026-09-28T00:00:00Z', marker: 'plan', slug: 'feat-a' }),
    'r/orders/archive/prove.md': order({ id: 'prove', date: '2026-10-01T00:00:00Z', marker: 'prove', slug: 'feat-b' }),
    'r/orders/claimed/ikke.md': order({ id: 'ikke', date: '2026-10-01T00:00:00Z', marker: 'ikke-nytt', slug: 'feat-a' }),
    'r/orders/none.md': order({ id: 'none', date: '2026-10-01T00:00:00Z', slug: 'feat-a' }),
    'r/orders/early.md': order({ id: 'early', date: '2026-09-27T23:59:59Z', marker: 'plan', slug: 'feat-a' }),
  });
  try {
    const row = measureUse({ queue: readOrders(coord), slugs: new Set(['feat-a']), cutoff: DEFAULT_CUTOFF });
    assert.equal(row.countable, true);
    assert.equal(row.denominator, 2, 'plan and prove - ikke-nytt is not new work, none and early are not measured');
    assert.equal(row.value, 1, 'only plan has a Voyage run with the same slug');
    assert.equal(row.facts.newWork, 2);
    assert.equal(row.facts.notNew, 1);
    assert.equal(row.facts.notMeasured, 2, 'the unmarked order and the one before 2026-09-28');
    assert.equal(row.facts.beforeMarker, 1);
    assert.equal(row.facts.unmarked, 1);
    assert.match(row.detail, /1 of 2 new-work orders/);
    assert.match(row.detail, /2 not measured/);
  } finally { cleanup(coord); }
});

test('use: work-class in the frontmatter no longer decides anything - only the marker does', () => {
  const coord = fixture({
    'r/orders/a.md': order({ id: 'a', date: '2026-09-29T00:00:00Z', workClass: 'new', slug: 'feat-a' }),
    'r/orders/b.md': order({ id: 'b', date: '2026-09-29T00:00:00Z', workClass: 'maintenance', marker: 'prove', slug: 'feat-a' }),
  });
  try {
    const row = measureUse({ queue: readOrders(coord), slugs: new Set(['feat-a']), cutoff: DEFAULT_CUTOFF });
    assert.equal(row.denominator, 1, 'b is new by its marker; a has no marker and is not measured');
    assert.equal(row.value, 1);
    assert.equal(row.facts.notMeasured, 1);
  } finally { cleanup(coord); }
});

test('use: only a marker on its own line counts - a mention in prose or an unknown value is not measured', () => {
  const coord = fixture({
    'r/orders/prose.md': order({ id: 'prose', date: '2026-09-29T00:00:00Z', extra: 'Count from the marker `<!-- loesningsvalg: plan -->` in the body.' }),
    'r/orders/odd.md': order({ id: 'odd', date: '2026-09-29T00:00:00Z', marker: 'kanskje' }),
    'r/orders/both.md': order({ id: 'both', date: '2026-09-29T00:00:00Z', marker: 'ikke-nytt', extra: 'Prose names `<!-- loesningsvalg: plan -->` too.' }),
  });
  try {
    const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: DEFAULT_CUTOFF });
    assert.equal(row.denominator, 0);
    assert.equal(row.facts.notNew, 1, 'both: the line marker wins over the prose mention');
    assert.equal(row.facts.notMeasured, 2, 'prose has no marker; kanskje is no known value');
    assert.equal(row.countable, false);
  } finally { cleanup(coord); }
});

test('use: a cutoff later than 2026-09-28 moves the start of the count', () => {
  const coord = fixture({
    'r/orders/a.md': order({ id: 'a', date: '2026-09-29T00:00:00Z', marker: 'plan' }),
    'r/orders/b.md': order({ id: 'b', date: '2026-10-02T00:00:00Z', marker: 'plan' }),
  });
  try {
    const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: '2026-10-01' });
    assert.equal(row.denominator, 1);
    assert.equal(row.facts.beforeMarker, 1);
    assert.match(row.source, /from 2026-10-01/);
  } finally { cleanup(coord); }
});

// --- 3 · INTERRUPTS ----------------------------------------------------------

test('interrupts: no user_input record means NOT COUNTABLE, with the line denominator stated', () => {
  const dir = fixture({
    'trekexecute-stats.jsonl': jsonl({ ts: 't', session_id: 's', success: true }, { ts: 't', event: 'main-merge-gate', payload: {} }),
    'trekplan-stats.jsonl': jsonl({ ts: 't', slug: 'x' }),
  });
  try {
    const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 3 } });
    assert.equal(row.countable, false);
    assert.equal(row.facts.lines, 3);
    assert.equal(row.facts.userInput, 0);
    assert.match(row.detail, /0 user_input records of 3 lines in 2 files/);
    assert.match(row.detail, /0 emitter call sites/);
  } finally { cleanup(dir); }
});

test('interrupts: user_input WITHOUT a slug or run id is still not countable per delivery', () => {
  const dir = fixture({
    'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { question: 'q' } }),
  });
  try {
    const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
    assert.equal(row.countable, false);
    assert.equal(row.facts.userInput, 1);
    assert.equal(row.facts.withDelivery, 0);
  } finally { cleanup(dir); }
});

test('interrupts: user_input carrying a slug or run id is countable, per delivery', () => {
  const dir = fixture({
    'trekexecute-stats.jsonl': jsonl(
      { ts: 't1', event: 'user_input', payload: { slug: 'feat-a' } },
      { ts: 't2', event: 'user_input', payload: { slug: 'feat-a' } },
      { ts: 't3', event: 'user_input', payload: { run_id: 'r-9' } },
    ),
  });
  try {
    const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
    assert.equal(row.countable, true);
    assert.equal(row.value, 3);
    assert.deepEqual(row.facts.perDelivery, { 'feat-a': 2, 'r-9': 1 });
  } finally { cleanup(dir); }
});

test('emitterCallSites: counts user_input emissions outside lib/stats/ and tests/ only', () => {
  const root = fixture({
    'commands/a.md': 'node ${CLAUDE_PLUGIN_ROOT}/lib/stats/event-emit.mjs \\\n  --event user_input \\\n',
    'commands/b.md': 'the emitter knows user_input as an event name - prose, not a call\n',
    'lib/util/x.mjs': "emitEvent('user_input', { slug });\n",
    'lib/stats/event-emit.mjs': "emit('user_input', {});\n",
    'tests/lib/y.test.mjs': "emit('user_input', {});\n",
    'hooks/scripts/h.mjs': "spawnSync('node', [EMIT, '--event', 'user_input']);\n",
  });
  try {
    const r = emitterCallSites(root);
    assert.deepEqual(r.sites.map((s) => s.file).sort(), ['commands/a.md', 'hooks/scripts/h.mjs', 'lib/util/x.mjs']);
    assert.equal(r.scanned, 4, 'commands/a.md, commands/b.md, lib/util/x.mjs, hooks/scripts/h.mjs');
  } finally { cleanup(root); }
});

// --- 4 · the signature-preserving mutant (M9) ----------------------------------

const MODULE = 'export function verdict(code) { return { status: code === 0 ? "PASSED" : "FAILED" }; }\n';
const STRONG_TEST = [
  "import { test } from 'node:test';",
  "import { strict as assert } from 'node:assert';",
  "import { verdict } from '../lib/m.mjs';",
  "test('verdict fells a failing exit', () => { assert.equal(verdict(1).status, 'FAILED'); });",
  '',
].join('\n');
const behaviourRegistry = {
  defects: [{
    id: 'D-X', probe: 'behaviour',
    check: [{ test: 'tests/t.test.mjs', name: 'verdict fells a failing exit', expect: 'fails', stubs: ['lib/m.mjs'] }],
  }],
  experiments: [],
};

test('buildM9Tree: every export keeps its name and returns {status:"PASSED"}; every named test only checks typeof', async () => {
  const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
  const m9 = buildM9Tree(root, behaviourRegistry);
  try {
    const mod = readFileSync(join(m9, 'lib', 'm.mjs'), 'utf8');
    assert.match(mod, /export function verdict\(\) \{ return \{ status: 'PASSED' \}; \}/);
    const t = readFileSync(join(m9, 'tests', 't.test.mjs'), 'utf8');
    assert.match(t, /verdict fells a failing exit/);
    assert.match(t, /typeof/);
    assert.doesNotMatch(t, /FAILED/, 'the mutant test asserts nothing about behaviour');
    assert.equal(readFileSync(join(root, 'lib', 'm.mjs'), 'utf8'), MODULE, 'the measured tree is never touched');
  } finally { cleanup(root, m9); }
});

test('mutants: a gate that CLOSES the probe on M9 has felled 0 of 1', () => {
  const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
  try {
    const row = measureMutants(root, {
      registry: behaviourRegistry,
      evaluate: () => ({ status: 'closed', detail: '' }),
      runTest: () => true,
    });
    assert.equal(row.countable, true);
    assert.equal(row.value, 0);
    assert.equal(row.denominator, 1);
    assert.match(row.detail, /D-X closed/);
  } finally { cleanup(root); }
});

test('mutants: a gate that refuses to close the probe on M9 has felled 1 of 1', () => {
  const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
  try {
    const row = measureMutants(root, {
      registry: behaviourRegistry,
      evaluate: () => ({ status: 'not-fellable', detail: 'still passes against a signature-preserving mutant' }),
      runTest: () => true,
    });
    assert.equal(row.value, 1);
    assert.equal(row.denominator, 1);
  } finally { cleanup(root); }
});

test('mutants: a mutant whose own tests do not pass is not LIVE, so the row cannot be counted', () => {
  // Without this control a broken fixture would read as "felled".
  const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
  try {
    const row = measureMutants(root, {
      registry: behaviourRegistry,
      evaluate: () => ({ status: 'not-fellable', detail: 'x' }),
      runTest: () => null,
    });
    assert.equal(row.countable, false);
    assert.equal(row.value, null);
    assert.match(row.detail, /not live/);
  } finally { cleanup(root); }
});

test('mutants: against the REAL gate and registry, M9 is live and the denominator is 1', () => {
  // Pins the measurement's shape, not today's value: M9's typeof-only tests
  // really do pass on the M9 tree (so "felled" can only come from the gate).
  const row = measureMutants(ROOT);
  assert.equal(row.countable, true, row.detail);
  assert.equal(row.denominator, 1);
  assert.ok(row.value === 0 || row.value === 1);
  assert.deepEqual(row.facts.entries.sort(), ['D-03', 'D-04']);
});

// --- the whole yardstick -------------------------------------------------------

function world({ orders = {}, stats = {} } = {}) {
  const coord = fixture(Object.keys(orders).length ? orders : { 'r/orders/.keep': '' });
  const data = fixture(Object.keys(stats).length ? stats : { '.keep': '' });
  return { coord, data };
}

test('measureAll + main: RED with exit 1 while any number is not countable', () => {
  const { coord, data } = world({
    orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z' }) },
    stats: { 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW')) },
  });
  try {
    const result = measureAll({ root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF });
    assert.equal(result.green, false);
    assert.equal(result.countable, 1);
    const text = render(result);
    assert.match(text, /1 of 3 numbers countable/);
    assert.match(text, /signature-preserving mutants felled: \d of 1/);
    assert.match(text, /verdict: NOT JUDGED/);
    for (const id of ['use', 'trust', 'interrupts', 'mutants']) assert.match(text, new RegExp(`\\| ${id} \\|`));
    assert.equal(main(['--root', ROOT, '--coord', coord, '--data', data, '--json']), 1);
  } finally { cleanup(coord, data); }
});

test('measureAll: all three countable and the mutant felled is still RED until the verdict can be judged', () => {
  const { coord, data } = world({
    orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-29T00:00:00Z', marker: 'plan', slug: 'a' }) },
    stats: {
      'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW', 'a')),
      'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { slug: 'a' } }),
    },
  });
  try {
    const result = measureAll({
      root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF,
      mutantOptions: { evaluate: () => ({ status: 'not-fellable', detail: '' }) },
    });
    assert.equal(result.countable, 3);
    assert.equal(result.rows.find((r) => r.id === 'mutants').value, 1);
    assert.equal(result.verdict.judged, false);
    assert.equal(result.green, false);
  } finally { cleanup(coord, data); }
});

test('main: an unknown argument or a malformed cutoff is a usage error (exit 2)', () => {
  assert.equal(main(['--nope']), 2);
  assert.equal(main(['--cutoff', '21.09.2026']), 2);
});