// tests/scripts/yardstick.test.mjs // The yardstick prints Voyage's three numbers (use · trust · interrupts) with // denominators read from their SOURCE, plus a fourth row that asks whether the // end-state gate itself can be fooled by a signature-preserving mutant. // // Like the gate's own test, these tests pin the MECHANICS in both directions // against throwaway fixtures - every number can be countable and can be not // countable, the mutant row can read felled and not felled. They deliberately // do NOT assert that the real data is red today: the yardstick is supposed to // turn green, and a test that pins today's distance would fail on the day the // work is done. Nothing here reads the real order queue or the real stats. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { DEFAULT_CUTOFF, loadStats, readOrders, measureUse, measureTrust, measureInterrupts, emitterCallSites, buildM9Tree, measureMutants, measureAll, render, main, } from '../../scripts/yardstick.mjs'; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); // --- helpers --------------------------------------------------------------- function fixture(files) { const dir = mkdtempSync(join(tmpdir(), 'yardstick-')); for (const [rel, body] of Object.entries(files)) { const p = join(dir, rel); mkdirSync(dirname(p), { recursive: true }); writeFileSync(p, body); } return dir; } const cleanup = (...dirs) => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); }; const jsonl = (...rows) => `${rows.map((r) => (typeof r === 'string' ? r : JSON.stringify(r))).join('\n')}\n`; function order({ id, date, workClass, slug, extra = '' }) { const lines = ['---', 'from: .claude', 'to: some-repo', `order-id: ${id}`, `subject: order ${id}`, `date: ${date}`]; if (workClass !== undefined) lines.push(`work-class: ${workClass}`); if (slug !== undefined) lines.push(`slug: ${slug}`); lines.push('---', '', `# ORDER ${id}`, extra, ''); return lines.join('\n'); } const review = (ts, verdict, slug = 's') => ({ ts, slug, verdict }); // --- the stats loader -------------------------------------------------------- test('loadStats: an invalid JSON line is COUNTED, never silently dropped', () => { const dir = fixture({ 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{not json', '', review('2026-09-02T00:00:00Z', 'BLOCK')), }); try { const stats = loadStats(dir); const f = stats.files.find((x) => x.name === 'trekreview-stats.jsonl'); assert.equal(f.lines, 3, 'blank lines are not lines; the invalid one is'); assert.equal(f.records.length, 2); assert.deepEqual(f.invalid, [2], 'the invalid line is reported by its line number'); } finally { cleanup(dir); } }); test('loadStats: a missing data directory is reported, not read as zero', () => { const stats = loadStats(join(tmpdir(), 'yardstick-no-such-dir-8f3a')); assert.equal(stats.missing, true); assert.deepEqual(stats.files, []); }); // --- 2 · TRUST --------------------------------------------------------------- test('trust: the ALLOW streak is read in ts order, not file order', () => { // File order ends on ALLOW, ALLOW; chronologically the last review is a BLOCK. const dir = fixture({ 'trekreview-stats.jsonl': jsonl( review('2026-09-01T00:00:00Z', 'ALLOW'), review('2026-09-05T00:00:00Z', 'BLOCK'), review('2026-09-02T00:00:00Z', 'ALLOW'), review('2026-09-03T00:00:00Z', 'ALLOW'), ), }); try { const row = measureTrust(loadStats(dir)); assert.equal(row.countable, true); assert.equal(row.denominator, 4); assert.equal(row.value, 0, 'the chronologically last review is a BLOCK'); assert.equal(row.facts.longestStreak, 3); assert.deepEqual(row.facts.verdicts, { ALLOW: 3, BLOCK: 1 }); assert.equal(row.facts.last.verdict, 'BLOCK'); } finally { cleanup(dir); } }); test('trust: both spacings of the verdict field count the same (it is read as JSON, not grepped)', () => { const dir = fixture({ 'trekreview-stats.jsonl': jsonl( '{"ts": "2026-09-01T00:00:00Z", "slug": "a", "verdict": "BLOCK"}', '{"ts":"2026-09-02T00:00:00Z","slug":"b","verdict":"ALLOW"}', ), }); try { const row = measureTrust(loadStats(dir)); assert.deepEqual(row.facts.verdicts, { BLOCK: 1, ALLOW: 1 }); assert.equal(row.value, 1); } finally { cleanup(dir); } }); test('trust: an invalid line is left out of n and said out loud', () => { const dir = fixture({ 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{"ts": oops'), }); try { const row = measureTrust(loadStats(dir)); assert.equal(row.denominator, 1); assert.equal(row.facts.invalid, 1); assert.match(row.detail, /1 invalid line/); } finally { cleanup(dir); } }); test('trust: an empty or missing review file is NOT countable', () => { const empty = fixture({ 'trekreview-stats.jsonl': '' }); const none = fixture({ 'trekplan-stats.jsonl': jsonl({ ts: '2026-09-01T00:00:00Z', slug: 'x' }) }); try { for (const dir of [empty, none]) { const row = measureTrust(loadStats(dir)); assert.equal(row.countable, false, dir); assert.equal(row.value, null); assert.ok(row.detail.length > 0, 'the row must say WHY it cannot be counted'); } } finally { cleanup(empty, none); } }); // --- 1 · USE ----------------------------------------------------------------- test('use: orders are read from orders/, archive/ and claimed/, and nothing else', () => { const coord = fixture({ 'repo-a/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }), 'repo-a/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-01T10:00:00Z' }), 'repo-b/orders/claimed/o3.md': order({ id: 'o3', date: '2026-09-22T10:00:00Z' }), 'repo-b/orders/claimed/o3.claim': 'claimed', 'repo-b/inbox/m1.md': order({ id: 'm1', date: '2026-09-22T10:00:00Z' }), 'repo-b/orders/broken.md': 'no frontmatter at all\n', }); try { const q = readOrders(coord); assert.equal(q.files, 4, 'three orders and one unreadable file, all .md under */orders/'); assert.equal(q.orders.length, 3); assert.equal(q.unreadable.length, 1); } finally { cleanup(coord); } }); test('use: a dot-mailbox is a mailbox - its orders count, and the detail names it', () => { // Measured 2026-09-22: a hand count over `coord/*/orders` came out lower than // the yardstick, because the shell's `*` skips dot-directories and the queue // does not. The detail names them, so the next re-measure can reconcile. const coord = fixture({ 'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }), '.pm/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-22T10:00:00Z' }), }); try { const q = readOrders(coord); assert.equal(q.orders.length, 2); const row = measureUse({ queue: q, slugs: new Set(), cutoff: DEFAULT_CUTOFF }); assert.match(row.detail, /2 order files in 2 mailboxes/); assert.match(row.detail, /dot-mailboxes .pm 1/); } finally { cleanup(coord); } }); test('use: no order carrying work-class means NOT COUNTABLE, not 0 %', () => { const coord = fixture({ 'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }), 'r/orders/o2.md': order({ id: 'o2', date: '2026-09-23T10:00:00Z' }), }); try { const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: DEFAULT_CUTOFF }); assert.equal(row.countable, false); assert.equal(row.value, null); assert.equal(row.denominator, 0); assert.equal(row.facts.scanned, 2); assert.equal(row.facts.withWorkClass, 0); assert.match(row.detail, /0 of 2 orders carry work-class/); } finally { cleanup(coord); } }); test('use: the denominator is new-work orders dated on or after the cutoff; the numerator is those with a Voyage run', () => { const coord = fixture({ 'r/orders/a.md': order({ id: 'a', date: '2026-09-21T00:00:00Z', workClass: 'new', slug: 'feat-a' }), 'r/orders/archive/b.md': order({ id: 'b', date: '2026-09-25T00:00:00Z', workClass: 'new', slug: 'feat-b' }), 'r/orders/c.md': order({ id: 'c', date: '2026-09-25T00:00:00Z', workClass: 'new' }), 'r/orders/d.md': order({ id: 'd', date: '2026-09-20T23:59:59Z', workClass: 'new', slug: 'feat-a' }), 'r/orders/e.md': order({ id: 'e', date: '2026-09-25T00:00:00Z', workClass: 'maintenance', slug: 'feat-a' }), }); try { const row = measureUse({ queue: readOrders(coord), slugs: new Set(['feat-a']), cutoff: '2026-09-21' }); assert.equal(row.countable, true); assert.equal(row.denominator, 3, 'a, b and c - d is before the cutoff, e is maintenance'); assert.equal(row.value, 1, 'only a has a Voyage run with the same slug; c has no slug to match'); assert.equal(row.facts.withWorkClass, 5); } finally { cleanup(coord); } }); // --- 3 · INTERRUPTS ---------------------------------------------------------- test('interrupts: no user_input record means NOT COUNTABLE, with the line denominator stated', () => { const dir = fixture({ 'trekexecute-stats.jsonl': jsonl({ ts: 't', session_id: 's', success: true }, { ts: 't', event: 'main-merge-gate', payload: {} }), 'trekplan-stats.jsonl': jsonl({ ts: 't', slug: 'x' }), }); try { const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 3 } }); assert.equal(row.countable, false); assert.equal(row.facts.lines, 3); assert.equal(row.facts.userInput, 0); assert.match(row.detail, /0 user_input records of 3 lines in 2 files/); assert.match(row.detail, /0 emitter call sites/); } finally { cleanup(dir); } }); test('interrupts: user_input WITHOUT a slug or run id is still not countable per delivery', () => { const dir = fixture({ 'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { question: 'q' } }), }); try { const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } }); assert.equal(row.countable, false); assert.equal(row.facts.userInput, 1); assert.equal(row.facts.withDelivery, 0); } finally { cleanup(dir); } }); test('interrupts: user_input carrying a slug or run id is countable, per delivery', () => { const dir = fixture({ 'trekexecute-stats.jsonl': jsonl( { ts: 't1', event: 'user_input', payload: { slug: 'feat-a' } }, { ts: 't2', event: 'user_input', payload: { slug: 'feat-a' } }, { ts: 't3', event: 'user_input', payload: { run_id: 'r-9' } }, ), }); try { const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } }); assert.equal(row.countable, true); assert.equal(row.value, 3); assert.deepEqual(row.facts.perDelivery, { 'feat-a': 2, 'r-9': 1 }); } finally { cleanup(dir); } }); test('emitterCallSites: counts user_input emissions outside lib/stats/ and tests/ only', () => { const root = fixture({ 'commands/a.md': 'node ${CLAUDE_PLUGIN_ROOT}/lib/stats/event-emit.mjs \\\n --event user_input \\\n', 'commands/b.md': 'the emitter knows user_input as an event name - prose, not a call\n', 'lib/util/x.mjs': "emitEvent('user_input', { slug });\n", 'lib/stats/event-emit.mjs': "emit('user_input', {});\n", 'tests/lib/y.test.mjs': "emit('user_input', {});\n", 'hooks/scripts/h.mjs': "spawnSync('node', [EMIT, '--event', 'user_input']);\n", }); try { const r = emitterCallSites(root); assert.deepEqual(r.sites.map((s) => s.file).sort(), ['commands/a.md', 'hooks/scripts/h.mjs', 'lib/util/x.mjs']); assert.equal(r.scanned, 4, 'commands/a.md, commands/b.md, lib/util/x.mjs, hooks/scripts/h.mjs'); } finally { cleanup(root); } }); // --- 4 · the signature-preserving mutant (M9) ---------------------------------- const MODULE = 'export function verdict(code) { return { status: code === 0 ? "PASSED" : "FAILED" }; }\n'; const STRONG_TEST = [ "import { test } from 'node:test';", "import { strict as assert } from 'node:assert';", "import { verdict } from '../lib/m.mjs';", "test('verdict fells a failing exit', () => { assert.equal(verdict(1).status, 'FAILED'); });", '', ].join('\n'); const behaviourRegistry = { defects: [{ id: 'D-X', probe: 'behaviour', check: [{ test: 'tests/t.test.mjs', name: 'verdict fells a failing exit', expect: 'fails', stubs: ['lib/m.mjs'] }], }], experiments: [], }; test('buildM9Tree: every export keeps its name and returns {status:"PASSED"}; every named test only checks typeof', async () => { const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST }); const m9 = buildM9Tree(root, behaviourRegistry); try { const mod = readFileSync(join(m9, 'lib', 'm.mjs'), 'utf8'); assert.match(mod, /export function verdict\(\) \{ return \{ status: 'PASSED' \}; \}/); const t = readFileSync(join(m9, 'tests', 't.test.mjs'), 'utf8'); assert.match(t, /verdict fells a failing exit/); assert.match(t, /typeof/); assert.doesNotMatch(t, /FAILED/, 'the mutant test asserts nothing about behaviour'); assert.equal(readFileSync(join(root, 'lib', 'm.mjs'), 'utf8'), MODULE, 'the measured tree is never touched'); } finally { cleanup(root, m9); } }); test('mutants: a gate that CLOSES the probe on M9 has felled 0 of 1', () => { const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST }); try { const row = measureMutants(root, { registry: behaviourRegistry, evaluate: () => ({ status: 'closed', detail: '' }), runTest: () => true, }); assert.equal(row.countable, true); assert.equal(row.value, 0); assert.equal(row.denominator, 1); assert.match(row.detail, /D-X closed/); } finally { cleanup(root); } }); test('mutants: a gate that refuses to close the probe on M9 has felled 1 of 1', () => { const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST }); try { const row = measureMutants(root, { registry: behaviourRegistry, evaluate: () => ({ status: 'not-fellable', detail: 'still passes against a signature-preserving mutant' }), runTest: () => true, }); assert.equal(row.value, 1); assert.equal(row.denominator, 1); } finally { cleanup(root); } }); test('mutants: a mutant whose own tests do not pass is not LIVE, so the row cannot be counted', () => { // Without this control a broken fixture would read as "felled". const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST }); try { const row = measureMutants(root, { registry: behaviourRegistry, evaluate: () => ({ status: 'not-fellable', detail: 'x' }), runTest: () => null, }); assert.equal(row.countable, false); assert.equal(row.value, null); assert.match(row.detail, /not live/); } finally { cleanup(root); } }); test('mutants: against the REAL gate and registry, M9 is live and the denominator is 1', () => { // Pins the measurement's shape, not today's value: M9's typeof-only tests // really do pass on the M9 tree (so "felled" can only come from the gate). const row = measureMutants(ROOT); assert.equal(row.countable, true, row.detail); assert.equal(row.denominator, 1); assert.ok(row.value === 0 || row.value === 1); assert.deepEqual(row.facts.entries.sort(), ['D-03', 'D-04']); }); // --- the whole yardstick ------------------------------------------------------- function world({ orders = {}, stats = {} } = {}) { const coord = fixture(Object.keys(orders).length ? orders : { 'r/orders/.keep': '' }); const data = fixture(Object.keys(stats).length ? stats : { '.keep': '' }); return { coord, data }; } test('measureAll + main: RED with exit 1 while any number is not countable', () => { const { coord, data } = world({ orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z' }) }, stats: { 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW')) }, }); try { const result = measureAll({ root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF }); assert.equal(result.green, false); assert.equal(result.countable, 1); const text = render(result); assert.match(text, /1 of 3 numbers countable/); assert.match(text, /signature-preserving mutants felled: \d of 1/); assert.match(text, /verdict: NOT JUDGED/); for (const id of ['use', 'trust', 'interrupts', 'mutants']) assert.match(text, new RegExp(`\\| ${id} \\|`)); assert.equal(main(['--root', ROOT, '--coord', coord, '--data', data, '--json']), 1); } finally { cleanup(coord, data); } }); test('measureAll: all three countable and the mutant felled is still RED until the verdict can be judged', () => { const { coord, data } = world({ orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z', workClass: 'new', slug: 'a' }) }, stats: { 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW', 'a')), 'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { slug: 'a' } }), }, }); try { const result = measureAll({ root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF, mutantOptions: { evaluate: () => ({ status: 'not-fellable', detail: '' }) }, }); assert.equal(result.countable, 3); assert.equal(result.rows.find((r) => r.id === 'mutants').value, 1); assert.equal(result.verdict.judged, false); assert.equal(result.green, false); } finally { cleanup(coord, data); } }); test('main: an unknown argument or a malformed cutoff is a usage error (exit 2)', () => { assert.equal(main(['--nope']), 2); assert.equal(main(['--cutoff', '21.09.2026']), 2); });