voyage/scripts/yardstick.mjs

#!/usr/bin/env node
// scripts/yardstick.mjs
// The yardstick - Voyage's three numbers, each with a denominator read from its SOURCE:
//
//   use         new work that went through Voyage   (the order queue: orders dated on or after
//                                                    the cutoff that carry `work-class: new`;
//                                                    counted = those with a Voyage run whose
//                                                    slug equals the order's `slug:`)
//   trust       reviews in a row with nothing that  (trekreview-stats.jsonl, read as JSON and
//               stops a release                      ordered by `ts` - file order is not
//                                                    chronological; value = the current ALLOW streak)
//   interrupts  operator interrupts per delivery    (every *-stats.jsonl: `user_input` records
//                                                    that carry a slug or run id)
//
// plus a fourth row that asks whether the end-state gate can itself be fooled:
//
//   mutants     signature-preserving mutants the    (M9: every export of the module a behaviour
//               gate fells                           probe binds keeps its name and returns
//                                                    {status:'PASSED'}; every named test only
//                                                    checks `typeof`. The gate fells it when it
//                                                    refuses to close the probe on that tree.)
//
// A number the source cannot give is NOT COUNTABLE - never 0. The verdict over the first five
// deliveries is not judged by anything yet, and says so. Exit 0 = every number countable, every
// mutant felled and the verdict judged and met; 1 = anything short of that; 2 = usage error.
// It only READS: the queue, the stats and the repo. The mutant is built in a temp directory.
//
// Usage: node scripts/yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>]
//                                   [--cutoff YYYY-MM-DD]

import {
  readFileSync, writeFileSync, existsSync, readdirSync, statSync, mkdirSync, mkdtempSync,
  realpathSync, rmSync,
} from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { join, resolve, dirname, relative, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { evaluateCheck, loadRegistry, TEST_TIMEOUT_MS } from './end-state-gate.mjs';

const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');

export const DEFAULT_CUTOFF = '2026-09-21';
export const DEFAULT_COORD = join(homedir(), '.claude', 'coord');
export const DEFAULT_DATA = process.env.CLAUDE_PLUGIN_DATA
  || join(homedir(), '.claude', 'plugins', 'data', 'voyage-ktg-plugin-marketplace');

const REVIEW_FILE = 'trekreview-stats.jsonl';
const toPosix = (p) => p.split(sep).join('/');

function walk(dir) {
  const out = [];
  for (const name of readdirSync(dir)) {
    const p = join(dir, name);
    if (statSync(p).isDirectory()) out.push(...walk(p));
    else out.push(p);
  }
  return out;
}

// --- the stats files -----------------------------------------------------------

/**
 * Every `*-stats.jsonl` in the data directory, parsed line by line. A blank line
 * is not a line; a line that is not JSON is a line, and is reported by number.
 */
export function loadStats(dataDir) {
  if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) {
    return { dir: dataDir, missing: true, files: [] };
  }
  const files = readdirSync(dataDir)
    .filter((n) => n.endsWith('-stats.jsonl'))
    .sort()
    .map((name) => {
      const path = join(dataDir, name);
      const records = [];
      const invalid = [];
      let lines = 0;
      readFileSync(path, 'utf8').split('\n').forEach((line, i) => {
        if (line.trim() === '') return;
        lines++;
        try {
          records.push(JSON.parse(line));
        } catch {
          invalid.push(i + 1);
        }
      });
      return { name, path, lines, records, invalid };
    });
  return { dir: dataDir, missing: false, files };
}

// The slugs of every Voyage run the stats know about.
function voyageSlugs(stats) {
  const slugs = new Set();
  for (const f of stats.files) {
    for (const r of f.records) {
      if (r && typeof r.slug === 'string' && r.slug !== '') slugs.add(r.slug);
    }
  }
  return slugs;
}

const notCountable = (detail, extra = {}) => ({ countable: false, value: null, detail, ...extra });

// --- 2 · TRUST -------------------------------------------------------------------

export function measureTrust(stats) {
  const base = { id: 'trust', label: 'reviews in a row with nothing that stops a release' };
  const file = stats.files.find((f) => f.name === REVIEW_FILE);
  if (!file) {
    return { ...base, source: join(stats.dir, REVIEW_FILE), denominator: 0, facts: {},
      ...notCountable(stats.missing ? `data directory ${stats.dir} not found` : `${REVIEW_FILE} not found`) };
  }
  const reviews = file.records.filter((r) => r && typeof r.verdict === 'string');
  const undated = reviews.filter((r) => Number.isNaN(Date.parse(r.ts))).length;
  // Stable sort on ts: the file is appended by whoever ran a review, in no fixed order.
  const ordered = reviews
    .map((r, i) => ({ r, i, t: Date.parse(r.ts) }))
    .filter((x) => !Number.isNaN(x.t))
    .sort((a, b) => a.t - b.t || a.i - b.i)
    .map((x) => x.r);
  const verdicts = {};
  for (const r of reviews) verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + 1;
  let longest = 0;
  let run = 0;
  for (const r of ordered) {
    run = r.verdict === 'ALLOW' ? run + 1 : 0;
    longest = Math.max(longest, run);
  }
  const last = ordered.at(-1) ?? null;
  const facts = {
    verdicts, longestStreak: longest, invalid: file.invalid.length, undated,
    last: last ? { ts: last.ts, slug: last.slug, verdict: last.verdict } : null,
  };
  const notes = [];
  if (file.invalid.length) notes.push(`${file.invalid.length} invalid line(s) left out (line ${file.invalid.join(', ')})`);
  if (undated) notes.push(`${undated} review(s) without a readable ts left out of the streak`);
  if (reviews.length === 0) {
    return { ...base, source: file.path, denominator: 0, facts, ...notCountable(['no review rows', ...notes].join('; ')) };
  }
  const counts = Object.entries(verdicts).map(([k, v]) => `${k} ${v}`).join(' · ');
  const detail = [
    `${counts}; longest ALLOW streak ${longest}; last review ${last ? `${last.ts} ${last.verdict} (${last.slug})` : 'n/a'}`,
    'ordered by ts, read as JSON',
    ...notes,
  ].join('; ');
  return { ...base, source: file.path, countable: true, value: run, denominator: reviews.length, facts, detail };
}

// --- 1 · USE ---------------------------------------------------------------------

// A minimal frontmatter reader: `key: value` lines between the first two `---`.
function frontmatter(text) {
  const m = String(text).match(/^---\r?\n([\s\S]*?)\r?\n---/);
  if (!m) return null;
  const fm = {};
  for (const line of m[1].split(/\r?\n/)) {
    const kv = line.match(/^([A-Za-z][\w-]*):\s*(.*?)\s*$/);
    if (kv) fm[kv[1]] = kv[2];
  }
  return fm;
}

/** Every order file under `<coord>/<repo>/orders/` - pending, `archive/` and `claimed/`. */
export function readOrders(coordDir) {
  const orders = [];
  const unreadable = [];
  let files = 0;
  if (existsSync(coordDir)) {
    for (const repo of readdirSync(coordDir).sort()) {
      const dir = join(coordDir, repo, 'orders');
      if (!existsSync(dir) || !statSync(dir).isDirectory()) continue;
      for (const path of walk(dir).filter((p) => p.endsWith('.md')).sort()) {
        files++;
        const fm = frontmatter(readFileSync(path, 'utf8'));
        if (fm && fm['order-id']) orders.push({ path, repo, fm });
        else unreadable.push(path);
      }
    }
  }
  return { dir: coordDir, files, orders, unreadable };
}

export function measureUse({ queue, slugs, cutoff }) {
  const base = {
    id: 'use', label: 'new work that went through Voyage',
    source: `${join(queue.dir, '*', 'orders')}/**/*.md (cutoff ${cutoff})`,
  };
  const withClass = queue.orders.filter((o) => o.fm['work-class'] !== undefined);
  const newWork = withClass.filter((o) => o.fm['work-class'] === 'new' && String(o.fm.date ?? '').slice(0, 10) >= cutoff);
  const through = newWork.filter((o) => o.fm.slug && slugs.has(o.fm.slug));
  const facts = {
    scanned: queue.orders.length, files: queue.files, unreadable: queue.unreadable.length,
    withWorkClass: withClass.length, newSinceCutoff: newWork.length, withVoyageRun: through.length,
  };
  const perMailbox = {};
  for (const o of queue.orders) perMailbox[o.repo] = (perMailbox[o.repo] ?? 0) + 1;
  const dot = Object.entries(perMailbox).filter(([k]) => k.startsWith('.'));
  const census = `${queue.files} order files in ${Object.keys(perMailbox).length} mailboxes`
    + (dot.length ? ` (dot-mailboxes ${dot.map(([k, v]) => `${k} ${v}`).join(' · ')} - a shell \`*\` skips them)` : '')
    + `; ${withClass.length} of ${queue.orders.length} orders carry work-class`
    + (queue.unreadable.length ? `; ${queue.unreadable.length} order file(s) without frontmatter` : '');
  if (newWork.length === 0) {
    return { ...base, denominator: 0, facts, ...notCountable(
      `${census}; 0 orders dated >= ${cutoff} with work-class: new - the denominator is empty`) };
  }
  const unmatched = newWork.filter((o) => !o.fm.slug).length;
  return {
    ...base, countable: true, value: through.length, denominator: newWork.length, facts,
    detail: `${census}; counted = a Voyage run in the stats with the order's slug`
      + (unmatched ? `; ${unmatched} new-work order(s) carry no slug and cannot match` : ''),
  };
}

// --- 3 · INTERRUPTS --------------------------------------------------------------

// A call site EMITS user_input: the CLI shim with `--event user_input`, or a
// call `emit…('user_input'`. A name in prose is not a call.
const CALL_SITE = /--event[\s'",=\\]+user_input\b|\bemit\w*\(\s*['"]user_input['"]/;
const SCAN = [
  { dir: 'commands', ext: '.md' },
  { dir: 'agents', ext: '.md' },
  { dir: 'hooks', ext: '' },
  { dir: 'lib', ext: '.mjs' },
];

/** Emitter call sites for user_input outside lib/stats/ (the emitter) and tests/. */
export function emitterCallSites(root) {
  const sites = [];
  let scanned = 0;
  for (const { dir, ext } of SCAN) {
    const abs = join(root, dir);
    if (!existsSync(abs)) continue;
    for (const file of walk(abs).sort()) {
      const rel = toPosix(relative(root, file));
      if (rel.startsWith('lib/stats/') || (ext && !rel.endsWith(ext))) continue;
      scanned++;
      readFileSync(file, 'utf8').split('\n').forEach((line, i) => {
        if (CALL_SITE.test(line)) sites.push({ file: rel, line: i + 1 });
      });
    }
  }
  return { sites, scanned };
}

const deliveryOf = (r) => {
  const p = r.payload && typeof r.payload === 'object' ? r.payload : {};
  return p.slug || p.run_id || r.slug || r.run_id || null;
};

export function measureInterrupts({ stats, callSites }) {
  const base = { id: 'interrupts', label: 'operator interrupts per delivery', source: join(stats.dir, '*-stats.jsonl') };
  const lines = stats.files.reduce((n, f) => n + f.lines, 0);
  const records = stats.files.flatMap((f) => f.records);
  const userInput = records.filter((r) => r && r.event === 'user_input');
  const perDelivery = {};
  for (const r of userInput) {
    const d = deliveryOf(r);
    if (d) perDelivery[d] = (perDelivery[d] ?? 0) + 1;
  }
  const withDelivery = Object.values(perDelivery).reduce((a, b) => a + b, 0);
  const facts = { lines, files: stats.files.length, userInput: userInput.length, withDelivery, perDelivery,
    callSites: callSites.sites.length, scannedForCallSites: callSites.scanned };
  const census = `${userInput.length} user_input records of ${lines} lines in ${stats.files.length} files`
    + `; ${withDelivery} carry a slug or run id`
    + `; ${callSites.sites.length} emitter call sites outside lib/stats/ and tests/ (${callSites.scanned} files scanned)`;
  if (withDelivery === 0) {
    return { ...base, denominator: 0, facts, ...notCountable(`${census} - an interrupt that names no delivery cannot be divided by one`) };
  }
  return {
    ...base, countable: true, value: withDelivery, denominator: Object.keys(perDelivery).length, facts,
    detail: `${census}; per delivery: ${Object.entries(perDelivery).map(([k, v]) => `${k} ${v}`).join(' · ')}`,
  };
}

// --- 4 · the signature-preserving mutant (M9) --------------------------------------

const EXPORT_NAME = /^\s*export\s+(?:async\s+)?(?:function\*?|const|let|var|class)\s+([A-Za-z_$][\w$]*)/gm;

// Every entry whose check RUNS a named test and names the module it binds.
function behaviourConditions(registry) {
  const out = [];
  for (const e of [...(registry.defects ?? []), ...(registry.experiments ?? [])]) {
    for (const c of Array.isArray(e.check) ? e.check : []) {
      if (c && c.test !== undefined && Array.isArray(c.stubs)) out.push({ id: e.id, entry: e, cond: c });
    }
  }
  return out;
}

const relImport = (fromFile, toFile) => {
  const r = toPosix(relative(dirname(fromFile), toFile));
  return r.startsWith('.') ? r : `./${r}`;
};

/**
 * Build M9 in a fresh temp directory: every module a behaviour probe binds is
 * replaced by one whose exports keep their names and always return
 * {status:'PASSED'}, and every named test is replaced by one that only checks
 * that each export is a function. Returns the directory. `root` is only read.
 */
export function buildM9Tree(root, registry) {
  const dir = mkdtempSync(join(tmpdir(), 'yardstick-m9-'));
  const tests = new Map();
  const modules = new Set();
  for (const { cond } of behaviourConditions(registry)) {
    if (!tests.has(cond.test)) tests.set(cond.test, { names: new Set(), stubs: new Set() });
    tests.get(cond.test).names.add(cond.name);
    for (const s of cond.stubs) { tests.get(cond.test).stubs.add(s); modules.add(s); }
  }
  const write = (rel, body) => {
    mkdirSync(dirname(join(dir, rel)), { recursive: true });
    writeFileSync(join(dir, rel), body);
  };
  for (const mod of modules) {
    const source = readFileSync(join(root, mod), 'utf8');
    const names = new Set([...source.matchAll(EXPORT_NAME)].map((m) => m[1]));
    const lines = [...names].map((n) => `export function ${n}() { return { status: 'PASSED' }; }`);
    if (/^\s*export\s+default\b/m.test(source)) lines.push("export default function () { return { status: 'PASSED' }; }");
    write(mod, `${lines.join('\n')}\n`);
  }
  for (const [testRel, { names, stubs }] of tests) {
    const lines = ["import { test } from 'node:test';", "import { strict as assert } from 'node:assert';"];
    [...stubs].forEach((s, i) => lines.push(`import * as m${i} from '${relImport(testRel, s)}';`));
    for (const name of names) {
      const checks = [...stubs].map((_, i) => `for (const v of Object.values(m${i})) assert.equal(typeof v, 'function');`);
      lines.push(`test(${JSON.stringify(name)}, () => { ${checks.join(' ')} });`);
    }
    write(testRel, `${lines.join('\n')}\n`);
  }
  return dir;
}

// Run ONE named test; true = passed, false = failed, null = it did not run.
function runNamedTest(root, testRel, name) {
  const env = { ...process.env };
  for (const key of Object.keys(env)) if (key.startsWith('NODE_TEST_')) delete env[key];
  const pattern = `^${String(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`;
  const r = spawnSync(process.execPath, ['--test', '--test-reporter=tap', `--test-name-pattern=${pattern}`, testRel],
    { cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env });
  for (const line of String(r.stdout ?? '').split('\n')) {
    const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
    if (m && m[2].replace(/\\(.)/g, '$1').trim() === name) return m[1] === undefined;
  }
  return null;
}

export function measureMutants(root, { registry, evaluate = evaluateCheck, runTest = runNamedTest } = {}) {
  const reg = registry ?? loadRegistry(root);
  const base = {
    id: 'mutants', label: 'signature-preserving mutants the end-state gate fells',
    source: 'scripts/end-state-gate.mjs behaviour probes vs M9 (exports return {status:"PASSED"}, tests check typeof)',
    denominator: 1,
  };
  const probes = behaviourConditions(reg);
  const entryIds = [...new Set(probes.map((p) => p.id))];
  if (probes.length === 0) {
    return { ...base, facts: { entries: [] }, ...notCountable('the registry has no behaviour probe to aim M9 at') };
  }
  let dir;
  try {
    dir = buildM9Tree(root, reg);
  } catch (err) {
    return { ...base, facts: { entries: entryIds }, ...notCountable(`M9 could not be built: ${err.message}`) };
  }
  try {
    return judgeM9(base, dir, probes, entryIds, evaluate, runTest);
  } finally {
    // Only ever the directory buildM9Tree made, under the system temp dir.
    if (realpathSync(dir).startsWith(realpathSync(tmpdir()))) rmSync(dir, { recursive: true, force: true });
  }
}

function judgeM9(base, dir, probes, entryIds, evaluate, runTest) {
  // Liveness: M9's own tests must PASS on the M9 tree, or "felled" would mean "broken fixture".
  const dead = probes.filter((p) => runTest(dir, p.cond.test, p.cond.name) !== true).map((p) => p.cond.name);
  if (dead.length > 0) {
    return { ...base, facts: { entries: entryIds },
      ...notCountable(`M9 is not live: ${dead.length} of its named tests do not pass on the M9 tree`) };
  }
  const results = [...new Map(probes.map((p) => [p.id, p.entry])).values()]
    .map((e) => ({ id: e.id, ...evaluate(dir, e.check) }));
  const closed = results.filter((r) => r.status === 'closed');
  const felled = closed.length === 0 ? 1 : 0;
  const per = results.map((r) => `${r.id} ${r.status === 'closed' ? 'closed' : `not closed (${r.status})`}`).join(', ');
  return {
    ...base, countable: true, value: felled, facts: { entries: entryIds },
    detail: `M9 on a temp tree: ${per} - ${felled ? 'the gate refuses to count a typeof-only test as behaviour'
      : 'a typeof-only test closes the probe, so the gate counts a signature as behaviour'}`,
  };
}

// --- the whole yardstick ---------------------------------------------------------

const VERDICT = {
  judged: false,
  met: false,
  detail: 'the verdict needs the first five deliveries through the new Voyage; nothing counts a delivery yet',
};

export function measureAll({ root = REPO_ROOT, coordDir = DEFAULT_COORD, dataDir = DEFAULT_DATA, cutoff = DEFAULT_CUTOFF, mutantOptions = {} } = {}) {
  const stats = loadStats(dataDir);
  const numbers = [
    measureUse({ queue: readOrders(coordDir), slugs: voyageSlugs(stats), cutoff }),
    measureTrust(stats),
    measureInterrupts({ stats, callSites: emitterCallSites(root) }),
  ];
  const mutants = measureMutants(root, mutantOptions);
  const countable = numbers.filter((r) => r.countable).length;
  const felled = mutants.countable && mutants.value === mutants.denominator;
  return {
    green: countable === numbers.length && felled && VERDICT.judged && VERDICT.met,
    countable,
    rows: [...numbers, mutants],
    verdict: VERDICT,
  };
}

const fmt = (v) => (v === null || v === undefined ? 'n/a' : String(v));

export function render(result) {
  const m = result.rows.find((r) => r.id === 'mutants');
  const out = [];
  out.push(`Voyage yardstick: ${result.green ? 'GREEN' : 'RED'} (${result.countable} of 3 numbers countable)`);
  out.push(`signature-preserving mutants felled: ${m.countable ? `${m.value} of ${m.denominator}` : 'n/a (not measurable)'}`);
  out.push(`verdict: ${result.verdict.judged ? (result.verdict.met ? 'MET' : 'NOT MET') : 'NOT JUDGED'} - ${result.verdict.detail}`);
  out.push('');
  out.push('| number | countable | value | denominator | source |');
  out.push('|---|---|---|---|---|');
  for (const r of result.rows) {
    out.push(`| ${r.id} | ${r.countable ? 'YES' : 'NO'} | ${fmt(r.value)} | ${fmt(r.denominator)} | ${r.source} |`);
  }
  for (const r of result.rows) {
    out.push('');
    out.push(`${r.id} - ${r.label}: ${r.detail}`);
  }
  return out.join('\n');
}

export function main(argv) {
  const opts = {};
  let json = false;
  const usage = 'usage: yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>] [--cutoff YYYY-MM-DD]';
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    const value = argv[i + 1];
    if (a === '--json') { json = true; continue; }
    if (['--root', '--coord', '--data', '--cutoff'].includes(a) && value !== undefined) {
      i++;
      if (a === '--cutoff') {
        if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
          process.stderr.write(`yardstick: --cutoff must be YYYY-MM-DD, got ${value}\n${usage}\n`);
          return 2;
        }
        opts.cutoff = value;
      } else {
        opts[{ '--root': 'root', '--coord': 'coordDir', '--data': 'dataDir' }[a]] = resolve(value);
      }
      continue;
    }
    process.stderr.write(`yardstick: unknown argument ${a}\n${usage}\n`);
    return 2;
  }
  const result = measureAll(opts);
  process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
  return result.green ? 0 : 1;
}

if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  process.exitCode = main(process.argv.slice(2));
}