feat(measure): the yardstick prints Voyage's three numbers with denominators from the source, and it is RED
One command, `node scripts/yardstick.mjs`, prints use, trust and
interrupts. Each row gives countable YES/NO, the value, the denominator and
the source it was read from. A fourth row asks whether the end-state gate can
be fooled by a mutant that keeps the module's signature. Exit is non-zero
while any number is not countable, the mutant is not felled, or the verdict
over the first five deliveries is not judged. Nothing counts a delivery yet,
so today it is RED for the right reasons:
use NO no order in the queue carries work-class
trust YES current ALLOW streak 0 of 37 (BLOCK 14, WARN 12, ALLOW 11,
last 2026-09-05 BLOCK, ordered by ts, read as JSON)
interrupts NO 0 user_input records of ~141k lines in 5 files, 0 emitter
call sites outside lib/stats/ and tests/
mutants 0 of 1 signature-preserving mutants felled: on M9 (every export
returns {status:'PASSED'}, every named test only checks typeof)
the gate closes both D-03 and D-04
It only measures. The gate, the emitter and the order writer are untouched.
The mutant row is meant to stay red until the gate learns to fell M9.
Choices, and why:
- Use counts over the order queue (pending, archive/ and claimed/), because
an order is the unit of work. The denominator is orders dated on or after a
cutoff (default 2026-09-21) that carry `work-class: new`. They count as
having gone through Voyage when a Voyage run in the stats has the order's
`slug:`. Both fields are the contract the order writer must meet. No order
carries them today, so the number is NOT COUNTABLE, not 0 %.
- Mailboxes whose names start with a dot are mailboxes too. A shell `*`
glob skips them, which is why a hand count of `coord/*/orders` comes out
lower. The row names them so a re-measure can reconcile.
- M9 must be LIVE before its row can count. Its typeof tests have to pass on
the M9 tree, checked by a runner independent of the gate, so a broken
fixture can never read as "felled".
Also: tests/fixtures/red-first/02243c6-always-allow-shim.mjs, the shim that
makes the red-first claim of 02243c6 reproducible (6 of 55). The recipe is in
its header.
Red first: the new test file failed to load before scripts/yardstick.mjs
existed, and a dot-mailbox test was red before the census learned about
dot-mailboxes. Suite 1161 -> 1183 (1181/0/2), also green on a clean tree
exported from the index.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5f1ae4b4c2
commit
1779411b49
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 907 additions and 0 deletions
472
scripts/yardstick.mjs
Normal file
472
scripts/yardstick.mjs
Normal file
|
|
@ -0,0 +1,472 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
// scripts/yardstick.mjs
|
||||||
|
// The yardstick - Voyage's three numbers, each with a denominator read from its SOURCE:
|
||||||
|
//
|
||||||
|
// use new work that went through Voyage (the order queue: orders dated on or after
|
||||||
|
// the cutoff that carry `work-class: new`;
|
||||||
|
// counted = those with a Voyage run whose
|
||||||
|
// slug equals the order's `slug:`)
|
||||||
|
// trust reviews in a row with nothing that (trekreview-stats.jsonl, read as JSON and
|
||||||
|
// stops a release ordered by `ts` - file order is not
|
||||||
|
// chronological; value = the current ALLOW streak)
|
||||||
|
// interrupts operator interrupts per delivery (every *-stats.jsonl: `user_input` records
|
||||||
|
// that carry a slug or run id)
|
||||||
|
//
|
||||||
|
// plus a fourth row that asks whether the end-state gate can itself be fooled:
|
||||||
|
//
|
||||||
|
// mutants signature-preserving mutants the (M9: every export of the module a behaviour
|
||||||
|
// gate fells probe binds keeps its name and returns
|
||||||
|
// {status:'PASSED'}; every named test only
|
||||||
|
// checks `typeof`. The gate fells it when it
|
||||||
|
// refuses to close the probe on that tree.)
|
||||||
|
//
|
||||||
|
// A number the source cannot give is NOT COUNTABLE - never 0. The verdict over the first five
|
||||||
|
// deliveries is not judged by anything yet, and says so. Exit 0 = every number countable, every
|
||||||
|
// mutant felled and the verdict judged and met; 1 = anything short of that; 2 = usage error.
|
||||||
|
// It only READS: the queue, the stats and the repo. The mutant is built in a temp directory.
|
||||||
|
//
|
||||||
|
// Usage: node scripts/yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>]
|
||||||
|
// [--cutoff YYYY-MM-DD]
|
||||||
|
|
||||||
|
import {
|
||||||
|
readFileSync, writeFileSync, existsSync, readdirSync, statSync, mkdirSync, mkdtempSync,
|
||||||
|
realpathSync, rmSync,
|
||||||
|
} from 'node:fs';
|
||||||
|
import { homedir, tmpdir } from 'node:os';
|
||||||
|
import { join, resolve, dirname, relative, sep } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { evaluateCheck, loadRegistry, TEST_TIMEOUT_MS } from './end-state-gate.mjs';
|
||||||
|
|
||||||
|
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
|
||||||
|
export const DEFAULT_CUTOFF = '2026-09-21';
|
||||||
|
export const DEFAULT_COORD = join(homedir(), '.claude', 'coord');
|
||||||
|
export const DEFAULT_DATA = process.env.CLAUDE_PLUGIN_DATA
|
||||||
|
|| join(homedir(), '.claude', 'plugins', 'data', 'voyage-ktg-plugin-marketplace');
|
||||||
|
|
||||||
|
const REVIEW_FILE = 'trekreview-stats.jsonl';
|
||||||
|
const toPosix = (p) => p.split(sep).join('/');
|
||||||
|
|
||||||
|
function walk(dir) {
|
||||||
|
const out = [];
|
||||||
|
for (const name of readdirSync(dir)) {
|
||||||
|
const p = join(dir, name);
|
||||||
|
if (statSync(p).isDirectory()) out.push(...walk(p));
|
||||||
|
else out.push(p);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the stats files -----------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every `*-stats.jsonl` in the data directory, parsed line by line. A blank line
|
||||||
|
* is not a line; a line that is not JSON is a line, and is reported by number.
|
||||||
|
*/
|
||||||
|
export function loadStats(dataDir) {
|
||||||
|
if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) {
|
||||||
|
return { dir: dataDir, missing: true, files: [] };
|
||||||
|
}
|
||||||
|
const files = readdirSync(dataDir)
|
||||||
|
.filter((n) => n.endsWith('-stats.jsonl'))
|
||||||
|
.sort()
|
||||||
|
.map((name) => {
|
||||||
|
const path = join(dataDir, name);
|
||||||
|
const records = [];
|
||||||
|
const invalid = [];
|
||||||
|
let lines = 0;
|
||||||
|
readFileSync(path, 'utf8').split('\n').forEach((line, i) => {
|
||||||
|
if (line.trim() === '') return;
|
||||||
|
lines++;
|
||||||
|
try {
|
||||||
|
records.push(JSON.parse(line));
|
||||||
|
} catch {
|
||||||
|
invalid.push(i + 1);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return { name, path, lines, records, invalid };
|
||||||
|
});
|
||||||
|
return { dir: dataDir, missing: false, files };
|
||||||
|
}
|
||||||
|
|
||||||
|
// The slugs of every Voyage run the stats know about.
|
||||||
|
function voyageSlugs(stats) {
|
||||||
|
const slugs = new Set();
|
||||||
|
for (const f of stats.files) {
|
||||||
|
for (const r of f.records) {
|
||||||
|
if (r && typeof r.slug === 'string' && r.slug !== '') slugs.add(r.slug);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return slugs;
|
||||||
|
}
|
||||||
|
|
||||||
|
const notCountable = (detail, extra = {}) => ({ countable: false, value: null, detail, ...extra });
|
||||||
|
|
||||||
|
// --- 2 · TRUST -------------------------------------------------------------------
|
||||||
|
|
||||||
|
export function measureTrust(stats) {
|
||||||
|
const base = { id: 'trust', label: 'reviews in a row with nothing that stops a release' };
|
||||||
|
const file = stats.files.find((f) => f.name === REVIEW_FILE);
|
||||||
|
if (!file) {
|
||||||
|
return { ...base, source: join(stats.dir, REVIEW_FILE), denominator: 0, facts: {},
|
||||||
|
...notCountable(stats.missing ? `data directory ${stats.dir} not found` : `${REVIEW_FILE} not found`) };
|
||||||
|
}
|
||||||
|
const reviews = file.records.filter((r) => r && typeof r.verdict === 'string');
|
||||||
|
const undated = reviews.filter((r) => Number.isNaN(Date.parse(r.ts))).length;
|
||||||
|
// Stable sort on ts: the file is appended by whoever ran a review, in no fixed order.
|
||||||
|
const ordered = reviews
|
||||||
|
.map((r, i) => ({ r, i, t: Date.parse(r.ts) }))
|
||||||
|
.filter((x) => !Number.isNaN(x.t))
|
||||||
|
.sort((a, b) => a.t - b.t || a.i - b.i)
|
||||||
|
.map((x) => x.r);
|
||||||
|
const verdicts = {};
|
||||||
|
for (const r of reviews) verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + 1;
|
||||||
|
let longest = 0;
|
||||||
|
let run = 0;
|
||||||
|
for (const r of ordered) {
|
||||||
|
run = r.verdict === 'ALLOW' ? run + 1 : 0;
|
||||||
|
longest = Math.max(longest, run);
|
||||||
|
}
|
||||||
|
const last = ordered.at(-1) ?? null;
|
||||||
|
const facts = {
|
||||||
|
verdicts, longestStreak: longest, invalid: file.invalid.length, undated,
|
||||||
|
last: last ? { ts: last.ts, slug: last.slug, verdict: last.verdict } : null,
|
||||||
|
};
|
||||||
|
const notes = [];
|
||||||
|
if (file.invalid.length) notes.push(`${file.invalid.length} invalid line(s) left out (line ${file.invalid.join(', ')})`);
|
||||||
|
if (undated) notes.push(`${undated} review(s) without a readable ts left out of the streak`);
|
||||||
|
if (reviews.length === 0) {
|
||||||
|
return { ...base, source: file.path, denominator: 0, facts, ...notCountable(['no review rows', ...notes].join('; ')) };
|
||||||
|
}
|
||||||
|
const counts = Object.entries(verdicts).map(([k, v]) => `${k} ${v}`).join(' · ');
|
||||||
|
const detail = [
|
||||||
|
`${counts}; longest ALLOW streak ${longest}; last review ${last ? `${last.ts} ${last.verdict} (${last.slug})` : 'n/a'}`,
|
||||||
|
'ordered by ts, read as JSON',
|
||||||
|
...notes,
|
||||||
|
].join('; ');
|
||||||
|
return { ...base, source: file.path, countable: true, value: run, denominator: reviews.length, facts, detail };
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 1 · USE ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
// A minimal frontmatter reader: `key: value` lines between the first two `---`.
|
||||||
|
function frontmatter(text) {
|
||||||
|
const m = String(text).match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
||||||
|
if (!m) return null;
|
||||||
|
const fm = {};
|
||||||
|
for (const line of m[1].split(/\r?\n/)) {
|
||||||
|
const kv = line.match(/^([A-Za-z][\w-]*):\s*(.*?)\s*$/);
|
||||||
|
if (kv) fm[kv[1]] = kv[2];
|
||||||
|
}
|
||||||
|
return fm;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Every order file under `<coord>/<repo>/orders/` - pending, `archive/` and `claimed/`. */
|
||||||
|
export function readOrders(coordDir) {
|
||||||
|
const orders = [];
|
||||||
|
const unreadable = [];
|
||||||
|
let files = 0;
|
||||||
|
if (existsSync(coordDir)) {
|
||||||
|
for (const repo of readdirSync(coordDir).sort()) {
|
||||||
|
const dir = join(coordDir, repo, 'orders');
|
||||||
|
if (!existsSync(dir) || !statSync(dir).isDirectory()) continue;
|
||||||
|
for (const path of walk(dir).filter((p) => p.endsWith('.md')).sort()) {
|
||||||
|
files++;
|
||||||
|
const fm = frontmatter(readFileSync(path, 'utf8'));
|
||||||
|
if (fm && fm['order-id']) orders.push({ path, repo, fm });
|
||||||
|
else unreadable.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { dir: coordDir, files, orders, unreadable };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function measureUse({ queue, slugs, cutoff }) {
|
||||||
|
const base = {
|
||||||
|
id: 'use', label: 'new work that went through Voyage',
|
||||||
|
source: `${join(queue.dir, '*', 'orders')}/**/*.md (cutoff ${cutoff})`,
|
||||||
|
};
|
||||||
|
const withClass = queue.orders.filter((o) => o.fm['work-class'] !== undefined);
|
||||||
|
const newWork = withClass.filter((o) => o.fm['work-class'] === 'new' && String(o.fm.date ?? '').slice(0, 10) >= cutoff);
|
||||||
|
const through = newWork.filter((o) => o.fm.slug && slugs.has(o.fm.slug));
|
||||||
|
const facts = {
|
||||||
|
scanned: queue.orders.length, files: queue.files, unreadable: queue.unreadable.length,
|
||||||
|
withWorkClass: withClass.length, newSinceCutoff: newWork.length, withVoyageRun: through.length,
|
||||||
|
};
|
||||||
|
const perMailbox = {};
|
||||||
|
for (const o of queue.orders) perMailbox[o.repo] = (perMailbox[o.repo] ?? 0) + 1;
|
||||||
|
const dot = Object.entries(perMailbox).filter(([k]) => k.startsWith('.'));
|
||||||
|
const census = `${queue.files} order files in ${Object.keys(perMailbox).length} mailboxes`
|
||||||
|
+ (dot.length ? ` (dot-mailboxes ${dot.map(([k, v]) => `${k} ${v}`).join(' · ')} - a shell \`*\` skips them)` : '')
|
||||||
|
+ `; ${withClass.length} of ${queue.orders.length} orders carry work-class`
|
||||||
|
+ (queue.unreadable.length ? `; ${queue.unreadable.length} order file(s) without frontmatter` : '');
|
||||||
|
if (newWork.length === 0) {
|
||||||
|
return { ...base, denominator: 0, facts, ...notCountable(
|
||||||
|
`${census}; 0 orders dated >= ${cutoff} with work-class: new - the denominator is empty`) };
|
||||||
|
}
|
||||||
|
const unmatched = newWork.filter((o) => !o.fm.slug).length;
|
||||||
|
return {
|
||||||
|
...base, countable: true, value: through.length, denominator: newWork.length, facts,
|
||||||
|
detail: `${census}; counted = a Voyage run in the stats with the order's slug`
|
||||||
|
+ (unmatched ? `; ${unmatched} new-work order(s) carry no slug and cannot match` : ''),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 3 · INTERRUPTS --------------------------------------------------------------
|
||||||
|
|
||||||
|
// A call site EMITS user_input: the CLI shim with `--event user_input`, or a
|
||||||
|
// call `emit…('user_input'`. A name in prose is not a call.
|
||||||
|
const CALL_SITE = /--event[\s'",=\\]+user_input\b|\bemit\w*\(\s*['"]user_input['"]/;
|
||||||
|
const SCAN = [
|
||||||
|
{ dir: 'commands', ext: '.md' },
|
||||||
|
{ dir: 'agents', ext: '.md' },
|
||||||
|
{ dir: 'hooks', ext: '' },
|
||||||
|
{ dir: 'lib', ext: '.mjs' },
|
||||||
|
];
|
||||||
|
|
||||||
|
/** Emitter call sites for user_input outside lib/stats/ (the emitter) and tests/. */
|
||||||
|
export function emitterCallSites(root) {
|
||||||
|
const sites = [];
|
||||||
|
let scanned = 0;
|
||||||
|
for (const { dir, ext } of SCAN) {
|
||||||
|
const abs = join(root, dir);
|
||||||
|
if (!existsSync(abs)) continue;
|
||||||
|
for (const file of walk(abs).sort()) {
|
||||||
|
const rel = toPosix(relative(root, file));
|
||||||
|
if (rel.startsWith('lib/stats/') || (ext && !rel.endsWith(ext))) continue;
|
||||||
|
scanned++;
|
||||||
|
readFileSync(file, 'utf8').split('\n').forEach((line, i) => {
|
||||||
|
if (CALL_SITE.test(line)) sites.push({ file: rel, line: i + 1 });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { sites, scanned };
|
||||||
|
}
|
||||||
|
|
||||||
|
const deliveryOf = (r) => {
|
||||||
|
const p = r.payload && typeof r.payload === 'object' ? r.payload : {};
|
||||||
|
return p.slug || p.run_id || r.slug || r.run_id || null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function measureInterrupts({ stats, callSites }) {
|
||||||
|
const base = { id: 'interrupts', label: 'operator interrupts per delivery', source: join(stats.dir, '*-stats.jsonl') };
|
||||||
|
const lines = stats.files.reduce((n, f) => n + f.lines, 0);
|
||||||
|
const records = stats.files.flatMap((f) => f.records);
|
||||||
|
const userInput = records.filter((r) => r && r.event === 'user_input');
|
||||||
|
const perDelivery = {};
|
||||||
|
for (const r of userInput) {
|
||||||
|
const d = deliveryOf(r);
|
||||||
|
if (d) perDelivery[d] = (perDelivery[d] ?? 0) + 1;
|
||||||
|
}
|
||||||
|
const withDelivery = Object.values(perDelivery).reduce((a, b) => a + b, 0);
|
||||||
|
const facts = { lines, files: stats.files.length, userInput: userInput.length, withDelivery, perDelivery,
|
||||||
|
callSites: callSites.sites.length, scannedForCallSites: callSites.scanned };
|
||||||
|
const census = `${userInput.length} user_input records of ${lines} lines in ${stats.files.length} files`
|
||||||
|
+ `; ${withDelivery} carry a slug or run id`
|
||||||
|
+ `; ${callSites.sites.length} emitter call sites outside lib/stats/ and tests/ (${callSites.scanned} files scanned)`;
|
||||||
|
if (withDelivery === 0) {
|
||||||
|
return { ...base, denominator: 0, facts, ...notCountable(`${census} - an interrupt that names no delivery cannot be divided by one`) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...base, countable: true, value: withDelivery, denominator: Object.keys(perDelivery).length, facts,
|
||||||
|
detail: `${census}; per delivery: ${Object.entries(perDelivery).map(([k, v]) => `${k} ${v}`).join(' · ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 4 · the signature-preserving mutant (M9) --------------------------------------
|
||||||
|
|
||||||
|
const EXPORT_NAME = /^\s*export\s+(?:async\s+)?(?:function\*?|const|let|var|class)\s+([A-Za-z_$][\w$]*)/gm;
|
||||||
|
|
||||||
|
// Every entry whose check RUNS a named test and names the module it binds.
|
||||||
|
function behaviourConditions(registry) {
|
||||||
|
const out = [];
|
||||||
|
for (const e of [...(registry.defects ?? []), ...(registry.experiments ?? [])]) {
|
||||||
|
for (const c of Array.isArray(e.check) ? e.check : []) {
|
||||||
|
if (c && c.test !== undefined && Array.isArray(c.stubs)) out.push({ id: e.id, entry: e, cond: c });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const relImport = (fromFile, toFile) => {
|
||||||
|
const r = toPosix(relative(dirname(fromFile), toFile));
|
||||||
|
return r.startsWith('.') ? r : `./${r}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build M9 in a fresh temp directory: every module a behaviour probe binds is
|
||||||
|
* replaced by one whose exports keep their names and always return
|
||||||
|
* {status:'PASSED'}, and every named test is replaced by one that only checks
|
||||||
|
* that each export is a function. Returns the directory. `root` is only read.
|
||||||
|
*/
|
||||||
|
export function buildM9Tree(root, registry) {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'yardstick-m9-'));
|
||||||
|
const tests = new Map();
|
||||||
|
const modules = new Set();
|
||||||
|
for (const { cond } of behaviourConditions(registry)) {
|
||||||
|
if (!tests.has(cond.test)) tests.set(cond.test, { names: new Set(), stubs: new Set() });
|
||||||
|
tests.get(cond.test).names.add(cond.name);
|
||||||
|
for (const s of cond.stubs) { tests.get(cond.test).stubs.add(s); modules.add(s); }
|
||||||
|
}
|
||||||
|
const write = (rel, body) => {
|
||||||
|
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||||
|
writeFileSync(join(dir, rel), body);
|
||||||
|
};
|
||||||
|
for (const mod of modules) {
|
||||||
|
const source = readFileSync(join(root, mod), 'utf8');
|
||||||
|
const names = new Set([...source.matchAll(EXPORT_NAME)].map((m) => m[1]));
|
||||||
|
const lines = [...names].map((n) => `export function ${n}() { return { status: 'PASSED' }; }`);
|
||||||
|
if (/^\s*export\s+default\b/m.test(source)) lines.push("export default function () { return { status: 'PASSED' }; }");
|
||||||
|
write(mod, `${lines.join('\n')}\n`);
|
||||||
|
}
|
||||||
|
for (const [testRel, { names, stubs }] of tests) {
|
||||||
|
const lines = ["import { test } from 'node:test';", "import { strict as assert } from 'node:assert';"];
|
||||||
|
[...stubs].forEach((s, i) => lines.push(`import * as m${i} from '${relImport(testRel, s)}';`));
|
||||||
|
for (const name of names) {
|
||||||
|
const checks = [...stubs].map((_, i) => `for (const v of Object.values(m${i})) assert.equal(typeof v, 'function');`);
|
||||||
|
lines.push(`test(${JSON.stringify(name)}, () => { ${checks.join(' ')} });`);
|
||||||
|
}
|
||||||
|
write(testRel, `${lines.join('\n')}\n`);
|
||||||
|
}
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run ONE named test; true = passed, false = failed, null = it did not run.
|
||||||
|
function runNamedTest(root, testRel, name) {
|
||||||
|
const env = { ...process.env };
|
||||||
|
for (const key of Object.keys(env)) if (key.startsWith('NODE_TEST_')) delete env[key];
|
||||||
|
const pattern = `^${String(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`;
|
||||||
|
const r = spawnSync(process.execPath, ['--test', '--test-reporter=tap', `--test-name-pattern=${pattern}`, testRel],
|
||||||
|
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env });
|
||||||
|
for (const line of String(r.stdout ?? '').split('\n')) {
|
||||||
|
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
||||||
|
if (m && m[2].replace(/\\(.)/g, '$1').trim() === name) return m[1] === undefined;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function measureMutants(root, { registry, evaluate = evaluateCheck, runTest = runNamedTest } = {}) {
|
||||||
|
const reg = registry ?? loadRegistry(root);
|
||||||
|
const base = {
|
||||||
|
id: 'mutants', label: 'signature-preserving mutants the end-state gate fells',
|
||||||
|
source: 'scripts/end-state-gate.mjs behaviour probes vs M9 (exports return {status:"PASSED"}, tests check typeof)',
|
||||||
|
denominator: 1,
|
||||||
|
};
|
||||||
|
const probes = behaviourConditions(reg);
|
||||||
|
const entryIds = [...new Set(probes.map((p) => p.id))];
|
||||||
|
if (probes.length === 0) {
|
||||||
|
return { ...base, facts: { entries: [] }, ...notCountable('the registry has no behaviour probe to aim M9 at') };
|
||||||
|
}
|
||||||
|
let dir;
|
||||||
|
try {
|
||||||
|
dir = buildM9Tree(root, reg);
|
||||||
|
} catch (err) {
|
||||||
|
return { ...base, facts: { entries: entryIds }, ...notCountable(`M9 could not be built: ${err.message}`) };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return judgeM9(base, dir, probes, entryIds, evaluate, runTest);
|
||||||
|
} finally {
|
||||||
|
// Only ever the directory buildM9Tree made, under the system temp dir.
|
||||||
|
if (realpathSync(dir).startsWith(realpathSync(tmpdir()))) rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function judgeM9(base, dir, probes, entryIds, evaluate, runTest) {
|
||||||
|
// Liveness: M9's own tests must PASS on the M9 tree, or "felled" would mean "broken fixture".
|
||||||
|
const dead = probes.filter((p) => runTest(dir, p.cond.test, p.cond.name) !== true).map((p) => p.cond.name);
|
||||||
|
if (dead.length > 0) {
|
||||||
|
return { ...base, facts: { entries: entryIds },
|
||||||
|
...notCountable(`M9 is not live: ${dead.length} of its named tests do not pass on the M9 tree`) };
|
||||||
|
}
|
||||||
|
const results = [...new Map(probes.map((p) => [p.id, p.entry])).values()]
|
||||||
|
.map((e) => ({ id: e.id, ...evaluate(dir, e.check) }));
|
||||||
|
const closed = results.filter((r) => r.status === 'closed');
|
||||||
|
const felled = closed.length === 0 ? 1 : 0;
|
||||||
|
const per = results.map((r) => `${r.id} ${r.status === 'closed' ? 'closed' : `not closed (${r.status})`}`).join(', ');
|
||||||
|
return {
|
||||||
|
...base, countable: true, value: felled, facts: { entries: entryIds },
|
||||||
|
detail: `M9 on a temp tree: ${per} - ${felled ? 'the gate refuses to count a typeof-only test as behaviour'
|
||||||
|
: 'a typeof-only test closes the probe, so the gate counts a signature as behaviour'}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the whole yardstick ---------------------------------------------------------
|
||||||
|
|
||||||
|
const VERDICT = {
|
||||||
|
judged: false,
|
||||||
|
met: false,
|
||||||
|
detail: 'the verdict needs the first five deliveries through the new Voyage; nothing counts a delivery yet',
|
||||||
|
};
|
||||||
|
|
||||||
|
export function measureAll({ root = REPO_ROOT, coordDir = DEFAULT_COORD, dataDir = DEFAULT_DATA, cutoff = DEFAULT_CUTOFF, mutantOptions = {} } = {}) {
|
||||||
|
const stats = loadStats(dataDir);
|
||||||
|
const numbers = [
|
||||||
|
measureUse({ queue: readOrders(coordDir), slugs: voyageSlugs(stats), cutoff }),
|
||||||
|
measureTrust(stats),
|
||||||
|
measureInterrupts({ stats, callSites: emitterCallSites(root) }),
|
||||||
|
];
|
||||||
|
const mutants = measureMutants(root, mutantOptions);
|
||||||
|
const countable = numbers.filter((r) => r.countable).length;
|
||||||
|
const felled = mutants.countable && mutants.value === mutants.denominator;
|
||||||
|
return {
|
||||||
|
green: countable === numbers.length && felled && VERDICT.judged && VERDICT.met,
|
||||||
|
countable,
|
||||||
|
rows: [...numbers, mutants],
|
||||||
|
verdict: VERDICT,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const fmt = (v) => (v === null || v === undefined ? 'n/a' : String(v));
|
||||||
|
|
||||||
|
export function render(result) {
|
||||||
|
const m = result.rows.find((r) => r.id === 'mutants');
|
||||||
|
const out = [];
|
||||||
|
out.push(`Voyage yardstick: ${result.green ? 'GREEN' : 'RED'} (${result.countable} of 3 numbers countable)`);
|
||||||
|
out.push(`signature-preserving mutants felled: ${m.countable ? `${m.value} of ${m.denominator}` : 'n/a (not measurable)'}`);
|
||||||
|
out.push(`verdict: ${result.verdict.judged ? (result.verdict.met ? 'MET' : 'NOT MET') : 'NOT JUDGED'} - ${result.verdict.detail}`);
|
||||||
|
out.push('');
|
||||||
|
out.push('| number | countable | value | denominator | source |');
|
||||||
|
out.push('|---|---|---|---|---|');
|
||||||
|
for (const r of result.rows) {
|
||||||
|
out.push(`| ${r.id} | ${r.countable ? 'YES' : 'NO'} | ${fmt(r.value)} | ${fmt(r.denominator)} | ${r.source} |`);
|
||||||
|
}
|
||||||
|
for (const r of result.rows) {
|
||||||
|
out.push('');
|
||||||
|
out.push(`${r.id} - ${r.label}: ${r.detail}`);
|
||||||
|
}
|
||||||
|
return out.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function main(argv) {
|
||||||
|
const opts = {};
|
||||||
|
let json = false;
|
||||||
|
const usage = 'usage: yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>] [--cutoff YYYY-MM-DD]';
|
||||||
|
for (let i = 0; i < argv.length; i++) {
|
||||||
|
const a = argv[i];
|
||||||
|
const value = argv[i + 1];
|
||||||
|
if (a === '--json') { json = true; continue; }
|
||||||
|
if (['--root', '--coord', '--data', '--cutoff'].includes(a) && value !== undefined) {
|
||||||
|
i++;
|
||||||
|
if (a === '--cutoff') {
|
||||||
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
|
||||||
|
process.stderr.write(`yardstick: --cutoff must be YYYY-MM-DD, got ${value}\n${usage}\n`);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
opts.cutoff = value;
|
||||||
|
} else {
|
||||||
|
opts[{ '--root': 'root', '--coord': 'coordDir', '--data': 'dataDir' }[a]] = resolve(value);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
process.stderr.write(`yardstick: unknown argument ${a}\n${usage}\n`);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
const result = measureAll(opts);
|
||||||
|
process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
|
||||||
|
return result.green ? 0 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||||
|
process.exitCode = main(process.argv.slice(2));
|
||||||
|
}
|
||||||
26
tests/fixtures/red-first/02243c6-always-allow-shim.mjs
vendored
Normal file
26
tests/fixtures/red-first/02243c6-always-allow-shim.mjs
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs
|
||||||
|
// The shim that makes the red-first claim of 02243c6 reproducible.
|
||||||
|
//
|
||||||
|
// 02243c6 ("the criteria runner screens with an ALLOWLIST") says six of its new
|
||||||
|
// tests were red before the fix. On the parent tree the test file does not even
|
||||||
|
// LOAD - it imports `allowedCommand`, which the parent does not export - so the
|
||||||
|
// red is a load error on the whole file, not six named failures. The six only
|
||||||
|
// appear once the missing export exists and allows everything. This file is
|
||||||
|
// that export, kept here so a re-measure does not have to guess it.
|
||||||
|
//
|
||||||
|
// Reproduce "6 of 55" (POSIX shell, from the repo root, needs git history):
|
||||||
|
//
|
||||||
|
// T="$(mktemp -d)"
|
||||||
|
// git archive 02243c6 | tar -x -C "$T"
|
||||||
|
// git show 02243c6^:lib/verification/criteria-runner.mjs > "$T/lib/verification/criteria-runner.mjs"
|
||||||
|
// cat tests/fixtures/red-first/02243c6-always-allow-shim.mjs >> "$T/lib/verification/criteria-runner.mjs"
|
||||||
|
// (cd "$T" && node --test --test-reporter=tap tests/lib/criteria-runner.test.mjs) | grep -E '^# (tests|fail)'
|
||||||
|
//
|
||||||
|
// Expected: `# tests 55` and `# fail 6` - four `allowedCommand` tests (all but
|
||||||
|
// "every known test runner is allowed", which an always-allow shim passes),
|
||||||
|
// `runCriteria: a command outside the allowlist is NOT RUN`, and the end-to-end
|
||||||
|
// canary test (28 evasions, 0 run). Measured 2026-09-22.
|
||||||
|
// It is a fixture, not a test: nothing in the suite runs it, because the suite
|
||||||
|
// must also pass on a `git archive` tree, where there is no history to rewind.
|
||||||
|
|
||||||
|
export function allowedCommand() { return { allowed: true, reason: '' }; }
|
||||||
409
tests/scripts/yardstick.test.mjs
Normal file
409
tests/scripts/yardstick.test.mjs
Normal file
|
|
@ -0,0 +1,409 @@
|
||||||
|
// tests/scripts/yardstick.test.mjs
|
||||||
|
// The yardstick prints Voyage's three numbers (use · trust · interrupts) with
|
||||||
|
// denominators read from their SOURCE, plus a fourth row that asks whether the
|
||||||
|
// end-state gate itself can be fooled by a signature-preserving mutant.
|
||||||
|
//
|
||||||
|
// Like the gate's own test, these tests pin the MECHANICS in both directions
|
||||||
|
// against throwaway fixtures - every number can be countable and can be not
|
||||||
|
// countable, the mutant row can read felled and not felled. They deliberately
|
||||||
|
// do NOT assert that the real data is red today: the yardstick is supposed to
|
||||||
|
// turn green, and a test that pins today's distance would fail on the day the
|
||||||
|
// work is done. Nothing here reads the real order queue or the real stats.
|
||||||
|
|
||||||
|
import { test } from 'node:test';
|
||||||
|
import { strict as assert } from 'node:assert';
|
||||||
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, dirname, resolve } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import {
|
||||||
|
DEFAULT_CUTOFF,
|
||||||
|
loadStats,
|
||||||
|
readOrders,
|
||||||
|
measureUse,
|
||||||
|
measureTrust,
|
||||||
|
measureInterrupts,
|
||||||
|
emitterCallSites,
|
||||||
|
buildM9Tree,
|
||||||
|
measureMutants,
|
||||||
|
measureAll,
|
||||||
|
render,
|
||||||
|
main,
|
||||||
|
} from '../../scripts/yardstick.mjs';
|
||||||
|
|
||||||
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||||
|
|
||||||
|
// --- helpers ---------------------------------------------------------------
|
||||||
|
|
||||||
|
function fixture(files) {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'yardstick-'));
|
||||||
|
for (const [rel, body] of Object.entries(files)) {
|
||||||
|
const p = join(dir, rel);
|
||||||
|
mkdirSync(dirname(p), { recursive: true });
|
||||||
|
writeFileSync(p, body);
|
||||||
|
}
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cleanup = (...dirs) => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); };
|
||||||
|
|
||||||
|
const jsonl = (...rows) => `${rows.map((r) => (typeof r === 'string' ? r : JSON.stringify(r))).join('\n')}\n`;
|
||||||
|
|
||||||
|
function order({ id, date, workClass, slug, extra = '' }) {
|
||||||
|
const lines = ['---', 'from: .claude', 'to: some-repo', `order-id: ${id}`, `subject: order ${id}`, `date: ${date}`];
|
||||||
|
if (workClass !== undefined) lines.push(`work-class: ${workClass}`);
|
||||||
|
if (slug !== undefined) lines.push(`slug: ${slug}`);
|
||||||
|
lines.push('---', '', `# ORDER ${id}`, extra, '');
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
const review = (ts, verdict, slug = 's') => ({ ts, slug, verdict });
|
||||||
|
|
||||||
|
// --- the stats loader --------------------------------------------------------
|
||||||
|
|
||||||
|
test('loadStats: an invalid JSON line is COUNTED, never silently dropped', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{not json', '', review('2026-09-02T00:00:00Z', 'BLOCK')),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const stats = loadStats(dir);
|
||||||
|
const f = stats.files.find((x) => x.name === 'trekreview-stats.jsonl');
|
||||||
|
assert.equal(f.lines, 3, 'blank lines are not lines; the invalid one is');
|
||||||
|
assert.equal(f.records.length, 2);
|
||||||
|
assert.deepEqual(f.invalid, [2], 'the invalid line is reported by its line number');
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('loadStats: a missing data directory is reported, not read as zero', () => {
|
||||||
|
const stats = loadStats(join(tmpdir(), 'yardstick-no-such-dir-8f3a'));
|
||||||
|
assert.equal(stats.missing, true);
|
||||||
|
assert.deepEqual(stats.files, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- 2 · TRUST ---------------------------------------------------------------
|
||||||
|
|
||||||
|
test('trust: the ALLOW streak is read in ts order, not file order', () => {
|
||||||
|
// File order ends on ALLOW, ALLOW; chronologically the last review is a BLOCK.
|
||||||
|
const dir = fixture({
|
||||||
|
'trekreview-stats.jsonl': jsonl(
|
||||||
|
review('2026-09-01T00:00:00Z', 'ALLOW'),
|
||||||
|
review('2026-09-05T00:00:00Z', 'BLOCK'),
|
||||||
|
review('2026-09-02T00:00:00Z', 'ALLOW'),
|
||||||
|
review('2026-09-03T00:00:00Z', 'ALLOW'),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureTrust(loadStats(dir));
|
||||||
|
assert.equal(row.countable, true);
|
||||||
|
assert.equal(row.denominator, 4);
|
||||||
|
assert.equal(row.value, 0, 'the chronologically last review is a BLOCK');
|
||||||
|
assert.equal(row.facts.longestStreak, 3);
|
||||||
|
assert.deepEqual(row.facts.verdicts, { ALLOW: 3, BLOCK: 1 });
|
||||||
|
assert.equal(row.facts.last.verdict, 'BLOCK');
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trust: both spacings of the verdict field count the same (it is read as JSON, not grepped)', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekreview-stats.jsonl': jsonl(
|
||||||
|
'{"ts": "2026-09-01T00:00:00Z", "slug": "a", "verdict": "BLOCK"}',
|
||||||
|
'{"ts":"2026-09-02T00:00:00Z","slug":"b","verdict":"ALLOW"}',
|
||||||
|
),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureTrust(loadStats(dir));
|
||||||
|
assert.deepEqual(row.facts.verdicts, { BLOCK: 1, ALLOW: 1 });
|
||||||
|
assert.equal(row.value, 1);
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trust: an invalid line is left out of n and said out loud', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{"ts": oops'),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureTrust(loadStats(dir));
|
||||||
|
assert.equal(row.denominator, 1);
|
||||||
|
assert.equal(row.facts.invalid, 1);
|
||||||
|
assert.match(row.detail, /1 invalid line/);
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trust: an empty or missing review file is NOT countable', () => {
|
||||||
|
const empty = fixture({ 'trekreview-stats.jsonl': '' });
|
||||||
|
const none = fixture({ 'trekplan-stats.jsonl': jsonl({ ts: '2026-09-01T00:00:00Z', slug: 'x' }) });
|
||||||
|
try {
|
||||||
|
for (const dir of [empty, none]) {
|
||||||
|
const row = measureTrust(loadStats(dir));
|
||||||
|
assert.equal(row.countable, false, dir);
|
||||||
|
assert.equal(row.value, null);
|
||||||
|
assert.ok(row.detail.length > 0, 'the row must say WHY it cannot be counted');
|
||||||
|
}
|
||||||
|
} finally { cleanup(empty, none); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- 1 · USE -----------------------------------------------------------------
|
||||||
|
|
||||||
|
test('use: orders are read from orders/, archive/ and claimed/, and nothing else', () => {
|
||||||
|
const coord = fixture({
|
||||||
|
'repo-a/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
'repo-a/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-01T10:00:00Z' }),
|
||||||
|
'repo-b/orders/claimed/o3.md': order({ id: 'o3', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
'repo-b/orders/claimed/o3.claim': 'claimed',
|
||||||
|
'repo-b/inbox/m1.md': order({ id: 'm1', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
'repo-b/orders/broken.md': 'no frontmatter at all\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const q = readOrders(coord);
|
||||||
|
assert.equal(q.files, 4, 'three orders and one unreadable file, all .md under */orders/');
|
||||||
|
assert.equal(q.orders.length, 3);
|
||||||
|
assert.equal(q.unreadable.length, 1);
|
||||||
|
} finally { cleanup(coord); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('use: a dot-mailbox is a mailbox - its orders count, and the detail names it', () => {
|
||||||
|
// Measured 2026-09-22: a hand count over `coord/*/orders` came out lower than
|
||||||
|
// the yardstick, because the shell's `*` skips dot-directories and the queue
|
||||||
|
// does not. The detail names them, so the next re-measure can reconcile.
|
||||||
|
const coord = fixture({
|
||||||
|
'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
'.pm/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const q = readOrders(coord);
|
||||||
|
assert.equal(q.orders.length, 2);
|
||||||
|
const row = measureUse({ queue: q, slugs: new Set(), cutoff: DEFAULT_CUTOFF });
|
||||||
|
assert.match(row.detail, /2 order files in 2 mailboxes/);
|
||||||
|
assert.match(row.detail, /dot-mailboxes .pm 1/);
|
||||||
|
} finally { cleanup(coord); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('use: no order carrying work-class means NOT COUNTABLE, not 0 %', () => {
|
||||||
|
const coord = fixture({
|
||||||
|
'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||||
|
'r/orders/o2.md': order({ id: 'o2', date: '2026-09-23T10:00:00Z' }),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: DEFAULT_CUTOFF });
|
||||||
|
assert.equal(row.countable, false);
|
||||||
|
assert.equal(row.value, null);
|
||||||
|
assert.equal(row.denominator, 0);
|
||||||
|
assert.equal(row.facts.scanned, 2);
|
||||||
|
assert.equal(row.facts.withWorkClass, 0);
|
||||||
|
assert.match(row.detail, /0 of 2 orders carry work-class/);
|
||||||
|
} finally { cleanup(coord); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('use: the denominator is new-work orders dated on or after the cutoff; the numerator is those with a Voyage run', () => {
|
||||||
|
const coord = fixture({
|
||||||
|
'r/orders/a.md': order({ id: 'a', date: '2026-09-21T00:00:00Z', workClass: 'new', slug: 'feat-a' }),
|
||||||
|
'r/orders/archive/b.md': order({ id: 'b', date: '2026-09-25T00:00:00Z', workClass: 'new', slug: 'feat-b' }),
|
||||||
|
'r/orders/c.md': order({ id: 'c', date: '2026-09-25T00:00:00Z', workClass: 'new' }),
|
||||||
|
'r/orders/d.md': order({ id: 'd', date: '2026-09-20T23:59:59Z', workClass: 'new', slug: 'feat-a' }),
|
||||||
|
'r/orders/e.md': order({ id: 'e', date: '2026-09-25T00:00:00Z', workClass: 'maintenance', slug: 'feat-a' }),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureUse({ queue: readOrders(coord), slugs: new Set(['feat-a']), cutoff: '2026-09-21' });
|
||||||
|
assert.equal(row.countable, true);
|
||||||
|
assert.equal(row.denominator, 3, 'a, b and c - d is before the cutoff, e is maintenance');
|
||||||
|
assert.equal(row.value, 1, 'only a has a Voyage run with the same slug; c has no slug to match');
|
||||||
|
assert.equal(row.facts.withWorkClass, 5);
|
||||||
|
} finally { cleanup(coord); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- 3 · INTERRUPTS ----------------------------------------------------------
|
||||||
|
|
||||||
|
test('interrupts: no user_input record means NOT COUNTABLE, with the line denominator stated', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekexecute-stats.jsonl': jsonl({ ts: 't', session_id: 's', success: true }, { ts: 't', event: 'main-merge-gate', payload: {} }),
|
||||||
|
'trekplan-stats.jsonl': jsonl({ ts: 't', slug: 'x' }),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 3 } });
|
||||||
|
assert.equal(row.countable, false);
|
||||||
|
assert.equal(row.facts.lines, 3);
|
||||||
|
assert.equal(row.facts.userInput, 0);
|
||||||
|
assert.match(row.detail, /0 user_input records of 3 lines in 2 files/);
|
||||||
|
assert.match(row.detail, /0 emitter call sites/);
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('interrupts: user_input WITHOUT a slug or run id is still not countable per delivery', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { question: 'q' } }),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
|
||||||
|
assert.equal(row.countable, false);
|
||||||
|
assert.equal(row.facts.userInput, 1);
|
||||||
|
assert.equal(row.facts.withDelivery, 0);
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('interrupts: user_input carrying a slug or run id is countable, per delivery', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'trekexecute-stats.jsonl': jsonl(
|
||||||
|
{ ts: 't1', event: 'user_input', payload: { slug: 'feat-a' } },
|
||||||
|
{ ts: 't2', event: 'user_input', payload: { slug: 'feat-a' } },
|
||||||
|
{ ts: 't3', event: 'user_input', payload: { run_id: 'r-9' } },
|
||||||
|
),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
|
||||||
|
assert.equal(row.countable, true);
|
||||||
|
assert.equal(row.value, 3);
|
||||||
|
assert.deepEqual(row.facts.perDelivery, { 'feat-a': 2, 'r-9': 1 });
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('emitterCallSites: counts user_input emissions outside lib/stats/ and tests/ only', () => {
|
||||||
|
const root = fixture({
|
||||||
|
'commands/a.md': 'node ${CLAUDE_PLUGIN_ROOT}/lib/stats/event-emit.mjs \\\n --event user_input \\\n',
|
||||||
|
'commands/b.md': 'the emitter knows user_input as an event name - prose, not a call\n',
|
||||||
|
'lib/util/x.mjs': "emitEvent('user_input', { slug });\n",
|
||||||
|
'lib/stats/event-emit.mjs': "emit('user_input', {});\n",
|
||||||
|
'tests/lib/y.test.mjs': "emit('user_input', {});\n",
|
||||||
|
'hooks/scripts/h.mjs': "spawnSync('node', [EMIT, '--event', 'user_input']);\n",
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const r = emitterCallSites(root);
|
||||||
|
assert.deepEqual(r.sites.map((s) => s.file).sort(), ['commands/a.md', 'hooks/scripts/h.mjs', 'lib/util/x.mjs']);
|
||||||
|
assert.equal(r.scanned, 4, 'commands/a.md, commands/b.md, lib/util/x.mjs, hooks/scripts/h.mjs');
|
||||||
|
} finally { cleanup(root); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- 4 · the signature-preserving mutant (M9) ----------------------------------
|
||||||
|
|
||||||
|
const MODULE = 'export function verdict(code) { return { status: code === 0 ? "PASSED" : "FAILED" }; }\n';
|
||||||
|
const STRONG_TEST = [
|
||||||
|
"import { test } from 'node:test';",
|
||||||
|
"import { strict as assert } from 'node:assert';",
|
||||||
|
"import { verdict } from '../lib/m.mjs';",
|
||||||
|
"test('verdict fells a failing exit', () => { assert.equal(verdict(1).status, 'FAILED'); });",
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
const behaviourRegistry = {
|
||||||
|
defects: [{
|
||||||
|
id: 'D-X', probe: 'behaviour',
|
||||||
|
check: [{ test: 'tests/t.test.mjs', name: 'verdict fells a failing exit', expect: 'fails', stubs: ['lib/m.mjs'] }],
|
||||||
|
}],
|
||||||
|
experiments: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
test('buildM9Tree: every export keeps its name and returns {status:"PASSED"}; every named test only checks typeof', async () => {
|
||||||
|
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||||
|
const m9 = buildM9Tree(root, behaviourRegistry);
|
||||||
|
try {
|
||||||
|
const mod = readFileSync(join(m9, 'lib', 'm.mjs'), 'utf8');
|
||||||
|
assert.match(mod, /export function verdict\(\) \{ return \{ status: 'PASSED' \}; \}/);
|
||||||
|
const t = readFileSync(join(m9, 'tests', 't.test.mjs'), 'utf8');
|
||||||
|
assert.match(t, /verdict fells a failing exit/);
|
||||||
|
assert.match(t, /typeof/);
|
||||||
|
assert.doesNotMatch(t, /FAILED/, 'the mutant test asserts nothing about behaviour');
|
||||||
|
assert.equal(readFileSync(join(root, 'lib', 'm.mjs'), 'utf8'), MODULE, 'the measured tree is never touched');
|
||||||
|
} finally { cleanup(root, m9); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mutants: a gate that CLOSES the probe on M9 has felled 0 of 1', () => {
|
||||||
|
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||||
|
try {
|
||||||
|
const row = measureMutants(root, {
|
||||||
|
registry: behaviourRegistry,
|
||||||
|
evaluate: () => ({ status: 'closed', detail: '' }),
|
||||||
|
runTest: () => true,
|
||||||
|
});
|
||||||
|
assert.equal(row.countable, true);
|
||||||
|
assert.equal(row.value, 0);
|
||||||
|
assert.equal(row.denominator, 1);
|
||||||
|
assert.match(row.detail, /D-X closed/);
|
||||||
|
} finally { cleanup(root); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mutants: a gate that refuses to close the probe on M9 has felled 1 of 1', () => {
|
||||||
|
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||||
|
try {
|
||||||
|
const row = measureMutants(root, {
|
||||||
|
registry: behaviourRegistry,
|
||||||
|
evaluate: () => ({ status: 'not-fellable', detail: 'still passes against a signature-preserving mutant' }),
|
||||||
|
runTest: () => true,
|
||||||
|
});
|
||||||
|
assert.equal(row.value, 1);
|
||||||
|
assert.equal(row.denominator, 1);
|
||||||
|
} finally { cleanup(root); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mutants: a mutant whose own tests do not pass is not LIVE, so the row cannot be counted', () => {
|
||||||
|
// Without this control a broken fixture would read as "felled".
|
||||||
|
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||||
|
try {
|
||||||
|
const row = measureMutants(root, {
|
||||||
|
registry: behaviourRegistry,
|
||||||
|
evaluate: () => ({ status: 'not-fellable', detail: 'x' }),
|
||||||
|
runTest: () => null,
|
||||||
|
});
|
||||||
|
assert.equal(row.countable, false);
|
||||||
|
assert.equal(row.value, null);
|
||||||
|
assert.match(row.detail, /not live/);
|
||||||
|
} finally { cleanup(root); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('mutants: against the REAL gate and registry, M9 is live and the denominator is 1', () => {
|
||||||
|
// Pins the measurement's shape, not today's value: M9's typeof-only tests
|
||||||
|
// really do pass on the M9 tree (so "felled" can only come from the gate).
|
||||||
|
const row = measureMutants(ROOT);
|
||||||
|
assert.equal(row.countable, true, row.detail);
|
||||||
|
assert.equal(row.denominator, 1);
|
||||||
|
assert.ok(row.value === 0 || row.value === 1);
|
||||||
|
assert.deepEqual(row.facts.entries.sort(), ['D-03', 'D-04']);
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- the whole yardstick -------------------------------------------------------
|
||||||
|
|
||||||
|
function world({ orders = {}, stats = {} } = {}) {
|
||||||
|
const coord = fixture(Object.keys(orders).length ? orders : { 'r/orders/.keep': '' });
|
||||||
|
const data = fixture(Object.keys(stats).length ? stats : { '.keep': '' });
|
||||||
|
return { coord, data };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('measureAll + main: RED with exit 1 while any number is not countable', () => {
|
||||||
|
const { coord, data } = world({
|
||||||
|
orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z' }) },
|
||||||
|
stats: { 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW')) },
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const result = measureAll({ root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF });
|
||||||
|
assert.equal(result.green, false);
|
||||||
|
assert.equal(result.countable, 1);
|
||||||
|
const text = render(result);
|
||||||
|
assert.match(text, /1 of 3 numbers countable/);
|
||||||
|
assert.match(text, /signature-preserving mutants felled: \d of 1/);
|
||||||
|
assert.match(text, /verdict: NOT JUDGED/);
|
||||||
|
for (const id of ['use', 'trust', 'interrupts', 'mutants']) assert.match(text, new RegExp(`\\| ${id} \\|`));
|
||||||
|
assert.equal(main(['--root', ROOT, '--coord', coord, '--data', data, '--json']), 1);
|
||||||
|
} finally { cleanup(coord, data); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('measureAll: all three countable and the mutant felled is still RED until the verdict can be judged', () => {
|
||||||
|
const { coord, data } = world({
|
||||||
|
orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z', workClass: 'new', slug: 'a' }) },
|
||||||
|
stats: {
|
||||||
|
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW', 'a')),
|
||||||
|
'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { slug: 'a' } }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const result = measureAll({
|
||||||
|
root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF,
|
||||||
|
mutantOptions: { evaluate: () => ({ status: 'not-fellable', detail: '' }) },
|
||||||
|
});
|
||||||
|
assert.equal(result.countable, 3);
|
||||||
|
assert.equal(result.rows.find((r) => r.id === 'mutants').value, 1);
|
||||||
|
assert.equal(result.verdict.judged, false);
|
||||||
|
assert.equal(result.green, false);
|
||||||
|
} finally { cleanup(coord, data); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('main: an unknown argument or a malformed cutoff is a usage error (exit 2)', () => {
|
||||||
|
assert.equal(main(['--nope']), 2);
|
||||||
|
assert.equal(main(['--cutoff', '21.09.2026']), 2);
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue