feat(measure): the yardstick prints Voyage's three numbers with denominators from the source, and it is RED
One command, `node scripts/yardstick.mjs`, prints use, trust and
interrupts. Each row gives countable YES/NO, the value, the denominator and
the source it was read from. A fourth row asks whether the end-state gate can
be fooled by a mutant that keeps the module's signature. Exit is non-zero
while any number is not countable, the mutant is not felled, or the verdict
over the first five deliveries is not judged. Nothing counts a delivery yet,
so today it is RED for the right reasons:
use NO no order in the queue carries work-class
trust YES current ALLOW streak 0 of 37 (BLOCK 14, WARN 12, ALLOW 11,
last 2026-09-05 BLOCK, ordered by ts, read as JSON)
interrupts NO 0 user_input records of ~141k lines in 5 files, 0 emitter
call sites outside lib/stats/ and tests/
mutants 0 of 1 signature-preserving mutants felled: on M9 (every export
returns {status:'PASSED'}, every named test only checks typeof)
the gate closes both D-03 and D-04
It only measures. The gate, the emitter and the order writer are untouched.
The mutant row is meant to stay red until the gate learns to fell M9.
Choices, and why:
- Use counts over the order queue (pending, archive/ and claimed/), because
an order is the unit of work. The denominator is orders dated on or after a
cutoff (default 2026-09-21) that carry `work-class: new`. They count as
having gone through Voyage when a Voyage run in the stats has the order's
`slug:`. Both fields are the contract the order writer must meet. No order
carries them today, so the number is NOT COUNTABLE, not 0 %.
- Mailboxes whose names start with a dot are mailboxes too. A shell `*`
glob skips them, which is why a hand count of `coord/*/orders` comes out
lower. The row names them so a re-measure can reconcile.
- M9 must be LIVE before its row can count. Its typeof tests have to pass on
the M9 tree, checked by a runner independent of the gate, so a broken
fixture can never read as "felled".
Also: tests/fixtures/red-first/02243c6-always-allow-shim.mjs, the shim that
makes the red-first claim of 02243c6 reproducible (6 of 55). The recipe is in
its header.
Red first: the new test file failed to load before scripts/yardstick.mjs
existed, and a dot-mailbox test was red before the census learned about
dot-mailboxes. Suite 1161 -> 1183 (1181/0/2), also green on a clean tree
exported from the index.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5f1ae4b4c2
commit
1779411b49
3 changed files with 907 additions and 0 deletions
26
tests/fixtures/red-first/02243c6-always-allow-shim.mjs
vendored
Normal file
26
tests/fixtures/red-first/02243c6-always-allow-shim.mjs
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs
|
||||
// The shim that makes the red-first claim of 02243c6 reproducible.
|
||||
//
|
||||
// 02243c6 ("the criteria runner screens with an ALLOWLIST") says six of its new
|
||||
// tests were red before the fix. On the parent tree the test file does not even
|
||||
// LOAD - it imports `allowedCommand`, which the parent does not export - so the
|
||||
// red is a load error on the whole file, not six named failures. The six only
|
||||
// appear once the missing export exists and allows everything. This file is
|
||||
// that export, kept here so a re-measure does not have to guess it.
|
||||
//
|
||||
// Reproduce "6 of 55" (POSIX shell, from the repo root, needs git history):
|
||||
//
|
||||
// T="$(mktemp -d)"
|
||||
// git archive 02243c6 | tar -x -C "$T"
|
||||
// git show 02243c6^:lib/verification/criteria-runner.mjs > "$T/lib/verification/criteria-runner.mjs"
|
||||
// cat tests/fixtures/red-first/02243c6-always-allow-shim.mjs >> "$T/lib/verification/criteria-runner.mjs"
|
||||
// (cd "$T" && node --test --test-reporter=tap tests/lib/criteria-runner.test.mjs) | grep -E '^# (tests|fail)'
|
||||
//
|
||||
// Expected: `# tests 55` and `# fail 6` - four `allowedCommand` tests (all but
|
||||
// "every known test runner is allowed", which an always-allow shim passes),
|
||||
// `runCriteria: a command outside the allowlist is NOT RUN`, and the end-to-end
|
||||
// canary test (28 evasions, 0 run). Measured 2026-09-22.
|
||||
// It is a fixture, not a test: nothing in the suite runs it, because the suite
|
||||
// must also pass on a `git archive` tree, where there is no history to rewind.
|
||||
|
||||
export function allowedCommand() { return { allowed: true, reason: '' }; }
|
||||
409
tests/scripts/yardstick.test.mjs
Normal file
409
tests/scripts/yardstick.test.mjs
Normal file
|
|
@ -0,0 +1,409 @@
|
|||
// tests/scripts/yardstick.test.mjs
|
||||
// The yardstick prints Voyage's three numbers (use · trust · interrupts) with
|
||||
// denominators read from their SOURCE, plus a fourth row that asks whether the
|
||||
// end-state gate itself can be fooled by a signature-preserving mutant.
|
||||
//
|
||||
// Like the gate's own test, these tests pin the MECHANICS in both directions
|
||||
// against throwaway fixtures - every number can be countable and can be not
|
||||
// countable, the mutant row can read felled and not felled. They deliberately
|
||||
// do NOT assert that the real data is red today: the yardstick is supposed to
|
||||
// turn green, and a test that pins today's distance would fail on the day the
|
||||
// work is done. Nothing here reads the real order queue or the real stats.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
DEFAULT_CUTOFF,
|
||||
loadStats,
|
||||
readOrders,
|
||||
measureUse,
|
||||
measureTrust,
|
||||
measureInterrupts,
|
||||
emitterCallSites,
|
||||
buildM9Tree,
|
||||
measureMutants,
|
||||
measureAll,
|
||||
render,
|
||||
main,
|
||||
} from '../../scripts/yardstick.mjs';
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
|
||||
// --- helpers ---------------------------------------------------------------
|
||||
|
||||
function fixture(files) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'yardstick-'));
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
const p = join(dir, rel);
|
||||
mkdirSync(dirname(p), { recursive: true });
|
||||
writeFileSync(p, body);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
const cleanup = (...dirs) => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); };
|
||||
|
||||
const jsonl = (...rows) => `${rows.map((r) => (typeof r === 'string' ? r : JSON.stringify(r))).join('\n')}\n`;
|
||||
|
||||
function order({ id, date, workClass, slug, extra = '' }) {
|
||||
const lines = ['---', 'from: .claude', 'to: some-repo', `order-id: ${id}`, `subject: order ${id}`, `date: ${date}`];
|
||||
if (workClass !== undefined) lines.push(`work-class: ${workClass}`);
|
||||
if (slug !== undefined) lines.push(`slug: ${slug}`);
|
||||
lines.push('---', '', `# ORDER ${id}`, extra, '');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
const review = (ts, verdict, slug = 's') => ({ ts, slug, verdict });
|
||||
|
||||
// --- the stats loader --------------------------------------------------------
|
||||
|
||||
test('loadStats: an invalid JSON line is COUNTED, never silently dropped', () => {
|
||||
const dir = fixture({
|
||||
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{not json', '', review('2026-09-02T00:00:00Z', 'BLOCK')),
|
||||
});
|
||||
try {
|
||||
const stats = loadStats(dir);
|
||||
const f = stats.files.find((x) => x.name === 'trekreview-stats.jsonl');
|
||||
assert.equal(f.lines, 3, 'blank lines are not lines; the invalid one is');
|
||||
assert.equal(f.records.length, 2);
|
||||
assert.deepEqual(f.invalid, [2], 'the invalid line is reported by its line number');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('loadStats: a missing data directory is reported, not read as zero', () => {
|
||||
const stats = loadStats(join(tmpdir(), 'yardstick-no-such-dir-8f3a'));
|
||||
assert.equal(stats.missing, true);
|
||||
assert.deepEqual(stats.files, []);
|
||||
});
|
||||
|
||||
// --- 2 · TRUST ---------------------------------------------------------------
|
||||
|
||||
test('trust: the ALLOW streak is read in ts order, not file order', () => {
|
||||
// File order ends on ALLOW, ALLOW; chronologically the last review is a BLOCK.
|
||||
const dir = fixture({
|
||||
'trekreview-stats.jsonl': jsonl(
|
||||
review('2026-09-01T00:00:00Z', 'ALLOW'),
|
||||
review('2026-09-05T00:00:00Z', 'BLOCK'),
|
||||
review('2026-09-02T00:00:00Z', 'ALLOW'),
|
||||
review('2026-09-03T00:00:00Z', 'ALLOW'),
|
||||
),
|
||||
});
|
||||
try {
|
||||
const row = measureTrust(loadStats(dir));
|
||||
assert.equal(row.countable, true);
|
||||
assert.equal(row.denominator, 4);
|
||||
assert.equal(row.value, 0, 'the chronologically last review is a BLOCK');
|
||||
assert.equal(row.facts.longestStreak, 3);
|
||||
assert.deepEqual(row.facts.verdicts, { ALLOW: 3, BLOCK: 1 });
|
||||
assert.equal(row.facts.last.verdict, 'BLOCK');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('trust: both spacings of the verdict field count the same (it is read as JSON, not grepped)', () => {
|
||||
const dir = fixture({
|
||||
'trekreview-stats.jsonl': jsonl(
|
||||
'{"ts": "2026-09-01T00:00:00Z", "slug": "a", "verdict": "BLOCK"}',
|
||||
'{"ts":"2026-09-02T00:00:00Z","slug":"b","verdict":"ALLOW"}',
|
||||
),
|
||||
});
|
||||
try {
|
||||
const row = measureTrust(loadStats(dir));
|
||||
assert.deepEqual(row.facts.verdicts, { BLOCK: 1, ALLOW: 1 });
|
||||
assert.equal(row.value, 1);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('trust: an invalid line is left out of n and said out loud', () => {
|
||||
const dir = fixture({
|
||||
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW'), '{"ts": oops'),
|
||||
});
|
||||
try {
|
||||
const row = measureTrust(loadStats(dir));
|
||||
assert.equal(row.denominator, 1);
|
||||
assert.equal(row.facts.invalid, 1);
|
||||
assert.match(row.detail, /1 invalid line/);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('trust: an empty or missing review file is NOT countable', () => {
|
||||
const empty = fixture({ 'trekreview-stats.jsonl': '' });
|
||||
const none = fixture({ 'trekplan-stats.jsonl': jsonl({ ts: '2026-09-01T00:00:00Z', slug: 'x' }) });
|
||||
try {
|
||||
for (const dir of [empty, none]) {
|
||||
const row = measureTrust(loadStats(dir));
|
||||
assert.equal(row.countable, false, dir);
|
||||
assert.equal(row.value, null);
|
||||
assert.ok(row.detail.length > 0, 'the row must say WHY it cannot be counted');
|
||||
}
|
||||
} finally { cleanup(empty, none); }
|
||||
});
|
||||
|
||||
// --- 1 · USE -----------------------------------------------------------------
|
||||
|
||||
test('use: orders are read from orders/, archive/ and claimed/, and nothing else', () => {
|
||||
const coord = fixture({
|
||||
'repo-a/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||
'repo-a/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-01T10:00:00Z' }),
|
||||
'repo-b/orders/claimed/o3.md': order({ id: 'o3', date: '2026-09-22T10:00:00Z' }),
|
||||
'repo-b/orders/claimed/o3.claim': 'claimed',
|
||||
'repo-b/inbox/m1.md': order({ id: 'm1', date: '2026-09-22T10:00:00Z' }),
|
||||
'repo-b/orders/broken.md': 'no frontmatter at all\n',
|
||||
});
|
||||
try {
|
||||
const q = readOrders(coord);
|
||||
assert.equal(q.files, 4, 'three orders and one unreadable file, all .md under */orders/');
|
||||
assert.equal(q.orders.length, 3);
|
||||
assert.equal(q.unreadable.length, 1);
|
||||
} finally { cleanup(coord); }
|
||||
});
|
||||
|
||||
test('use: a dot-mailbox is a mailbox - its orders count, and the detail names it', () => {
|
||||
// Measured 2026-09-22: a hand count over `coord/*/orders` came out lower than
|
||||
// the yardstick, because the shell's `*` skips dot-directories and the queue
|
||||
// does not. The detail names them, so the next re-measure can reconcile.
|
||||
const coord = fixture({
|
||||
'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||
'.pm/orders/archive/o2.md': order({ id: 'o2', date: '2026-09-22T10:00:00Z' }),
|
||||
});
|
||||
try {
|
||||
const q = readOrders(coord);
|
||||
assert.equal(q.orders.length, 2);
|
||||
const row = measureUse({ queue: q, slugs: new Set(), cutoff: DEFAULT_CUTOFF });
|
||||
assert.match(row.detail, /2 order files in 2 mailboxes/);
|
||||
assert.match(row.detail, /dot-mailboxes .pm 1/);
|
||||
} finally { cleanup(coord); }
|
||||
});
|
||||
|
||||
test('use: no order carrying work-class means NOT COUNTABLE, not 0 %', () => {
|
||||
const coord = fixture({
|
||||
'r/orders/o1.md': order({ id: 'o1', date: '2026-09-22T10:00:00Z' }),
|
||||
'r/orders/o2.md': order({ id: 'o2', date: '2026-09-23T10:00:00Z' }),
|
||||
});
|
||||
try {
|
||||
const row = measureUse({ queue: readOrders(coord), slugs: new Set(), cutoff: DEFAULT_CUTOFF });
|
||||
assert.equal(row.countable, false);
|
||||
assert.equal(row.value, null);
|
||||
assert.equal(row.denominator, 0);
|
||||
assert.equal(row.facts.scanned, 2);
|
||||
assert.equal(row.facts.withWorkClass, 0);
|
||||
assert.match(row.detail, /0 of 2 orders carry work-class/);
|
||||
} finally { cleanup(coord); }
|
||||
});
|
||||
|
||||
test('use: the denominator is new-work orders dated on or after the cutoff; the numerator is those with a Voyage run', () => {
|
||||
const coord = fixture({
|
||||
'r/orders/a.md': order({ id: 'a', date: '2026-09-21T00:00:00Z', workClass: 'new', slug: 'feat-a' }),
|
||||
'r/orders/archive/b.md': order({ id: 'b', date: '2026-09-25T00:00:00Z', workClass: 'new', slug: 'feat-b' }),
|
||||
'r/orders/c.md': order({ id: 'c', date: '2026-09-25T00:00:00Z', workClass: 'new' }),
|
||||
'r/orders/d.md': order({ id: 'd', date: '2026-09-20T23:59:59Z', workClass: 'new', slug: 'feat-a' }),
|
||||
'r/orders/e.md': order({ id: 'e', date: '2026-09-25T00:00:00Z', workClass: 'maintenance', slug: 'feat-a' }),
|
||||
});
|
||||
try {
|
||||
const row = measureUse({ queue: readOrders(coord), slugs: new Set(['feat-a']), cutoff: '2026-09-21' });
|
||||
assert.equal(row.countable, true);
|
||||
assert.equal(row.denominator, 3, 'a, b and c - d is before the cutoff, e is maintenance');
|
||||
assert.equal(row.value, 1, 'only a has a Voyage run with the same slug; c has no slug to match');
|
||||
assert.equal(row.facts.withWorkClass, 5);
|
||||
} finally { cleanup(coord); }
|
||||
});
|
||||
|
||||
// --- 3 · INTERRUPTS ----------------------------------------------------------
|
||||
|
||||
test('interrupts: no user_input record means NOT COUNTABLE, with the line denominator stated', () => {
|
||||
const dir = fixture({
|
||||
'trekexecute-stats.jsonl': jsonl({ ts: 't', session_id: 's', success: true }, { ts: 't', event: 'main-merge-gate', payload: {} }),
|
||||
'trekplan-stats.jsonl': jsonl({ ts: 't', slug: 'x' }),
|
||||
});
|
||||
try {
|
||||
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 3 } });
|
||||
assert.equal(row.countable, false);
|
||||
assert.equal(row.facts.lines, 3);
|
||||
assert.equal(row.facts.userInput, 0);
|
||||
assert.match(row.detail, /0 user_input records of 3 lines in 2 files/);
|
||||
assert.match(row.detail, /0 emitter call sites/);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('interrupts: user_input WITHOUT a slug or run id is still not countable per delivery', () => {
|
||||
const dir = fixture({
|
||||
'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { question: 'q' } }),
|
||||
});
|
||||
try {
|
||||
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
|
||||
assert.equal(row.countable, false);
|
||||
assert.equal(row.facts.userInput, 1);
|
||||
assert.equal(row.facts.withDelivery, 0);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('interrupts: user_input carrying a slug or run id is countable, per delivery', () => {
|
||||
const dir = fixture({
|
||||
'trekexecute-stats.jsonl': jsonl(
|
||||
{ ts: 't1', event: 'user_input', payload: { slug: 'feat-a' } },
|
||||
{ ts: 't2', event: 'user_input', payload: { slug: 'feat-a' } },
|
||||
{ ts: 't3', event: 'user_input', payload: { run_id: 'r-9' } },
|
||||
),
|
||||
});
|
||||
try {
|
||||
const row = measureInterrupts({ stats: loadStats(dir), callSites: { sites: [], scanned: 1 } });
|
||||
assert.equal(row.countable, true);
|
||||
assert.equal(row.value, 3);
|
||||
assert.deepEqual(row.facts.perDelivery, { 'feat-a': 2, 'r-9': 1 });
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('emitterCallSites: counts user_input emissions outside lib/stats/ and tests/ only', () => {
|
||||
const root = fixture({
|
||||
'commands/a.md': 'node ${CLAUDE_PLUGIN_ROOT}/lib/stats/event-emit.mjs \\\n --event user_input \\\n',
|
||||
'commands/b.md': 'the emitter knows user_input as an event name - prose, not a call\n',
|
||||
'lib/util/x.mjs': "emitEvent('user_input', { slug });\n",
|
||||
'lib/stats/event-emit.mjs': "emit('user_input', {});\n",
|
||||
'tests/lib/y.test.mjs': "emit('user_input', {});\n",
|
||||
'hooks/scripts/h.mjs': "spawnSync('node', [EMIT, '--event', 'user_input']);\n",
|
||||
});
|
||||
try {
|
||||
const r = emitterCallSites(root);
|
||||
assert.deepEqual(r.sites.map((s) => s.file).sort(), ['commands/a.md', 'hooks/scripts/h.mjs', 'lib/util/x.mjs']);
|
||||
assert.equal(r.scanned, 4, 'commands/a.md, commands/b.md, lib/util/x.mjs, hooks/scripts/h.mjs');
|
||||
} finally { cleanup(root); }
|
||||
});
|
||||
|
||||
// --- 4 · the signature-preserving mutant (M9) ----------------------------------
|
||||
|
||||
const MODULE = 'export function verdict(code) { return { status: code === 0 ? "PASSED" : "FAILED" }; }\n';
|
||||
const STRONG_TEST = [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"import { verdict } from '../lib/m.mjs';",
|
||||
"test('verdict fells a failing exit', () => { assert.equal(verdict(1).status, 'FAILED'); });",
|
||||
'',
|
||||
].join('\n');
|
||||
const behaviourRegistry = {
|
||||
defects: [{
|
||||
id: 'D-X', probe: 'behaviour',
|
||||
check: [{ test: 'tests/t.test.mjs', name: 'verdict fells a failing exit', expect: 'fails', stubs: ['lib/m.mjs'] }],
|
||||
}],
|
||||
experiments: [],
|
||||
};
|
||||
|
||||
test('buildM9Tree: every export keeps its name and returns {status:"PASSED"}; every named test only checks typeof', async () => {
|
||||
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||
const m9 = buildM9Tree(root, behaviourRegistry);
|
||||
try {
|
||||
const mod = readFileSync(join(m9, 'lib', 'm.mjs'), 'utf8');
|
||||
assert.match(mod, /export function verdict\(\) \{ return \{ status: 'PASSED' \}; \}/);
|
||||
const t = readFileSync(join(m9, 'tests', 't.test.mjs'), 'utf8');
|
||||
assert.match(t, /verdict fells a failing exit/);
|
||||
assert.match(t, /typeof/);
|
||||
assert.doesNotMatch(t, /FAILED/, 'the mutant test asserts nothing about behaviour');
|
||||
assert.equal(readFileSync(join(root, 'lib', 'm.mjs'), 'utf8'), MODULE, 'the measured tree is never touched');
|
||||
} finally { cleanup(root, m9); }
|
||||
});
|
||||
|
||||
test('mutants: a gate that CLOSES the probe on M9 has felled 0 of 1', () => {
|
||||
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||
try {
|
||||
const row = measureMutants(root, {
|
||||
registry: behaviourRegistry,
|
||||
evaluate: () => ({ status: 'closed', detail: '' }),
|
||||
runTest: () => true,
|
||||
});
|
||||
assert.equal(row.countable, true);
|
||||
assert.equal(row.value, 0);
|
||||
assert.equal(row.denominator, 1);
|
||||
assert.match(row.detail, /D-X closed/);
|
||||
} finally { cleanup(root); }
|
||||
});
|
||||
|
||||
test('mutants: a gate that refuses to close the probe on M9 has felled 1 of 1', () => {
|
||||
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||
try {
|
||||
const row = measureMutants(root, {
|
||||
registry: behaviourRegistry,
|
||||
evaluate: () => ({ status: 'not-fellable', detail: 'still passes against a signature-preserving mutant' }),
|
||||
runTest: () => true,
|
||||
});
|
||||
assert.equal(row.value, 1);
|
||||
assert.equal(row.denominator, 1);
|
||||
} finally { cleanup(root); }
|
||||
});
|
||||
|
||||
test('mutants: a mutant whose own tests do not pass is not LIVE, so the row cannot be counted', () => {
|
||||
// Without this control a broken fixture would read as "felled".
|
||||
const root = fixture({ 'lib/m.mjs': MODULE, 'tests/t.test.mjs': STRONG_TEST });
|
||||
try {
|
||||
const row = measureMutants(root, {
|
||||
registry: behaviourRegistry,
|
||||
evaluate: () => ({ status: 'not-fellable', detail: 'x' }),
|
||||
runTest: () => null,
|
||||
});
|
||||
assert.equal(row.countable, false);
|
||||
assert.equal(row.value, null);
|
||||
assert.match(row.detail, /not live/);
|
||||
} finally { cleanup(root); }
|
||||
});
|
||||
|
||||
test('mutants: against the REAL gate and registry, M9 is live and the denominator is 1', () => {
|
||||
// Pins the measurement's shape, not today's value: M9's typeof-only tests
|
||||
// really do pass on the M9 tree (so "felled" can only come from the gate).
|
||||
const row = measureMutants(ROOT);
|
||||
assert.equal(row.countable, true, row.detail);
|
||||
assert.equal(row.denominator, 1);
|
||||
assert.ok(row.value === 0 || row.value === 1);
|
||||
assert.deepEqual(row.facts.entries.sort(), ['D-03', 'D-04']);
|
||||
});
|
||||
|
||||
// --- the whole yardstick -------------------------------------------------------
|
||||
|
||||
function world({ orders = {}, stats = {} } = {}) {
|
||||
const coord = fixture(Object.keys(orders).length ? orders : { 'r/orders/.keep': '' });
|
||||
const data = fixture(Object.keys(stats).length ? stats : { '.keep': '' });
|
||||
return { coord, data };
|
||||
}
|
||||
|
||||
test('measureAll + main: RED with exit 1 while any number is not countable', () => {
|
||||
const { coord, data } = world({
|
||||
orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z' }) },
|
||||
stats: { 'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW')) },
|
||||
});
|
||||
try {
|
||||
const result = measureAll({ root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF });
|
||||
assert.equal(result.green, false);
|
||||
assert.equal(result.countable, 1);
|
||||
const text = render(result);
|
||||
assert.match(text, /1 of 3 numbers countable/);
|
||||
assert.match(text, /signature-preserving mutants felled: \d of 1/);
|
||||
assert.match(text, /verdict: NOT JUDGED/);
|
||||
for (const id of ['use', 'trust', 'interrupts', 'mutants']) assert.match(text, new RegExp(`\\| ${id} \\|`));
|
||||
assert.equal(main(['--root', ROOT, '--coord', coord, '--data', data, '--json']), 1);
|
||||
} finally { cleanup(coord, data); }
|
||||
});
|
||||
|
||||
test('measureAll: all three countable and the mutant felled is still RED until the verdict can be judged', () => {
|
||||
const { coord, data } = world({
|
||||
orders: { 'r/orders/o.md': order({ id: 'o', date: '2026-09-22T00:00:00Z', workClass: 'new', slug: 'a' }) },
|
||||
stats: {
|
||||
'trekreview-stats.jsonl': jsonl(review('2026-09-01T00:00:00Z', 'ALLOW', 'a')),
|
||||
'trekexecute-stats.jsonl': jsonl({ ts: 't', event: 'user_input', payload: { slug: 'a' } }),
|
||||
},
|
||||
});
|
||||
try {
|
||||
const result = measureAll({
|
||||
root: ROOT, coordDir: coord, dataDir: data, cutoff: DEFAULT_CUTOFF,
|
||||
mutantOptions: { evaluate: () => ({ status: 'not-fellable', detail: '' }) },
|
||||
});
|
||||
assert.equal(result.countable, 3);
|
||||
assert.equal(result.rows.find((r) => r.id === 'mutants').value, 1);
|
||||
assert.equal(result.verdict.judged, false);
|
||||
assert.equal(result.green, false);
|
||||
} finally { cleanup(coord, data); }
|
||||
});
|
||||
|
||||
test('main: an unknown argument or a malformed cutoff is a usage error (exit 2)', () => {
|
||||
assert.equal(main(['--nope']), 2);
|
||||
assert.equal(main(['--cutoff', '21.09.2026']), 2);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue