feat(end-state): a gate that counts the distance to "Voyage is finished" - red
Voyage now has a destination that can fail: finish the phase-2 pipeline, close the open
experiments, freeze. This adds the gate that measures the distance to it. Nothing is
fixed or removed here; every defect and every removal is its own change.
scripts/end-state-gate.mjs counts four tallies; finished = all four at 0.
defects listed in scripts/end-state-registry.json, each with a runnable check
experiments listed in the registry, each with a runnable check
dormant-agents spawnable agents no commands/*.md references (structural, word-bounded;
files carrying the reference-document marker are excluded)
decisions open operator decisions in the local STATE.md, by a fixed marker
Exit 0 green, 1 red, 2 usage/registry error. `--json` for machines.
Fail-closed by construction: an entry with no check, or whose check cannot run (missing
file, empty glob, missing section), is NOT FELLABLE and counts as open; a missing STATE.md,
a missing section, or an unmarked list item in it makes the decisions tally n/a, which
keeps the gate red. The decisions row is therefore n/a in any clean clone.
Result today (clean export of the index): RED, 0 of 4 tallies at 0 -
defects 7 of 7, experiments 3 of 3, dormant agents 1 of 20, decisions n/a.
With the local STATE present: decisions 11 of 12.
Verified in both directions: 18 fixture tests pin each tally at 0 and above 0. Each of the
10 registry checks was measured open on this tree and closed after a simulated fix on a
throwaway copy (10 of 10). That run caught a too-broad D-03 check - Phase 4's legitimate
entry-condition skip also matched - which is now scoped to the Phase 7 section.
Suite 1041 -> 1059 (1057/0/2), also run on the clean export.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
012db1fe8d
commit
553c288925
3 changed files with 717 additions and 0 deletions
361
tests/scripts/end-state-gate.test.mjs
Normal file
361
tests/scripts/end-state-gate.test.mjs
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
// tests/scripts/end-state-gate.test.mjs
|
||||
// The end-state gate counts the distance to "Voyage is finished" as four tallies:
|
||||
// open pipeline defects, unresolved experiments/opt-ins, dormant agents, and open
|
||||
// operator decisions. "Finished" = all four at 0.
|
||||
//
|
||||
// These tests pin the gate's MECHANICS in both directions — every tally can be 0 and
|
||||
// can be > 0 — against throwaway fixture trees. They deliberately do NOT assert that the
|
||||
// real repo is red today: the gate is supposed to go green, and a test that pins the
|
||||
// current distance would fail on the day the work is done.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
evaluateCondition,
|
||||
evaluateCheck,
|
||||
measure,
|
||||
loadRegistry,
|
||||
} from '../../scripts/end-state-gate.mjs';
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const GATE = join(ROOT, 'scripts', 'end-state-gate.mjs');
|
||||
|
||||
function fixture(files) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'end-state-gate-'));
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
const p = join(dir, rel);
|
||||
mkdirSync(dirname(p), { recursive: true });
|
||||
writeFileSync(p, body);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
const DECISIONS = {
|
||||
file: 'STATE.md',
|
||||
section: '## Open decisions',
|
||||
open: '^- \\[ \\] ',
|
||||
closed: '^- \\[x\\] ',
|
||||
};
|
||||
const AGENTS = {
|
||||
dir: 'agents',
|
||||
spawnSites: 'commands/*.md',
|
||||
referenceMarker: 'Reference document, not a spawnable capability',
|
||||
};
|
||||
|
||||
function registry(overrides = {}) {
|
||||
return {
|
||||
defects: [],
|
||||
experiments: [],
|
||||
agents: AGENTS,
|
||||
decisions: DECISIONS,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const rowById = (result, id) => result.rows.find((r) => r.id === id);
|
||||
|
||||
// --- conditions -----------------------------------------------------------
|
||||
|
||||
test('evaluateCondition: match / no-match on a single file, multiline regex', () => {
|
||||
const dir = fixture({ 'a.md': 'intro\n### Heading\nbody\n' });
|
||||
try {
|
||||
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Heading', flags: 'm', expect: 'match' }), true);
|
||||
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Other', flags: 'm', expect: 'match' }), false);
|
||||
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Other', flags: 'm', expect: 'no-match' }), true);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('evaluateCondition: globs (dir/*.ext, dir/**/*.ext) and path arrays match ANY file', () => {
|
||||
const dir = fixture({
|
||||
'commands/x.md': 'nothing here\n',
|
||||
'lib/deep/y.mjs': 'const FLAG = process.env.SOME_FLAG;\n',
|
||||
});
|
||||
try {
|
||||
assert.equal(evaluateCondition(dir, { path: 'commands/*.md', pattern: 'SOME_FLAG', expect: 'match' }), false);
|
||||
assert.equal(evaluateCondition(dir, { path: 'lib/**/*.mjs', pattern: 'SOME_FLAG', expect: 'match' }), true);
|
||||
assert.equal(evaluateCondition(dir, { path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'SOME_FLAG', expect: 'match' }), true);
|
||||
assert.equal(evaluateCondition(dir, { path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'SOME_FLAG', expect: 'no-match' }), false);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('evaluateCondition: section scoping reads only from the heading to the next "## "', () => {
|
||||
const dir = fixture({
|
||||
'doc.md': '## 5. Design\nno status here\n## 6. PoC\n> **STATUS: RUN AND DECLINED**\n',
|
||||
});
|
||||
try {
|
||||
const inFive = { path: 'doc.md', section: '## 5.', pattern: '^> \\*\\*STATUS:', flags: 'm', expect: 'no-match' };
|
||||
const inSix = { ...inFive, section: '## 6.' };
|
||||
assert.equal(evaluateCondition(dir, inFive), true, 'section 5 has no status block');
|
||||
assert.equal(evaluateCondition(dir, inSix), false, 'section 6 has one');
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('evaluateCondition: a literal NUL byte is detectable', () => {
|
||||
const dir = fixture({ 'nul.mjs': "const SEP = '\u0000';\n", 'clean.mjs': "const SEP = '\\x00';\n" });
|
||||
try {
|
||||
assert.equal(evaluateCondition(dir, { path: 'nul.mjs', pattern: '\\u0000', expect: 'match' }), true);
|
||||
assert.equal(evaluateCondition(dir, { path: 'clean.mjs', pattern: '\\u0000', expect: 'match' }), false);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('evaluateCondition: a missing file or empty glob throws (never a silent "no match")', () => {
|
||||
const dir = fixture({ 'a.md': 'x\n' });
|
||||
try {
|
||||
assert.throws(() => evaluateCondition(dir, { path: 'missing.md', pattern: 'x', expect: 'no-match' }), /missing\.md/);
|
||||
assert.throws(() => evaluateCondition(dir, { path: 'nodir/*.md', pattern: 'x', expect: 'no-match' }), /nodir/);
|
||||
assert.throws(() => evaluateCondition(dir, { path: 'a.md', section: '## 9.', pattern: 'x', expect: 'no-match' }), /## 9\./);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('evaluateCheck: open only when ALL conditions hold; null check is not fellable', () => {
|
||||
const dir = fixture({
|
||||
'r.md': 'run the `### Bridge agent` block\n',
|
||||
'fixed.md': 'run the `### Bridge agent` block\n### Bridge agent\n',
|
||||
});
|
||||
const dangling = (file) => [
|
||||
{ path: file, pattern: '`### Bridge agent`', expect: 'match' },
|
||||
{ path: file, pattern: '^### Bridge agent', flags: 'm', expect: 'no-match' },
|
||||
];
|
||||
try {
|
||||
assert.equal(evaluateCheck(dir, dangling('r.md')).status, 'open');
|
||||
assert.equal(evaluateCheck(dir, dangling('fixed.md')).status, 'closed');
|
||||
assert.equal(evaluateCheck(dir, null).status, 'not-fellable');
|
||||
const broken = evaluateCheck(dir, dangling('gone.md'));
|
||||
assert.equal(broken.status, 'not-fellable');
|
||||
assert.match(broken.detail, /gone\.md/);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// --- the four tallies, both directions -------------------------------------
|
||||
|
||||
test('defects tally: > 0 while a check holds, 0 once fixed; not-fellable counts as open', () => {
|
||||
const defect = {
|
||||
id: 'D-X',
|
||||
summary: 'removed tool still referenced',
|
||||
check: [{ path: 'commands/plan.md', pattern: '\\bTeamCreate\\b', expect: 'match' }],
|
||||
};
|
||||
const unfellable = { id: 'D-Y', summary: 'no runnable check yet', check: null };
|
||||
const base = { 'STATE.md': '## Open decisions\n', 'agents/a.md': 'x\n', 'commands/plan.md': 'uses a\n' };
|
||||
|
||||
const red = fixture({ ...base, 'commands/plan.md': 'uses a and TeamCreate\n' });
|
||||
const green = fixture(base);
|
||||
try {
|
||||
let row = rowById(measure(red, registry({ defects: [defect] })), 'defects');
|
||||
assert.equal(row.open, 1);
|
||||
assert.equal(row.total, 1);
|
||||
row = rowById(measure(green, registry({ defects: [defect] })), 'defects');
|
||||
assert.equal(row.open, 0);
|
||||
row = rowById(measure(green, registry({ defects: [defect, unfellable] })), 'defects');
|
||||
assert.equal(row.open, 1, 'a defect without a runnable check stays open');
|
||||
assert.equal(row.items.find((i) => i.id === 'D-Y').status, 'not-fellable');
|
||||
} finally {
|
||||
rmSync(red, { recursive: true, force: true });
|
||||
rmSync(green, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('experiments tally: an env-gated opt-in is open until its gate is gone', () => {
|
||||
const exp = {
|
||||
id: 'E-X',
|
||||
summary: 'default-off loop',
|
||||
check: [{ path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'LOOP_ENABLED', expect: 'match' }],
|
||||
};
|
||||
const base = { 'STATE.md': '## Open decisions\n', 'agents/a.md': 'x\n', 'commands/r.md': 'uses a\n' };
|
||||
const red = fixture({ ...base, 'lib/util/cap.mjs': 'if (process.env.LOOP_ENABLED) {}\n' });
|
||||
const green = fixture({ ...base, 'lib/util/cap.mjs': 'export const x = 1;\n' });
|
||||
try {
|
||||
assert.equal(rowById(measure(red, registry({ experiments: [exp] })), 'experiments').open, 1);
|
||||
assert.equal(rowById(measure(green, registry({ experiments: [exp] })), 'experiments').open, 0);
|
||||
} finally {
|
||||
rmSync(red, { recursive: true, force: true });
|
||||
rmSync(green, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('dormant-agents tally: unreferenced spawnable agent counts; reference docs never do', () => {
|
||||
const files = {
|
||||
'STATE.md': '## Open decisions\n',
|
||||
'agents/used-agent.md': '---\nname: used-agent\ndescription: |\n does work\n---\n',
|
||||
'agents/idle-agent.md': '---\nname: idle-agent\ndescription: |\n never wired\n---\n',
|
||||
'agents/some-orchestrator.md': '---\nname: some-orchestrator\ndescription: Reference document, not a spawnable capability — docs\n---\n',
|
||||
'commands/run.md': 'Launch the **used-agent** agent.\n',
|
||||
};
|
||||
const red = fixture(files);
|
||||
const green = fixture({ ...files, 'commands/other.md': 'Then spawn `idle-agent`.\n' });
|
||||
try {
|
||||
let row = rowById(measure(red, registry()), 'dormant-agents');
|
||||
assert.equal(row.open, 1);
|
||||
assert.equal(row.total, 2, 'denominator = spawnable agents (reference docs excluded)');
|
||||
assert.deepEqual(row.items.filter((i) => i.status === 'open').map((i) => i.id), ['idle-agent']);
|
||||
row = rowById(measure(green, registry()), 'dormant-agents');
|
||||
assert.equal(row.open, 0);
|
||||
} finally {
|
||||
rmSync(red, { recursive: true, force: true });
|
||||
rmSync(green, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('dormant-agents tally: a name inside a longer word is not a reference', () => {
|
||||
const dir = fixture({
|
||||
'STATE.md': '## Open decisions\n',
|
||||
'agents/scout.md': '---\nname: scout\ndescription: |\n x\n---\n',
|
||||
'commands/run.md': 'the scouting phase and research-scouts\n',
|
||||
});
|
||||
try {
|
||||
assert.equal(rowById(measure(dir, registry()), 'dormant-agents').open, 1);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('decisions tally: counts open markers inside the section only', () => {
|
||||
const state = [
|
||||
'# STATE',
|
||||
'- [ ] outside the section, ignored',
|
||||
'## Open decisions',
|
||||
'- [ ] first open',
|
||||
'- [x] already decided',
|
||||
'- [ ] second open',
|
||||
'## Next section',
|
||||
'- [ ] also ignored',
|
||||
'',
|
||||
].join('\n');
|
||||
const red = fixture({ 'STATE.md': state, 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||
const green = fixture({
|
||||
'STATE.md': '## Open decisions\n- [x] done\n## Next\n',
|
||||
'agents/a.md': 'x\n',
|
||||
'commands/r.md': 'a\n',
|
||||
});
|
||||
try {
|
||||
let row = rowById(measure(red, registry()), 'decisions');
|
||||
assert.equal(row.open, 2);
|
||||
assert.equal(row.total, 3);
|
||||
row = rowById(measure(green, registry()), 'decisions');
|
||||
assert.equal(row.open, 0);
|
||||
assert.equal(row.total, 1);
|
||||
} finally {
|
||||
rmSync(red, { recursive: true, force: true });
|
||||
rmSync(green, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('decisions tally: a missing STATE.md or section is NOT MEASURABLE (null), never 0', () => {
|
||||
const noState = fixture({ 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||
const noSection = fixture({ 'STATE.md': '# STATE\n- [ ] x\n', 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||
try {
|
||||
for (const dir of [noState, noSection]) {
|
||||
const result = measure(dir, registry());
|
||||
const row = rowById(result, 'decisions');
|
||||
assert.equal(row.open, null);
|
||||
assert.ok(row.detail.length > 0, 'the reason is reported');
|
||||
assert.equal(result.green, false, 'an unmeasurable tally keeps the gate red');
|
||||
}
|
||||
} finally {
|
||||
rmSync(noState, { recursive: true, force: true });
|
||||
rmSync(noSection, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('decisions tally: an unmarked list item in the section is format drift → NOT MEASURABLE', () => {
|
||||
// A STATE written without the marker convention must not read as "0 open".
|
||||
for (const item of ['1. **Some decision?** prose', '- plain bullet decision', '* star bullet']) {
|
||||
const dir = fixture({
|
||||
'STATE.md': `## Open decisions\n- [ ] marked\n${item}\n## Next\n`,
|
||||
'agents/a.md': 'x\n',
|
||||
'commands/r.md': 'a\n',
|
||||
});
|
||||
try {
|
||||
const row = rowById(measure(dir, registry()), 'decisions');
|
||||
assert.equal(row.open, null, `unmarked "${item}" must make the tally unmeasurable`);
|
||||
assert.match(row.detail, /unmarked/);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
});
|
||||
|
||||
// --- verdict + CLI ---------------------------------------------------------
|
||||
|
||||
function writeRegistry(dir, reg) {
|
||||
mkdirSync(join(dir, 'scripts'), { recursive: true });
|
||||
writeFileSync(join(dir, 'scripts', 'end-state-registry.json'), JSON.stringify(reg, null, 2));
|
||||
}
|
||||
|
||||
function runGate(dir, extra = []) {
|
||||
return spawnSync(process.execPath, [GATE, '--root', dir, ...extra], { encoding: 'utf8' });
|
||||
}
|
||||
|
||||
test('CLI: all four tallies at 0 → green, exit 0; --json carries the rows', () => {
|
||||
const dir = fixture({
|
||||
'STATE.md': '## Open decisions\n- [x] done\n',
|
||||
'agents/a.md': '---\nname: a\n---\n',
|
||||
'commands/r.md': 'spawn `a`\n',
|
||||
});
|
||||
try {
|
||||
writeRegistry(dir, registry());
|
||||
const r = runGate(dir, ['--json']);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
const out = JSON.parse(r.stdout);
|
||||
assert.equal(out.green, true);
|
||||
assert.deepEqual(out.rows.map((x) => x.id), ['defects', 'experiments', 'dormant-agents', 'decisions']);
|
||||
assert.ok(out.rows.every((x) => x.open === 0 && x.target === 0 && typeof x.source === 'string'));
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('CLI: any tally > 0 → red, exit 1; the table names every row', () => {
|
||||
const dir = fixture({
|
||||
'STATE.md': '## Open decisions\n- [ ] undecided\n',
|
||||
'agents/a.md': '---\nname: a\n---\n',
|
||||
'commands/r.md': 'spawn `a`\n',
|
||||
});
|
||||
try {
|
||||
writeRegistry(dir, registry());
|
||||
const r = runGate(dir);
|
||||
assert.equal(r.status, 1, r.stderr);
|
||||
for (const label of ['defects', 'experiments', 'dormant-agents', 'decisions']) {
|
||||
assert.match(r.stdout, new RegExp(label));
|
||||
}
|
||||
assert.match(r.stdout, /RED/);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('CLI: an unreadable registry is a usage error (exit 2), not a verdict', () => {
|
||||
const dir = fixture({ 'STATE.md': '## Open decisions\n' });
|
||||
try {
|
||||
const r = runGate(dir);
|
||||
assert.equal(r.status, 2);
|
||||
assert.match(r.stderr, /end-state-registry\.json/);
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// --- the real registry: shape, never state ---------------------------------
|
||||
|
||||
test('real registry: every counted entry has an id, a summary and a check that RUNS here', () => {
|
||||
const reg = loadRegistry(ROOT);
|
||||
const entries = [...reg.defects, ...reg.experiments];
|
||||
assert.ok(reg.defects.length >= 1 && reg.experiments.length >= 1);
|
||||
const ids = new Set();
|
||||
for (const e of entries) {
|
||||
assert.match(e.id, /^[DE]-\d{2}$/, `bad id ${e.id}`);
|
||||
assert.ok(!ids.has(e.id), `duplicate id ${e.id}`);
|
||||
ids.add(e.id);
|
||||
assert.ok(typeof e.summary === 'string' && e.summary.length > 10, `${e.id} needs a summary`);
|
||||
// A check that cannot evaluate against this very repo is a typo, not a finding.
|
||||
assert.notEqual(evaluateCheck(ROOT, e.check).status, 'not-fellable', `${e.id} check does not run here`);
|
||||
}
|
||||
});
|
||||
|
||||
test('real repo: four rows, open is a count or null, exit code agrees with the rows', () => {
|
||||
const r = spawnSync(process.execPath, [GATE, '--json'], { encoding: 'utf8', cwd: ROOT });
|
||||
assert.ok(r.status === 0 || r.status === 1, r.stderr);
|
||||
const out = JSON.parse(r.stdout);
|
||||
assert.equal(out.rows.length, 4);
|
||||
for (const row of out.rows) {
|
||||
assert.ok(row.open === null || (Number.isInteger(row.open) && row.open >= 0), `${row.id}: ${row.open}`);
|
||||
}
|
||||
const allZero = out.rows.every((row) => row.open === 0);
|
||||
assert.equal(out.green, allZero);
|
||||
assert.equal(r.status, allZero ? 0 : 1);
|
||||
assert.ok(existsSync(join(ROOT, 'scripts', 'end-state-registry.json')));
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue