feat(end-state): a gate that counts the distance to "Voyage is finished" - red
Voyage now has a destination that can fail: finish the phase-2 pipeline, close the open
experiments, freeze. This adds the gate that measures the distance to it. Nothing is
fixed or removed here; every defect and every removal is its own change.
scripts/end-state-gate.mjs counts four tallies; finished = all four at 0.
defects listed in scripts/end-state-registry.json, each with a runnable check
experiments listed in the registry, each with a runnable check
dormant-agents spawnable agents no commands/*.md references (structural, word-bounded;
files carrying the reference-document marker are excluded)
decisions open operator decisions in the local STATE.md, by a fixed marker
Exit 0 green, 1 red, 2 usage/registry error. `--json` for machines.
Fail-closed by construction: an entry with no check, or whose check cannot run (missing
file, empty glob, missing section), is NOT FELLABLE and counts as open; a missing STATE.md,
a missing section, or an unmarked list item in it makes the decisions tally n/a, which
keeps the gate red. The decisions row is therefore n/a in any clean clone.
Result today (clean export of the index): RED, 0 of 4 tallies at 0 -
defects 7 of 7, experiments 3 of 3, dormant agents 1 of 20, decisions n/a.
With the local STATE present: decisions 11 of 12.
Verified in both directions: 18 fixture tests pin each tally at 0 and above 0. Each of the
10 registry checks was measured open on this tree and closed after a simulated fix on a
throwaway copy (10 of 10). That run caught a too-broad D-03 check - Phase 4's legitimate
entry-condition skip also matched - which is now scoped to the Phase 7 section.
Suite 1041 -> 1059 (1057/0/2), also run on the clean export.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
012db1fe8d
commit
553c288925
3 changed files with 717 additions and 0 deletions
256
scripts/end-state-gate.mjs
Normal file
256
scripts/end-state-gate.mjs
Normal file
|
|
@ -0,0 +1,256 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
// scripts/end-state-gate.mjs
|
||||||
|
// End-state gate — counts the distance to "Voyage is finished" as four tallies:
|
||||||
|
//
|
||||||
|
// defects open pipeline defects (listed in the registry, each with a runnable check)
|
||||||
|
// experiments unresolved experiments/opt-ins (listed in the registry, each with a runnable check)
|
||||||
|
// dormant-agents spawnable agents no command references (measured structurally from agents/)
|
||||||
|
// decisions open operator decisions (counted in STATE.md by a fixed marker)
|
||||||
|
//
|
||||||
|
// "Finished" = all four at 0. Exit 0 = green, 1 = red (any tally > 0 or not measurable),
|
||||||
|
// 2 = usage or registry error. The denominators come from the tracked registry and the
|
||||||
|
// tracked agents/ directory — never from free-text greps over prose.
|
||||||
|
//
|
||||||
|
// A registry entry is OPEN while ALL of its conditions hold. An entry whose check is null,
|
||||||
|
// or whose check cannot run (missing file, empty glob, missing section), is NOT FELLABLE
|
||||||
|
// and is counted as open — a check that cannot fire must never read as "fixed".
|
||||||
|
//
|
||||||
|
// Usage: node scripts/end-state-gate.mjs [--json] [--root <dir>]
|
||||||
|
|
||||||
|
import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
|
||||||
|
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
export const REGISTRY_PATH = 'scripts/end-state-registry.json';
|
||||||
|
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||||
|
|
||||||
|
const toPosix = (p) => p.split(sep).join('/');
|
||||||
|
|
||||||
|
function walk(dir) {
|
||||||
|
const out = [];
|
||||||
|
for (const name of readdirSync(dir)) {
|
||||||
|
const p = join(dir, name);
|
||||||
|
if (statSync(p).isDirectory()) out.push(...walk(p));
|
||||||
|
else out.push(p);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Supported path forms: an exact file, `dir/*.ext`, `dir/**/*.ext`.
|
||||||
|
export function expandPath(root, pattern) {
|
||||||
|
if (!pattern.includes('*')) {
|
||||||
|
const abs = join(root, pattern);
|
||||||
|
if (!existsSync(abs)) throw new Error(`path not found: ${pattern}`);
|
||||||
|
return [abs];
|
||||||
|
}
|
||||||
|
const m = pattern.match(/^(.+?)\/(\*\*\/)?\*(\.[A-Za-z0-9]+)$/);
|
||||||
|
if (!m) throw new Error(`unsupported glob: ${pattern}`);
|
||||||
|
const [, base, deep, ext] = m;
|
||||||
|
const absBase = join(root, base);
|
||||||
|
if (!existsSync(absBase) || !statSync(absBase).isDirectory()) {
|
||||||
|
throw new Error(`glob base not found: ${pattern}`);
|
||||||
|
}
|
||||||
|
const files = deep
|
||||||
|
? walk(absBase)
|
||||||
|
: readdirSync(absBase).map((f) => join(absBase, f)).filter((f) => statSync(f).isFile());
|
||||||
|
const hits = files.filter((f) => f.endsWith(ext)).sort();
|
||||||
|
if (hits.length === 0) throw new Error(`glob matched no files: ${pattern}`);
|
||||||
|
return hits;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The text from the first line starting with `heading` up to the next `## ` heading.
|
||||||
|
function sectionOf(text, heading, label) {
|
||||||
|
const lines = text.split('\n');
|
||||||
|
const start = lines.findIndex((l) => l.startsWith(heading));
|
||||||
|
if (start === -1) throw new Error(`section "${heading}" not found in ${label}`);
|
||||||
|
let end = lines.length;
|
||||||
|
for (let i = start + 1; i < lines.length; i++) {
|
||||||
|
if (lines[i].startsWith('## ')) { end = i; break; }
|
||||||
|
}
|
||||||
|
return lines.slice(start, end).join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateCondition(root, cond) {
|
||||||
|
if (cond.expect !== 'match' && cond.expect !== 'no-match') {
|
||||||
|
throw new Error(`expect must be "match" or "no-match", got ${JSON.stringify(cond.expect)}`);
|
||||||
|
}
|
||||||
|
if (String(cond.flags ?? '').includes('g')) throw new Error('the g flag is not allowed');
|
||||||
|
const paths = Array.isArray(cond.path) ? cond.path : [cond.path];
|
||||||
|
const files = paths.flatMap((p) => expandPath(root, p));
|
||||||
|
const anyMatch = files.some((f) => {
|
||||||
|
let text = readFileSync(f, 'utf8');
|
||||||
|
if (cond.section) text = sectionOf(text, cond.section, toPosix(relative(root, f)));
|
||||||
|
return new RegExp(cond.pattern, cond.flags ?? '').test(text);
|
||||||
|
});
|
||||||
|
return cond.expect === 'match' ? anyMatch : !anyMatch;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function evaluateCheck(root, check) {
|
||||||
|
if (check === null || check === undefined) {
|
||||||
|
return { status: 'not-fellable', detail: 'no runnable check' };
|
||||||
|
}
|
||||||
|
if (!Array.isArray(check) || check.length === 0) {
|
||||||
|
return { status: 'not-fellable', detail: 'check must be a non-empty list of conditions' };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
// Evaluate every condition (no short-circuit) so a broken one is always reported.
|
||||||
|
const results = check.map((c) => evaluateCondition(root, c));
|
||||||
|
return { status: results.every(Boolean) ? 'open' : 'closed', detail: '' };
|
||||||
|
} catch (err) {
|
||||||
|
return { status: 'not-fellable', detail: err.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function tallyEntries(root, entries) {
|
||||||
|
const items = entries.map((e) => ({ id: e.id, summary: e.summary, ...evaluateCheck(root, e.check) }));
|
||||||
|
return {
|
||||||
|
open: items.filter((i) => i.status !== 'closed').length,
|
||||||
|
total: items.length,
|
||||||
|
items,
|
||||||
|
detail: '',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||||
|
|
||||||
|
function tallyDormantAgents(root, cfg) {
|
||||||
|
try {
|
||||||
|
const agentFiles = expandPath(root, `${cfg.dir}/*.md`);
|
||||||
|
const sites = expandPath(root, cfg.spawnSites).map((f) => readFileSync(f, 'utf8')).join('\n');
|
||||||
|
const items = [];
|
||||||
|
for (const file of agentFiles) {
|
||||||
|
const text = readFileSync(file, 'utf8');
|
||||||
|
const fm = text.startsWith('---') ? text.split('\n---')[0] : '';
|
||||||
|
if (cfg.referenceMarker && fm.includes(cfg.referenceMarker)) continue;
|
||||||
|
const name = (fm.match(/^name:\s*(\S+)\s*$/m) || [])[1] || basename(file, '.md');
|
||||||
|
const ref = new RegExp(`(?<![A-Za-z0-9-])${escapeRe(name)}(?![A-Za-z0-9-])`);
|
||||||
|
const referenced = ref.test(sites);
|
||||||
|
items.push({
|
||||||
|
id: name,
|
||||||
|
summary: toPosix(relative(root, file)),
|
||||||
|
status: referenced ? 'closed' : 'open',
|
||||||
|
detail: referenced ? '' : `not referenced by any ${cfg.spawnSites}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
open: items.filter((i) => i.status === 'open').length,
|
||||||
|
total: items.length,
|
||||||
|
items,
|
||||||
|
detail: 'spawnable = agent files without the reference-document marker; referenced = name appears (word-bounded) in a spawn site',
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return { open: null, total: null, items: [], detail: `not measurable: ${err.message}` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function tallyDecisions(root, cfg) {
|
||||||
|
const abs = join(root, cfg.file);
|
||||||
|
if (!existsSync(abs)) {
|
||||||
|
return { open: null, total: null, items: [], detail: `not measurable: ${cfg.file} not found (it is local-only)` };
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const section = sectionOf(readFileSync(abs, 'utf8'), cfg.section, cfg.file);
|
||||||
|
const lines = section.split('\n');
|
||||||
|
const openRe = new RegExp(cfg.open);
|
||||||
|
const closedRe = new RegExp(cfg.closed);
|
||||||
|
const open = lines.filter((l) => openRe.test(l));
|
||||||
|
const closed = lines.filter((l) => closedRe.test(l));
|
||||||
|
// Format drift guard: a list item that carries neither marker would silently read as "0 open".
|
||||||
|
const unmarked = lines.filter((l) => /^\s*(\d+\.|[-*]) /.test(l) && !openRe.test(l) && !closedRe.test(l));
|
||||||
|
if (unmarked.length > 0) {
|
||||||
|
return {
|
||||||
|
open: null,
|
||||||
|
total: null,
|
||||||
|
items: [],
|
||||||
|
detail: `not measurable: ${unmarked.length} unmarked list item(s) in ${cfg.section} — mark each with the open/decided marker`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
open: open.length,
|
||||||
|
total: open.length + closed.length,
|
||||||
|
items: open.map((l, i) => ({ id: `#${i + 1}`, summary: l.replace(openRe, '').slice(0, 100), status: 'open', detail: '' })),
|
||||||
|
detail: `marker: open = /${cfg.open}/, decided = /${cfg.closed}/`,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return { open: null, total: null, items: [], detail: `not measurable: ${err.message}` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function measure(root, registry) {
|
||||||
|
const d = registry.decisions;
|
||||||
|
const rows = [
|
||||||
|
{ id: 'defects', label: 'open pipeline defects', source: `${REGISTRY_PATH}#defects`, ...tallyEntries(root, registry.defects) },
|
||||||
|
{ id: 'experiments', label: 'unresolved experiments/opt-ins', source: `${REGISTRY_PATH}#experiments`, ...tallyEntries(root, registry.experiments) },
|
||||||
|
{ id: 'dormant-agents', label: 'dormant agents', source: `${registry.agents.dir}/*.md vs ${registry.agents.spawnSites}`, ...tallyDormantAgents(root, registry.agents) },
|
||||||
|
{ id: 'decisions', label: 'open operator decisions', source: `${d.file} ${d.section.replace(/^#+\s*/, '§ ')}`, ...tallyDecisions(root, d) },
|
||||||
|
].map((r) => ({ ...r, target: 0 }));
|
||||||
|
return { green: rows.every((r) => r.open === 0), rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadRegistry(root) {
|
||||||
|
const abs = join(root, REGISTRY_PATH);
|
||||||
|
let reg;
|
||||||
|
try {
|
||||||
|
reg = JSON.parse(readFileSync(abs, 'utf8'));
|
||||||
|
} catch (err) {
|
||||||
|
throw new Error(`cannot read ${REGISTRY_PATH}: ${err.message}`);
|
||||||
|
}
|
||||||
|
for (const key of ['defects', 'experiments']) {
|
||||||
|
if (!Array.isArray(reg[key])) throw new Error(`${REGISTRY_PATH}: "${key}" must be a list`);
|
||||||
|
}
|
||||||
|
for (const key of ['agents', 'decisions']) {
|
||||||
|
if (!reg[key] || typeof reg[key] !== 'object') throw new Error(`${REGISTRY_PATH}: "${key}" must be an object`);
|
||||||
|
}
|
||||||
|
return reg;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function render(result) {
|
||||||
|
const zero = result.rows.filter((r) => r.open === 0).length;
|
||||||
|
const out = [];
|
||||||
|
out.push(`Voyage end-state gate: ${result.green ? 'GREEN' : 'RED'} (${zero} of ${result.rows.length} tallies at 0)`);
|
||||||
|
out.push('');
|
||||||
|
out.push('| tally | open | of | target | source |');
|
||||||
|
out.push('|---|---|---|---|---|');
|
||||||
|
for (const r of result.rows) {
|
||||||
|
const open = r.open === null ? 'n/a' : String(r.open);
|
||||||
|
const total = r.total === null ? 'n/a' : String(r.total);
|
||||||
|
out.push(`| ${r.id} | ${open} | ${total} | ${r.target} | ${r.source} |`);
|
||||||
|
}
|
||||||
|
for (const r of result.rows) {
|
||||||
|
out.push('');
|
||||||
|
out.push(`${r.id} — ${r.label}${r.detail ? ` (${r.detail})` : ''}`);
|
||||||
|
for (const i of r.items) {
|
||||||
|
if (i.status === 'closed') continue;
|
||||||
|
const tag = i.status === 'not-fellable' ? 'NOT FELLABLE, counted open' : 'open';
|
||||||
|
out.push(` [${tag}] ${i.id}: ${i.summary}${i.detail ? ` — ${i.detail}` : ''}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function main(argv) {
|
||||||
|
let json = false;
|
||||||
|
let root = REPO_ROOT;
|
||||||
|
for (let i = 0; i < argv.length; i++) {
|
||||||
|
if (argv[i] === '--json') json = true;
|
||||||
|
else if (argv[i] === '--root' && argv[i + 1]) root = resolve(argv[++i]);
|
||||||
|
else {
|
||||||
|
process.stderr.write(`end-state-gate: unknown argument ${argv[i]}\nusage: end-state-gate.mjs [--json] [--root <dir>]\n`);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let registry;
|
||||||
|
try {
|
||||||
|
registry = loadRegistry(root);
|
||||||
|
} catch (err) {
|
||||||
|
process.stderr.write(`end-state-gate: ${err.message}\n`);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
const result = measure(root, registry);
|
||||||
|
process.stdout.write((json ? JSON.stringify(result, null, 2) : render(result)) + '\n');
|
||||||
|
return result.green ? 0 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||||
|
process.exitCode = main(process.argv.slice(2));
|
||||||
|
}
|
||||||
100
scripts/end-state-registry.json
Normal file
100
scripts/end-state-registry.json
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
{
|
||||||
|
"defects": [
|
||||||
|
{
|
||||||
|
"id": "D-01",
|
||||||
|
"summary": "commands/trekplan.md still references TeamCreate/TeamDelete (allowed-tools and the execute-with-team path); both tools were removed in Claude Code 2.1.178, so that path always falls back to sequential",
|
||||||
|
"closesWhen": "trekplan.md no longer names TeamCreate or TeamDelete",
|
||||||
|
"check": [
|
||||||
|
{ "path": "commands/trekplan.md", "pattern": "\\bTeam(Create|Delete)\\b", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-02",
|
||||||
|
"summary": "commands/trekresearch.md tells the swarm engine to run the `### Bridge agent` block, but that heading was removed together with gemini-bridge",
|
||||||
|
"closesWhen": "the dangling reference is gone (or the heading exists again)",
|
||||||
|
"check": [
|
||||||
|
{ "path": "commands/trekresearch.md", "pattern": "`### Bridge agent`", "expect": "match" },
|
||||||
|
{ "path": "commands/trekresearch.md", "pattern": "^### Bridge agent", "flags": "m", "expect": "no-match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-03",
|
||||||
|
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
||||||
|
"closesWhen": "Phase 7 no longer skips trekplans (proxy: the fix must also make that path run the plan's Verification; Phase 4's entry-condition skip is legitimate and out of scope)",
|
||||||
|
"check": [
|
||||||
|
{ "path": "commands/trekexecute.md", "section": "## Phase 7 —", "pattern": "^\\*\\*Skip for trekplans\\.\\*\\*", "flags": "m", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-04",
|
||||||
|
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
||||||
|
"closesWhen": "the rubric no longer asks it to judge 'passes', or the reviewer can execute the command",
|
||||||
|
"check": [
|
||||||
|
{ "path": "agents/brief-conformance-reviewer.md", "pattern": "exists and passes", "expect": "match" },
|
||||||
|
{ "path": "agents/brief-conformance-reviewer.md", "pattern": "^tools:.*\"Bash\"", "flags": "m", "expect": "no-match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-05",
|
||||||
|
"summary": "commands/trekplan.md allowed-tools lists TaskCreate/TaskUpdate, which Claude Code 2.1.233 no longer offers on Opus 4.8 / Sonnet 5 / Fable 5 and newer by default (runtime effect not measured)",
|
||||||
|
"closesWhen": "trekplan.md allowed-tools no longer lists TaskCreate or TaskUpdate",
|
||||||
|
"check": [
|
||||||
|
{ "path": "commands/trekplan.md", "pattern": "^allowed-tools:.*\\bTask(Create|Update)\\b", "flags": "m", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-06",
|
||||||
|
"summary": "lib/review/gold-scorer.mjs contains a literal NUL byte, so git treats the file as binary and hides it from diffs and --numstat",
|
||||||
|
"closesWhen": "the file contains no NUL byte (write the separator as an escape sequence)",
|
||||||
|
"check": [
|
||||||
|
{ "path": "lib/review/gold-scorer.mjs", "pattern": "\\u0000", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "D-07",
|
||||||
|
"summary": "CLAUDE.md's command table says every /trek* command runs on opus, but since v5.9.0 no command pins model: and the orchestrator follows the session model",
|
||||||
|
"closesWhen": "the table stops claiming opus per command, or the commands pin it again",
|
||||||
|
"check": [
|
||||||
|
{ "path": "CLAUDE.md", "pattern": "^\\| `/trek[a-z]+` \\|.*\\| opus \\|\\s*$", "flags": "m", "expect": "match" },
|
||||||
|
{ "path": "commands/*.md", "pattern": "^model:", "flags": "m", "expect": "no-match" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"experiments": [
|
||||||
|
{
|
||||||
|
"id": "E-01",
|
||||||
|
"summary": "STORM dimension discovery + bounded research loop ships default-off behind VOYAGE_STORM_ENABLED; adoption is gated on a pre-registered measurement that has not run",
|
||||||
|
"closesWhen": "the env gate is gone from the pipeline surface: adopted as default, or the loop is removed",
|
||||||
|
"check": [
|
||||||
|
{ "path": ["commands/*.md", "agents/*.md", "lib/**/*.mjs", "hooks/**/*.mjs"], "pattern": "VOYAGE_STORM_ENABLED", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "E-02",
|
||||||
|
"summary": "`/trekresearch --engine deep-research` delegates to Claude Code's /deep-research, which is manual-only since 2.1.218, so the opt-in always falls back to the swarm",
|
||||||
|
"closesWhen": "the --engine opt-in is removed from trekresearch.md",
|
||||||
|
"check": [
|
||||||
|
{ "path": "commands/trekresearch.md", "pattern": "--engine\\b", "expect": "match" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "E-03",
|
||||||
|
"summary": "the delegated-orchestration head-to-head (T1 §5) is designed but has no recorded decision",
|
||||||
|
"closesWhen": "T1 §5 carries a STATUS block (run, or declined)",
|
||||||
|
"check": [
|
||||||
|
{ "path": "docs/T1-cc26-delegated-orchestration.md", "section": "## 5.", "pattern": "^> \\*\\*STATUS:", "flags": "m", "expect": "no-match" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"agents": {
|
||||||
|
"dir": "agents",
|
||||||
|
"spawnSites": "commands/*.md",
|
||||||
|
"referenceMarker": "Reference document, not a spawnable capability"
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"file": "STATE.md",
|
||||||
|
"section": "## Åpne operatørbeslutninger",
|
||||||
|
"open": "^- \\[ \\] ",
|
||||||
|
"closed": "^- \\[x\\] "
|
||||||
|
}
|
||||||
|
}
|
||||||
361
tests/scripts/end-state-gate.test.mjs
Normal file
361
tests/scripts/end-state-gate.test.mjs
Normal file
|
|
@ -0,0 +1,361 @@
|
||||||
|
// tests/scripts/end-state-gate.test.mjs
|
||||||
|
// The end-state gate counts the distance to "Voyage is finished" as four tallies:
|
||||||
|
// open pipeline defects, unresolved experiments/opt-ins, dormant agents, and open
|
||||||
|
// operator decisions. "Finished" = all four at 0.
|
||||||
|
//
|
||||||
|
// These tests pin the gate's MECHANICS in both directions — every tally can be 0 and
|
||||||
|
// can be > 0 — against throwaway fixture trees. They deliberately do NOT assert that the
|
||||||
|
// real repo is red today: the gate is supposed to go green, and a test that pins the
|
||||||
|
// current distance would fail on the day the work is done.
|
||||||
|
|
||||||
|
import { test } from 'node:test';
|
||||||
|
import { strict as assert } from 'node:assert';
|
||||||
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, dirname, resolve } from 'node:path';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import {
|
||||||
|
evaluateCondition,
|
||||||
|
evaluateCheck,
|
||||||
|
measure,
|
||||||
|
loadRegistry,
|
||||||
|
} from '../../scripts/end-state-gate.mjs';
|
||||||
|
|
||||||
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||||
|
const GATE = join(ROOT, 'scripts', 'end-state-gate.mjs');
|
||||||
|
|
||||||
|
function fixture(files) {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'end-state-gate-'));
|
||||||
|
for (const [rel, body] of Object.entries(files)) {
|
||||||
|
const p = join(dir, rel);
|
||||||
|
mkdirSync(dirname(p), { recursive: true });
|
||||||
|
writeFileSync(p, body);
|
||||||
|
}
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DECISIONS = {
|
||||||
|
file: 'STATE.md',
|
||||||
|
section: '## Open decisions',
|
||||||
|
open: '^- \\[ \\] ',
|
||||||
|
closed: '^- \\[x\\] ',
|
||||||
|
};
|
||||||
|
const AGENTS = {
|
||||||
|
dir: 'agents',
|
||||||
|
spawnSites: 'commands/*.md',
|
||||||
|
referenceMarker: 'Reference document, not a spawnable capability',
|
||||||
|
};
|
||||||
|
|
||||||
|
function registry(overrides = {}) {
|
||||||
|
return {
|
||||||
|
defects: [],
|
||||||
|
experiments: [],
|
||||||
|
agents: AGENTS,
|
||||||
|
decisions: DECISIONS,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const rowById = (result, id) => result.rows.find((r) => r.id === id);
|
||||||
|
|
||||||
|
// --- conditions -----------------------------------------------------------
|
||||||
|
|
||||||
|
test('evaluateCondition: match / no-match on a single file, multiline regex', () => {
|
||||||
|
const dir = fixture({ 'a.md': 'intro\n### Heading\nbody\n' });
|
||||||
|
try {
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Heading', flags: 'm', expect: 'match' }), true);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Other', flags: 'm', expect: 'match' }), false);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'a.md', pattern: '^### Other', flags: 'm', expect: 'no-match' }), true);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('evaluateCondition: globs (dir/*.ext, dir/**/*.ext) and path arrays match ANY file', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'commands/x.md': 'nothing here\n',
|
||||||
|
'lib/deep/y.mjs': 'const FLAG = process.env.SOME_FLAG;\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'commands/*.md', pattern: 'SOME_FLAG', expect: 'match' }), false);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'lib/**/*.mjs', pattern: 'SOME_FLAG', expect: 'match' }), true);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'SOME_FLAG', expect: 'match' }), true);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'SOME_FLAG', expect: 'no-match' }), false);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('evaluateCondition: section scoping reads only from the heading to the next "## "', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'doc.md': '## 5. Design\nno status here\n## 6. PoC\n> **STATUS: RUN AND DECLINED**\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const inFive = { path: 'doc.md', section: '## 5.', pattern: '^> \\*\\*STATUS:', flags: 'm', expect: 'no-match' };
|
||||||
|
const inSix = { ...inFive, section: '## 6.' };
|
||||||
|
assert.equal(evaluateCondition(dir, inFive), true, 'section 5 has no status block');
|
||||||
|
assert.equal(evaluateCondition(dir, inSix), false, 'section 6 has one');
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('evaluateCondition: a literal NUL byte is detectable', () => {
|
||||||
|
const dir = fixture({ 'nul.mjs': "const SEP = '\u0000';\n", 'clean.mjs': "const SEP = '\\x00';\n" });
|
||||||
|
try {
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'nul.mjs', pattern: '\\u0000', expect: 'match' }), true);
|
||||||
|
assert.equal(evaluateCondition(dir, { path: 'clean.mjs', pattern: '\\u0000', expect: 'match' }), false);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('evaluateCondition: a missing file or empty glob throws (never a silent "no match")', () => {
|
||||||
|
const dir = fixture({ 'a.md': 'x\n' });
|
||||||
|
try {
|
||||||
|
assert.throws(() => evaluateCondition(dir, { path: 'missing.md', pattern: 'x', expect: 'no-match' }), /missing\.md/);
|
||||||
|
assert.throws(() => evaluateCondition(dir, { path: 'nodir/*.md', pattern: 'x', expect: 'no-match' }), /nodir/);
|
||||||
|
assert.throws(() => evaluateCondition(dir, { path: 'a.md', section: '## 9.', pattern: 'x', expect: 'no-match' }), /## 9\./);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('evaluateCheck: open only when ALL conditions hold; null check is not fellable', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'r.md': 'run the `### Bridge agent` block\n',
|
||||||
|
'fixed.md': 'run the `### Bridge agent` block\n### Bridge agent\n',
|
||||||
|
});
|
||||||
|
const dangling = (file) => [
|
||||||
|
{ path: file, pattern: '`### Bridge agent`', expect: 'match' },
|
||||||
|
{ path: file, pattern: '^### Bridge agent', flags: 'm', expect: 'no-match' },
|
||||||
|
];
|
||||||
|
try {
|
||||||
|
assert.equal(evaluateCheck(dir, dangling('r.md')).status, 'open');
|
||||||
|
assert.equal(evaluateCheck(dir, dangling('fixed.md')).status, 'closed');
|
||||||
|
assert.equal(evaluateCheck(dir, null).status, 'not-fellable');
|
||||||
|
const broken = evaluateCheck(dir, dangling('gone.md'));
|
||||||
|
assert.equal(broken.status, 'not-fellable');
|
||||||
|
assert.match(broken.detail, /gone\.md/);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- the four tallies, both directions -------------------------------------
|
||||||
|
|
||||||
|
test('defects tally: > 0 while a check holds, 0 once fixed; not-fellable counts as open', () => {
|
||||||
|
const defect = {
|
||||||
|
id: 'D-X',
|
||||||
|
summary: 'removed tool still referenced',
|
||||||
|
check: [{ path: 'commands/plan.md', pattern: '\\bTeamCreate\\b', expect: 'match' }],
|
||||||
|
};
|
||||||
|
const unfellable = { id: 'D-Y', summary: 'no runnable check yet', check: null };
|
||||||
|
const base = { 'STATE.md': '## Open decisions\n', 'agents/a.md': 'x\n', 'commands/plan.md': 'uses a\n' };
|
||||||
|
|
||||||
|
const red = fixture({ ...base, 'commands/plan.md': 'uses a and TeamCreate\n' });
|
||||||
|
const green = fixture(base);
|
||||||
|
try {
|
||||||
|
let row = rowById(measure(red, registry({ defects: [defect] })), 'defects');
|
||||||
|
assert.equal(row.open, 1);
|
||||||
|
assert.equal(row.total, 1);
|
||||||
|
row = rowById(measure(green, registry({ defects: [defect] })), 'defects');
|
||||||
|
assert.equal(row.open, 0);
|
||||||
|
row = rowById(measure(green, registry({ defects: [defect, unfellable] })), 'defects');
|
||||||
|
assert.equal(row.open, 1, 'a defect without a runnable check stays open');
|
||||||
|
assert.equal(row.items.find((i) => i.id === 'D-Y').status, 'not-fellable');
|
||||||
|
} finally {
|
||||||
|
rmSync(red, { recursive: true, force: true });
|
||||||
|
rmSync(green, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('experiments tally: an env-gated opt-in is open until its gate is gone', () => {
|
||||||
|
const exp = {
|
||||||
|
id: 'E-X',
|
||||||
|
summary: 'default-off loop',
|
||||||
|
check: [{ path: ['commands/*.md', 'lib/**/*.mjs'], pattern: 'LOOP_ENABLED', expect: 'match' }],
|
||||||
|
};
|
||||||
|
const base = { 'STATE.md': '## Open decisions\n', 'agents/a.md': 'x\n', 'commands/r.md': 'uses a\n' };
|
||||||
|
const red = fixture({ ...base, 'lib/util/cap.mjs': 'if (process.env.LOOP_ENABLED) {}\n' });
|
||||||
|
const green = fixture({ ...base, 'lib/util/cap.mjs': 'export const x = 1;\n' });
|
||||||
|
try {
|
||||||
|
assert.equal(rowById(measure(red, registry({ experiments: [exp] })), 'experiments').open, 1);
|
||||||
|
assert.equal(rowById(measure(green, registry({ experiments: [exp] })), 'experiments').open, 0);
|
||||||
|
} finally {
|
||||||
|
rmSync(red, { recursive: true, force: true });
|
||||||
|
rmSync(green, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('dormant-agents tally: unreferenced spawnable agent counts; reference docs never do', () => {
|
||||||
|
const files = {
|
||||||
|
'STATE.md': '## Open decisions\n',
|
||||||
|
'agents/used-agent.md': '---\nname: used-agent\ndescription: |\n does work\n---\n',
|
||||||
|
'agents/idle-agent.md': '---\nname: idle-agent\ndescription: |\n never wired\n---\n',
|
||||||
|
'agents/some-orchestrator.md': '---\nname: some-orchestrator\ndescription: Reference document, not a spawnable capability — docs\n---\n',
|
||||||
|
'commands/run.md': 'Launch the **used-agent** agent.\n',
|
||||||
|
};
|
||||||
|
const red = fixture(files);
|
||||||
|
const green = fixture({ ...files, 'commands/other.md': 'Then spawn `idle-agent`.\n' });
|
||||||
|
try {
|
||||||
|
let row = rowById(measure(red, registry()), 'dormant-agents');
|
||||||
|
assert.equal(row.open, 1);
|
||||||
|
assert.equal(row.total, 2, 'denominator = spawnable agents (reference docs excluded)');
|
||||||
|
assert.deepEqual(row.items.filter((i) => i.status === 'open').map((i) => i.id), ['idle-agent']);
|
||||||
|
row = rowById(measure(green, registry()), 'dormant-agents');
|
||||||
|
assert.equal(row.open, 0);
|
||||||
|
} finally {
|
||||||
|
rmSync(red, { recursive: true, force: true });
|
||||||
|
rmSync(green, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('dormant-agents tally: a name inside a longer word is not a reference', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'STATE.md': '## Open decisions\n',
|
||||||
|
'agents/scout.md': '---\nname: scout\ndescription: |\n x\n---\n',
|
||||||
|
'commands/run.md': 'the scouting phase and research-scouts\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
assert.equal(rowById(measure(dir, registry()), 'dormant-agents').open, 1);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('decisions tally: counts open markers inside the section only', () => {
|
||||||
|
const state = [
|
||||||
|
'# STATE',
|
||||||
|
'- [ ] outside the section, ignored',
|
||||||
|
'## Open decisions',
|
||||||
|
'- [ ] first open',
|
||||||
|
'- [x] already decided',
|
||||||
|
'- [ ] second open',
|
||||||
|
'## Next section',
|
||||||
|
'- [ ] also ignored',
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
const red = fixture({ 'STATE.md': state, 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||||
|
const green = fixture({
|
||||||
|
'STATE.md': '## Open decisions\n- [x] done\n## Next\n',
|
||||||
|
'agents/a.md': 'x\n',
|
||||||
|
'commands/r.md': 'a\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
let row = rowById(measure(red, registry()), 'decisions');
|
||||||
|
assert.equal(row.open, 2);
|
||||||
|
assert.equal(row.total, 3);
|
||||||
|
row = rowById(measure(green, registry()), 'decisions');
|
||||||
|
assert.equal(row.open, 0);
|
||||||
|
assert.equal(row.total, 1);
|
||||||
|
} finally {
|
||||||
|
rmSync(red, { recursive: true, force: true });
|
||||||
|
rmSync(green, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('decisions tally: a missing STATE.md or section is NOT MEASURABLE (null), never 0', () => {
|
||||||
|
const noState = fixture({ 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||||
|
const noSection = fixture({ 'STATE.md': '# STATE\n- [ ] x\n', 'agents/a.md': 'x\n', 'commands/r.md': 'a\n' });
|
||||||
|
try {
|
||||||
|
for (const dir of [noState, noSection]) {
|
||||||
|
const result = measure(dir, registry());
|
||||||
|
const row = rowById(result, 'decisions');
|
||||||
|
assert.equal(row.open, null);
|
||||||
|
assert.ok(row.detail.length > 0, 'the reason is reported');
|
||||||
|
assert.equal(result.green, false, 'an unmeasurable tally keeps the gate red');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rmSync(noState, { recursive: true, force: true });
|
||||||
|
rmSync(noSection, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('decisions tally: an unmarked list item in the section is format drift → NOT MEASURABLE', () => {
|
||||||
|
// A STATE written without the marker convention must not read as "0 open".
|
||||||
|
for (const item of ['1. **Some decision?** prose', '- plain bullet decision', '* star bullet']) {
|
||||||
|
const dir = fixture({
|
||||||
|
'STATE.md': `## Open decisions\n- [ ] marked\n${item}\n## Next\n`,
|
||||||
|
'agents/a.md': 'x\n',
|
||||||
|
'commands/r.md': 'a\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const row = rowById(measure(dir, registry()), 'decisions');
|
||||||
|
assert.equal(row.open, null, `unmarked "${item}" must make the tally unmeasurable`);
|
||||||
|
assert.match(row.detail, /unmarked/);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- verdict + CLI ---------------------------------------------------------
|
||||||
|
|
||||||
|
function writeRegistry(dir, reg) {
|
||||||
|
mkdirSync(join(dir, 'scripts'), { recursive: true });
|
||||||
|
writeFileSync(join(dir, 'scripts', 'end-state-registry.json'), JSON.stringify(reg, null, 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
function runGate(dir, extra = []) {
|
||||||
|
return spawnSync(process.execPath, [GATE, '--root', dir, ...extra], { encoding: 'utf8' });
|
||||||
|
}
|
||||||
|
|
||||||
|
test('CLI: all four tallies at 0 → green, exit 0; --json carries the rows', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'STATE.md': '## Open decisions\n- [x] done\n',
|
||||||
|
'agents/a.md': '---\nname: a\n---\n',
|
||||||
|
'commands/r.md': 'spawn `a`\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
writeRegistry(dir, registry());
|
||||||
|
const r = runGate(dir, ['--json']);
|
||||||
|
assert.equal(r.status, 0, r.stderr);
|
||||||
|
const out = JSON.parse(r.stdout);
|
||||||
|
assert.equal(out.green, true);
|
||||||
|
assert.deepEqual(out.rows.map((x) => x.id), ['defects', 'experiments', 'dormant-agents', 'decisions']);
|
||||||
|
assert.ok(out.rows.every((x) => x.open === 0 && x.target === 0 && typeof x.source === 'string'));
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('CLI: any tally > 0 → red, exit 1; the table names every row', () => {
|
||||||
|
const dir = fixture({
|
||||||
|
'STATE.md': '## Open decisions\n- [ ] undecided\n',
|
||||||
|
'agents/a.md': '---\nname: a\n---\n',
|
||||||
|
'commands/r.md': 'spawn `a`\n',
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
writeRegistry(dir, registry());
|
||||||
|
const r = runGate(dir);
|
||||||
|
assert.equal(r.status, 1, r.stderr);
|
||||||
|
for (const label of ['defects', 'experiments', 'dormant-agents', 'decisions']) {
|
||||||
|
assert.match(r.stdout, new RegExp(label));
|
||||||
|
}
|
||||||
|
assert.match(r.stdout, /RED/);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('CLI: an unreadable registry is a usage error (exit 2), not a verdict', () => {
|
||||||
|
const dir = fixture({ 'STATE.md': '## Open decisions\n' });
|
||||||
|
try {
|
||||||
|
const r = runGate(dir);
|
||||||
|
assert.equal(r.status, 2);
|
||||||
|
assert.match(r.stderr, /end-state-registry\.json/);
|
||||||
|
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- the real registry: shape, never state ---------------------------------
|
||||||
|
|
||||||
|
test('real registry: every counted entry has an id, a summary and a check that RUNS here', () => {
|
||||||
|
const reg = loadRegistry(ROOT);
|
||||||
|
const entries = [...reg.defects, ...reg.experiments];
|
||||||
|
assert.ok(reg.defects.length >= 1 && reg.experiments.length >= 1);
|
||||||
|
const ids = new Set();
|
||||||
|
for (const e of entries) {
|
||||||
|
assert.match(e.id, /^[DE]-\d{2}$/, `bad id ${e.id}`);
|
||||||
|
assert.ok(!ids.has(e.id), `duplicate id ${e.id}`);
|
||||||
|
ids.add(e.id);
|
||||||
|
assert.ok(typeof e.summary === 'string' && e.summary.length > 10, `${e.id} needs a summary`);
|
||||||
|
// A check that cannot evaluate against this very repo is a typo, not a finding.
|
||||||
|
assert.notEqual(evaluateCheck(ROOT, e.check).status, 'not-fellable', `${e.id} check does not run here`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('real repo: four rows, open is a count or null, exit code agrees with the rows', () => {
|
||||||
|
const r = spawnSync(process.execPath, [GATE, '--json'], { encoding: 'utf8', cwd: ROOT });
|
||||||
|
assert.ok(r.status === 0 || r.status === 1, r.stderr);
|
||||||
|
const out = JSON.parse(r.stdout);
|
||||||
|
assert.equal(out.rows.length, 4);
|
||||||
|
for (const row of out.rows) {
|
||||||
|
assert.ok(row.open === null || (Number.isInteger(row.open) && row.open >= 0), `${row.id}: ${row.open}`);
|
||||||
|
}
|
||||||
|
const allZero = out.rows.every((row) => row.open === 0);
|
||||||
|
assert.equal(out.green, allZero);
|
||||||
|
assert.equal(r.status, allZero ? 0 : 1);
|
||||||
|
assert.ok(existsSync(join(ROOT, 'scripts', 'end-state-registry.json')));
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue