feat(end-state): a gate that counts the distance to "Voyage is finished" - red
Voyage now has a destination that can fail: finish the phase-2 pipeline, close the open
experiments, freeze. This adds the gate that measures the distance to it. Nothing is
fixed or removed here; every defect and every removal is its own change.
scripts/end-state-gate.mjs counts four tallies; finished = all four at 0.
defects listed in scripts/end-state-registry.json, each with a runnable check
experiments listed in the registry, each with a runnable check
dormant-agents spawnable agents no commands/*.md references (structural, word-bounded;
files carrying the reference-document marker are excluded)
decisions open operator decisions in the local STATE.md, by a fixed marker
Exit 0 green, 1 red, 2 usage/registry error. `--json` for machines.
Fail-closed by construction: an entry with no check, or whose check cannot run (missing
file, empty glob, missing section), is NOT FELLABLE and counts as open; a missing STATE.md,
a missing section, or an unmarked list item in it makes the decisions tally n/a, which
keeps the gate red. The decisions row is therefore n/a in any clean clone.
Result today (clean export of the index): RED, 0 of 4 tallies at 0 -
defects 7 of 7, experiments 3 of 3, dormant agents 1 of 20, decisions n/a.
With the local STATE present: decisions 11 of 12.
Verified in both directions: 18 fixture tests pin each tally at 0 and above 0. Each of the
10 registry checks was measured open on this tree and closed after a simulated fix on a
throwaway copy (10 of 10). That run caught a too-broad D-03 check - Phase 4's legitimate
entry-condition skip also matched - which is now scoped to the Phase 7 section.
Suite 1041 -> 1059 (1057/0/2), also run on the clean export.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
012db1fe8d
commit
553c288925
3 changed files with 717 additions and 0 deletions
256
scripts/end-state-gate.mjs
Normal file
256
scripts/end-state-gate.mjs
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/end-state-gate.mjs
|
||||
// End-state gate — counts the distance to "Voyage is finished" as four tallies:
|
||||
//
|
||||
// defects open pipeline defects (listed in the registry, each with a runnable check)
|
||||
// experiments unresolved experiments/opt-ins (listed in the registry, each with a runnable check)
|
||||
// dormant-agents spawnable agents no command references (measured structurally from agents/)
|
||||
// decisions open operator decisions (counted in STATE.md by a fixed marker)
|
||||
//
|
||||
// "Finished" = all four at 0. Exit 0 = green, 1 = red (any tally > 0 or not measurable),
|
||||
// 2 = usage or registry error. The denominators come from the tracked registry and the
|
||||
// tracked agents/ directory — never from free-text greps over prose.
|
||||
//
|
||||
// A registry entry is OPEN while ALL of its conditions hold. An entry whose check is null,
|
||||
// or whose check cannot run (missing file, empty glob, missing section), is NOT FELLABLE
|
||||
// and is counted as open — a check that cannot fire must never read as "fixed".
|
||||
//
|
||||
// Usage: node scripts/end-state-gate.mjs [--json] [--root <dir>]
|
||||
|
||||
import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
|
||||
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const REGISTRY_PATH = 'scripts/end-state-registry.json';
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
|
||||
function walk(dir) {
|
||||
const out = [];
|
||||
for (const name of readdirSync(dir)) {
|
||||
const p = join(dir, name);
|
||||
if (statSync(p).isDirectory()) out.push(...walk(p));
|
||||
else out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Supported path forms: an exact file, `dir/*.ext`, `dir/**/*.ext`.
|
||||
export function expandPath(root, pattern) {
|
||||
if (!pattern.includes('*')) {
|
||||
const abs = join(root, pattern);
|
||||
if (!existsSync(abs)) throw new Error(`path not found: ${pattern}`);
|
||||
return [abs];
|
||||
}
|
||||
const m = pattern.match(/^(.+?)\/(\*\*\/)?\*(\.[A-Za-z0-9]+)$/);
|
||||
if (!m) throw new Error(`unsupported glob: ${pattern}`);
|
||||
const [, base, deep, ext] = m;
|
||||
const absBase = join(root, base);
|
||||
if (!existsSync(absBase) || !statSync(absBase).isDirectory()) {
|
||||
throw new Error(`glob base not found: ${pattern}`);
|
||||
}
|
||||
const files = deep
|
||||
? walk(absBase)
|
||||
: readdirSync(absBase).map((f) => join(absBase, f)).filter((f) => statSync(f).isFile());
|
||||
const hits = files.filter((f) => f.endsWith(ext)).sort();
|
||||
if (hits.length === 0) throw new Error(`glob matched no files: ${pattern}`);
|
||||
return hits;
|
||||
}
|
||||
|
||||
// The text from the first line starting with `heading` up to the next `## ` heading.
|
||||
function sectionOf(text, heading, label) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.findIndex((l) => l.startsWith(heading));
|
||||
if (start === -1) throw new Error(`section "${heading}" not found in ${label}`);
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('## ')) { end = i; break; }
|
||||
}
|
||||
return lines.slice(start, end).join('\n');
|
||||
}
|
||||
|
||||
export function evaluateCondition(root, cond) {
|
||||
if (cond.expect !== 'match' && cond.expect !== 'no-match') {
|
||||
throw new Error(`expect must be "match" or "no-match", got ${JSON.stringify(cond.expect)}`);
|
||||
}
|
||||
if (String(cond.flags ?? '').includes('g')) throw new Error('the g flag is not allowed');
|
||||
const paths = Array.isArray(cond.path) ? cond.path : [cond.path];
|
||||
const files = paths.flatMap((p) => expandPath(root, p));
|
||||
const anyMatch = files.some((f) => {
|
||||
let text = readFileSync(f, 'utf8');
|
||||
if (cond.section) text = sectionOf(text, cond.section, toPosix(relative(root, f)));
|
||||
return new RegExp(cond.pattern, cond.flags ?? '').test(text);
|
||||
});
|
||||
return cond.expect === 'match' ? anyMatch : !anyMatch;
|
||||
}
|
||||
|
||||
export function evaluateCheck(root, check) {
|
||||
if (check === null || check === undefined) {
|
||||
return { status: 'not-fellable', detail: 'no runnable check' };
|
||||
}
|
||||
if (!Array.isArray(check) || check.length === 0) {
|
||||
return { status: 'not-fellable', detail: 'check must be a non-empty list of conditions' };
|
||||
}
|
||||
try {
|
||||
// Evaluate every condition (no short-circuit) so a broken one is always reported.
|
||||
const results = check.map((c) => evaluateCondition(root, c));
|
||||
return { status: results.every(Boolean) ? 'open' : 'closed', detail: '' };
|
||||
} catch (err) {
|
||||
return { status: 'not-fellable', detail: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
function tallyEntries(root, entries) {
|
||||
const items = entries.map((e) => ({ id: e.id, summary: e.summary, ...evaluateCheck(root, e.check) }));
|
||||
return {
|
||||
open: items.filter((i) => i.status !== 'closed').length,
|
||||
total: items.length,
|
||||
items,
|
||||
detail: '',
|
||||
};
|
||||
}
|
||||
|
||||
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
function tallyDormantAgents(root, cfg) {
|
||||
try {
|
||||
const agentFiles = expandPath(root, `${cfg.dir}/*.md`);
|
||||
const sites = expandPath(root, cfg.spawnSites).map((f) => readFileSync(f, 'utf8')).join('\n');
|
||||
const items = [];
|
||||
for (const file of agentFiles) {
|
||||
const text = readFileSync(file, 'utf8');
|
||||
const fm = text.startsWith('---') ? text.split('\n---')[0] : '';
|
||||
if (cfg.referenceMarker && fm.includes(cfg.referenceMarker)) continue;
|
||||
const name = (fm.match(/^name:\s*(\S+)\s*$/m) || [])[1] || basename(file, '.md');
|
||||
const ref = new RegExp(`(?<![A-Za-z0-9-])${escapeRe(name)}(?![A-Za-z0-9-])`);
|
||||
const referenced = ref.test(sites);
|
||||
items.push({
|
||||
id: name,
|
||||
summary: toPosix(relative(root, file)),
|
||||
status: referenced ? 'closed' : 'open',
|
||||
detail: referenced ? '' : `not referenced by any ${cfg.spawnSites}`,
|
||||
});
|
||||
}
|
||||
return {
|
||||
open: items.filter((i) => i.status === 'open').length,
|
||||
total: items.length,
|
||||
items,
|
||||
detail: 'spawnable = agent files without the reference-document marker; referenced = name appears (word-bounded) in a spawn site',
|
||||
};
|
||||
} catch (err) {
|
||||
return { open: null, total: null, items: [], detail: `not measurable: ${err.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
function tallyDecisions(root, cfg) {
|
||||
const abs = join(root, cfg.file);
|
||||
if (!existsSync(abs)) {
|
||||
return { open: null, total: null, items: [], detail: `not measurable: ${cfg.file} not found (it is local-only)` };
|
||||
}
|
||||
try {
|
||||
const section = sectionOf(readFileSync(abs, 'utf8'), cfg.section, cfg.file);
|
||||
const lines = section.split('\n');
|
||||
const openRe = new RegExp(cfg.open);
|
||||
const closedRe = new RegExp(cfg.closed);
|
||||
const open = lines.filter((l) => openRe.test(l));
|
||||
const closed = lines.filter((l) => closedRe.test(l));
|
||||
// Format drift guard: a list item that carries neither marker would silently read as "0 open".
|
||||
const unmarked = lines.filter((l) => /^\s*(\d+\.|[-*]) /.test(l) && !openRe.test(l) && !closedRe.test(l));
|
||||
if (unmarked.length > 0) {
|
||||
return {
|
||||
open: null,
|
||||
total: null,
|
||||
items: [],
|
||||
detail: `not measurable: ${unmarked.length} unmarked list item(s) in ${cfg.section} — mark each with the open/decided marker`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
open: open.length,
|
||||
total: open.length + closed.length,
|
||||
items: open.map((l, i) => ({ id: `#${i + 1}`, summary: l.replace(openRe, '').slice(0, 100), status: 'open', detail: '' })),
|
||||
detail: `marker: open = /${cfg.open}/, decided = /${cfg.closed}/`,
|
||||
};
|
||||
} catch (err) {
|
||||
return { open: null, total: null, items: [], detail: `not measurable: ${err.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
export function measure(root, registry) {
|
||||
const d = registry.decisions;
|
||||
const rows = [
|
||||
{ id: 'defects', label: 'open pipeline defects', source: `${REGISTRY_PATH}#defects`, ...tallyEntries(root, registry.defects) },
|
||||
{ id: 'experiments', label: 'unresolved experiments/opt-ins', source: `${REGISTRY_PATH}#experiments`, ...tallyEntries(root, registry.experiments) },
|
||||
{ id: 'dormant-agents', label: 'dormant agents', source: `${registry.agents.dir}/*.md vs ${registry.agents.spawnSites}`, ...tallyDormantAgents(root, registry.agents) },
|
||||
{ id: 'decisions', label: 'open operator decisions', source: `${d.file} ${d.section.replace(/^#+\s*/, '§ ')}`, ...tallyDecisions(root, d) },
|
||||
].map((r) => ({ ...r, target: 0 }));
|
||||
return { green: rows.every((r) => r.open === 0), rows };
|
||||
}
|
||||
|
||||
export function loadRegistry(root) {
|
||||
const abs = join(root, REGISTRY_PATH);
|
||||
let reg;
|
||||
try {
|
||||
reg = JSON.parse(readFileSync(abs, 'utf8'));
|
||||
} catch (err) {
|
||||
throw new Error(`cannot read ${REGISTRY_PATH}: ${err.message}`);
|
||||
}
|
||||
for (const key of ['defects', 'experiments']) {
|
||||
if (!Array.isArray(reg[key])) throw new Error(`${REGISTRY_PATH}: "${key}" must be a list`);
|
||||
}
|
||||
for (const key of ['agents', 'decisions']) {
|
||||
if (!reg[key] || typeof reg[key] !== 'object') throw new Error(`${REGISTRY_PATH}: "${key}" must be an object`);
|
||||
}
|
||||
return reg;
|
||||
}
|
||||
|
||||
export function render(result) {
|
||||
const zero = result.rows.filter((r) => r.open === 0).length;
|
||||
const out = [];
|
||||
out.push(`Voyage end-state gate: ${result.green ? 'GREEN' : 'RED'} (${zero} of ${result.rows.length} tallies at 0)`);
|
||||
out.push('');
|
||||
out.push('| tally | open | of | target | source |');
|
||||
out.push('|---|---|---|---|---|');
|
||||
for (const r of result.rows) {
|
||||
const open = r.open === null ? 'n/a' : String(r.open);
|
||||
const total = r.total === null ? 'n/a' : String(r.total);
|
||||
out.push(`| ${r.id} | ${open} | ${total} | ${r.target} | ${r.source} |`);
|
||||
}
|
||||
for (const r of result.rows) {
|
||||
out.push('');
|
||||
out.push(`${r.id} — ${r.label}${r.detail ? ` (${r.detail})` : ''}`);
|
||||
for (const i of r.items) {
|
||||
if (i.status === 'closed') continue;
|
||||
const tag = i.status === 'not-fellable' ? 'NOT FELLABLE, counted open' : 'open';
|
||||
out.push(` [${tag}] ${i.id}: ${i.summary}${i.detail ? ` — ${i.detail}` : ''}`);
|
||||
}
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let json = false;
|
||||
let root = REPO_ROOT;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--json') json = true;
|
||||
else if (argv[i] === '--root' && argv[i + 1]) root = resolve(argv[++i]);
|
||||
else {
|
||||
process.stderr.write(`end-state-gate: unknown argument ${argv[i]}\nusage: end-state-gate.mjs [--json] [--root <dir>]\n`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
let registry;
|
||||
try {
|
||||
registry = loadRegistry(root);
|
||||
} catch (err) {
|
||||
process.stderr.write(`end-state-gate: ${err.message}\n`);
|
||||
return 2;
|
||||
}
|
||||
const result = measure(root, registry);
|
||||
process.stdout.write((json ? JSON.stringify(result, null, 2) : render(result)) + '\n');
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
100
scripts/end-state-registry.json
Normal file
100
scripts/end-state-registry.json
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
{
|
||||
"defects": [
|
||||
{
|
||||
"id": "D-01",
|
||||
"summary": "commands/trekplan.md still references TeamCreate/TeamDelete (allowed-tools and the execute-with-team path); both tools were removed in Claude Code 2.1.178, so that path always falls back to sequential",
|
||||
"closesWhen": "trekplan.md no longer names TeamCreate or TeamDelete",
|
||||
"check": [
|
||||
{ "path": "commands/trekplan.md", "pattern": "\\bTeam(Create|Delete)\\b", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-02",
|
||||
"summary": "commands/trekresearch.md tells the swarm engine to run the `### Bridge agent` block, but that heading was removed together with gemini-bridge",
|
||||
"closesWhen": "the dangling reference is gone (or the heading exists again)",
|
||||
"check": [
|
||||
{ "path": "commands/trekresearch.md", "pattern": "`### Bridge agent`", "expect": "match" },
|
||||
{ "path": "commands/trekresearch.md", "pattern": "^### Bridge agent", "flags": "m", "expect": "no-match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-03",
|
||||
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
||||
"closesWhen": "Phase 7 no longer skips trekplans (proxy: the fix must also make that path run the plan's Verification; Phase 4's entry-condition skip is legitimate and out of scope)",
|
||||
"check": [
|
||||
{ "path": "commands/trekexecute.md", "section": "## Phase 7 —", "pattern": "^\\*\\*Skip for trekplans\\.\\*\\*", "flags": "m", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-04",
|
||||
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
||||
"closesWhen": "the rubric no longer asks it to judge 'passes', or the reviewer can execute the command",
|
||||
"check": [
|
||||
{ "path": "agents/brief-conformance-reviewer.md", "pattern": "exists and passes", "expect": "match" },
|
||||
{ "path": "agents/brief-conformance-reviewer.md", "pattern": "^tools:.*\"Bash\"", "flags": "m", "expect": "no-match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-05",
|
||||
"summary": "commands/trekplan.md allowed-tools lists TaskCreate/TaskUpdate, which Claude Code 2.1.233 no longer offers on Opus 4.8 / Sonnet 5 / Fable 5 and newer by default (runtime effect not measured)",
|
||||
"closesWhen": "trekplan.md allowed-tools no longer lists TaskCreate or TaskUpdate",
|
||||
"check": [
|
||||
{ "path": "commands/trekplan.md", "pattern": "^allowed-tools:.*\\bTask(Create|Update)\\b", "flags": "m", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-06",
|
||||
"summary": "lib/review/gold-scorer.mjs contains a literal NUL byte, so git treats the file as binary and hides it from diffs and --numstat",
|
||||
"closesWhen": "the file contains no NUL byte (write the separator as an escape sequence)",
|
||||
"check": [
|
||||
{ "path": "lib/review/gold-scorer.mjs", "pattern": "\\u0000", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-07",
|
||||
"summary": "CLAUDE.md's command table says every /trek* command runs on opus, but since v5.9.0 no command pins model: and the orchestrator follows the session model",
|
||||
"closesWhen": "the table stops claiming opus per command, or the commands pin it again",
|
||||
"check": [
|
||||
{ "path": "CLAUDE.md", "pattern": "^\\| `/trek[a-z]+` \\|.*\\| opus \\|\\s*$", "flags": "m", "expect": "match" },
|
||||
{ "path": "commands/*.md", "pattern": "^model:", "flags": "m", "expect": "no-match" }
|
||||
]
|
||||
}
|
||||
],
|
||||
"experiments": [
|
||||
{
|
||||
"id": "E-01",
|
||||
"summary": "STORM dimension discovery + bounded research loop ships default-off behind VOYAGE_STORM_ENABLED; adoption is gated on a pre-registered measurement that has not run",
|
||||
"closesWhen": "the env gate is gone from the pipeline surface: adopted as default, or the loop is removed",
|
||||
"check": [
|
||||
{ "path": ["commands/*.md", "agents/*.md", "lib/**/*.mjs", "hooks/**/*.mjs"], "pattern": "VOYAGE_STORM_ENABLED", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "E-02",
|
||||
"summary": "`/trekresearch --engine deep-research` delegates to Claude Code's /deep-research, which is manual-only since 2.1.218, so the opt-in always falls back to the swarm",
|
||||
"closesWhen": "the --engine opt-in is removed from trekresearch.md",
|
||||
"check": [
|
||||
{ "path": "commands/trekresearch.md", "pattern": "--engine\\b", "expect": "match" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "E-03",
|
||||
"summary": "the delegated-orchestration head-to-head (T1 §5) is designed but has no recorded decision",
|
||||
"closesWhen": "T1 §5 carries a STATUS block (run, or declined)",
|
||||
"check": [
|
||||
{ "path": "docs/T1-cc26-delegated-orchestration.md", "section": "## 5.", "pattern": "^> \\*\\*STATUS:", "flags": "m", "expect": "no-match" }
|
||||
]
|
||||
}
|
||||
],
|
||||
"agents": {
|
||||
"dir": "agents",
|
||||
"spawnSites": "commands/*.md",
|
||||
"referenceMarker": "Reference document, not a spawnable capability"
|
||||
},
|
||||
"decisions": {
|
||||
"file": "STATE.md",
|
||||
"section": "## Åpne operatørbeslutninger",
|
||||
"open": "^- \\[ \\] ",
|
||||
"closed": "^- \\[x\\] "
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue