chore(cleanup): R-b R1 — Voyage's own stats, events and OTel export removed
Removes Voyage's measurement, statistics and export apparatus (decision 189; condition 175 met in step 1) and the R-a leftovers: lib/stats/, lib/exporters/, the otel-export and post-bash-stats hooks, yardstick, end-state and comparison gates, synthesis-measure, gold-scorer, docs/observability.md, examples/observability/, docs/T1-cc26-delegated-orchestration.md and their tests and fixtures. The six command stats phases and the /trekexecute main-merge event blocks go; intent-approval --stamp no longer emits brief-approved. README and docs point to Claude Code's own OpenTelemetry. Files removed: 52. Files edited: 37. npm test: 2930 tests (2928 pass / 0 fail / 2 skipped) before, 2600 tests (2598 pass / 0 fail / 2 skipped) after. removed-surfaces group R1 green. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
92010d20de
commit
21480e8627
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
89 changed files with 94 additions and 10290 deletions
|
|
@ -1,342 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/comparison-gate.mjs
|
||||
// The comparison gate - the lifted target in PLAN.md (operator, 2026-09-27): Voyage + the
|
||||
// operator's way of working is level with or ahead of the related frameworks in every area,
|
||||
// and every Voyage part a built-in Claude Code primitive covers is removed or has a measured
|
||||
// reason to stay. Two counts, each with a denominator read from its SOURCE:
|
||||
//
|
||||
// comparison areas where no reference is ahead (the areas: the first column of the table
|
||||
// under `source.heading` in `source.loc`;
|
||||
// counted = areas where every reference
|
||||
// cell's verdict is `likt` or `foran`)
|
||||
// tidy covered parts that are resolved (the parts: `git ls-files` over PART_SCOPE,
|
||||
// one part per eval case; counted = parts a
|
||||
// primitive covers that are `fjernet` or
|
||||
// carry a `maalt-grunn` with evidence)
|
||||
//
|
||||
// It fells the table on MISSING EVIDENCE, never on a verdict it disagrees with: a cited file
|
||||
// must exist inside its root and hold the cited lines, a pinned clone must sit at its pin, a
|
||||
// web citation needs a verbatim quote, and an absence needs zero hits plus a known-positive
|
||||
// control. A verdict is a reader's judgement; green also requires `reader.kind: uavhengig`.
|
||||
//
|
||||
// Exit 0 = both counts full, no evidence problem, independent reader; 1 = anything short of
|
||||
// that, including a table that does not exist (NOT MEASURED, never 0); 2 = usage error.
|
||||
// It only READS. The table is local-only (docs/comparison.local.json, `*.local.*`).
|
||||
//
|
||||
// Usage: node scripts/comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]
|
||||
|
||||
import { readFileSync, existsSync, statSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, resolve, dirname, relative, isAbsolute, sep } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { isMain } from '../lib/util/is-main.mjs';
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export const DEFAULT_DATA = join(REPO_ROOT, 'docs', 'comparison.local.json');
|
||||
export const PART_SCOPE = ['commands', 'agents', 'hooks', 'lib', 'scripts', 'templates', 'evals', 'settings.json', 'verify.sh'];
|
||||
export const VERDICTS = ['foran', 'likt', 'bak'];
|
||||
const LEVEL_OR_AHEAD = new Set(['foran', 'likt']);
|
||||
export const COVERAGE = ['dekket', 'delvis', 'ikke-dekket'];
|
||||
export const RESOLUTIONS = ['maalt-grunn', 'aapen', 'fjernet'];
|
||||
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
const nonEmpty = (s) => typeof s === 'string' && s.trim() !== '';
|
||||
|
||||
// `ROOT:path:N` or `ROOT:path:N-M`, lines counted from 1. Anything else is not a loc.
|
||||
export function parseLoc(text) {
|
||||
const m = /^([A-Z]{2,}):(.+):(\d+)(?:-(\d+))?$/.exec(String(text ?? ''));
|
||||
if (!m) return null;
|
||||
const start = Number(m[3]);
|
||||
const end = m[4] === undefined ? start : Number(m[4]);
|
||||
if (start < 1 || end < start) return null;
|
||||
return { root: m[1], path: m[2], start, end };
|
||||
}
|
||||
|
||||
// The first column of the first table after `heading`, header and separator rows excluded.
|
||||
export function parseSourceAreas(markdown, heading) {
|
||||
const all = String(markdown).split('\n');
|
||||
const at = all.findIndex((l) => l.trim() === heading);
|
||||
if (at === -1) return null;
|
||||
const rows = [];
|
||||
let inTable = false;
|
||||
for (const line of all.slice(at + 1)) {
|
||||
if (/^#{1,6}\s/.test(line)) break;
|
||||
if (line.trim().startsWith('|')) {
|
||||
inTable = true;
|
||||
rows.push(line);
|
||||
} else if (inTable) break;
|
||||
}
|
||||
return rows.slice(2).map((r) => r.split('|')[1].trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
// One part per tracked file in PART_SCOPE, except eval cases: one part per `evals/<case>/`.
|
||||
export function listParts(root) {
|
||||
const out = execFileSync('git', ['ls-files', '-z', '--', ...PART_SCOPE], { cwd: root, encoding: 'utf-8' });
|
||||
const parts = new Set();
|
||||
for (const f of out.split('\0').filter(Boolean)) {
|
||||
const segs = f.split('/');
|
||||
parts.add(segs[0] === 'evals' && segs.length > 2 ? `evals/${segs[1]}/` : f);
|
||||
}
|
||||
return [...parts].sort();
|
||||
}
|
||||
|
||||
function resolveRoot(value) {
|
||||
if (!nonEmpty(value)) return null;
|
||||
if (value === '~' || value.startsWith('~/')) return join(homedir(), value.slice(2));
|
||||
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
|
||||
}
|
||||
|
||||
function lineCount(file) {
|
||||
const text = readFileSync(file, 'utf-8');
|
||||
if (text === '') return 0;
|
||||
return text.split('\n').length - (text.endsWith('\n') ? 1 : 0);
|
||||
}
|
||||
|
||||
// Problems with one evidence item, each prefixed with `where`.
|
||||
function checkEvidence(item, where, roots) {
|
||||
if (!item || typeof item !== 'object') return [`${where}: an evidence item must be an object`];
|
||||
const problems = [];
|
||||
if (!nonEmpty(item.says)) problems.push(`${where}: evidence needs \`says\``);
|
||||
if ('loc' in item) {
|
||||
const loc = parseLoc(item.loc);
|
||||
if (!loc) return [...problems, `${where}: ${item.loc} is not ROOT:path:N[-M]`];
|
||||
const root = roots[loc.root];
|
||||
if (!root) return [...problems, `${where}: ${item.loc} names an unknown root ${loc.root}`];
|
||||
const file = resolve(root, loc.path);
|
||||
const rel = relative(root, file);
|
||||
if (rel.startsWith('..') || isAbsolute(rel)) return [...problems, `${where}: ${item.loc} points outside its root`];
|
||||
if (!existsSync(file) || !statSync(file).isFile()) return [...problems, `${where}: ${item.loc} does not exist`];
|
||||
const n = lineCount(file);
|
||||
if (loc.end > n) problems.push(`${where}: ${item.loc} cites past the end of a file of ${n} lines`);
|
||||
return problems;
|
||||
}
|
||||
if ('url' in item) {
|
||||
if (!/^https:\/\/\S+$/.test(String(item.url))) problems.push(`${where}: ${item.url} is not an https URL`);
|
||||
if (!nonEmpty(item.quote)) problems.push(`${where}: a web citation needs a verbatim \`quote\``);
|
||||
return problems;
|
||||
}
|
||||
if ('absence' in item) {
|
||||
const a = item.absence ?? {};
|
||||
if (!nonEmpty(a.query)) problems.push(`${where}: an absence needs the \`query\` that was run`);
|
||||
if (a.hits !== 0) problems.push(`${where}: an absence must report hits: 0, got ${JSON.stringify(a.hits)}`);
|
||||
if (!nonEmpty(a.known_positive)) problems.push(`${where}: an absence needs a \`known_positive\` control`);
|
||||
return problems;
|
||||
}
|
||||
return [...problems, `${where}: evidence must be one of loc, url or absence`];
|
||||
}
|
||||
|
||||
function checkEvidenceList(list, where, roots) {
|
||||
if (!Array.isArray(list) || list.length === 0) return [`${where}: no evidence`];
|
||||
return list.flatMap((item, i) => checkEvidence(item, `${where}[${i}]`, roots));
|
||||
}
|
||||
|
||||
function pinProblems(references, roots) {
|
||||
const problems = [];
|
||||
for (const ref of references) {
|
||||
if (!nonEmpty(ref.pin)) continue;
|
||||
const root = roots[ref.id];
|
||||
if (!root || !existsSync(join(root, '.git'))) {
|
||||
problems.push(`reference ${ref.id}: pinned ${ref.pin} but ${root ?? 'no root'} is not a git clone`);
|
||||
continue;
|
||||
}
|
||||
let head = '';
|
||||
try {
|
||||
head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf-8' }).trim();
|
||||
} catch { /* reported below */ }
|
||||
if (!head.startsWith(ref.pin)) problems.push(`reference ${ref.id}: pinned ${ref.pin} but the clone's HEAD is ${head.slice(0, 7) || 'unreadable'}`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function evaluateComparison(data, roots) {
|
||||
const problems = [];
|
||||
const references = Array.isArray(data.references) ? data.references : [];
|
||||
if (references.length === 0) problems.push('no references to compare against');
|
||||
problems.push(...pinProblems(references, roots));
|
||||
|
||||
let sourceAreas = null;
|
||||
const src = parseLoc(`${data.source?.loc ?? ''}:1`);
|
||||
const srcFile = src && roots[src.root] ? resolve(roots[src.root], src.path) : null;
|
||||
if (!srcFile || !existsSync(srcFile)) {
|
||||
problems.push(`source ${data.source?.loc ?? '(none)'}: not readable - the denominator cannot be read`);
|
||||
} else {
|
||||
sourceAreas = parseSourceAreas(readFileSync(srcFile, 'utf-8'), data.source.heading);
|
||||
if (!sourceAreas || sourceAreas.length === 0) {
|
||||
problems.push(`source ${data.source.loc}: no table under ${JSON.stringify(data.source.heading)}`);
|
||||
sourceAreas = null;
|
||||
}
|
||||
}
|
||||
|
||||
const areas = Array.isArray(data.areas) ? data.areas : [];
|
||||
const byTitle = new Map(areas.map((a) => [a.title, a]));
|
||||
const counted = [];
|
||||
const behind = [];
|
||||
const perReference = Object.fromEntries(references.map((r) => [r.id, 0]));
|
||||
const contested = [];
|
||||
for (const title of sourceAreas ?? []) {
|
||||
if (!byTitle.has(title)) problems.push(`area ${title}: in the source but missing from the table`);
|
||||
}
|
||||
for (const area of areas) {
|
||||
const id = area.id ?? area.title;
|
||||
if (sourceAreas && !sourceAreas.includes(area.title)) {
|
||||
problems.push(`area ${area.title}: not in the source - not counted`);
|
||||
continue;
|
||||
}
|
||||
problems.push(...checkEvidenceList(area.own?.evidence, `${id} own`, roots));
|
||||
const behindRefs = [];
|
||||
for (const ref of references) {
|
||||
const cell = area.cells?.[ref.id];
|
||||
if (!cell) { problems.push(`${id} ${ref.id}: cell missing`); continue; }
|
||||
if (!VERDICTS.includes(cell.verdict)) problems.push(`${id} ${ref.id}: verdict ${JSON.stringify(cell.verdict)} is not one of ${VERDICTS.join('|')}`);
|
||||
if (!nonEmpty(cell.why)) problems.push(`${id} ${ref.id}: a verdict needs \`why\``);
|
||||
problems.push(...checkEvidenceList(cell.evidence, `${id} ${ref.id}`, roots));
|
||||
if (cell.contested) contested.push(`${id} (${ref.id})`);
|
||||
if (!LEVEL_OR_AHEAD.has(cell.verdict)) behindRefs.push(ref.id);
|
||||
else perReference[ref.id]++;
|
||||
}
|
||||
if (behindRefs.length) behind.push({ area: id, references: behindRefs });
|
||||
else counted.push(id);
|
||||
}
|
||||
const total = sourceAreas ? sourceAreas.length : null;
|
||||
return {
|
||||
problems,
|
||||
comparison: {
|
||||
total,
|
||||
level_or_ahead: total === null ? null : counted.length,
|
||||
behind,
|
||||
per_reference: perReference,
|
||||
contested,
|
||||
references: references.map((r) => `${r.id}${r.pin ? `@${r.pin}` : ''} ${r.name ?? ''}`.trim()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateTidy(tidy, parts, roots) {
|
||||
const problems = [];
|
||||
if (!tidy || typeof tidy !== 'object') {
|
||||
return { problems: ['tidy: missing - the tidy denominator is not measured'], tidy: { parts: parts.length, covered: null, resolved: null } };
|
||||
}
|
||||
const primitives = Array.isArray(tidy.primitives) ? tidy.primitives : [];
|
||||
const known = new Set();
|
||||
for (const p of primitives) {
|
||||
known.add(p.id);
|
||||
if (!/^https:\/\/\S+$/.test(String(p.doc ?? ''))) problems.push(`primitive ${p.id}: needs a documentation URL`);
|
||||
if (!nonEmpty(p.quote)) problems.push(`primitive ${p.id}: needs a verbatim \`quote\` from its documentation`);
|
||||
}
|
||||
const listed = Array.isArray(tidy.parts) ? tidy.parts : [];
|
||||
const tree = new Set(parts);
|
||||
const seen = new Set();
|
||||
let covered = 0;
|
||||
let resolved = 0;
|
||||
for (const part of listed) {
|
||||
const where = `part ${part.path}`;
|
||||
if (seen.has(part.path)) problems.push(`${where}: listed twice`);
|
||||
seen.add(part.path);
|
||||
if (!COVERAGE.includes(part.coverage)) { problems.push(`${where}: coverage ${JSON.stringify(part.coverage)} is not one of ${COVERAGE.join('|')}`); continue; }
|
||||
if (part.resolution !== 'fjernet' && !tree.has(part.path)) problems.push(`${where}: not in the tree and not marked fjernet`);
|
||||
if (part.coverage === 'ikke-dekket') continue;
|
||||
covered++;
|
||||
const by = Array.isArray(part.covered_by) ? part.covered_by : [];
|
||||
if (by.length === 0) problems.push(`${where}: ${part.coverage} needs \`covered_by\``);
|
||||
for (const id of by) if (!known.has(id)) problems.push(`${where}: covered by an unknown primitive ${id}`);
|
||||
if (!RESOLUTIONS.includes(part.resolution)) {
|
||||
problems.push(`${where}: resolution ${JSON.stringify(part.resolution)} is not one of ${RESOLUTIONS.join('|')}`);
|
||||
} else if (part.resolution === 'maalt-grunn') {
|
||||
if (!part.reason) problems.push(`${where}: maalt-grunn needs a \`reason\` with evidence`);
|
||||
else {
|
||||
const p = checkEvidence(part.reason, `${where} reason`, roots);
|
||||
problems.push(...p);
|
||||
if (p.length === 0) resolved++;
|
||||
}
|
||||
} else if (part.resolution === 'fjernet') {
|
||||
if (tree.has(part.path)) problems.push(`${where}: marked fjernet but still in the tree`);
|
||||
else if (!nonEmpty(part.commit)) problems.push(`${where}: fjernet needs the \`commit\` that removed it`);
|
||||
else resolved++;
|
||||
}
|
||||
}
|
||||
for (const p of parts) if (!seen.has(p)) problems.push(`part ${p}: in the tree but missing from the tidy list`);
|
||||
return { problems, tidy: { parts: parts.length, covered, resolved, primitives: primitives.length } };
|
||||
}
|
||||
|
||||
export function evaluate(data, { parts = [] } = {}) {
|
||||
const roots = {};
|
||||
for (const [id, value] of Object.entries(data?.roots ?? {})) roots[id] = resolveRoot(value);
|
||||
const cmp = evaluateComparison(data ?? {}, roots);
|
||||
const tdy = evaluateTidy(data?.tidy, parts, roots);
|
||||
const problems = [...cmp.problems, ...tdy.problems];
|
||||
const reader = data?.reader ?? {};
|
||||
const reasons = [];
|
||||
if (problems.length) reasons.push(`${problems.length} evidence problem(s)`);
|
||||
const c = cmp.comparison;
|
||||
if (c.total === null) reasons.push('the comparison denominator is not measured');
|
||||
else if (c.level_or_ahead < c.total) reasons.push(`${c.level_or_ahead} of ${c.total} areas level or ahead`);
|
||||
if (reader.kind !== 'uavhengig') reasons.push(`the verdicts are by ${reader.kind ?? 'nobody'}, not an independent reader`);
|
||||
if (tdy.tidy.covered === null) reasons.push('the tidy list is not measured');
|
||||
else if (tdy.tidy.resolved < tdy.tidy.covered) reasons.push(`${tdy.tidy.resolved} of ${tdy.tidy.covered} covered parts resolved`);
|
||||
return {
|
||||
measured: true,
|
||||
reader,
|
||||
comparison: c,
|
||||
tidy: tdy.tidy,
|
||||
problems,
|
||||
reasons,
|
||||
green: reasons.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function render(r) {
|
||||
if (!r.measured) return `Comparison gate: NOT MEASURED - ${r.reasons.join('; ')}\nRED`;
|
||||
const c = r.comparison;
|
||||
const out = [];
|
||||
out.push(`Comparison gate - areas where no reference is ahead of Voyage + the operator's way of working`);
|
||||
out.push(` references: ${c.references.join(' · ')}`);
|
||||
out.push(` level or ahead: ${c.level_or_ahead ?? 'NOT MEASURED'} of ${c.total ?? '?'}`);
|
||||
for (const b of c.behind) out.push(` behind: ${b.area} (${b.references.join(', ')})`);
|
||||
out.push(` per reference: ${Object.entries(c.per_reference).map(([id, n]) => `${id}: ${n} of ${c.total ?? '?'}`).join(' · ')}`);
|
||||
if (c.contested.length) out.push(` contested cells: ${c.contested.join(', ')}`);
|
||||
out.push(` reader: ${r.reader.kind ?? 'none'}${r.reader.who ? ` - ${r.reader.who}` : ''}${r.reader.date ? ` (${r.reader.date})` : ''}`
|
||||
+ `${r.reader.kind === 'uavhengig' ? '' : ' - green requires an independent reader'}`);
|
||||
const t = r.tidy;
|
||||
out.push(`Tidy - Voyage parts a documented Claude Code primitive covers`);
|
||||
out.push(` parts in the tree: ${t.parts} · covered: ${t.covered ?? 'NOT MEASURED'} · resolved: ${t.resolved ?? '-'} of ${t.covered ?? '-'}`);
|
||||
out.push(`Evidence problems: ${r.problems.length}`);
|
||||
for (const p of r.problems) out.push(` - ${p}`);
|
||||
out.push(r.green ? 'GREEN' : `RED - ${r.reasons.join('; ')}`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let json = false;
|
||||
let dataFile = DEFAULT_DATA;
|
||||
let partsFrom = REPO_ROOT;
|
||||
const usage = 'usage: comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]';
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
const value = argv[i + 1];
|
||||
if (a === '--json') { json = true; continue; }
|
||||
if (a === '--data' && value !== undefined) { dataFile = resolve(value); i++; continue; }
|
||||
if (a === '--parts-from' && value !== undefined) { partsFrom = resolve(value); i++; continue; }
|
||||
process.stderr.write(`comparison-gate: unknown argument ${a}\n${usage}\n`);
|
||||
return 2;
|
||||
}
|
||||
let result;
|
||||
if (!existsSync(dataFile)) {
|
||||
result = { measured: false, green: false, reasons: [`${toPosix(relative(REPO_ROOT, dataFile))} does not exist`] };
|
||||
} else {
|
||||
let data = null;
|
||||
try {
|
||||
data = JSON.parse(readFileSync(dataFile, 'utf-8'));
|
||||
} catch (err) {
|
||||
result = { measured: false, green: false, reasons: [`${dataFile} is not valid JSON: ${err.message}`] };
|
||||
}
|
||||
if (data) result = evaluate(data, { parts: listParts(partsFrom) });
|
||||
}
|
||||
process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (isMain(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
|
|
@ -1,585 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/end-state-gate.mjs
|
||||
// End-state gate — counts the distance to "Voyage is finished" as five tallies:
|
||||
//
|
||||
// defects open pipeline defects (listed in the registry, each with a runnable check)
|
||||
// experiments unresolved experiments/opt-ins (listed in the registry, each with a runnable check)
|
||||
// dormant-agents spawnable agents no command spawns (measured structurally, see spawnedNames)
|
||||
// decisions open operator decisions (counted in STATE.md by a fixed marker)
|
||||
// feat-after-freeze feat commits after the freeze tag (measured from git)
|
||||
//
|
||||
// "Finished" = all five at 0 AND the registry is intact against its frozen denominator
|
||||
// (tests/fixtures/end-state-frozen.json: an entry may close, never disappear or change its
|
||||
// check). Exit 0 = green, 1 = red (a tally > 0, a tally not measurable, or the registry
|
||||
// tampered with), 2 = usage or registry error.
|
||||
//
|
||||
// A registry entry is OPEN while ALL of its conditions hold. An entry whose check is null,
|
||||
// or whose check cannot run (missing file, empty glob, missing section), is NOT FELLABLE and
|
||||
// is counted as open — a check that cannot fire must never read as "fixed".
|
||||
//
|
||||
// Usage: node scripts/end-state-gate.mjs [--json] [--root <dir>]
|
||||
|
||||
import {
|
||||
readFileSync, writeFileSync, existsSync, readdirSync, statSync, realpathSync,
|
||||
mkdirSync, mkdtempSync, symlinkSync, rmSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { isMain } from '../lib/util/is-main.mjs';
|
||||
|
||||
export const REGISTRY_PATH = 'scripts/end-state-registry.json';
|
||||
export const FROZEN_PATH = 'tests/fixtures/end-state-frozen.json';
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
|
||||
function walk(dir) {
|
||||
const out = [];
|
||||
for (const name of readdirSync(dir)) {
|
||||
const p = join(dir, name);
|
||||
if (statSync(p).isDirectory()) out.push(...walk(p));
|
||||
else out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Supported path forms: an exact file, `dir/*.ext`, `dir/**/*.ext`.
|
||||
export function expandPath(root, pattern) {
|
||||
if (!pattern.includes('*')) {
|
||||
const abs = join(root, pattern);
|
||||
if (!existsSync(abs)) throw new Error(`path not found: ${pattern}`);
|
||||
return [abs];
|
||||
}
|
||||
const m = pattern.match(/^(.+?)\/(\*\*\/)?\*(\.[A-Za-z0-9]+)$/);
|
||||
if (!m) throw new Error(`unsupported glob: ${pattern}`);
|
||||
const [, base, deep, ext] = m;
|
||||
const absBase = join(root, base);
|
||||
if (!existsSync(absBase) || !statSync(absBase).isDirectory()) {
|
||||
throw new Error(`glob base not found: ${pattern}`);
|
||||
}
|
||||
const files = deep
|
||||
? walk(absBase)
|
||||
: readdirSync(absBase).map((f) => join(absBase, f)).filter((f) => statSync(f).isFile());
|
||||
const hits = files.filter((f) => f.endsWith(ext)).sort();
|
||||
if (hits.length === 0) throw new Error(`glob matched no files: ${pattern}`);
|
||||
return hits;
|
||||
}
|
||||
|
||||
// The text from the first line starting with `heading` up to the next `## ` heading.
|
||||
function sectionOf(text, heading, label) {
|
||||
const lines = text.split('\n');
|
||||
const start = lines.findIndex((l) => l.startsWith(heading));
|
||||
if (start === -1) throw new Error(`section "${heading}" not found in ${label}`);
|
||||
let end = lines.length;
|
||||
for (let i = start + 1; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('## ')) { end = i; break; }
|
||||
}
|
||||
return lines.slice(start, end).join('\n');
|
||||
}
|
||||
|
||||
export const TEST_TIMEOUT_MS = 120_000;
|
||||
|
||||
const escapeRegExp = (t) => String(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
// Every `export` name a module declares, so a stub can offer the same surface
|
||||
// and none of the behaviour.
|
||||
const EXPORT_NAME = /^\s*export\s+(?:async\s+)?(?:function\*?|const|let|var|class)\s+([A-Za-z_$][\w$]*)/gm;
|
||||
|
||||
/**
|
||||
* A stub of a module: the same export names, every one of them inert.
|
||||
*
|
||||
* This is the mutant a behaviour probe must FELL. A test that still passes
|
||||
* when the module it names does nothing is not measuring that module — the
|
||||
* empty test that closed D-03 and D-04 on a tree without `lib/verification/`
|
||||
* is the limiting case (measured 2026-09-18).
|
||||
*/
|
||||
export function stubModule(source) {
|
||||
const names = new Set();
|
||||
for (const m of String(source).matchAll(EXPORT_NAME)) names.add(m[1]);
|
||||
const lines = [...names].map((n) => `export const ${n} = undefined;`);
|
||||
if (/^\s*export\s+default\b/m.test(source)) lines.push('export default undefined;');
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
// Build `dest` as a shallow copy of `root`: every entry is a SYMLINK to the
|
||||
// original, except the files in `files`, which are written for real (and whose
|
||||
// parent directories are therefore real directories too). Node resolves an
|
||||
// ESM import through the realpath, so a symlinked test file would import the
|
||||
// ORIGINAL module and never see the stub — only the files that must differ are
|
||||
// materialised. Nothing is ever written inside the measured tree.
|
||||
function overlay(root, dest, files) {
|
||||
const realDirs = new Set();
|
||||
for (const rel of files.keys()) {
|
||||
const parts = rel.split('/');
|
||||
for (let i = 0; i < parts.length - 1; i++) realDirs.add(parts.slice(0, i + 1).join('/'));
|
||||
}
|
||||
const build = (relDir) => {
|
||||
const absSrc = relDir === '' ? root : join(root, relDir);
|
||||
const absDst = relDir === '' ? dest : join(dest, relDir);
|
||||
mkdirSync(absDst, { recursive: true });
|
||||
if (!existsSync(absSrc)) return;
|
||||
for (const name of readdirSync(absSrc)) {
|
||||
const childRel = relDir === '' ? name : `${relDir}/${name}`;
|
||||
if (realDirs.has(childRel)) { build(childRel); continue; }
|
||||
if (files.has(childRel)) continue;
|
||||
symlinkSync(join(absSrc, name), join(absDst, name));
|
||||
}
|
||||
};
|
||||
build('');
|
||||
for (const [rel, body] of files) writeFileSync(join(dest, rel), body);
|
||||
}
|
||||
|
||||
// Run ONE named test out of one test file and report whether it passed.
|
||||
// `null` = the runner produced no TAP line for that name (it did not run at
|
||||
// all), which is never a pass.
|
||||
function runNamedTest(root, testRel, name) {
|
||||
// The gate itself often runs UNDER the test runner (the gate's own test
|
||||
// spawns it). `NODE_TEST_CONTEXT` inherited into this child makes it report
|
||||
// over the parent's IPC channel instead of stdout, and the TAP line would
|
||||
// never arrive — a probe that silently stops felling anything.
|
||||
const env = { ...process.env };
|
||||
for (const key of Object.keys(env)) {
|
||||
if (key.startsWith('NODE_TEST_')) delete env[key];
|
||||
}
|
||||
const r = spawnSync(
|
||||
process.execPath,
|
||||
['--test', '--test-reporter=tap', `--test-name-pattern=^${escapeRegExp(name)}$`, testRel],
|
||||
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env },
|
||||
);
|
||||
if (r.error) throw new Error(`test runner did not run ${testRel}: ${r.error.message}`);
|
||||
// The file itself reports as a subtest too, so the summary counters are
|
||||
// ambiguous; the TAP line carrying the exact name is not. TAP escapes `#`
|
||||
// and `\`, so unescape before comparing.
|
||||
for (const line of String(r.stdout ?? '').split('\n')) {
|
||||
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
||||
if (!m) continue;
|
||||
if (m[2].replace(/\\(.)/g, '$1').trim() !== name) continue;
|
||||
return m[1] === undefined;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A condition that RUNS a named test, reads its result, and requires that the
|
||||
* same test FELLS a stub of the module it claims to bind.
|
||||
*
|
||||
* A grep condition can only ask whether a string is present, so a "behaviour"
|
||||
* probe built from one is a phrase probe pointed at code: measured
|
||||
* 2026-09-18, a two-line file exporting `runPlanVerification` and
|
||||
* `formatCriteriaEvidence` closed D-03 and D-04 on a tree where nothing was
|
||||
* fixed. Running a NAMED test closed that hole and opened a smaller one one
|
||||
* level up: two EMPTY tests with the right names closed them again. A name is
|
||||
* always forgeable; felling a mutant is not. So the named test is run twice —
|
||||
* once on the tree, once against a stub of each module in `stubs` — and it
|
||||
* must not pass the second time.
|
||||
*
|
||||
* `expect: "passes"` holds when the named test passes; `"fails"` holds when it
|
||||
* does not. A test file that is missing, a name that matches nothing, a runner
|
||||
* that cannot start, a missing or unfelled stub target: all THROW, which the
|
||||
* caller counts as NOT FELLABLE and therefore open. A check that cannot fire
|
||||
* is never "fixed".
|
||||
*/
|
||||
export function evaluateTestCondition(root, cond) {
|
||||
if (cond.expect !== 'passes' && cond.expect !== 'fails') {
|
||||
throw new Error(`a test condition's expect must be "passes" or "fails", got ${JSON.stringify(cond.expect)}`);
|
||||
}
|
||||
if (typeof cond.name !== 'string' || cond.name.trim() === '') {
|
||||
throw new Error('a test condition must name exactly one test');
|
||||
}
|
||||
if (!Array.isArray(cond.stubs) || cond.stubs.length === 0) {
|
||||
throw new Error('a test condition must declare `stubs`: the module(s) whose stub the named test must fell');
|
||||
}
|
||||
const file = join(root, cond.test);
|
||||
if (!existsSync(file)) throw new Error(`test file not found: ${cond.test}`);
|
||||
|
||||
for (const target of cond.stubs) {
|
||||
const abs = join(root, target);
|
||||
if (!existsSync(abs)) throw new Error(`stub target not found: ${target}`);
|
||||
const sandbox = mkdtempSync(join(tmpdir(), 'end-state-mutant-'));
|
||||
try {
|
||||
overlay(root, sandbox, new Map([
|
||||
[cond.test, readFileSync(file, 'utf8')],
|
||||
[target, stubModule(readFileSync(abs, 'utf8'))],
|
||||
]));
|
||||
if (runNamedTest(sandbox, cond.test, cond.name) === true) {
|
||||
throw new Error(
|
||||
`"${cond.name}" still passes against a stub of ${target} — it binds the name, not the behaviour`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
// Only ever a directory this function made, under the system temp dir.
|
||||
if (sandbox.startsWith(realpathSync(tmpdir()))) rmSync(sandbox, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const passed = runNamedTest(root, cond.test, cond.name);
|
||||
if (passed === null) throw new Error(`no test named "${cond.name}" ran in ${cond.test}`);
|
||||
return cond.expect === 'passes' ? passed : !passed;
|
||||
}
|
||||
|
||||
export function evaluateCondition(root, cond) {
|
||||
if (cond.test !== undefined) return evaluateTestCondition(root, cond);
|
||||
if (cond.expect !== 'match' && cond.expect !== 'no-match') {
|
||||
throw new Error(`expect must be "match" or "no-match", got ${JSON.stringify(cond.expect)}`);
|
||||
}
|
||||
if (String(cond.flags ?? '').includes('g')) throw new Error('the g flag is not allowed');
|
||||
const paths = Array.isArray(cond.path) ? cond.path : [cond.path];
|
||||
const files = paths.flatMap((p) => expandPath(root, p));
|
||||
const anyMatch = files.some((f) => {
|
||||
let text = readFileSync(f, 'utf8');
|
||||
if (cond.section) text = sectionOf(text, cond.section, toPosix(relative(root, f)));
|
||||
return new RegExp(cond.pattern, cond.flags ?? '').test(text);
|
||||
});
|
||||
return cond.expect === 'match' ? anyMatch : !anyMatch;
|
||||
}
|
||||
|
||||
export function evaluateCheck(root, check) {
|
||||
if (check === null || check === undefined) {
|
||||
return { status: 'not-fellable', detail: 'no runnable check' };
|
||||
}
|
||||
if (!Array.isArray(check) || check.length === 0) {
|
||||
return { status: 'not-fellable', detail: 'check must be a non-empty list of conditions' };
|
||||
}
|
||||
try {
|
||||
// Evaluate every condition (no short-circuit) so a broken one is always reported.
|
||||
const results = check.map((c) => evaluateCondition(root, c));
|
||||
return { status: results.every(Boolean) ? 'open' : 'closed', detail: '' };
|
||||
} catch (err) {
|
||||
return { status: 'not-fellable', detail: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
// --- the frozen denominator -------------------------------------------------
|
||||
|
||||
// JSON with object keys sorted, so a signature does not depend on key order.
|
||||
function canonical(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
|
||||
if (value && typeof value === 'object') {
|
||||
return `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(',')}}`;
|
||||
}
|
||||
return JSON.stringify(value ?? null);
|
||||
}
|
||||
|
||||
export function checkSignature(check) {
|
||||
return createHash('sha256').update(canonical(check)).digest('hex');
|
||||
}
|
||||
|
||||
export function loadFrozen(root) {
|
||||
return JSON.parse(readFileSync(join(root, FROZEN_PATH), 'utf8'));
|
||||
}
|
||||
|
||||
// The judged party must not own the ledger: every frozen entry must still be present with
|
||||
// the same check, and the counting configuration must not move. New entries are allowed.
|
||||
export function verifyIntegrity(registry, frozen) {
|
||||
const violations = [];
|
||||
for (const kind of ['defects', 'experiments']) {
|
||||
const byId = new Map();
|
||||
for (const e of registry[kind] || []) {
|
||||
if (byId.has(e.id)) violations.push(`${kind}: duplicate id ${e.id}`);
|
||||
byId.set(e.id, e);
|
||||
}
|
||||
for (const [id, sig] of Object.entries(frozen[kind] || {})) {
|
||||
const e = byId.get(id);
|
||||
if (!e) {
|
||||
violations.push(`${kind}: ${id} is frozen but missing from the registry (an entry may close, never disappear)`);
|
||||
} else if (checkSignature(e.check) !== sig) {
|
||||
violations.push(`${kind}: ${id} has a changed check (signature ${checkSignature(e.check).slice(0, 12)}, frozen ${sig.slice(0, 12)})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of ['agents', 'decisions', 'freeze']) {
|
||||
if (canonical(registry[key]) !== canonical(frozen[key])) {
|
||||
violations.push(`${key}: configuration differs from the frozen manifest`);
|
||||
}
|
||||
}
|
||||
const sites = [].concat(registry.agents?.spawnSites ?? []);
|
||||
if (sites.some((s) => /(^|\/)agents\//.test(String(s)))) {
|
||||
violations.push('agents: spawnSites may never include agents/ (an agent file would reference itself)');
|
||||
}
|
||||
return { ok: violations.length === 0, violations };
|
||||
}
|
||||
|
||||
// --- tallies ----------------------------------------------------------------
|
||||
|
||||
const notMeasured = (detail) => ({ open: null, total: null, items: [], detail });
|
||||
|
||||
// What each probe kind can and cannot prove. An entry declares its kind; the
|
||||
// gate states the limit out loud, because a closed check is only ever as strong
|
||||
// as the thing it reads. Every kind a registry entry may declare belongs here —
|
||||
// the entry-shape test rejects a kind the gate cannot explain.
|
||||
// What no probe in this file can prove. /trekexecute and /trekreview are
|
||||
// markdown a model interprets, so no unit test can observe that Phase 7 or
|
||||
// Phase 4.5 was performed. The gate says this out loud rather than letting a
|
||||
// green defect row imply it.
|
||||
export const WIRING_NOTE =
|
||||
'wiring: a behaviour probe proves the capability WORKS. That /trekexecute Phase 7 and ' +
|
||||
'/trekreview Phase 4.5 actually CALL it is pinned by TEXT in tests/lib/doc-consistency.test.mjs, ' +
|
||||
'not proven deterministically - real proof is a headless plugin-eval run against a fixture plan (week 40).';
|
||||
|
||||
export const PROBE_NOTES = {
|
||||
phrase:
|
||||
'a phrase probe closes on rewording: a closed phrase probe is evidence, not proof of behaviour',
|
||||
behaviour:
|
||||
'a behaviour probe RUNS a named test AND re-runs it against a stub of the module it binds, which it must fell: neither a stub that exports the right symbols nor an empty test with the right name closes it; it proves the capability WORKS, not that a prose phase calls it',
|
||||
byte:
|
||||
"a byte probe reads the file's bytes, so it closes only on a real change to them",
|
||||
};
|
||||
|
||||
function tallyEntries(root, entries) {
|
||||
const items = entries.map((e) => ({ id: e.id, summary: e.summary, probe: e.probe, ...evaluateCheck(root, e.check) }));
|
||||
const kinds = Object.keys(PROBE_NOTES).filter((k) => items.some((i) => i.probe === k));
|
||||
return {
|
||||
open: items.filter((i) => i.status !== 'closed').length,
|
||||
total: items.length,
|
||||
items,
|
||||
detail: kinds.map((k) => PROBE_NOTES[k]).join('; '),
|
||||
};
|
||||
}
|
||||
|
||||
// Lines that can carry an instruction: HTML comments and fenced code are blanked (line numbers
|
||||
// are kept so "the next line" still means the next line). Block quotes need no stripping: every
|
||||
// spawn form below is anchored at the start of the line, and a quoted line starts with `>`.
|
||||
function instructionLines(markdown) {
|
||||
const uncommented = markdown.replace(/<!--[\s\S]*?-->/g, (m) => m.replace(/[^\n]/g, ''));
|
||||
const out = [];
|
||||
let fenced = false;
|
||||
for (const line of uncommented.split('\n')) {
|
||||
if (/^\s*(```|~~~)/.test(line)) { fenced = !fenced; out.push(''); continue; }
|
||||
out.push(fenced ? '' : line);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const NAME = '[a-z0-9][a-z0-9-]*';
|
||||
const ROSTER_ROW = new RegExp(`^\\s*\\|\\s*\`(${NAME})\`\\s*\\|`);
|
||||
const LAUNCH_LINE = /^\s*(?:[-*]\s+|\d+\.\s+)?(?:\*\*)?(?:launch|spawn)\b(?:\*\*)?(.*)$/i;
|
||||
const LAUNCH_NAME = new RegExp(`\`(?:voyage:)?(${NAME})\`|\\*\\*(${NAME})\\*\\*|voyage:(${NAME})`, 'g');
|
||||
const AGENT_BLOCK = new RegExp(`^\\s*\\*\\*(${NAME})\\*\\*`);
|
||||
|
||||
// The agent names a command markdown file actually spawns. A spawn instruction is one of:
|
||||
// 1. a row of a table whose header's first cell is `Agent` (a roster the step launches);
|
||||
// 2. an imperative line that STARTS with Launch/Spawn and names the agent as `name`,
|
||||
// **name** or voyage:name;
|
||||
// 3. a line starting with **name** whose next non-empty line starts with `Prompt:`.
|
||||
// Prose, negations, HTML comments, fenced code and block quotes never count.
|
||||
export function spawnedNames(markdown) {
|
||||
const lines = instructionLines(markdown);
|
||||
const names = new Set();
|
||||
let inTable = false;
|
||||
let roster = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (/^\s*\|/.test(line)) {
|
||||
if (!inTable) {
|
||||
inTable = true;
|
||||
roster = /^\s*\|\s*Agent\s*\|/.test(line);
|
||||
continue;
|
||||
}
|
||||
const row = roster && line.match(ROSTER_ROW);
|
||||
if (row) names.add(row[1]);
|
||||
continue;
|
||||
}
|
||||
inTable = false;
|
||||
const launch = line.match(LAUNCH_LINE);
|
||||
if (launch) {
|
||||
for (const m of launch[1].matchAll(LAUNCH_NAME)) names.add(m[1] || m[2] || m[3]);
|
||||
continue;
|
||||
}
|
||||
const block = line.match(AGENT_BLOCK);
|
||||
if (block) {
|
||||
let j = i + 1;
|
||||
while (j < lines.length && lines[j].trim() === '') j++;
|
||||
if (j < lines.length && /^\s*Prompt:/.test(lines[j])) names.add(block[1]);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function tallyDormantAgents(root, cfg) {
|
||||
try {
|
||||
const agentFiles = expandPath(root, `${cfg.dir}/*.md`);
|
||||
const spawned = new Set();
|
||||
for (const f of expandPath(root, cfg.spawnSites)) {
|
||||
for (const n of spawnedNames(readFileSync(f, 'utf8'))) spawned.add(n);
|
||||
}
|
||||
const items = [];
|
||||
for (const file of agentFiles) {
|
||||
const text = readFileSync(file, 'utf8');
|
||||
const fm = text.startsWith('---') ? text.split('\n---')[0] : '';
|
||||
if (cfg.referenceMarker && fm.includes(cfg.referenceMarker)) continue;
|
||||
const name = (fm.match(/^name:\s*(\S+)\s*$/m) || [])[1] || basename(file, '.md');
|
||||
const isSpawned = spawned.has(name);
|
||||
items.push({
|
||||
id: name,
|
||||
summary: toPosix(relative(root, file)),
|
||||
status: isSpawned ? 'closed' : 'open',
|
||||
detail: isSpawned ? '' : `no spawn instruction in ${cfg.spawnSites}`,
|
||||
});
|
||||
}
|
||||
return {
|
||||
open: items.filter((i) => i.status === 'open').length,
|
||||
total: items.length,
|
||||
items,
|
||||
detail: 'spawnable = agent files without the reference-document marker; spawned = named in a spawn instruction (an `Agent` table row, a line starting with Launch/Spawn, or a **name** block followed by Prompt:); prose, comments, fences and quotes do not count',
|
||||
};
|
||||
} catch (err) {
|
||||
return notMeasured(`not measurable: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function tallyDecisions(root, cfg) {
|
||||
const abs = join(root, cfg.file);
|
||||
if (!existsSync(abs)) {
|
||||
return notMeasured(`not measurable: ${cfg.file} not found (it is local-only)`);
|
||||
}
|
||||
try {
|
||||
const section = sectionOf(readFileSync(abs, 'utf8'), cfg.section, cfg.file);
|
||||
const lines = section.split('\n');
|
||||
const openRe = new RegExp(cfg.open);
|
||||
const closedRe = new RegExp(cfg.closed);
|
||||
const open = lines.filter((l) => openRe.test(l));
|
||||
const closed = lines.filter((l) => closedRe.test(l));
|
||||
// Format drift guard: a list item that carries neither marker would silently read as "0 open".
|
||||
const unmarked = lines.filter((l) => /^\s*(\d+\.|[-*]) /.test(l) && !openRe.test(l) && !closedRe.test(l));
|
||||
if (unmarked.length > 0) {
|
||||
return notMeasured(`not measurable: ${unmarked.length} unmarked list item(s) in ${cfg.section} — mark each with the open/decided marker`);
|
||||
}
|
||||
return {
|
||||
open: open.length,
|
||||
total: open.length + closed.length,
|
||||
items: open.map((l, i) => ({ id: `#${i + 1}`, summary: l.replace(openRe, '').slice(0, 100), status: 'open', detail: '' })),
|
||||
detail: `marker: open = /${cfg.open}/, decided = /${cfg.closed}/; this counts ticks — a ticked line is not verified to be an actual decision`,
|
||||
};
|
||||
} catch (err) {
|
||||
return notMeasured(`not measurable: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function tallyFeatAfterFreeze(root, cfg) {
|
||||
const git = (...args) => spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' });
|
||||
const top = git('rev-parse', '--show-toplevel');
|
||||
if (top.status !== 0) return notMeasured(`not measured: ${toPosix(root)} is not a git work tree`);
|
||||
let atTop = false;
|
||||
try { atTop = realpathSync(top.stdout.trim()) === realpathSync(root); } catch { atTop = false; }
|
||||
if (!atTop) return notMeasured('not measured: the gate root is not the top of its git work tree');
|
||||
const ref = `refs/tags/${cfg.tag}`;
|
||||
if (git('rev-parse', '-q', '--verify', ref).status !== 0) {
|
||||
return notMeasured(`not measured: no freeze tag "${cfg.tag}" yet — open until the freeze is declared`);
|
||||
}
|
||||
const log = git('log', '--format=%s', `${ref}..HEAD`);
|
||||
if (log.status !== 0) return notMeasured(`not measured: git log failed: ${log.stderr.trim()}`);
|
||||
const subjects = log.stdout.split('\n').filter(Boolean);
|
||||
const featRe = new RegExp(cfg.featPattern);
|
||||
const feats = subjects.filter((s) => featRe.test(s));
|
||||
return {
|
||||
open: feats.length,
|
||||
total: subjects.length,
|
||||
items: feats.map((s, i) => ({ id: `#${i + 1}`, summary: s, status: 'open', detail: '' })),
|
||||
detail: `commits after ${cfg.tag} whose subject matches /${cfg.featPattern}/`,
|
||||
};
|
||||
}
|
||||
|
||||
export function measure(root, registry, frozen) {
|
||||
const d = registry.decisions;
|
||||
const f = registry.freeze;
|
||||
const rows = [
|
||||
{ id: 'defects', label: 'open pipeline defects', source: `${REGISTRY_PATH}#defects`, ...tallyEntries(root, registry.defects) },
|
||||
{ id: 'experiments', label: 'unresolved experiments/opt-ins', source: `${REGISTRY_PATH}#experiments`, ...tallyEntries(root, registry.experiments) },
|
||||
{ id: 'dormant-agents', label: 'dormant agents', source: `${registry.agents.dir}/*.md vs ${registry.agents.spawnSites}`, ...tallyDormantAgents(root, registry.agents) },
|
||||
{ id: 'decisions', label: 'open operator decisions', source: `${d.file} ${d.section.replace(/^#+\s*/, '§ ')}`, ...tallyDecisions(root, d) },
|
||||
{ id: 'feat-after-freeze', label: 'feat commits after the freeze tag', source: `git log ${f.tag}..HEAD`, ...tallyFeatAfterFreeze(root, f) },
|
||||
].map((r) => ({ ...r, target: 0 }));
|
||||
let integrity;
|
||||
try {
|
||||
integrity = { ...verifyIntegrity(registry, frozen ?? loadFrozen(root)), detail: '' };
|
||||
} catch (err) {
|
||||
integrity = { ok: null, violations: [], detail: `not measurable: cannot read ${FROZEN_PATH}: ${err.message}` };
|
||||
}
|
||||
return { green: rows.every((r) => r.open === 0) && integrity.ok === true, rows, integrity, wiring: WIRING_NOTE };
|
||||
}
|
||||
|
||||
export function loadRegistry(root) {
|
||||
const abs = join(root, REGISTRY_PATH);
|
||||
let reg;
|
||||
try {
|
||||
reg = JSON.parse(readFileSync(abs, 'utf8'));
|
||||
} catch (err) {
|
||||
throw new Error(`cannot read ${REGISTRY_PATH}: ${err.message}`);
|
||||
}
|
||||
for (const key of ['defects', 'experiments']) {
|
||||
if (!Array.isArray(reg[key])) throw new Error(`${REGISTRY_PATH}: "${key}" must be a list`);
|
||||
}
|
||||
for (const key of ['agents', 'decisions', 'freeze']) {
|
||||
if (!reg[key] || typeof reg[key] !== 'object') throw new Error(`${REGISTRY_PATH}: "${key}" must be an object`);
|
||||
}
|
||||
return reg;
|
||||
}
|
||||
|
||||
export function render(result) {
|
||||
const zero = result.rows.filter((r) => r.open === 0).length;
|
||||
const out = [];
|
||||
out.push(`Voyage end-state gate: ${result.green ? 'GREEN' : 'RED'} (${zero} of ${result.rows.length} tallies at 0)`);
|
||||
const integrity = result.integrity;
|
||||
if (integrity?.ok === true) {
|
||||
out.push(`registry integrity: intact against ${FROZEN_PATH}`);
|
||||
} else if (integrity?.ok === false) {
|
||||
out.push('registry integrity: VIOLATED — the gate cannot be green while the ledger differs from its frozen denominator');
|
||||
for (const v of integrity.violations) out.push(` - ${v}`);
|
||||
} else if (integrity) {
|
||||
out.push(`registry integrity: n/a — ${integrity.detail}`);
|
||||
}
|
||||
out.push(result.wiring ?? WIRING_NOTE);
|
||||
out.push('');
|
||||
out.push('| tally | open | of | target | source |');
|
||||
out.push('|---|---|---|---|---|');
|
||||
for (const r of result.rows) {
|
||||
const open = r.open === null ? 'n/a' : String(r.open);
|
||||
const total = r.total === null ? 'n/a' : String(r.total);
|
||||
out.push(`| ${r.id} | ${open} | ${total} | ${r.target} | ${r.source} |`);
|
||||
}
|
||||
for (const r of result.rows) {
|
||||
out.push('');
|
||||
out.push(`${r.id} — ${r.label}${r.detail ? ` (${r.detail})` : ''}`);
|
||||
for (const i of r.items) {
|
||||
if (i.status === 'closed') continue;
|
||||
let tag = i.status === 'not-fellable' ? 'NOT FELLABLE, counted open' : 'open';
|
||||
if (i.probe) tag += ` · ${i.probe} probe`;
|
||||
out.push(` [${tag}] ${i.id}: ${i.summary}${i.detail ? ` — ${i.detail}` : ''}`);
|
||||
}
|
||||
const closed = r.items.filter((i) => i.status === 'closed' && i.probe);
|
||||
if (closed.length > 0) {
|
||||
out.push(` closed: ${closed.map((i) => `${i.id} (${i.probe} probe)`).join(', ')}`);
|
||||
}
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let json = false;
|
||||
let root = REPO_ROOT;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--json') json = true;
|
||||
else if (argv[i] === '--root' && argv[i + 1]) root = resolve(argv[++i]);
|
||||
else {
|
||||
process.stderr.write(`end-state-gate: unknown argument ${argv[i]}\nusage: end-state-gate.mjs [--json] [--root <dir>]\n`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
let registry;
|
||||
try {
|
||||
registry = loadRegistry(root);
|
||||
} catch (err) {
|
||||
process.stderr.write(`end-state-gate: ${err.message}\n`);
|
||||
return 2;
|
||||
}
|
||||
const result = measure(root, registry);
|
||||
process.stdout.write((json ? JSON.stringify(result, null, 2) : render(result)) + '\n');
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (isMain(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
{
|
||||
"defects": [
|
||||
{
|
||||
"id": "D-01",
|
||||
"summary": "commands/trekplan.md still references TeamCreate/TeamDelete (allowed-tools and the execute-with-team path); both tools were removed in Claude Code 2.1.178, so that path always falls back to sequential",
|
||||
"closesWhen": "trekplan.md no longer names TeamCreate or TeamDelete",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": "commands/trekplan.md",
|
||||
"pattern": "\\bTeam(Create|Delete)\\b",
|
||||
"expect": "match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-02",
|
||||
"summary": "commands/trekresearch.md tells the swarm engine to run the `### Bridge agent` block, but that heading was removed together with gemini-bridge",
|
||||
"closesWhen": "the dangling reference is gone (or the heading exists again)",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": "commands/trekresearch.md",
|
||||
"pattern": "`### Bridge agent`",
|
||||
"expect": "match"
|
||||
},
|
||||
{
|
||||
"path": "commands/trekresearch.md",
|
||||
"pattern": "^### Bridge agent",
|
||||
"flags": "m",
|
||||
"expect": "no-match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-03",
|
||||
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
||||
"closesWhen": "a named test PROVES the single-session path fells a run on a criterion that does not hold (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it, and the named test must itself FELL a stub of lib/verification/criteria-runner.mjs, so an EMPTY test with the right name does not close it either; that Phase 7 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||
"probe": "behaviour",
|
||||
"check": [
|
||||
{
|
||||
"test": "tests/lib/criteria-runner.test.mjs",
|
||||
"name": "runPlanVerification: a plan whose success criterion FAILS fells the run",
|
||||
"expect": "fails",
|
||||
"stubs": [
|
||||
"lib/verification/criteria-runner.mjs"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-04",
|
||||
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
||||
"closesWhen": "a named test PROVES the evidence block the conformance reviewer judges is BUILT IN CODE from real exit codes (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it, and the named test must itself FELL a stub of lib/verification/criteria-runner.mjs, so an EMPTY test with the right name does not close it either; that Phase 4.5 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||
"probe": "behaviour",
|
||||
"check": [
|
||||
{
|
||||
"test": "tests/lib/criteria-runner.test.mjs",
|
||||
"name": "formatCriteriaEvidence: one row per criterion, with command and exit code",
|
||||
"expect": "fails",
|
||||
"stubs": [
|
||||
"lib/verification/criteria-runner.mjs"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-05",
|
||||
"summary": "commands/trekplan.md allowed-tools lists TaskCreate/TaskUpdate, which Claude Code 2.1.233 no longer offers on Opus 4.8 / Sonnet 5 / Fable 5 and newer by default (runtime effect not measured)",
|
||||
"closesWhen": "trekplan.md allowed-tools no longer lists TaskCreate or TaskUpdate",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": "commands/trekplan.md",
|
||||
"pattern": "^allowed-tools:.*\\bTask(Create|Update)\\b",
|
||||
"flags": "m",
|
||||
"expect": "match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-06",
|
||||
"summary": "lib/review/gold-scorer.mjs contains a literal NUL byte, so git treats the file as binary and hides it from diffs and --numstat",
|
||||
"closesWhen": "the file contains no NUL byte (write the separator as an escape sequence)",
|
||||
"probe": "byte",
|
||||
"check": [
|
||||
{
|
||||
"path": "lib/review/gold-scorer.mjs",
|
||||
"pattern": "\\u0000",
|
||||
"expect": "match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-07",
|
||||
"summary": "CLAUDE.md, README.md and commands/trekresearch.md claim the /trek* orchestrators run on opus (README even ships a sed recipe for `model: opus` lines no command has), but since v5.9.0 no command pins model: and the orchestrator follows the session model",
|
||||
"closesWhen": "none of the three files claims an opus orchestrator any more, or the commands pin opus again",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": [
|
||||
"CLAUDE.md",
|
||||
"README.md",
|
||||
"commands/trekresearch.md"
|
||||
],
|
||||
"pattern": "(^\\| `/trek[a-z]+` \\|.*\\| opus \\|\\s*$)|(\\^model: opus\\$)|(default for `/trekbrief`[\\s\\S]{0,80}?is `opus`)|(orchestrator runs on Opus)",
|
||||
"flags": "m",
|
||||
"expect": "match"
|
||||
},
|
||||
{
|
||||
"path": "commands/*.md",
|
||||
"pattern": "^model:",
|
||||
"flags": "m",
|
||||
"expect": "no-match"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"experiments": [
|
||||
{
|
||||
"id": "E-01",
|
||||
"summary": "STORM dimension discovery + bounded research loop ships default-off behind VOYAGE_STORM_ENABLED; adoption is gated on a pre-registered measurement that has not run",
|
||||
"closesWhen": "the env gate is gone from the pipeline surface: adopted as default, or the loop is removed",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": [
|
||||
"commands/*.md",
|
||||
"agents/*.md",
|
||||
"lib/**/*.mjs",
|
||||
"hooks/**/*.mjs"
|
||||
],
|
||||
"pattern": "VOYAGE_STORM_ENABLED",
|
||||
"expect": "match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "E-02",
|
||||
"summary": "`/trekresearch --engine deep-research` delegates to Claude Code's /deep-research, which is manual-only since 2.1.218, so the opt-in always falls back to the swarm",
|
||||
"closesWhen": "the --engine opt-in is removed from trekresearch.md",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": "commands/trekresearch.md",
|
||||
"pattern": "--engine\\b",
|
||||
"expect": "match"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "E-03",
|
||||
"summary": "the delegated-orchestration head-to-head (T1 §5) is designed but has no recorded decision",
|
||||
"closesWhen": "T1 §5 carries a STATUS block (run, or declined)",
|
||||
"probe": "phrase",
|
||||
"check": [
|
||||
{
|
||||
"path": "docs/T1-cc26-delegated-orchestration.md",
|
||||
"section": "## 5.",
|
||||
"pattern": "^> \\*\\*STATUS:",
|
||||
"flags": "m",
|
||||
"expect": "no-match"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"agents": {
|
||||
"dir": "agents",
|
||||
"spawnSites": "commands/*.md",
|
||||
"referenceMarker": "Reference document, not a spawnable capability"
|
||||
},
|
||||
"decisions": {
|
||||
"file": "STATE.md",
|
||||
"section": "## Åpne operatørbeslutninger",
|
||||
"open": "^- \\[ \\] ",
|
||||
"closed": "^- \\[x\\] "
|
||||
},
|
||||
"freeze": {
|
||||
"tag": "end-state-freeze",
|
||||
"featPattern": "^feat(\\(|!|:)"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/gen-expected-prom.mjs
|
||||
// Regenerate tests/fixtures/expected.prom snapshot from tests/fixtures/stats-sample.jsonl.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/gen-expected-prom.mjs > tests/fixtures/expected.prom
|
||||
//
|
||||
// When the snapshot is stale (e.g. after intentional format change or new
|
||||
// stats-sample row), regenerate via the command above and inspect the diff.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { transformToPrometheus } from '../lib/exporters/textfile-format.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const SAMPLE_PATH = join(__dirname, '..', 'tests', 'fixtures', 'stats-sample.jsonl');
|
||||
|
||||
const text = readFileSync(SAMPLE_PATH, 'utf-8');
|
||||
const records = text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
process.stdout.write(transformToPrometheus(records));
|
||||
|
|
@ -198,7 +198,7 @@ export function decideVerdict(sourcesGain, dimensionsGain) {
|
|||
* APPENDS; nothing read it.
|
||||
*
|
||||
* The record cannot carry the dimension names: names are free prose, and
|
||||
* lib/exporters/field-allowlist.mjs denies prose by omission. So the run attests
|
||||
* the record denies prose by omission. So the run attests
|
||||
* membership with `dimensions_baseline_preserved`, a low-cardinality boolean set
|
||||
* in Phase 4.5, and this gate refuses to call activation OK without it. An
|
||||
* ABSENT attestation is not an attestation — legacy rows fail here rather than
|
||||
|
|
|
|||
|
|
@ -1,298 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/synthesis-measure.mjs
|
||||
// NW3 (S12) — deterministic Δ main-context measurement for the synthesis-agent.
|
||||
//
|
||||
// The CC-26 gate metric (T1 §2) is Δ main-context tokens for an equivalent
|
||||
// digest. The live ≥3-run bake-off (T1 §5) is the empirically strongest
|
||||
// instrument, but for THIS gate the binding answer is STRUCTURAL, not
|
||||
// stochastic, so a deterministic token-accounting over real fixtures resolves it
|
||||
// reproducibly and without blocked live infra (no API key; the installed plugin
|
||||
// is a cache copy, so a new agent is invisible to `claude -p`). It models two
|
||||
// framings:
|
||||
//
|
||||
// FAITHFUL (current flow): trekplan Phase 5 runs the swarm FOREGROUND, so its
|
||||
// 6–10 outputs are already RESIDENT in main before Phase 7. Delegating only
|
||||
// the synthesis read cannot evict them → main holds base+out+dig in BOTH
|
||||
// arms → Δ ≈ 0. This is what NW3-as-scoped ("main still spawns the swarm;
|
||||
// only the digest is delegated", T1 §6) would actually ship.
|
||||
//
|
||||
// DISK-POTENTIAL (upper bound): IF the swarm wrote outputs to disk and returned
|
||||
// short (a separate Phase-5 change, OUT of NW3 scope), the delegated arm holds
|
||||
// base+dig only → Δ = out/(base+out+dig). BASE-sensitive; swept, not asserted.
|
||||
//
|
||||
// POSITIVE adopt requires Δ ≥ 30% AND quality ≥ inline (T1 §5).
|
||||
//
|
||||
// Zero deps. Node stdlib only. Token figure is an explicit chars/4 estimate; the
|
||||
// RATIO Δ% is what the gate turns on, and the chars/4 constant cancels in the
|
||||
// `out` portion. BASE is environment-dependent (system prompt + plugin listings
|
||||
// + CLAUDE.md) and NOT API-measured this session → swept across a documented band.
|
||||
|
||||
import { readFileSync, readdirSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
||||
import { join, dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { isMain } from '../lib/util/is-main.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = resolve(HERE, '..');
|
||||
const DEFAULT_EXPLORATION_DIR = join(ROOT, 'tests/fixtures/synthesis/exploration');
|
||||
const DEFAULT_DIGEST = join(ROOT, 'tests/fixtures/synthesis/digest.json');
|
||||
const DEFAULT_OUT = join(ROOT, 'docs/T1-synthesis-poc-results.md');
|
||||
|
||||
// T1 §5 thresholds.
|
||||
export const POSITIVE_THRESHOLD = 0.30;
|
||||
export const NEGATIVE_THRESHOLD = 0.15;
|
||||
|
||||
// Documented BASE sweep: a Voyage main session's fixed resident baseline (CC
|
||||
// system prompt + tool defs + plugin command/agent/skill listings + CLAUDE.md).
|
||||
// Genuinely environment-dependent; not API-measured this session.
|
||||
export const BASE_SWEEP = Object.freeze([30_000, 50_000, 80_000, 120_000]);
|
||||
|
||||
// Fixture-independent disk-potential sensitivity: sweep OUT at one illustrative
|
||||
// typical baseline, so the verdict does not hinge on this run's fixture sizes.
|
||||
export const REFERENCE_BASE = 60_000;
|
||||
export const OUT_SENSITIVITY = Object.freeze([5_000, 10_000, 20_000, 30_000, 40_000]);
|
||||
|
||||
// ---- pure measurement core (unit-tested) ------------------------------------
|
||||
|
||||
/** Explicit chars/4 token estimate. @param {string} text @returns {number} */
|
||||
export function estimateTokens(text) {
|
||||
if (typeof text !== 'string' || text.length === 0) return 0;
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens resident in MAIN at synthesis-complete, per arm.
|
||||
* @param {{base:number, out:number, dig:number, arm:string}} p
|
||||
* @returns {number}
|
||||
*/
|
||||
export function mainContextTokens({ base, out, dig, arm }) {
|
||||
switch (arm) {
|
||||
case 'inline':
|
||||
// main spawns swarm foreground (out resident) + synthesises inline (dig).
|
||||
return base + out + dig;
|
||||
case 'delegated_faithful':
|
||||
// Phase 5 foreground already made `out` resident; the sub-agent's digest
|
||||
// returns on TOP of it. Delegating Phase 7 evicts nothing.
|
||||
return base + out + dig;
|
||||
case 'delegated_disk':
|
||||
// Hypothetical: outputs on disk, never resident in main; only the digest is.
|
||||
return base + dig;
|
||||
default:
|
||||
throw new Error(`mainContextTokens: unknown arm "${arm}"`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Fractional reduction (A−B)/A. Divide-by-zero guarded to 0. */
|
||||
export function deltaPct(armA, armB) {
|
||||
if (!armA) return 0;
|
||||
return (armA - armB) / armA;
|
||||
}
|
||||
|
||||
/** T1 §5 verdict. Quality loss vetoes a token win. */
|
||||
export function decideVerdict(delta, qualityOK) {
|
||||
if (!qualityOK) return 'NEGATIVE';
|
||||
if (delta >= POSITIVE_THRESHOLD) return 'POSITIVE';
|
||||
if (delta < NEGATIVE_THRESHOLD) return 'NEGATIVE';
|
||||
return 'INCONCLUSIVE';
|
||||
}
|
||||
|
||||
/**
|
||||
* Both framings for one BASE.
|
||||
* @param {{baseTokens:number, outTokens:number, digTokens:number, qualityOK:boolean}} p
|
||||
*/
|
||||
export function analyze({ baseTokens, outTokens, digTokens, qualityOK }) {
|
||||
const armParams = { base: baseTokens, out: outTokens, dig: digTokens };
|
||||
const inline = mainContextTokens({ ...armParams, arm: 'inline' });
|
||||
const faithfulB = mainContextTokens({ ...armParams, arm: 'delegated_faithful' });
|
||||
const diskB = mainContextTokens({ ...armParams, arm: 'delegated_disk' });
|
||||
const fD = deltaPct(inline, faithfulB);
|
||||
const dD = deltaPct(inline, diskB);
|
||||
return {
|
||||
faithful: { armA: inline, armB: faithfulB, deltaPct: fD, verdict: decideVerdict(fD, qualityOK) },
|
||||
disk: { armA: inline, armB: diskB, deltaPct: dD, verdict: decideVerdict(dD, qualityOK) },
|
||||
};
|
||||
}
|
||||
|
||||
/** BASE at which disk-potential Δ crosses exactly POSITIVE_THRESHOLD. */
|
||||
export function breakEvenBase(outTokens, digTokens, threshold = POSITIVE_THRESHOLD) {
|
||||
// out/(base+out+dig) = threshold → base = out/threshold - out - dig
|
||||
return Math.round(outTokens / threshold - outTokens - digTokens);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
function pct(x) { return `${(x * 100).toFixed(1)}%`; }
|
||||
|
||||
function loadExploration(dir) {
|
||||
const files = readdirSync(dir)
|
||||
.filter((f) => f.endsWith('.md') || f.endsWith('.txt'))
|
||||
.sort();
|
||||
return files.map((f) => {
|
||||
const text = readFileSync(join(dir, f), 'utf-8');
|
||||
return { name: f, chars: text.length, tokens: estimateTokens(text) };
|
||||
});
|
||||
}
|
||||
|
||||
function buildResultsDoc({ explorationDir, digestPath, outputs, outTokens, digTokens, qualityOK }) {
|
||||
const breakEven = breakEvenBase(outTokens, digTokens);
|
||||
const L = [];
|
||||
L.push('# T1 — Synthesis-agent PoC: Δ main-context measurement (NW3 / S12)');
|
||||
L.push('');
|
||||
L.push('**Status:** Measurement complete — verdict below. **Method:** deterministic token-');
|
||||
L.push('accounting over real exploration fixtures (the live ≥3-run bake-off of T1 §5 is the');
|
||||
L.push('stronger instrument but is (a) environment-blocked here — no `ANTHROPIC_API_KEY`, and the');
|
||||
L.push('installed plugin is a cache copy so a fresh `synthesis-agent` is invisible to `claude -p`;');
|
||||
L.push('and (b) unnecessary, because the binding answer is STRUCTURAL, not stochastic).');
|
||||
L.push('**Resolves:** decision-matrix §W1 / CC-26 narrow PoC (`docs/T1-cc26-delegated-orchestration.md` §6).');
|
||||
L.push('**Reproduce:** `node scripts/synthesis-measure.mjs` (regenerates this file).');
|
||||
L.push('');
|
||||
L.push('> Verifiseringsplikt: token figures are an explicit **chars/4 estimate** (labelled), not a');
|
||||
L.push('> tokenizer count. The gate turns on the RATIO Δ%, in which the per-token constant cancels');
|
||||
L.push('> for the `out` term. BASE (the fixed main-session baseline) is environment-dependent and');
|
||||
L.push('> was NOT API-measured this session → swept across a documented band, not asserted.');
|
||||
L.push('');
|
||||
L.push('## 1. The decisive structural finding (BASE-independent)');
|
||||
L.push('');
|
||||
L.push('trekplan runs the Phase 5 exploration swarm **foreground** (foreground is the only mode');
|
||||
L.push('since v2.4.0; `commands/trekplan.md`). Foreground Agent/Task results are delivered back');
|
||||
L.push('into the main transcript, so after Phase 5 the 6–10 exploration outputs are **already');
|
||||
L.push('resident in main**. Raw outputs are never written to disk (`trekplan.md:569` reserves the');
|
||||
L.push('"do NOT write to disk" rule for the synthesis text only). Phase 7 synthesis therefore');
|
||||
L.push('*reasons over already-resident context*. Delegating **only** the Phase-7 read to a');
|
||||
L.push('synthesis-agent — "main still spawns the swarm; only the digest is delegated" (T1 §6) —');
|
||||
L.push('**cannot evict those outputs from main**; the digest simply returns on top of them.');
|
||||
L.push('');
|
||||
L.push('⇒ **Δ main-context (faithful flow) ≈ 0** — independent of every token count below. The');
|
||||
L.push('≥30% saving is only realizable by ALSO moving Phase-5 delivery off-main (swarm-writes-to-');
|
||||
L.push('disk, or a nested orchestrator owning the swarm), which is the wholesale change T1 §7');
|
||||
L.push('explicitly declined and is OUT of NW3 scope.');
|
||||
L.push('');
|
||||
L.push('## 2. Fixtures (measured)');
|
||||
L.push('');
|
||||
L.push(`- Exploration dir: \`${explorationDir.replace(ROOT + '/', '')}\``);
|
||||
L.push(`- Digest: \`${digestPath.replace(ROOT + '/', '')}\``);
|
||||
L.push('');
|
||||
L.push('| exploration output | chars | est. tokens |');
|
||||
L.push('|--------------------|-------|-------------|');
|
||||
for (const o of outputs) L.push(`| ${o.name} | ${o.chars} | ${o.tokens} |`);
|
||||
L.push(`| **OUT (Σ resident in main)** | — | **${outTokens}** |`);
|
||||
L.push(`| digest (DIG) | — | ${digTokens} |`);
|
||||
L.push('');
|
||||
L.push('## 3. Δ main-context — both framings, swept over BASE');
|
||||
L.push('');
|
||||
L.push('`inline` = base+out+dig · `delegated (faithful)` = base+out+dig (out already resident) ·');
|
||||
L.push('`delegated (disk-potential)` = base+dig (out off-main).');
|
||||
L.push('');
|
||||
L.push('| BASE (est.) | inline | faithful Δ | faithful verdict | disk-potential Δ | disk verdict |');
|
||||
L.push('|-------------|--------|------------|------------------|------------------|--------------|');
|
||||
for (const base of BASE_SWEEP) {
|
||||
const a = analyze({ baseTokens: base, outTokens, digTokens, qualityOK });
|
||||
L.push(
|
||||
`| ${base} | ${a.faithful.armA} | ${pct(a.faithful.deltaPct)} | ${a.faithful.verdict} ` +
|
||||
`| ${pct(a.disk.deltaPct)} | ${a.disk.verdict} |`,
|
||||
);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`Break-even BASE for the disk-potential upper bound to reach the 30% adopt bar: ` +
|
||||
`**~${breakEven.toLocaleString('en-US')} tokens** (below this BASE the *hypothetical* disk path ` +
|
||||
`would clear 30%; at/above it, even the upper bound fails). A real Voyage main session's BASE ` +
|
||||
`(CC system prompt + plugin command/agent/skill listings + CLAUDE.md) is large, so the disk ` +
|
||||
`upper bound is itself fragile.`);
|
||||
L.push('');
|
||||
L.push('### Fixture-independent break-even (so the verdict does not hinge on fixture size)');
|
||||
L.push('');
|
||||
L.push('disk-potential Δ = out/(base+out+dig), so it clears the 30% adopt bar **iff**');
|
||||
L.push('`out / base > 0.30/0.70 ≈ 0.43` — the combined exploration output must exceed ~43% of the');
|
||||
L.push('fixed main baseline. The table below sweeps OUT at an illustrative typical `BASE = ' +
|
||||
`${REFERENCE_BASE.toLocaleString('en-US')}\` (independent of this run's fixtures):`);
|
||||
L.push('');
|
||||
L.push('| OUT (Σ exploration tokens) | disk-potential Δ @ ref BASE | clears 30%? |');
|
||||
L.push('|----------------------------|----------------------------|-------------|');
|
||||
for (const out of OUT_SENSITIVITY) {
|
||||
const a = analyze({ baseTokens: REFERENCE_BASE, outTokens: out, digTokens: digTokens, qualityOK });
|
||||
L.push(`| ${out.toLocaleString('en-US')} | ${pct(a.disk.deltaPct)} | ${a.disk.deltaPct >= POSITIVE_THRESHOLD ? 'yes' : 'no'} |`);
|
||||
}
|
||||
L.push('');
|
||||
L.push(`This run's fixtures total **OUT = ${outTokens} tokens** across ${outputs.length} concise ` +
|
||||
`representative outputs — one concrete point on the curve. Even a generously large real swarm ` +
|
||||
`(OUT in the tens of thousands) only clears 30% when the main baseline is unusually small, and ` +
|
||||
`*never* in the faithful flow (Δ=0). The verdict is therefore robust to fixture size.`);
|
||||
L.push('');
|
||||
L.push('## 4. Quality');
|
||||
L.push('');
|
||||
L.push('The digest-output contract (`lib/plan/synthesis-digest-schema.mjs`) pins the same Phase-7');
|
||||
L.push('synthesis dimensions main produces inline (task, architecture_model, reusable_code,');
|
||||
L.push('contradictions, risks, gaps, source-tagged findings). A delegated digest that validates is');
|
||||
L.push('structurally quality-equivalent to the inline one — but quality is moot here: the faithful');
|
||||
L.push('Δ is ~0, so there is no token win for quality to defend.');
|
||||
L.push('');
|
||||
L.push('## 5. Verdict');
|
||||
L.push('');
|
||||
const faithfulVerdict = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK }).faithful;
|
||||
L.push('**DECLINED per measurement.** NW3-as-scoped yields **Δ main-context ≈ 0%** (faithful flow,');
|
||||
L.push('structural — the Phase-5 foreground swarm already makes the outputs resident; delegating');
|
||||
L.push('Phase 7 evicts nothing). The disk-potential upper bound is reachable only via an out-of-');
|
||||
L.push('scope Phase-5 change and is itself BASE-fragile.');
|
||||
L.push('');
|
||||
L.push(`RESULT: NEGATIVE (Δ_faithful = ${pct(faithfulVerdict.deltaPct)} < ${pct(NEGATIVE_THRESHOLD)} adopt-floor)`);
|
||||
L.push('');
|
||||
L.push('## 6. Disposition');
|
||||
L.push('');
|
||||
L.push('- `agents/synthesis-agent.md` ships **dormant** (a documented, schema-conformant deliverable);');
|
||||
L.push(' `commands/trekplan.md` Phase 7 is **NOT** wired to it.');
|
||||
L.push('- If main-context relief is later wanted, the prerequisite is a Phase-5 redesign (swarm-');
|
||||
L.push(' writes-to-disk / nested orchestrator) — a separate, larger decision (re-open CC-26 §7).');
|
||||
L.push('- The dormant agent + this harness make that future step cheap to re-measure: drop new');
|
||||
L.push(' fixtures in and re-run.');
|
||||
L.push('');
|
||||
return L.join('\n') + '\n';
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const o = { explorationDir: DEFAULT_EXPLORATION_DIR, digest: DEFAULT_DIGEST, out: DEFAULT_OUT, qualityOK: true, json: false };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--exploration') o.explorationDir = resolve(argv[++i]);
|
||||
else if (a === '--digest') o.digest = resolve(argv[++i]);
|
||||
else if (a === '--out') o.out = resolve(argv[++i]);
|
||||
else if (a === '--quality-fail') o.qualityOK = false;
|
||||
else if (a === '--json') o.json = true;
|
||||
else if (a === '--help' || a === '-h') { o.help = true; }
|
||||
else { process.stderr.write(`Unknown argument: ${a}\n`); process.exit(2); }
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
function mainCli() {
|
||||
const o = parseArgs(process.argv.slice(2));
|
||||
if (o.help) {
|
||||
process.stdout.write('Usage: synthesis-measure.mjs [--exploration DIR] [--digest FILE] [--out FILE] [--quality-fail] [--json]\n');
|
||||
process.exit(0);
|
||||
}
|
||||
if (!existsSync(o.explorationDir)) { process.stderr.write(`exploration dir not found: ${o.explorationDir}\n`); process.exit(2); }
|
||||
if (!existsSync(o.digest)) { process.stderr.write(`digest not found: ${o.digest}\n`); process.exit(2); }
|
||||
|
||||
const outputs = loadExploration(o.explorationDir);
|
||||
const outTokens = outputs.reduce((s, x) => s + x.tokens, 0);
|
||||
const digTokens = estimateTokens(readFileSync(o.digest, 'utf-8'));
|
||||
|
||||
if (o.json) {
|
||||
const rows = BASE_SWEEP.map((base) => ({ base, ...analyze({ baseTokens: base, outTokens, digTokens, qualityOK: o.qualityOK }) }));
|
||||
process.stdout.write(JSON.stringify({ outTokens, digTokens, breakEvenBase: breakEvenBase(outTokens, digTokens), rows }, null, 2) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const doc = buildResultsDoc({
|
||||
explorationDir: o.explorationDir, digestPath: o.digest,
|
||||
outputs, outTokens, digTokens, qualityOK: o.qualityOK,
|
||||
});
|
||||
if (!existsSync(dirname(o.out))) mkdirSync(dirname(o.out), { recursive: true });
|
||||
writeFileSync(o.out, doc, 'utf-8');
|
||||
const faithful = analyze({ baseTokens: BASE_SWEEP[1], outTokens, digTokens, qualityOK: o.qualityOK }).faithful;
|
||||
process.stderr.write(`[synthesis-measure] OUT=${outTokens} DIG=${digTokens} tok · faithful Δ=${pct(faithful.deltaPct)} → ${faithful.verdict} · wrote ${o.out}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (isMain(import.meta.url)) {
|
||||
mainCli();
|
||||
}
|
||||
|
|
@ -1,557 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/yardstick.mjs
|
||||
// The yardstick - Voyage's three numbers, each with a denominator read from its SOURCE:
|
||||
//
|
||||
// use new work that went through Voyage (the order queue joined to trekbrief-stats.jsonl
|
||||
// by order id. New work = an order whose
|
||||
// `<!-- loesningsvalg: plan|prove -->` marker
|
||||
// says so, dated on or after the latest of the
|
||||
// cutoff, MARKER_SINCE and USE_COUNTING_SINCE.
|
||||
// YES = a /trekbrief record names the order's
|
||||
// id (its file stem) under one slug that names
|
||||
// no other order. NO = the order's own
|
||||
// `<!-- arbeidsform: ordre -->` line. No record,
|
||||
// or a disputed one, is not measured, never a
|
||||
// no. USE_COUNTING_SINCE unset = counting has
|
||||
// not started: not countable)
|
||||
// trust reviews in a row with nothing that (trekreview-stats.jsonl, read as JSON and
|
||||
// stops a release ordered by `ts` - file order is not
|
||||
// chronological; value = the current ALLOW streak)
|
||||
// interrupts operator interrupts per delivery (every *-stats.jsonl: `user_input` records
|
||||
// that carry a slug or run id)
|
||||
//
|
||||
// plus a fourth row that asks whether the end-state gate can itself be fooled:
|
||||
//
|
||||
// mutants signature-preserving mutants the (M9: every export of the module a behaviour
|
||||
// gate fells probe binds keeps its name and returns
|
||||
// {status:'PASSED'}; every named test only
|
||||
// checks `typeof`. The gate fells it when it
|
||||
// refuses to close the probe on that tree.)
|
||||
//
|
||||
// What `use` can and cannot see (PM 04.10): the "no" side is only what orders
|
||||
// declare, so undeclared non-Voyage work is not measured and the value covers the
|
||||
// measured share only. Known false-positive class: an `arbeidsform: voyage`
|
||||
// order claimed while an unrelated /trekbrief runs would be named by that
|
||||
// record; PM chose no repo-mailbox change (the record must itself name the id) -
|
||||
// a strong link would need the claim to name the claiming session.
|
||||
//
|
||||
// A number the source cannot give is NOT COUNTABLE - never 0. The verdict over the first five
|
||||
// deliveries is not judged by anything yet, and says so. Exit 0 = every number countable, every
|
||||
// mutant felled and the verdict judged and met; 1 = anything short of that; 2 = usage error.
|
||||
// It only READS: the queue, the stats and the repo. The mutant is built in a temp directory.
|
||||
//
|
||||
// Usage: node scripts/yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>]
|
||||
// [--cutoff YYYY-MM-DD]
|
||||
|
||||
import {
|
||||
readFileSync, writeFileSync, existsSync, readdirSync, statSync, mkdirSync, mkdtempSync,
|
||||
realpathSync, rmSync,
|
||||
} from 'node:fs';
|
||||
import { homedir, tmpdir } from 'node:os';
|
||||
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { evaluateCheck, loadRegistry, TEST_TIMEOUT_MS } from './end-state-gate.mjs';
|
||||
import { isMain } from '../lib/util/is-main.mjs';
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export const DEFAULT_CUTOFF = '2026-09-21';
|
||||
// The day pre-order-loesningsvalg.sh began refusing an order without the marker (PM 02.10).
|
||||
export const MARKER_SINCE = '2026-09-28';
|
||||
// The day `use` starts counting. Unset until the operator declares the work done
|
||||
// (operator 04.10): until then `use` is NOT COUNTABLE and every order not measured.
|
||||
export const USE_COUNTING_SINCE = null;
|
||||
export const DEFAULT_COORD = join(homedir(), '.claude', 'coord');
|
||||
export const DEFAULT_DATA = process.env.CLAUDE_PLUGIN_DATA
|
||||
|| join(homedir(), '.claude', 'plugins', 'data', 'voyage-ktg-plugin-marketplace');
|
||||
|
||||
const REVIEW_FILE = 'trekreview-stats.jsonl';
|
||||
const BRIEF_FILE = 'trekbrief-stats.jsonl';
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
|
||||
function walk(dir) {
|
||||
const out = [];
|
||||
for (const name of readdirSync(dir)) {
|
||||
const p = join(dir, name);
|
||||
if (statSync(p).isDirectory()) out.push(...walk(p));
|
||||
else out.push(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// --- the stats files -----------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Every `*-stats.jsonl` in the data directory, parsed line by line. A blank line
|
||||
* is not a line; a line that is not JSON is a line, and is reported by number.
|
||||
*/
|
||||
export function loadStats(dataDir) {
|
||||
if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) {
|
||||
return { dir: dataDir, missing: true, files: [] };
|
||||
}
|
||||
const files = readdirSync(dataDir)
|
||||
.filter((n) => n.endsWith('-stats.jsonl'))
|
||||
.sort()
|
||||
.map((name) => {
|
||||
const path = join(dataDir, name);
|
||||
const records = [];
|
||||
const invalid = [];
|
||||
let lines = 0;
|
||||
readFileSync(path, 'utf8').split('\n').forEach((line, i) => {
|
||||
if (line.trim() === '') return;
|
||||
lines++;
|
||||
try {
|
||||
records.push(JSON.parse(line));
|
||||
} catch {
|
||||
invalid.push(i + 1);
|
||||
}
|
||||
});
|
||||
return { name, path, lines, records, invalid };
|
||||
});
|
||||
return { dir: dataDir, missing: false, files };
|
||||
}
|
||||
|
||||
/**
|
||||
* The orders /trekbrief runs named: order id -> its slugs, and slug -> its order
|
||||
* ids. Only trekbrief-stats.jsonl - the one record that carries order_id.
|
||||
*/
|
||||
export function orderRuns(stats) {
|
||||
const byOrder = new Map();
|
||||
const bySlug = new Map();
|
||||
const file = stats.files.find((f) => f.name === BRIEF_FILE);
|
||||
for (const r of file ? file.records : []) {
|
||||
if (!r || typeof r.order_id !== 'string' || r.order_id === '') continue;
|
||||
const slug = typeof r.slug === 'string' ? r.slug : '';
|
||||
if (!byOrder.has(r.order_id)) byOrder.set(r.order_id, new Set());
|
||||
byOrder.get(r.order_id).add(slug);
|
||||
if (!bySlug.has(slug)) bySlug.set(slug, new Set());
|
||||
bySlug.get(slug).add(r.order_id);
|
||||
}
|
||||
return { byOrder, bySlug };
|
||||
}
|
||||
|
||||
const notCountable = (detail, extra = {}) => ({ countable: false, value: null, detail, ...extra });
|
||||
|
||||
// --- 2 · TRUST -------------------------------------------------------------------
|
||||
|
||||
export function measureTrust(stats) {
|
||||
const base = { id: 'trust', label: 'reviews in a row with nothing that stops a release' };
|
||||
const file = stats.files.find((f) => f.name === REVIEW_FILE);
|
||||
if (!file) {
|
||||
return { ...base, source: join(stats.dir, REVIEW_FILE), denominator: 0, facts: {},
|
||||
...notCountable(stats.missing ? `data directory ${stats.dir} not found` : `${REVIEW_FILE} not found`) };
|
||||
}
|
||||
const reviews = file.records.filter((r) => r && typeof r.verdict === 'string');
|
||||
const undated = reviews.filter((r) => Number.isNaN(Date.parse(r.ts))).length;
|
||||
// Stable sort on ts: the file is appended by whoever ran a review, in no fixed order.
|
||||
const ordered = reviews
|
||||
.map((r, i) => ({ r, i, t: Date.parse(r.ts) }))
|
||||
.filter((x) => !Number.isNaN(x.t))
|
||||
.sort((a, b) => a.t - b.t || a.i - b.i)
|
||||
.map((x) => x.r);
|
||||
const verdicts = {};
|
||||
for (const r of reviews) verdicts[r.verdict] = (verdicts[r.verdict] ?? 0) + 1;
|
||||
let longest = 0;
|
||||
let run = 0;
|
||||
for (const r of ordered) {
|
||||
run = r.verdict === 'ALLOW' ? run + 1 : 0;
|
||||
longest = Math.max(longest, run);
|
||||
}
|
||||
const last = ordered.at(-1) ?? null;
|
||||
const facts = {
|
||||
verdicts, longestStreak: longest, invalid: file.invalid.length, undated,
|
||||
last: last ? { ts: last.ts, slug: last.slug, verdict: last.verdict } : null,
|
||||
};
|
||||
const notes = [];
|
||||
if (file.invalid.length) notes.push(`${file.invalid.length} invalid line(s) left out (line ${file.invalid.join(', ')})`);
|
||||
if (undated) notes.push(`${undated} review(s) without a readable ts left out of the streak`);
|
||||
if (reviews.length === 0) {
|
||||
return { ...base, source: file.path, denominator: 0, facts, ...notCountable(['no review rows', ...notes].join('; ')) };
|
||||
}
|
||||
const counts = Object.entries(verdicts).map(([k, v]) => `${k} ${v}`).join(' · ');
|
||||
const detail = [
|
||||
`${counts}; longest ALLOW streak ${longest}; last review ${last ? `${last.ts} ${last.verdict} (${last.slug})` : 'n/a'}`,
|
||||
'ordered by ts, read as JSON',
|
||||
...notes,
|
||||
].join('; ');
|
||||
return { ...base, source: file.path, countable: true, value: run, denominator: reviews.length, facts, detail };
|
||||
}
|
||||
|
||||
// --- 1 · USE ---------------------------------------------------------------------
|
||||
|
||||
// A minimal frontmatter reader: `key: value` lines between the first two `---`.
|
||||
function frontmatter(text) {
|
||||
const m = String(text).match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
||||
if (!m) return null;
|
||||
const fm = {};
|
||||
for (const line of m[1].split(/\r?\n/)) {
|
||||
const kv = line.match(/^([A-Za-z][\w-]*):\s*(.*?)\s*$/);
|
||||
if (kv) fm[kv[1]] = kv[2];
|
||||
}
|
||||
return fm;
|
||||
}
|
||||
|
||||
// The solution-choice marker dispatch writes on its own line in the order body. A mention
|
||||
// inside prose (mid-line, in backticks) is not a marker.
|
||||
const MARKER = /^<!--\s*loesningsvalg:\s*([^;\s>]+)/m;
|
||||
const NEW_WORK = new Set(['plan', 'prove']);
|
||||
const NOT_NEW = new Set(['ikke-nytt']);
|
||||
// The work-form marker dispatch writes on its own line; `ordre` declares the work is not done through Voyage.
|
||||
const ARBEIDSFORM = /^<!--\s*arbeidsform:\s*([^;\s>]+)/m;
|
||||
|
||||
/** Every order file under `<coord>/<repo>/orders/` - pending, `archive/` and `claimed/`. */
|
||||
export function readOrders(coordDir) {
|
||||
const orders = [];
|
||||
const unreadable = [];
|
||||
let files = 0;
|
||||
if (existsSync(coordDir)) {
|
||||
for (const repo of readdirSync(coordDir).sort()) {
|
||||
const dir = join(coordDir, repo, 'orders');
|
||||
if (!existsSync(dir) || !statSync(dir).isDirectory()) continue;
|
||||
for (const path of walk(dir).filter((p) => p.endsWith('.md')).sort()) {
|
||||
files++;
|
||||
const text = readFileSync(path, 'utf8');
|
||||
const fm = frontmatter(text);
|
||||
if (fm && fm['order-id']) {
|
||||
orders.push({
|
||||
path, repo, fm, id: basename(path, '.md'),
|
||||
marker: text.match(MARKER)?.[1], arbeidsform: text.match(ARBEIDSFORM)?.[1],
|
||||
});
|
||||
}
|
||||
else unreadable.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { dir: coordDir, files, orders, unreadable };
|
||||
}
|
||||
|
||||
export function measureUse({ queue, runs, cutoff, countingSince }) {
|
||||
const started = typeof countingSince === 'string' && countingSince !== '';
|
||||
const from = [cutoff, MARKER_SINCE, ...(started ? [countingSince] : [])].reduce((a, b) => (b > a ? b : a));
|
||||
const glob = `${join(queue.dir, '*', 'orders')}/**/*.md`;
|
||||
const base = {
|
||||
id: 'use', label: 'new work that went through Voyage',
|
||||
source: started
|
||||
? `${glob} joined to ${BRIEF_FILE} by order id (loesningsvalg marker, from ${from})`
|
||||
: `${glob} joined to ${BRIEF_FILE} by order id (counting not started - USE_COUNTING_SINCE is not set)`,
|
||||
};
|
||||
const named = queue.orders.filter((o) => runs.byOrder.has(o.id)).length;
|
||||
const withMarker = queue.orders.filter((o) => o.marker !== undefined);
|
||||
const perMailbox = {};
|
||||
for (const o of queue.orders) perMailbox[o.repo] = (perMailbox[o.repo] ?? 0) + 1;
|
||||
const dot = Object.entries(perMailbox).filter(([k]) => k.startsWith('.'));
|
||||
const files = `${queue.files} order files in ${Object.keys(perMailbox).length} mailboxes`
|
||||
+ (dot.length ? ` (dot-mailboxes ${dot.map(([k, v]) => `${k} ${v}`).join(' · ')} - a shell \`*\` skips them)` : '');
|
||||
const unreadableNote = queue.unreadable.length ? `; ${queue.unreadable.length} order file(s) without frontmatter` : '';
|
||||
if (!started) {
|
||||
const facts = { scanned: queue.orders.length, files: queue.files, unreadable: queue.unreadable.length,
|
||||
withMarker: withMarker.length, notMeasured: queue.orders.length, withVoyageRun: 0,
|
||||
ordersNamedByARecord: named, countingSince: null };
|
||||
return { ...base, denominator: 0, facts, ...notCountable(
|
||||
`${files}${unreadableNote}; counting has not started (USE_COUNTING_SINCE is not set) - all ${queue.orders.length} orders not measured`
|
||||
+ `; ${named} order(s) already named by a trekbrief record`) };
|
||||
}
|
||||
const inWindow = queue.orders.filter((o) => String(o.fm.date ?? '').slice(0, 10) >= from);
|
||||
const newWork = inWindow.filter((o) => NEW_WORK.has(o.marker));
|
||||
const notNew = inWindow.filter((o) => NOT_NEW.has(o.marker));
|
||||
const unmarked = inWindow.length - newWork.length - notNew.length;
|
||||
const beforeMarker = queue.orders.length - inWindow.length;
|
||||
const yes = [];
|
||||
let declaredNo = 0;
|
||||
let noRecord = 0;
|
||||
let disputed = 0;
|
||||
for (const o of newWork) {
|
||||
if (o.arbeidsform === 'ordre') { declaredNo++; continue; }
|
||||
const slugs = runs.byOrder.get(o.id);
|
||||
if (!slugs) { noRecord++; continue; }
|
||||
const [slug] = slugs;
|
||||
if (slugs.size !== 1 || runs.bySlug.get(slug).size !== 1) { disputed++; continue; }
|
||||
yes.push(o.id);
|
||||
}
|
||||
const measured = yes.length + declaredNo;
|
||||
const facts = {
|
||||
scanned: queue.orders.length, files: queue.files, unreadable: queue.unreadable.length,
|
||||
withMarker: withMarker.length, newWork: newWork.length, notNew: notNew.length,
|
||||
unmarked, beforeMarker, notMeasured: unmarked + beforeMarker + noRecord + disputed,
|
||||
withVoyageRun: yes.length, declaredNo, noRecord, disputed, matched: yes,
|
||||
ordersNamedByARecord: named, countingSince,
|
||||
};
|
||||
const census = `${files}; ${withMarker.length} of ${queue.orders.length} orders carry a loesningsvalg marker`
|
||||
+ `; from ${from}: ${newWork.length} new (plan|prove) · ${notNew.length} not new (ikke-nytt)`
|
||||
+ `; ${facts.notMeasured} not measured (${beforeMarker} before ${from} · ${unmarked} unmarked or unknown value`
|
||||
+ ` · ${noRecord} new without a trekbrief record · ${disputed} disputed)${unreadableNote}`;
|
||||
if (newWork.length === 0) {
|
||||
return { ...base, denominator: 0, facts, ...notCountable(
|
||||
`${census}; 0 new-work orders dated >= ${from} - the denominator is empty`) };
|
||||
}
|
||||
if (measured === 0) {
|
||||
return { ...base, denominator: 0, facts, ...notCountable(
|
||||
`${census}; none of ${newWork.length} new-work orders is measured - a yes needs a trekbrief record naming the order,`
|
||||
+ ' a no the order\'s own arbeidsform: ordre') };
|
||||
}
|
||||
return {
|
||||
...base, countable: true, value: yes.length, denominator: measured, facts,
|
||||
detail: `${census}; of ${newWork.length} new-work orders: ${yes.length} through Voyage (a trekbrief record names the order)`
|
||||
+ ` · ${declaredNo} declared outside Voyage (arbeidsform: ordre); value = ${yes.length} of ${measured} measured`
|
||||
+ ' - the measured share only: undeclared non-Voyage work is not measured, never a no',
|
||||
};
|
||||
}
|
||||
|
||||
// --- 3 · INTERRUPTS --------------------------------------------------------------
|
||||
|
||||
// A call site EMITS user_input: the CLI shim with `--event user_input`, or a
|
||||
// call `emit…('user_input'`. A name in prose is not a call.
|
||||
const CALL_SITE = /--event[\s'",=\\]+user_input\b|\bemit\w*\(\s*['"]user_input['"]/;
|
||||
const SCAN = [
|
||||
{ dir: 'commands', ext: '.md' },
|
||||
{ dir: 'agents', ext: '.md' },
|
||||
{ dir: 'hooks', ext: '' },
|
||||
{ dir: 'lib', ext: '.mjs' },
|
||||
];
|
||||
|
||||
/** Emitter call sites for user_input outside lib/stats/ (the emitter) and tests/. */
|
||||
export function emitterCallSites(root) {
|
||||
const sites = [];
|
||||
let scanned = 0;
|
||||
for (const { dir, ext } of SCAN) {
|
||||
const abs = join(root, dir);
|
||||
if (!existsSync(abs)) continue;
|
||||
for (const file of walk(abs).sort()) {
|
||||
const rel = toPosix(relative(root, file));
|
||||
if (rel.startsWith('lib/stats/') || (ext && !rel.endsWith(ext))) continue;
|
||||
scanned++;
|
||||
readFileSync(file, 'utf8').split('\n').forEach((line, i) => {
|
||||
if (CALL_SITE.test(line)) sites.push({ file: rel, line: i + 1 });
|
||||
});
|
||||
}
|
||||
}
|
||||
return { sites, scanned };
|
||||
}
|
||||
|
||||
const deliveryOf = (r) => {
|
||||
const p = r.payload && typeof r.payload === 'object' ? r.payload : {};
|
||||
return p.slug || p.run_id || r.slug || r.run_id || null;
|
||||
};
|
||||
|
||||
export function measureInterrupts({ stats, callSites }) {
|
||||
const base = { id: 'interrupts', label: 'operator interrupts per delivery', source: join(stats.dir, '*-stats.jsonl') };
|
||||
const lines = stats.files.reduce((n, f) => n + f.lines, 0);
|
||||
const records = stats.files.flatMap((f) => f.records);
|
||||
const userInput = records.filter((r) => r && r.event === 'user_input');
|
||||
const perDelivery = {};
|
||||
for (const r of userInput) {
|
||||
const d = deliveryOf(r);
|
||||
if (d) perDelivery[d] = (perDelivery[d] ?? 0) + 1;
|
||||
}
|
||||
const withDelivery = Object.values(perDelivery).reduce((a, b) => a + b, 0);
|
||||
const facts = { lines, files: stats.files.length, userInput: userInput.length, withDelivery, perDelivery,
|
||||
callSites: callSites.sites.length, scannedForCallSites: callSites.scanned };
|
||||
const census = `${userInput.length} user_input records of ${lines} lines in ${stats.files.length} files`
|
||||
+ `; ${withDelivery} carry a slug or run id`
|
||||
+ `; ${callSites.sites.length} emitter call sites outside lib/stats/ and tests/ (${callSites.scanned} files scanned)`;
|
||||
if (withDelivery === 0) {
|
||||
return { ...base, denominator: 0, facts, ...notCountable(`${census} - an interrupt that names no delivery cannot be divided by one`) };
|
||||
}
|
||||
return {
|
||||
...base, countable: true, value: withDelivery, denominator: Object.keys(perDelivery).length, facts,
|
||||
detail: `${census}; per delivery: ${Object.entries(perDelivery).map(([k, v]) => `${k} ${v}`).join(' · ')}`,
|
||||
};
|
||||
}
|
||||
|
||||
// --- 4 · the signature-preserving mutant (M9) --------------------------------------
|
||||
|
||||
const EXPORT_NAME = /^\s*export\s+(?:async\s+)?(?:function\*?|const|let|var|class)\s+([A-Za-z_$][\w$]*)/gm;
|
||||
|
||||
// Every entry whose check RUNS a named test and names the module it binds.
|
||||
function behaviourConditions(registry) {
|
||||
const out = [];
|
||||
for (const e of [...(registry.defects ?? []), ...(registry.experiments ?? [])]) {
|
||||
for (const c of Array.isArray(e.check) ? e.check : []) {
|
||||
if (c && c.test !== undefined && Array.isArray(c.stubs)) out.push({ id: e.id, entry: e, cond: c });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const relImport = (fromFile, toFile) => {
|
||||
const r = toPosix(relative(dirname(fromFile), toFile));
|
||||
return r.startsWith('.') ? r : `./${r}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Build M9 in a fresh temp directory: every module a behaviour probe binds is
|
||||
* replaced by one whose exports keep their names and always return
|
||||
* {status:'PASSED'}, and every named test is replaced by one that only checks
|
||||
* that each export is a function. Returns the directory. `root` is only read.
|
||||
*/
|
||||
export function buildM9Tree(root, registry) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'yardstick-m9-'));
|
||||
const tests = new Map();
|
||||
const modules = new Set();
|
||||
for (const { cond } of behaviourConditions(registry)) {
|
||||
if (!tests.has(cond.test)) tests.set(cond.test, { names: new Set(), stubs: new Set() });
|
||||
tests.get(cond.test).names.add(cond.name);
|
||||
for (const s of cond.stubs) { tests.get(cond.test).stubs.add(s); modules.add(s); }
|
||||
}
|
||||
const write = (rel, body) => {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), body);
|
||||
};
|
||||
for (const mod of modules) {
|
||||
const source = readFileSync(join(root, mod), 'utf8');
|
||||
const names = new Set([...source.matchAll(EXPORT_NAME)].map((m) => m[1]));
|
||||
const lines = [...names].map((n) => `export function ${n}() { return { status: 'PASSED' }; }`);
|
||||
if (/^\s*export\s+default\b/m.test(source)) lines.push("export default function () { return { status: 'PASSED' }; }");
|
||||
write(mod, `${lines.join('\n')}\n`);
|
||||
}
|
||||
for (const [testRel, { names, stubs }] of tests) {
|
||||
const lines = ["import { test } from 'node:test';", "import { strict as assert } from 'node:assert';"];
|
||||
[...stubs].forEach((s, i) => lines.push(`import * as m${i} from '${relImport(testRel, s)}';`));
|
||||
for (const name of names) {
|
||||
const checks = [...stubs].map((_, i) => `for (const v of Object.values(m${i})) assert.equal(typeof v, 'function');`);
|
||||
lines.push(`test(${JSON.stringify(name)}, () => { ${checks.join(' ')} });`);
|
||||
}
|
||||
write(testRel, `${lines.join('\n')}\n`);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
// Run ONE named test; true = passed, false = failed, null = it did not run.
|
||||
function runNamedTest(root, testRel, name) {
|
||||
const env = { ...process.env };
|
||||
for (const key of Object.keys(env)) if (key.startsWith('NODE_TEST_')) delete env[key];
|
||||
const pattern = `^${String(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`;
|
||||
const r = spawnSync(process.execPath, ['--test', '--test-reporter=tap', `--test-name-pattern=${pattern}`, testRel],
|
||||
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env });
|
||||
for (const line of String(r.stdout ?? '').split('\n')) {
|
||||
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
||||
if (m && m[2].replace(/\\(.)/g, '$1').trim() === name) return m[1] === undefined;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function measureMutants(root, { registry, evaluate = evaluateCheck, runTest = runNamedTest } = {}) {
|
||||
const reg = registry ?? loadRegistry(root);
|
||||
const base = {
|
||||
id: 'mutants', label: 'signature-preserving mutants the end-state gate fells',
|
||||
source: 'scripts/end-state-gate.mjs behaviour probes vs M9 (exports return {status:"PASSED"}, tests check typeof)',
|
||||
denominator: 1,
|
||||
};
|
||||
const probes = behaviourConditions(reg);
|
||||
const entryIds = [...new Set(probes.map((p) => p.id))];
|
||||
if (probes.length === 0) {
|
||||
return { ...base, facts: { entries: [] }, ...notCountable('the registry has no behaviour probe to aim M9 at') };
|
||||
}
|
||||
let dir;
|
||||
try {
|
||||
dir = buildM9Tree(root, reg);
|
||||
} catch (err) {
|
||||
return { ...base, facts: { entries: entryIds }, ...notCountable(`M9 could not be built: ${err.message}`) };
|
||||
}
|
||||
try {
|
||||
return judgeM9(base, dir, probes, entryIds, evaluate, runTest);
|
||||
} finally {
|
||||
// Only ever the directory buildM9Tree made, under the system temp dir.
|
||||
if (realpathSync(dir).startsWith(realpathSync(tmpdir()))) rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function judgeM9(base, dir, probes, entryIds, evaluate, runTest) {
|
||||
// Liveness: M9's own tests must PASS on the M9 tree, or "felled" would mean "broken fixture".
|
||||
const dead = probes.filter((p) => runTest(dir, p.cond.test, p.cond.name) !== true).map((p) => p.cond.name);
|
||||
if (dead.length > 0) {
|
||||
return { ...base, facts: { entries: entryIds },
|
||||
...notCountable(`M9 is not live: ${dead.length} of its named tests do not pass on the M9 tree`) };
|
||||
}
|
||||
const results = [...new Map(probes.map((p) => [p.id, p.entry])).values()]
|
||||
.map((e) => ({ id: e.id, ...evaluate(dir, e.check) }));
|
||||
const closed = results.filter((r) => r.status === 'closed');
|
||||
const felled = closed.length === 0 ? 1 : 0;
|
||||
const per = results.map((r) => `${r.id} ${r.status === 'closed' ? 'closed' : `not closed (${r.status})`}`).join(', ');
|
||||
return {
|
||||
...base, countable: true, value: felled, facts: { entries: entryIds },
|
||||
detail: `M9 on a temp tree: ${per} - ${felled ? 'the gate refuses to count a typeof-only test as behaviour'
|
||||
: 'a typeof-only test closes the probe, so the gate counts a signature as behaviour'}`,
|
||||
};
|
||||
}
|
||||
|
||||
// --- the whole yardstick ---------------------------------------------------------
|
||||
|
||||
const VERDICT = {
|
||||
judged: false,
|
||||
met: false,
|
||||
detail: 'the verdict needs the first five deliveries through the new Voyage; nothing counts a delivery yet',
|
||||
};
|
||||
|
||||
export function measureAll({ root = REPO_ROOT, coordDir = DEFAULT_COORD, dataDir = DEFAULT_DATA, cutoff = DEFAULT_CUTOFF, countingSince = USE_COUNTING_SINCE, mutantOptions = {} } = {}) {
|
||||
const stats = loadStats(dataDir);
|
||||
const numbers = [
|
||||
measureUse({ queue: readOrders(coordDir), runs: orderRuns(stats), cutoff, countingSince }),
|
||||
measureTrust(stats),
|
||||
measureInterrupts({ stats, callSites: emitterCallSites(root) }),
|
||||
];
|
||||
const mutants = measureMutants(root, mutantOptions);
|
||||
const countable = numbers.filter((r) => r.countable).length;
|
||||
const felled = mutants.countable && mutants.value === mutants.denominator;
|
||||
return {
|
||||
green: countable === numbers.length && felled && VERDICT.judged && VERDICT.met,
|
||||
countable,
|
||||
rows: [...numbers, mutants],
|
||||
verdict: VERDICT,
|
||||
};
|
||||
}
|
||||
|
||||
const fmt = (v) => (v === null || v === undefined ? 'n/a' : String(v));
|
||||
|
||||
export function render(result) {
|
||||
const m = result.rows.find((r) => r.id === 'mutants');
|
||||
const out = [];
|
||||
out.push(`Voyage yardstick: ${result.green ? 'GREEN' : 'RED'} (${result.countable} of 3 numbers countable)`);
|
||||
out.push(`signature-preserving mutants felled: ${m.countable ? `${m.value} of ${m.denominator}` : 'n/a (not measurable)'}`);
|
||||
out.push(`verdict: ${result.verdict.judged ? (result.verdict.met ? 'MET' : 'NOT MET') : 'NOT JUDGED'} - ${result.verdict.detail}`);
|
||||
out.push('');
|
||||
out.push('| number | countable | value | denominator | source |');
|
||||
out.push('|---|---|---|---|---|');
|
||||
for (const r of result.rows) {
|
||||
out.push(`| ${r.id} | ${r.countable ? 'YES' : 'NO'} | ${fmt(r.value)} | ${fmt(r.denominator)} | ${r.source} |`);
|
||||
}
|
||||
for (const r of result.rows) {
|
||||
out.push('');
|
||||
out.push(`${r.id} - ${r.label}: ${r.detail}`);
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
const opts = {};
|
||||
let json = false;
|
||||
const usage = 'usage: yardstick.mjs [--json] [--root <repo>] [--coord <dir>] [--data <dir>] [--cutoff YYYY-MM-DD]';
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
const value = argv[i + 1];
|
||||
if (a === '--json') { json = true; continue; }
|
||||
if (['--root', '--coord', '--data', '--cutoff'].includes(a) && value !== undefined) {
|
||||
i++;
|
||||
if (a === '--cutoff') {
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
|
||||
process.stderr.write(`yardstick: --cutoff must be YYYY-MM-DD, got ${value}\n${usage}\n`);
|
||||
return 2;
|
||||
}
|
||||
opts.cutoff = value;
|
||||
} else {
|
||||
opts[{ '--root': 'root', '--coord': 'coordDir', '--data': 'dataDir' }[a]] = resolve(value);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
process.stderr.write(`yardstick: unknown argument ${a}\n${usage}\n`);
|
||||
return 2;
|
||||
}
|
||||
const result = measureAll(opts);
|
||||
process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (isMain(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue