feat(measure): the comparison gate counts the 14 areas against five frameworks and the tidy denominator, and it is RED [skip-docs]
The lifted target (operator, 2026-09-27; PM order 44111111): Voyage + the
operator's way of working level with or ahead of related frameworks in all
14 areas, and every Voyage part a documented Claude Code primitive covers
removed or with a measured reason to stay.
scripts/comparison-gate.mjs reads both denominators from their source, never
from the table: the areas from the first column of the PM report's table,
the parts from git ls-files (one part per eval case). It fells the table on
missing evidence, never on a verdict: a cited file must exist inside its
root and hold the cited lines, a pinned clone must sit at its pin, a web
citation needs a verbatim quote, an absence needs zero hits and a
known-positive control. Green also requires an independent reader. The
table itself is local-only (docs/comparison.local.json).
Measured today on 7439dac: RED, 0 evidence problems; 2 of 14 areas against
all five references (Superpowers 11, Pocock 14, agent-skills 11, Cherny 10,
playbook 2); tidy 5 of 64 covered parts resolved.
23 tests in tests/scripts/comparison-gate.test.mjs; six mutants felled (no
evidence check, no line check, no reader check, denominator from the table,
no tidy completeness, no pin check). Suite 1278 -> 1301 (1299/0/2).
[skip-docs]: an operator measurement instrument like scripts/yardstick.mjs,
whose table is local-only - no user-facing behaviour changes, so README and
CLAUDE.md have nothing true to say about it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
7439dac927
commit
e9a4abc34f
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 763 additions and 0 deletions
341
scripts/comparison-gate.mjs
Normal file
341
scripts/comparison-gate.mjs
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/comparison-gate.mjs
|
||||
// The comparison gate - the lifted target in PLAN.md (operator, 2026-09-27): Voyage + the
|
||||
// operator's way of working is level with or ahead of the related frameworks in every area,
|
||||
// and every Voyage part a built-in Claude Code primitive covers is removed or has a measured
|
||||
// reason to stay. Two counts, each with a denominator read from its SOURCE:
|
||||
//
|
||||
// comparison areas where no reference is ahead (the areas: the first column of the table
|
||||
// under `source.heading` in `source.loc`;
|
||||
// counted = areas where every reference
|
||||
// cell's verdict is `likt` or `foran`)
|
||||
// tidy covered parts that are resolved (the parts: `git ls-files` over PART_SCOPE,
|
||||
// one part per eval case; counted = parts a
|
||||
// primitive covers that are `fjernet` or
|
||||
// carry a `maalt-grunn` with evidence)
|
||||
//
|
||||
// It fells the table on MISSING EVIDENCE, never on a verdict it disagrees with: a cited file
|
||||
// must exist inside its root and hold the cited lines, a pinned clone must sit at its pin, a
|
||||
// web citation needs a verbatim quote, and an absence needs zero hits plus a known-positive
|
||||
// control. A verdict is a reader's judgement; green also requires `reader.kind: uavhengig`.
|
||||
//
|
||||
// Exit 0 = both counts full, no evidence problem, independent reader; 1 = anything short of
|
||||
// that, including a table that does not exist (NOT MEASURED, never 0); 2 = usage error.
|
||||
// It only READS. The table is local-only (docs/comparison.local.json, `*.local.*`).
|
||||
//
|
||||
// Usage: node scripts/comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]
|
||||
|
||||
import { readFileSync, existsSync, statSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, resolve, dirname, relative, isAbsolute, sep } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export const DEFAULT_DATA = join(REPO_ROOT, 'docs', 'comparison.local.json');
|
||||
export const PART_SCOPE = ['commands', 'agents', 'hooks', 'lib', 'scripts', 'templates', 'evals', 'settings.json', 'verify.sh'];
|
||||
export const VERDICTS = ['foran', 'likt', 'bak'];
|
||||
const LEVEL_OR_AHEAD = new Set(['foran', 'likt']);
|
||||
export const COVERAGE = ['dekket', 'delvis', 'ikke-dekket'];
|
||||
export const RESOLUTIONS = ['maalt-grunn', 'aapen', 'fjernet'];
|
||||
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
const nonEmpty = (s) => typeof s === 'string' && s.trim() !== '';
|
||||
|
||||
// `ROOT:path:N` or `ROOT:path:N-M`, lines counted from 1. Anything else is not a loc.
|
||||
export function parseLoc(text) {
|
||||
const m = /^([A-Z]{2,}):(.+):(\d+)(?:-(\d+))?$/.exec(String(text ?? ''));
|
||||
if (!m) return null;
|
||||
const start = Number(m[3]);
|
||||
const end = m[4] === undefined ? start : Number(m[4]);
|
||||
if (start < 1 || end < start) return null;
|
||||
return { root: m[1], path: m[2], start, end };
|
||||
}
|
||||
|
||||
// The first column of the first table after `heading`, header and separator rows excluded.
|
||||
export function parseSourceAreas(markdown, heading) {
|
||||
const all = String(markdown).split('\n');
|
||||
const at = all.findIndex((l) => l.trim() === heading);
|
||||
if (at === -1) return null;
|
||||
const rows = [];
|
||||
let inTable = false;
|
||||
for (const line of all.slice(at + 1)) {
|
||||
if (/^#{1,6}\s/.test(line)) break;
|
||||
if (line.trim().startsWith('|')) {
|
||||
inTable = true;
|
||||
rows.push(line);
|
||||
} else if (inTable) break;
|
||||
}
|
||||
return rows.slice(2).map((r) => r.split('|')[1].trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
// One part per tracked file in PART_SCOPE, except eval cases: one part per `evals/<case>/`.
|
||||
export function listParts(root) {
|
||||
const out = execFileSync('git', ['ls-files', '-z', '--', ...PART_SCOPE], { cwd: root, encoding: 'utf-8' });
|
||||
const parts = new Set();
|
||||
for (const f of out.split('\0').filter(Boolean)) {
|
||||
const segs = f.split('/');
|
||||
parts.add(segs[0] === 'evals' && segs.length > 2 ? `evals/${segs[1]}/` : f);
|
||||
}
|
||||
return [...parts].sort();
|
||||
}
|
||||
|
||||
function resolveRoot(value) {
|
||||
if (!nonEmpty(value)) return null;
|
||||
if (value === '~' || value.startsWith('~/')) return join(homedir(), value.slice(2));
|
||||
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
|
||||
}
|
||||
|
||||
function lineCount(file) {
|
||||
const text = readFileSync(file, 'utf-8');
|
||||
if (text === '') return 0;
|
||||
return text.split('\n').length - (text.endsWith('\n') ? 1 : 0);
|
||||
}
|
||||
|
||||
// Problems with one evidence item, each prefixed with `where`.
|
||||
function checkEvidence(item, where, roots) {
|
||||
if (!item || typeof item !== 'object') return [`${where}: an evidence item must be an object`];
|
||||
const problems = [];
|
||||
if (!nonEmpty(item.says)) problems.push(`${where}: evidence needs \`says\``);
|
||||
if ('loc' in item) {
|
||||
const loc = parseLoc(item.loc);
|
||||
if (!loc) return [...problems, `${where}: ${item.loc} is not ROOT:path:N[-M]`];
|
||||
const root = roots[loc.root];
|
||||
if (!root) return [...problems, `${where}: ${item.loc} names an unknown root ${loc.root}`];
|
||||
const file = resolve(root, loc.path);
|
||||
const rel = relative(root, file);
|
||||
if (rel.startsWith('..') || isAbsolute(rel)) return [...problems, `${where}: ${item.loc} points outside its root`];
|
||||
if (!existsSync(file) || !statSync(file).isFile()) return [...problems, `${where}: ${item.loc} does not exist`];
|
||||
const n = lineCount(file);
|
||||
if (loc.end > n) problems.push(`${where}: ${item.loc} cites past the end of a file of ${n} lines`);
|
||||
return problems;
|
||||
}
|
||||
if ('url' in item) {
|
||||
if (!/^https:\/\/\S+$/.test(String(item.url))) problems.push(`${where}: ${item.url} is not an https URL`);
|
||||
if (!nonEmpty(item.quote)) problems.push(`${where}: a web citation needs a verbatim \`quote\``);
|
||||
return problems;
|
||||
}
|
||||
if ('absence' in item) {
|
||||
const a = item.absence ?? {};
|
||||
if (!nonEmpty(a.query)) problems.push(`${where}: an absence needs the \`query\` that was run`);
|
||||
if (a.hits !== 0) problems.push(`${where}: an absence must report hits: 0, got ${JSON.stringify(a.hits)}`);
|
||||
if (!nonEmpty(a.known_positive)) problems.push(`${where}: an absence needs a \`known_positive\` control`);
|
||||
return problems;
|
||||
}
|
||||
return [...problems, `${where}: evidence must be one of loc, url or absence`];
|
||||
}
|
||||
|
||||
function checkEvidenceList(list, where, roots) {
|
||||
if (!Array.isArray(list) || list.length === 0) return [`${where}: no evidence`];
|
||||
return list.flatMap((item, i) => checkEvidence(item, `${where}[${i}]`, roots));
|
||||
}
|
||||
|
||||
function pinProblems(references, roots) {
|
||||
const problems = [];
|
||||
for (const ref of references) {
|
||||
if (!nonEmpty(ref.pin)) continue;
|
||||
const root = roots[ref.id];
|
||||
if (!root || !existsSync(join(root, '.git'))) {
|
||||
problems.push(`reference ${ref.id}: pinned ${ref.pin} but ${root ?? 'no root'} is not a git clone`);
|
||||
continue;
|
||||
}
|
||||
let head = '';
|
||||
try {
|
||||
head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf-8' }).trim();
|
||||
} catch { /* reported below */ }
|
||||
if (!head.startsWith(ref.pin)) problems.push(`reference ${ref.id}: pinned ${ref.pin} but the clone's HEAD is ${head.slice(0, 7) || 'unreadable'}`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function evaluateComparison(data, roots) {
|
||||
const problems = [];
|
||||
const references = Array.isArray(data.references) ? data.references : [];
|
||||
if (references.length === 0) problems.push('no references to compare against');
|
||||
problems.push(...pinProblems(references, roots));
|
||||
|
||||
let sourceAreas = null;
|
||||
const src = parseLoc(`${data.source?.loc ?? ''}:1`);
|
||||
const srcFile = src && roots[src.root] ? resolve(roots[src.root], src.path) : null;
|
||||
if (!srcFile || !existsSync(srcFile)) {
|
||||
problems.push(`source ${data.source?.loc ?? '(none)'}: not readable - the denominator cannot be read`);
|
||||
} else {
|
||||
sourceAreas = parseSourceAreas(readFileSync(srcFile, 'utf-8'), data.source.heading);
|
||||
if (!sourceAreas || sourceAreas.length === 0) {
|
||||
problems.push(`source ${data.source.loc}: no table under ${JSON.stringify(data.source.heading)}`);
|
||||
sourceAreas = null;
|
||||
}
|
||||
}
|
||||
|
||||
const areas = Array.isArray(data.areas) ? data.areas : [];
|
||||
const byTitle = new Map(areas.map((a) => [a.title, a]));
|
||||
const counted = [];
|
||||
const behind = [];
|
||||
const perReference = Object.fromEntries(references.map((r) => [r.id, 0]));
|
||||
const contested = [];
|
||||
for (const title of sourceAreas ?? []) {
|
||||
if (!byTitle.has(title)) problems.push(`area ${title}: in the source but missing from the table`);
|
||||
}
|
||||
for (const area of areas) {
|
||||
const id = area.id ?? area.title;
|
||||
if (sourceAreas && !sourceAreas.includes(area.title)) {
|
||||
problems.push(`area ${area.title}: not in the source - not counted`);
|
||||
continue;
|
||||
}
|
||||
problems.push(...checkEvidenceList(area.own?.evidence, `${id} own`, roots));
|
||||
const behindRefs = [];
|
||||
for (const ref of references) {
|
||||
const cell = area.cells?.[ref.id];
|
||||
if (!cell) { problems.push(`${id} ${ref.id}: cell missing`); continue; }
|
||||
if (!VERDICTS.includes(cell.verdict)) problems.push(`${id} ${ref.id}: verdict ${JSON.stringify(cell.verdict)} is not one of ${VERDICTS.join('|')}`);
|
||||
if (!nonEmpty(cell.why)) problems.push(`${id} ${ref.id}: a verdict needs \`why\``);
|
||||
problems.push(...checkEvidenceList(cell.evidence, `${id} ${ref.id}`, roots));
|
||||
if (cell.contested) contested.push(`${id} (${ref.id})`);
|
||||
if (!LEVEL_OR_AHEAD.has(cell.verdict)) behindRefs.push(ref.id);
|
||||
else perReference[ref.id]++;
|
||||
}
|
||||
if (behindRefs.length) behind.push({ area: id, references: behindRefs });
|
||||
else counted.push(id);
|
||||
}
|
||||
const total = sourceAreas ? sourceAreas.length : null;
|
||||
return {
|
||||
problems,
|
||||
comparison: {
|
||||
total,
|
||||
level_or_ahead: total === null ? null : counted.length,
|
||||
behind,
|
||||
per_reference: perReference,
|
||||
contested,
|
||||
references: references.map((r) => `${r.id}${r.pin ? `@${r.pin}` : ''} ${r.name ?? ''}`.trim()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateTidy(tidy, parts, roots) {
|
||||
const problems = [];
|
||||
if (!tidy || typeof tidy !== 'object') {
|
||||
return { problems: ['tidy: missing - the tidy denominator is not measured'], tidy: { parts: parts.length, covered: null, resolved: null } };
|
||||
}
|
||||
const primitives = Array.isArray(tidy.primitives) ? tidy.primitives : [];
|
||||
const known = new Set();
|
||||
for (const p of primitives) {
|
||||
known.add(p.id);
|
||||
if (!/^https:\/\/\S+$/.test(String(p.doc ?? ''))) problems.push(`primitive ${p.id}: needs a documentation URL`);
|
||||
if (!nonEmpty(p.quote)) problems.push(`primitive ${p.id}: needs a verbatim \`quote\` from its documentation`);
|
||||
}
|
||||
const listed = Array.isArray(tidy.parts) ? tidy.parts : [];
|
||||
const tree = new Set(parts);
|
||||
const seen = new Set();
|
||||
let covered = 0;
|
||||
let resolved = 0;
|
||||
for (const part of listed) {
|
||||
const where = `part ${part.path}`;
|
||||
if (seen.has(part.path)) problems.push(`${where}: listed twice`);
|
||||
seen.add(part.path);
|
||||
if (!COVERAGE.includes(part.coverage)) { problems.push(`${where}: coverage ${JSON.stringify(part.coverage)} is not one of ${COVERAGE.join('|')}`); continue; }
|
||||
if (part.resolution !== 'fjernet' && !tree.has(part.path)) problems.push(`${where}: not in the tree and not marked fjernet`);
|
||||
if (part.coverage === 'ikke-dekket') continue;
|
||||
covered++;
|
||||
const by = Array.isArray(part.covered_by) ? part.covered_by : [];
|
||||
if (by.length === 0) problems.push(`${where}: ${part.coverage} needs \`covered_by\``);
|
||||
for (const id of by) if (!known.has(id)) problems.push(`${where}: covered by an unknown primitive ${id}`);
|
||||
if (!RESOLUTIONS.includes(part.resolution)) {
|
||||
problems.push(`${where}: resolution ${JSON.stringify(part.resolution)} is not one of ${RESOLUTIONS.join('|')}`);
|
||||
} else if (part.resolution === 'maalt-grunn') {
|
||||
if (!part.reason) problems.push(`${where}: maalt-grunn needs a \`reason\` with evidence`);
|
||||
else {
|
||||
const p = checkEvidence(part.reason, `${where} reason`, roots);
|
||||
problems.push(...p);
|
||||
if (p.length === 0) resolved++;
|
||||
}
|
||||
} else if (part.resolution === 'fjernet') {
|
||||
if (tree.has(part.path)) problems.push(`${where}: marked fjernet but still in the tree`);
|
||||
else if (!nonEmpty(part.commit)) problems.push(`${where}: fjernet needs the \`commit\` that removed it`);
|
||||
else resolved++;
|
||||
}
|
||||
}
|
||||
for (const p of parts) if (!seen.has(p)) problems.push(`part ${p}: in the tree but missing from the tidy list`);
|
||||
return { problems, tidy: { parts: parts.length, covered, resolved, primitives: primitives.length } };
|
||||
}
|
||||
|
||||
export function evaluate(data, { parts = [] } = {}) {
|
||||
const roots = {};
|
||||
for (const [id, value] of Object.entries(data?.roots ?? {})) roots[id] = resolveRoot(value);
|
||||
const cmp = evaluateComparison(data ?? {}, roots);
|
||||
const tdy = evaluateTidy(data?.tidy, parts, roots);
|
||||
const problems = [...cmp.problems, ...tdy.problems];
|
||||
const reader = data?.reader ?? {};
|
||||
const reasons = [];
|
||||
if (problems.length) reasons.push(`${problems.length} evidence problem(s)`);
|
||||
const c = cmp.comparison;
|
||||
if (c.total === null) reasons.push('the comparison denominator is not measured');
|
||||
else if (c.level_or_ahead < c.total) reasons.push(`${c.level_or_ahead} of ${c.total} areas level or ahead`);
|
||||
if (reader.kind !== 'uavhengig') reasons.push(`the verdicts are by ${reader.kind ?? 'nobody'}, not an independent reader`);
|
||||
if (tdy.tidy.covered === null) reasons.push('the tidy list is not measured');
|
||||
else if (tdy.tidy.resolved < tdy.tidy.covered) reasons.push(`${tdy.tidy.resolved} of ${tdy.tidy.covered} covered parts resolved`);
|
||||
return {
|
||||
measured: true,
|
||||
reader,
|
||||
comparison: c,
|
||||
tidy: tdy.tidy,
|
||||
problems,
|
||||
reasons,
|
||||
green: reasons.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function render(r) {
|
||||
if (!r.measured) return `Comparison gate: NOT MEASURED - ${r.reasons.join('; ')}\nRED`;
|
||||
const c = r.comparison;
|
||||
const out = [];
|
||||
out.push(`Comparison gate - areas where no reference is ahead of Voyage + the operator's way of working`);
|
||||
out.push(` references: ${c.references.join(' · ')}`);
|
||||
out.push(` level or ahead: ${c.level_or_ahead ?? 'NOT MEASURED'} of ${c.total ?? '?'}`);
|
||||
for (const b of c.behind) out.push(` behind: ${b.area} (${b.references.join(', ')})`);
|
||||
out.push(` per reference: ${Object.entries(c.per_reference).map(([id, n]) => `${id}: ${n} of ${c.total ?? '?'}`).join(' · ')}`);
|
||||
if (c.contested.length) out.push(` contested cells: ${c.contested.join(', ')}`);
|
||||
out.push(` reader: ${r.reader.kind ?? 'none'}${r.reader.who ? ` - ${r.reader.who}` : ''}${r.reader.date ? ` (${r.reader.date})` : ''}`
|
||||
+ `${r.reader.kind === 'uavhengig' ? '' : ' - green requires an independent reader'}`);
|
||||
const t = r.tidy;
|
||||
out.push(`Tidy - Voyage parts a documented Claude Code primitive covers`);
|
||||
out.push(` parts in the tree: ${t.parts} · covered: ${t.covered ?? 'NOT MEASURED'} · resolved: ${t.resolved ?? '-'} of ${t.covered ?? '-'}`);
|
||||
out.push(`Evidence problems: ${r.problems.length}`);
|
||||
for (const p of r.problems) out.push(` - ${p}`);
|
||||
out.push(r.green ? 'GREEN' : `RED - ${r.reasons.join('; ')}`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let json = false;
|
||||
let dataFile = DEFAULT_DATA;
|
||||
let partsFrom = REPO_ROOT;
|
||||
const usage = 'usage: comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]';
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
const value = argv[i + 1];
|
||||
if (a === '--json') { json = true; continue; }
|
||||
if (a === '--data' && value !== undefined) { dataFile = resolve(value); i++; continue; }
|
||||
if (a === '--parts-from' && value !== undefined) { partsFrom = resolve(value); i++; continue; }
|
||||
process.stderr.write(`comparison-gate: unknown argument ${a}\n${usage}\n`);
|
||||
return 2;
|
||||
}
|
||||
let result;
|
||||
if (!existsSync(dataFile)) {
|
||||
result = { measured: false, green: false, reasons: [`${toPosix(relative(REPO_ROOT, dataFile))} does not exist`] };
|
||||
} else {
|
||||
let data = null;
|
||||
try {
|
||||
data = JSON.parse(readFileSync(dataFile, 'utf-8'));
|
||||
} catch (err) {
|
||||
result = { measured: false, green: false, reasons: [`${dataFile} is not valid JSON: ${err.message}`] };
|
||||
}
|
||||
if (data) result = evaluate(data, { parts: listParts(partsFrom) });
|
||||
}
|
||||
process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue