feat(measure): the comparison gate counts the 14 areas against five frameworks and the tidy denominator, and it is RED [skip-docs]
The lifted target (operator, 2026-09-27; PM order 44111111): Voyage + the
operator's way of working level with or ahead of related frameworks in all
14 areas, and every Voyage part a documented Claude Code primitive covers
removed or with a measured reason to stay.
scripts/comparison-gate.mjs reads both denominators from their source, never
from the table: the areas from the first column of the PM report's table,
the parts from git ls-files (one part per eval case). It fells the table on
missing evidence, never on a verdict: a cited file must exist inside its
root and hold the cited lines, a pinned clone must sit at its pin, a web
citation needs a verbatim quote, an absence needs zero hits and a
known-positive control. Green also requires an independent reader. The
table itself is local-only (docs/comparison.local.json).
Measured today on 7439dac: RED, 0 evidence problems; 2 of 14 areas against
all five references (Superpowers 11, Pocock 14, agent-skills 11, Cherny 10,
playbook 2); tidy 5 of 64 covered parts resolved.
23 tests in tests/scripts/comparison-gate.test.mjs; six mutants felled (no
evidence check, no line check, no reader check, denominator from the table,
no tidy completeness, no pin check). Suite 1278 -> 1301 (1299/0/2).
[skip-docs]: an operator measurement instrument like scripts/yardstick.mjs,
whose table is local-only - no user-facing behaviour changes, so README and
CLAUDE.md have nothing true to say about it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
7439dac927
commit
e9a4abc34f
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 763 additions and 0 deletions
341
scripts/comparison-gate.mjs
Normal file
341
scripts/comparison-gate.mjs
Normal file
|
|
@ -0,0 +1,341 @@
|
|||
#!/usr/bin/env node
|
||||
// scripts/comparison-gate.mjs
|
||||
// The comparison gate - the lifted target in PLAN.md (operator, 2026-09-27): Voyage + the
|
||||
// operator's way of working is level with or ahead of the related frameworks in every area,
|
||||
// and every Voyage part a built-in Claude Code primitive covers is removed or has a measured
|
||||
// reason to stay. Two counts, each with a denominator read from its SOURCE:
|
||||
//
|
||||
// comparison areas where no reference is ahead (the areas: the first column of the table
|
||||
// under `source.heading` in `source.loc`;
|
||||
// counted = areas where every reference
|
||||
// cell's verdict is `likt` or `foran`)
|
||||
// tidy covered parts that are resolved (the parts: `git ls-files` over PART_SCOPE,
|
||||
// one part per eval case; counted = parts a
|
||||
// primitive covers that are `fjernet` or
|
||||
// carry a `maalt-grunn` with evidence)
|
||||
//
|
||||
// It fells the table on MISSING EVIDENCE, never on a verdict it disagrees with: a cited file
|
||||
// must exist inside its root and hold the cited lines, a pinned clone must sit at its pin, a
|
||||
// web citation needs a verbatim quote, and an absence needs zero hits plus a known-positive
|
||||
// control. A verdict is a reader's judgement; green also requires `reader.kind: uavhengig`.
|
||||
//
|
||||
// Exit 0 = both counts full, no evidence problem, independent reader; 1 = anything short of
|
||||
// that, including a table that does not exist (NOT MEASURED, never 0); 2 = usage error.
|
||||
// It only READS. The table is local-only (docs/comparison.local.json, `*.local.*`).
|
||||
//
|
||||
// Usage: node scripts/comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]
|
||||
|
||||
import { readFileSync, existsSync, statSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, resolve, dirname, relative, isAbsolute, sep } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export const DEFAULT_DATA = join(REPO_ROOT, 'docs', 'comparison.local.json');
|
||||
export const PART_SCOPE = ['commands', 'agents', 'hooks', 'lib', 'scripts', 'templates', 'evals', 'settings.json', 'verify.sh'];
|
||||
export const VERDICTS = ['foran', 'likt', 'bak'];
|
||||
const LEVEL_OR_AHEAD = new Set(['foran', 'likt']);
|
||||
export const COVERAGE = ['dekket', 'delvis', 'ikke-dekket'];
|
||||
export const RESOLUTIONS = ['maalt-grunn', 'aapen', 'fjernet'];
|
||||
|
||||
const toPosix = (p) => p.split(sep).join('/');
|
||||
const nonEmpty = (s) => typeof s === 'string' && s.trim() !== '';
|
||||
|
||||
// `ROOT:path:N` or `ROOT:path:N-M`, lines counted from 1. Anything else is not a loc.
|
||||
export function parseLoc(text) {
|
||||
const m = /^([A-Z]{2,}):(.+):(\d+)(?:-(\d+))?$/.exec(String(text ?? ''));
|
||||
if (!m) return null;
|
||||
const start = Number(m[3]);
|
||||
const end = m[4] === undefined ? start : Number(m[4]);
|
||||
if (start < 1 || end < start) return null;
|
||||
return { root: m[1], path: m[2], start, end };
|
||||
}
|
||||
|
||||
// The first column of the first table after `heading`, header and separator rows excluded.
|
||||
export function parseSourceAreas(markdown, heading) {
|
||||
const all = String(markdown).split('\n');
|
||||
const at = all.findIndex((l) => l.trim() === heading);
|
||||
if (at === -1) return null;
|
||||
const rows = [];
|
||||
let inTable = false;
|
||||
for (const line of all.slice(at + 1)) {
|
||||
if (/^#{1,6}\s/.test(line)) break;
|
||||
if (line.trim().startsWith('|')) {
|
||||
inTable = true;
|
||||
rows.push(line);
|
||||
} else if (inTable) break;
|
||||
}
|
||||
return rows.slice(2).map((r) => r.split('|')[1].trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
// One part per tracked file in PART_SCOPE, except eval cases: one part per `evals/<case>/`.
|
||||
export function listParts(root) {
|
||||
const out = execFileSync('git', ['ls-files', '-z', '--', ...PART_SCOPE], { cwd: root, encoding: 'utf-8' });
|
||||
const parts = new Set();
|
||||
for (const f of out.split('\0').filter(Boolean)) {
|
||||
const segs = f.split('/');
|
||||
parts.add(segs[0] === 'evals' && segs.length > 2 ? `evals/${segs[1]}/` : f);
|
||||
}
|
||||
return [...parts].sort();
|
||||
}
|
||||
|
||||
function resolveRoot(value) {
|
||||
if (!nonEmpty(value)) return null;
|
||||
if (value === '~' || value.startsWith('~/')) return join(homedir(), value.slice(2));
|
||||
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
|
||||
}
|
||||
|
||||
function lineCount(file) {
|
||||
const text = readFileSync(file, 'utf-8');
|
||||
if (text === '') return 0;
|
||||
return text.split('\n').length - (text.endsWith('\n') ? 1 : 0);
|
||||
}
|
||||
|
||||
// Problems with one evidence item, each prefixed with `where`.
|
||||
function checkEvidence(item, where, roots) {
|
||||
if (!item || typeof item !== 'object') return [`${where}: an evidence item must be an object`];
|
||||
const problems = [];
|
||||
if (!nonEmpty(item.says)) problems.push(`${where}: evidence needs \`says\``);
|
||||
if ('loc' in item) {
|
||||
const loc = parseLoc(item.loc);
|
||||
if (!loc) return [...problems, `${where}: ${item.loc} is not ROOT:path:N[-M]`];
|
||||
const root = roots[loc.root];
|
||||
if (!root) return [...problems, `${where}: ${item.loc} names an unknown root ${loc.root}`];
|
||||
const file = resolve(root, loc.path);
|
||||
const rel = relative(root, file);
|
||||
if (rel.startsWith('..') || isAbsolute(rel)) return [...problems, `${where}: ${item.loc} points outside its root`];
|
||||
if (!existsSync(file) || !statSync(file).isFile()) return [...problems, `${where}: ${item.loc} does not exist`];
|
||||
const n = lineCount(file);
|
||||
if (loc.end > n) problems.push(`${where}: ${item.loc} cites past the end of a file of ${n} lines`);
|
||||
return problems;
|
||||
}
|
||||
if ('url' in item) {
|
||||
if (!/^https:\/\/\S+$/.test(String(item.url))) problems.push(`${where}: ${item.url} is not an https URL`);
|
||||
if (!nonEmpty(item.quote)) problems.push(`${where}: a web citation needs a verbatim \`quote\``);
|
||||
return problems;
|
||||
}
|
||||
if ('absence' in item) {
|
||||
const a = item.absence ?? {};
|
||||
if (!nonEmpty(a.query)) problems.push(`${where}: an absence needs the \`query\` that was run`);
|
||||
if (a.hits !== 0) problems.push(`${where}: an absence must report hits: 0, got ${JSON.stringify(a.hits)}`);
|
||||
if (!nonEmpty(a.known_positive)) problems.push(`${where}: an absence needs a \`known_positive\` control`);
|
||||
return problems;
|
||||
}
|
||||
return [...problems, `${where}: evidence must be one of loc, url or absence`];
|
||||
}
|
||||
|
||||
function checkEvidenceList(list, where, roots) {
|
||||
if (!Array.isArray(list) || list.length === 0) return [`${where}: no evidence`];
|
||||
return list.flatMap((item, i) => checkEvidence(item, `${where}[${i}]`, roots));
|
||||
}
|
||||
|
||||
function pinProblems(references, roots) {
|
||||
const problems = [];
|
||||
for (const ref of references) {
|
||||
if (!nonEmpty(ref.pin)) continue;
|
||||
const root = roots[ref.id];
|
||||
if (!root || !existsSync(join(root, '.git'))) {
|
||||
problems.push(`reference ${ref.id}: pinned ${ref.pin} but ${root ?? 'no root'} is not a git clone`);
|
||||
continue;
|
||||
}
|
||||
let head = '';
|
||||
try {
|
||||
head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf-8' }).trim();
|
||||
} catch { /* reported below */ }
|
||||
if (!head.startsWith(ref.pin)) problems.push(`reference ${ref.id}: pinned ${ref.pin} but the clone's HEAD is ${head.slice(0, 7) || 'unreadable'}`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function evaluateComparison(data, roots) {
|
||||
const problems = [];
|
||||
const references = Array.isArray(data.references) ? data.references : [];
|
||||
if (references.length === 0) problems.push('no references to compare against');
|
||||
problems.push(...pinProblems(references, roots));
|
||||
|
||||
let sourceAreas = null;
|
||||
const src = parseLoc(`${data.source?.loc ?? ''}:1`);
|
||||
const srcFile = src && roots[src.root] ? resolve(roots[src.root], src.path) : null;
|
||||
if (!srcFile || !existsSync(srcFile)) {
|
||||
problems.push(`source ${data.source?.loc ?? '(none)'}: not readable - the denominator cannot be read`);
|
||||
} else {
|
||||
sourceAreas = parseSourceAreas(readFileSync(srcFile, 'utf-8'), data.source.heading);
|
||||
if (!sourceAreas || sourceAreas.length === 0) {
|
||||
problems.push(`source ${data.source.loc}: no table under ${JSON.stringify(data.source.heading)}`);
|
||||
sourceAreas = null;
|
||||
}
|
||||
}
|
||||
|
||||
const areas = Array.isArray(data.areas) ? data.areas : [];
|
||||
const byTitle = new Map(areas.map((a) => [a.title, a]));
|
||||
const counted = [];
|
||||
const behind = [];
|
||||
const perReference = Object.fromEntries(references.map((r) => [r.id, 0]));
|
||||
const contested = [];
|
||||
for (const title of sourceAreas ?? []) {
|
||||
if (!byTitle.has(title)) problems.push(`area ${title}: in the source but missing from the table`);
|
||||
}
|
||||
for (const area of areas) {
|
||||
const id = area.id ?? area.title;
|
||||
if (sourceAreas && !sourceAreas.includes(area.title)) {
|
||||
problems.push(`area ${area.title}: not in the source - not counted`);
|
||||
continue;
|
||||
}
|
||||
problems.push(...checkEvidenceList(area.own?.evidence, `${id} own`, roots));
|
||||
const behindRefs = [];
|
||||
for (const ref of references) {
|
||||
const cell = area.cells?.[ref.id];
|
||||
if (!cell) { problems.push(`${id} ${ref.id}: cell missing`); continue; }
|
||||
if (!VERDICTS.includes(cell.verdict)) problems.push(`${id} ${ref.id}: verdict ${JSON.stringify(cell.verdict)} is not one of ${VERDICTS.join('|')}`);
|
||||
if (!nonEmpty(cell.why)) problems.push(`${id} ${ref.id}: a verdict needs \`why\``);
|
||||
problems.push(...checkEvidenceList(cell.evidence, `${id} ${ref.id}`, roots));
|
||||
if (cell.contested) contested.push(`${id} (${ref.id})`);
|
||||
if (!LEVEL_OR_AHEAD.has(cell.verdict)) behindRefs.push(ref.id);
|
||||
else perReference[ref.id]++;
|
||||
}
|
||||
if (behindRefs.length) behind.push({ area: id, references: behindRefs });
|
||||
else counted.push(id);
|
||||
}
|
||||
const total = sourceAreas ? sourceAreas.length : null;
|
||||
return {
|
||||
problems,
|
||||
comparison: {
|
||||
total,
|
||||
level_or_ahead: total === null ? null : counted.length,
|
||||
behind,
|
||||
per_reference: perReference,
|
||||
contested,
|
||||
references: references.map((r) => `${r.id}${r.pin ? `@${r.pin}` : ''} ${r.name ?? ''}`.trim()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateTidy(tidy, parts, roots) {
|
||||
const problems = [];
|
||||
if (!tidy || typeof tidy !== 'object') {
|
||||
return { problems: ['tidy: missing - the tidy denominator is not measured'], tidy: { parts: parts.length, covered: null, resolved: null } };
|
||||
}
|
||||
const primitives = Array.isArray(tidy.primitives) ? tidy.primitives : [];
|
||||
const known = new Set();
|
||||
for (const p of primitives) {
|
||||
known.add(p.id);
|
||||
if (!/^https:\/\/\S+$/.test(String(p.doc ?? ''))) problems.push(`primitive ${p.id}: needs a documentation URL`);
|
||||
if (!nonEmpty(p.quote)) problems.push(`primitive ${p.id}: needs a verbatim \`quote\` from its documentation`);
|
||||
}
|
||||
const listed = Array.isArray(tidy.parts) ? tidy.parts : [];
|
||||
const tree = new Set(parts);
|
||||
const seen = new Set();
|
||||
let covered = 0;
|
||||
let resolved = 0;
|
||||
for (const part of listed) {
|
||||
const where = `part ${part.path}`;
|
||||
if (seen.has(part.path)) problems.push(`${where}: listed twice`);
|
||||
seen.add(part.path);
|
||||
if (!COVERAGE.includes(part.coverage)) { problems.push(`${where}: coverage ${JSON.stringify(part.coverage)} is not one of ${COVERAGE.join('|')}`); continue; }
|
||||
if (part.resolution !== 'fjernet' && !tree.has(part.path)) problems.push(`${where}: not in the tree and not marked fjernet`);
|
||||
if (part.coverage === 'ikke-dekket') continue;
|
||||
covered++;
|
||||
const by = Array.isArray(part.covered_by) ? part.covered_by : [];
|
||||
if (by.length === 0) problems.push(`${where}: ${part.coverage} needs \`covered_by\``);
|
||||
for (const id of by) if (!known.has(id)) problems.push(`${where}: covered by an unknown primitive ${id}`);
|
||||
if (!RESOLUTIONS.includes(part.resolution)) {
|
||||
problems.push(`${where}: resolution ${JSON.stringify(part.resolution)} is not one of ${RESOLUTIONS.join('|')}`);
|
||||
} else if (part.resolution === 'maalt-grunn') {
|
||||
if (!part.reason) problems.push(`${where}: maalt-grunn needs a \`reason\` with evidence`);
|
||||
else {
|
||||
const p = checkEvidence(part.reason, `${where} reason`, roots);
|
||||
problems.push(...p);
|
||||
if (p.length === 0) resolved++;
|
||||
}
|
||||
} else if (part.resolution === 'fjernet') {
|
||||
if (tree.has(part.path)) problems.push(`${where}: marked fjernet but still in the tree`);
|
||||
else if (!nonEmpty(part.commit)) problems.push(`${where}: fjernet needs the \`commit\` that removed it`);
|
||||
else resolved++;
|
||||
}
|
||||
}
|
||||
for (const p of parts) if (!seen.has(p)) problems.push(`part ${p}: in the tree but missing from the tidy list`);
|
||||
return { problems, tidy: { parts: parts.length, covered, resolved, primitives: primitives.length } };
|
||||
}
|
||||
|
||||
export function evaluate(data, { parts = [] } = {}) {
|
||||
const roots = {};
|
||||
for (const [id, value] of Object.entries(data?.roots ?? {})) roots[id] = resolveRoot(value);
|
||||
const cmp = evaluateComparison(data ?? {}, roots);
|
||||
const tdy = evaluateTidy(data?.tidy, parts, roots);
|
||||
const problems = [...cmp.problems, ...tdy.problems];
|
||||
const reader = data?.reader ?? {};
|
||||
const reasons = [];
|
||||
if (problems.length) reasons.push(`${problems.length} evidence problem(s)`);
|
||||
const c = cmp.comparison;
|
||||
if (c.total === null) reasons.push('the comparison denominator is not measured');
|
||||
else if (c.level_or_ahead < c.total) reasons.push(`${c.level_or_ahead} of ${c.total} areas level or ahead`);
|
||||
if (reader.kind !== 'uavhengig') reasons.push(`the verdicts are by ${reader.kind ?? 'nobody'}, not an independent reader`);
|
||||
if (tdy.tidy.covered === null) reasons.push('the tidy list is not measured');
|
||||
else if (tdy.tidy.resolved < tdy.tidy.covered) reasons.push(`${tdy.tidy.resolved} of ${tdy.tidy.covered} covered parts resolved`);
|
||||
return {
|
||||
measured: true,
|
||||
reader,
|
||||
comparison: c,
|
||||
tidy: tdy.tidy,
|
||||
problems,
|
||||
reasons,
|
||||
green: reasons.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function render(r) {
|
||||
if (!r.measured) return `Comparison gate: NOT MEASURED - ${r.reasons.join('; ')}\nRED`;
|
||||
const c = r.comparison;
|
||||
const out = [];
|
||||
out.push(`Comparison gate - areas where no reference is ahead of Voyage + the operator's way of working`);
|
||||
out.push(` references: ${c.references.join(' · ')}`);
|
||||
out.push(` level or ahead: ${c.level_or_ahead ?? 'NOT MEASURED'} of ${c.total ?? '?'}`);
|
||||
for (const b of c.behind) out.push(` behind: ${b.area} (${b.references.join(', ')})`);
|
||||
out.push(` per reference: ${Object.entries(c.per_reference).map(([id, n]) => `${id}: ${n} of ${c.total ?? '?'}`).join(' · ')}`);
|
||||
if (c.contested.length) out.push(` contested cells: ${c.contested.join(', ')}`);
|
||||
out.push(` reader: ${r.reader.kind ?? 'none'}${r.reader.who ? ` - ${r.reader.who}` : ''}${r.reader.date ? ` (${r.reader.date})` : ''}`
|
||||
+ `${r.reader.kind === 'uavhengig' ? '' : ' - green requires an independent reader'}`);
|
||||
const t = r.tidy;
|
||||
out.push(`Tidy - Voyage parts a documented Claude Code primitive covers`);
|
||||
out.push(` parts in the tree: ${t.parts} · covered: ${t.covered ?? 'NOT MEASURED'} · resolved: ${t.resolved ?? '-'} of ${t.covered ?? '-'}`);
|
||||
out.push(`Evidence problems: ${r.problems.length}`);
|
||||
for (const p of r.problems) out.push(` - ${p}`);
|
||||
out.push(r.green ? 'GREEN' : `RED - ${r.reasons.join('; ')}`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let json = false;
|
||||
let dataFile = DEFAULT_DATA;
|
||||
let partsFrom = REPO_ROOT;
|
||||
const usage = 'usage: comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]';
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
const value = argv[i + 1];
|
||||
if (a === '--json') { json = true; continue; }
|
||||
if (a === '--data' && value !== undefined) { dataFile = resolve(value); i++; continue; }
|
||||
if (a === '--parts-from' && value !== undefined) { partsFrom = resolve(value); i++; continue; }
|
||||
process.stderr.write(`comparison-gate: unknown argument ${a}\n${usage}\n`);
|
||||
return 2;
|
||||
}
|
||||
let result;
|
||||
if (!existsSync(dataFile)) {
|
||||
result = { measured: false, green: false, reasons: [`${toPosix(relative(REPO_ROOT, dataFile))} does not exist`] };
|
||||
} else {
|
||||
let data = null;
|
||||
try {
|
||||
data = JSON.parse(readFileSync(dataFile, 'utf-8'));
|
||||
} catch (err) {
|
||||
result = { measured: false, green: false, reasons: [`${dataFile} is not valid JSON: ${err.message}`] };
|
||||
}
|
||||
if (data) result = evaluate(data, { parts: listParts(partsFrom) });
|
||||
}
|
||||
process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
|
||||
return result.green ? 0 : 1;
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
422
tests/scripts/comparison-gate.test.mjs
Normal file
422
tests/scripts/comparison-gate.test.mjs
Normal file
|
|
@ -0,0 +1,422 @@
|
|||
// tests/scripts/comparison-gate.test.mjs
|
||||
// The comparison gate counts the areas where Voyage + the operator's way of working is
|
||||
// level with or ahead of every reference framework, and fells the table when a claim
|
||||
// has no evidence behind it. It also counts the "tidy" denominator: every Voyage part a
|
||||
// documented Claude Code primitive covers, and how many of those are resolved.
|
||||
//
|
||||
// Like the yardstick's tests, these pin the MECHANICS in both directions against
|
||||
// throwaway fixtures. They deliberately do NOT assert that the real table is red today:
|
||||
// the gate is supposed to turn green, and a test that pins today's distance would fail
|
||||
// on the day the work is done. Nothing here reads the real table or the real source.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import {
|
||||
parseLoc,
|
||||
parseSourceAreas,
|
||||
listParts,
|
||||
evaluate,
|
||||
render,
|
||||
main,
|
||||
} from '../../scripts/comparison-gate.mjs';
|
||||
|
||||
// --- helpers ---------------------------------------------------------------
|
||||
|
||||
const lines = (n) => `${Array.from({ length: n }, (_, i) => `line ${i + 1}`).join('\n')}\n`;
|
||||
|
||||
const SOURCE = [
|
||||
'# report',
|
||||
'',
|
||||
'## 2. Per kapabilitet',
|
||||
'',
|
||||
'| Kapabilitet | A | B |',
|
||||
'|---|---|---|',
|
||||
'| Alpha | x | y |',
|
||||
'| Beta | x | y |',
|
||||
'',
|
||||
'## 3. Next section',
|
||||
'',
|
||||
'| Not | an | area |',
|
||||
'|---|---|---|',
|
||||
'| Gamma | x | y |',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
function world() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'comparison-gate-'));
|
||||
const files = {
|
||||
'vy/own.md': lines(5),
|
||||
'pm/source.md': SOURCE,
|
||||
'pm/why.md': lines(3),
|
||||
'sp/ref.md': lines(10),
|
||||
};
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
const p = join(dir, rel);
|
||||
mkdirSync(dirname(p), { recursive: true });
|
||||
writeFileSync(p, body);
|
||||
}
|
||||
return { dir, roots: { VY: join(dir, 'vy'), PM: join(dir, 'pm'), SP: join(dir, 'sp') } };
|
||||
}
|
||||
|
||||
const cleanup = (w) => rmSync(w.dir, { recursive: true, force: true });
|
||||
|
||||
// A table that is fully green: every area level or ahead, every cell with evidence,
|
||||
// an independent reader, and the one covered part resolved with a measured reason.
|
||||
function greenData(w) {
|
||||
return {
|
||||
schema: 'voyage-comparison/1',
|
||||
source: { loc: 'PM:source.md', heading: '## 2. Per kapabilitet' },
|
||||
roots: w.roots,
|
||||
references: [{ id: 'SP', name: 'obra/superpowers' }],
|
||||
reader: { kind: 'uavhengig', who: 'fixture', date: '2026-10-02' },
|
||||
areas: [
|
||||
{
|
||||
id: 'alpha',
|
||||
title: 'Alpha',
|
||||
own: { evidence: [{ loc: 'VY:own.md:1-2', says: 'own side' }] },
|
||||
cells: {
|
||||
SP: { verdict: 'likt', why: 'level', evidence: [{ loc: 'SP:ref.md:3-10', says: 'ref side' }] },
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'beta',
|
||||
title: 'Beta',
|
||||
own: { evidence: [{ loc: 'VY:own.md:5', says: 'own side' }] },
|
||||
cells: {
|
||||
SP: {
|
||||
verdict: 'foran',
|
||||
why: 'the reference has nothing here',
|
||||
evidence: [{
|
||||
absence: { query: 'grep -rn beta .', hits: 0, known_positive: 'grep -rn line . finds 10' },
|
||||
says: 'no beta in the reference',
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
tidy: {
|
||||
primitives: [{ id: 'subagents', name: 'Subagents', doc: 'https://code.claude.com/docs/en/sub-agents', quote: 'q' }],
|
||||
parts: [
|
||||
{
|
||||
path: 'commands/x.md', covered_by: ['subagents'], coverage: 'delvis', why: 'w',
|
||||
resolution: 'maalt-grunn', reason: { loc: 'PM:why.md:1-3', says: 'measured' },
|
||||
},
|
||||
{ path: 'lib/y.mjs', covered_by: [], coverage: 'ikke-dekket', why: 'w', resolution: null, reason: null },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const PARTS = ['commands/x.md', 'lib/y.mjs'];
|
||||
const run = (w, data) => evaluate(data, { parts: PARTS });
|
||||
const problemText = (r) => r.problems.join('\n');
|
||||
|
||||
// --- parsing -----------------------------------------------------------------
|
||||
|
||||
test('parseLoc: root, path and a line or a line range', () => {
|
||||
assert.deepEqual(parseLoc('SP:skills/a/SKILL.md:38-88'), { root: 'SP', path: 'skills/a/SKILL.md', start: 38, end: 88 });
|
||||
assert.deepEqual(parseLoc('VY:lib/x.mjs:7'), { root: 'VY', path: 'lib/x.mjs', start: 7, end: 7 });
|
||||
assert.equal(parseLoc('lib/x.mjs:7'), null, 'a loc without a root is not evidence');
|
||||
assert.equal(parseLoc('VY:lib/x.mjs'), null, 'a loc without a line is not evidence');
|
||||
assert.equal(parseLoc('VY:lib/x.mjs:9-3'), null, 'a range that runs backwards is not evidence');
|
||||
assert.equal(parseLoc('VY:lib/x.mjs:0'), null, 'lines count from 1');
|
||||
});
|
||||
|
||||
test('parseSourceAreas: reads the first column of the table under the heading, and only that table', () => {
|
||||
assert.deepEqual(parseSourceAreas(SOURCE, '## 2. Per kapabilitet'), ['Alpha', 'Beta']);
|
||||
assert.equal(parseSourceAreas(SOURCE, '## 9. Missing'), null);
|
||||
});
|
||||
|
||||
test('listParts: tracked files in the scope, eval cases as one part each, nothing outside the scope', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'comparison-gate-parts-'));
|
||||
try {
|
||||
for (const rel of ['commands/a.md', 'lib/u/b.mjs', 'evals/case-1/eval.json', 'evals/case-1/fixture/x.md',
|
||||
'evals/case-2/eval.json', 'docs/not-a-part.md', 'settings.json', 'untracked/z.md']) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), 'x\n');
|
||||
}
|
||||
execFileSync('git', ['init', '-q'], { cwd: dir });
|
||||
execFileSync('git', ['add', 'commands', 'lib', 'evals', 'docs', 'settings.json'], { cwd: dir });
|
||||
assert.deepEqual(listParts(dir), ['commands/a.md', 'evals/case-1/', 'evals/case-2/', 'lib/u/b.mjs', 'settings.json']);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- the comparison ------------------------------------------------------------
|
||||
|
||||
test('a fully evidenced table, every area level or ahead, read by an independent reader, is green', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const r = run(w, greenData(w));
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.equal(r.comparison.level_or_ahead, 2);
|
||||
assert.equal(r.comparison.total, 2);
|
||||
assert.equal(r.green, true);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a cell with NO evidence fells the table, whatever its verdict says', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.evidence = [];
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /alpha.*SP.*no evidence/);
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('an area with no evidence on our own side fells the table', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
delete data.areas[1].own;
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /beta.*own.*no evidence/);
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a cited line past the end of the file is not evidence', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.evidence[0].loc = 'SP:ref.md:3-11';
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /SP:ref\.md:3-11.*10 lines/);
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a cited file that does not exist, or that escapes its root, is not evidence', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.evidence[0].loc = 'SP:missing.md:1';
|
||||
data.areas[0].own.evidence[0].loc = 'VY:../pm/source.md:1';
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /SP:missing\.md:1.*does not exist/);
|
||||
assert.match(problemText(r), /VY:\.\.\/pm\/source\.md:1.*outside its root/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('an absence without a known-positive control is not evidence, and neither is one with hits', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
delete data.areas[1].cells.SP.evidence[0].absence.known_positive;
|
||||
let r = run(w, data);
|
||||
assert.match(problemText(r), /beta.*SP.*known_positive/);
|
||||
const again = greenData(w);
|
||||
again.areas[1].cells.SP.evidence[0].absence.hits = 2;
|
||||
r = run(w, again);
|
||||
assert.match(problemText(r), /beta.*SP.*hits/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a web citation needs a verbatim quote', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.evidence = [{ url: 'https://example.org/playbook#review', says: 'x' }];
|
||||
let r = run(w, data);
|
||||
assert.match(problemText(r), /alpha.*SP.*quote/);
|
||||
data.areas[0].cells.SP.evidence = [{ url: 'https://example.org/playbook#review', quote: 'every change is reviewed', says: 'x' }];
|
||||
r = run(w, data);
|
||||
assert.deepEqual(r.problems, []);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('the denominator comes from the SOURCE: an area the source lists but the table drops fells it', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas.pop();
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /Beta.*missing from the table/);
|
||||
assert.equal(r.comparison.total, 2, 'the total is the source count, not the table count');
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('an area the source does not list is not counted and is reported', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas.push({ ...data.areas[0], id: 'gamma', title: 'Gamma' });
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /Gamma.*not in the source/);
|
||||
assert.equal(r.comparison.total, 2);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a missing reference cell fells the table; an unknown verdict fells it', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.references.push({ id: 'PS', name: 'mattpocock/skills' });
|
||||
let r = run(w, data);
|
||||
assert.match(problemText(r), /alpha.*PS.*missing/);
|
||||
const again = greenData(w);
|
||||
again.areas[0].cells.SP.verdict = 'probably fine';
|
||||
r = run(w, again);
|
||||
assert.match(problemText(r), /alpha.*SP.*verdict/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('one reference ahead of us in one area: that area is not counted, and it is named', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.verdict = 'bak';
|
||||
const r = run(w, data);
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.equal(r.comparison.level_or_ahead, 1);
|
||||
assert.deepEqual(r.comparison.behind, [{ area: 'alpha', references: ['SP'] }]);
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('the count per reference shows which framework pulls the total down', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.references.push({ id: 'PS', name: 'mattpocock/skills' });
|
||||
for (const a of data.areas) a.cells.PS = { ...a.cells.SP };
|
||||
data.areas[0].cells.PS = { ...data.areas[0].cells.SP, verdict: 'bak' };
|
||||
const r = run(w, data);
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.deepEqual(r.comparison.per_reference, { SP: 2, PS: 1 });
|
||||
assert.equal(r.comparison.level_or_ahead, 1);
|
||||
assert.match(render(r), /PS: 1 of 2/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('14 of 14 is not green until an INDEPENDENT reader judged it', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.reader.kind = 'pm-rapport';
|
||||
const r = run(w, data);
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.equal(r.comparison.level_or_ahead, 2);
|
||||
assert.equal(r.green, false);
|
||||
assert.match(r.reasons.join('\n'), /independent reader/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('a pinned reference whose clone has moved is not evidence for the pinned revision', () => {
|
||||
const w = world();
|
||||
try {
|
||||
execFileSync('git', ['init', '-q'], { cwd: w.roots.SP });
|
||||
execFileSync('git', ['add', '.'], { cwd: w.roots.SP });
|
||||
execFileSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'x'], { cwd: w.roots.SP });
|
||||
const data = greenData(w);
|
||||
data.references[0].pin = '0000000';
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /SP.*pinned 0000000.*HEAD/);
|
||||
data.references[0].pin = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: w.roots.SP, encoding: 'utf-8' }).trim();
|
||||
assert.deepEqual(run(w, data).problems, []);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
// --- tidy ------------------------------------------------------------------------
|
||||
|
||||
test('tidy: a tracked part the list leaves out fells it; the denominator is the tree, not the list', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const r = evaluate(greenData(w), { parts: [...PARTS, 'agents/new.md'] });
|
||||
assert.match(problemText(r), /agents\/new\.md.*missing from the tidy list/);
|
||||
assert.equal(r.tidy.parts, 3);
|
||||
assert.equal(r.green, false);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('tidy: a covered part counts as resolved only with a measured reason or as removed from the tree', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.tidy.parts[0].resolution = 'aapen';
|
||||
data.tidy.parts[0].reason = null;
|
||||
let r = run(w, data);
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.equal(r.tidy.covered, 1);
|
||||
assert.equal(r.tidy.resolved, 0);
|
||||
assert.equal(r.green, false);
|
||||
|
||||
const noReason = greenData(w);
|
||||
noReason.tidy.parts[0].reason = null;
|
||||
r = run(w, noReason);
|
||||
assert.match(problemText(r), /commands\/x\.md.*maalt-grunn.*reason/);
|
||||
|
||||
const removed = greenData(w);
|
||||
removed.tidy.parts[0] = { ...removed.tidy.parts[0], resolution: 'fjernet', reason: null, commit: 'abc1234' };
|
||||
r = run(w, removed);
|
||||
assert.match(problemText(r), /commands\/x\.md.*fjernet.*still in the tree/);
|
||||
r = evaluate(removed, { parts: ['lib/y.mjs'] });
|
||||
assert.deepEqual(r.problems, []);
|
||||
assert.equal(r.tidy.resolved, 1);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('tidy: a primitive without a documentation URL, or a part covered by an unknown primitive, fells it', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.tidy.primitives[0].doc = 'somewhere';
|
||||
data.tidy.parts[0].covered_by = ['subagents', 'nonexistent'];
|
||||
const r = run(w, data);
|
||||
assert.match(problemText(r), /subagents.*documentation URL/);
|
||||
assert.match(problemText(r), /commands\/x\.md.*nonexistent/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
// --- output and CLI ------------------------------------------------------------------
|
||||
|
||||
test('render names the count, the behind areas and the reader', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const data = greenData(w);
|
||||
data.areas[0].cells.SP.verdict = 'bak';
|
||||
data.reader.kind = 'pm-rapport';
|
||||
const text = render(run(w, data));
|
||||
assert.match(text, /1 of 2/);
|
||||
assert.match(text, /alpha \(SP\)/);
|
||||
assert.match(text, /pm-rapport/);
|
||||
assert.match(text, /RED/);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('main: a missing table is NOT MEASURED and exits 1, never 0; an unknown flag exits 2', () => {
|
||||
const w = world();
|
||||
try {
|
||||
assert.equal(main(['--data', join(w.dir, 'nope.json'), '--json']), 1);
|
||||
assert.equal(main(['--nope']), 2);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
|
||||
test('main: exit 0 only for a green table, 1 for a red one', () => {
|
||||
const w = world();
|
||||
try {
|
||||
const file = join(w.dir, 'table.json');
|
||||
writeFileSync(file, JSON.stringify(greenData(w)));
|
||||
// --parts-from pins the denominator to a fixture repo instead of the real tree
|
||||
const repo = join(w.dir, 'repo');
|
||||
for (const rel of PARTS) {
|
||||
mkdirSync(dirname(join(repo, rel)), { recursive: true });
|
||||
writeFileSync(join(repo, rel), 'x\n');
|
||||
}
|
||||
execFileSync('git', ['init', '-q'], { cwd: repo });
|
||||
execFileSync('git', ['add', '.'], { cwd: repo });
|
||||
assert.equal(main(['--data', file, '--parts-from', repo, '--json']), 0);
|
||||
const red = greenData(w);
|
||||
red.areas[0].cells.SP.evidence = [];
|
||||
writeFileSync(file, JSON.stringify(red));
|
||||
assert.equal(main(['--data', file, '--parts-from', repo, '--json']), 1);
|
||||
} finally { cleanup(w); }
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue