#!/usr/bin/env node // scripts/comparison-gate.mjs // The comparison gate - the lifted target in PLAN.md (operator, 2026-09-27): Voyage + the // operator's way of working is level with or ahead of the related frameworks in every area, // and every Voyage part a built-in Claude Code primitive covers is removed or has a measured // reason to stay. Two counts, each with a denominator read from its SOURCE: // // comparison areas where no reference is ahead (the areas: the first column of the table // under `source.heading` in `source.loc`; // counted = areas where every reference // cell's verdict is `likt` or `foran`) // tidy covered parts that are resolved (the parts: `git ls-files` over PART_SCOPE, // one part per eval case; counted = parts a // primitive covers that are `fjernet` or // carry a `maalt-grunn` with evidence) // // It fells the table on MISSING EVIDENCE, never on a verdict it disagrees with: a cited file // must exist inside its root and hold the cited lines, a pinned clone must sit at its pin, a // web citation needs a verbatim quote, and an absence needs zero hits plus a known-positive // control. A verdict is a reader's judgement; green also requires `reader.kind: uavhengig`. // // Exit 0 = both counts full, no evidence problem, independent reader; 1 = anything short of // that, including a table that does not exist (NOT MEASURED, never 0); 2 = usage error. // It only READS. The table is local-only (docs/comparison.local.json, `*.local.*`). // // Usage: node scripts/comparison-gate.mjs [--json] [--data ] [--parts-from ] import { readFileSync, existsSync, statSync } from 'node:fs'; import { homedir } from 'node:os'; import { join, resolve, dirname, relative, isAbsolute, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; import { isMain } from '../lib/util/is-main.mjs'; const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); export const DEFAULT_DATA = join(REPO_ROOT, 'docs', 'comparison.local.json'); export const PART_SCOPE = ['commands', 'agents', 'hooks', 'lib', 'scripts', 'templates', 'evals', 'settings.json', 'verify.sh']; export const VERDICTS = ['foran', 'likt', 'bak']; const LEVEL_OR_AHEAD = new Set(['foran', 'likt']); export const COVERAGE = ['dekket', 'delvis', 'ikke-dekket']; export const RESOLUTIONS = ['maalt-grunn', 'aapen', 'fjernet']; const toPosix = (p) => p.split(sep).join('/'); const nonEmpty = (s) => typeof s === 'string' && s.trim() !== ''; // `ROOT:path:N` or `ROOT:path:N-M`, lines counted from 1. Anything else is not a loc. export function parseLoc(text) { const m = /^([A-Z]{2,}):(.+):(\d+)(?:-(\d+))?$/.exec(String(text ?? '')); if (!m) return null; const start = Number(m[3]); const end = m[4] === undefined ? start : Number(m[4]); if (start < 1 || end < start) return null; return { root: m[1], path: m[2], start, end }; } // The first column of the first table after `heading`, header and separator rows excluded. export function parseSourceAreas(markdown, heading) { const all = String(markdown).split('\n'); const at = all.findIndex((l) => l.trim() === heading); if (at === -1) return null; const rows = []; let inTable = false; for (const line of all.slice(at + 1)) { if (/^#{1,6}\s/.test(line)) break; if (line.trim().startsWith('|')) { inTable = true; rows.push(line); } else if (inTable) break; } return rows.slice(2).map((r) => r.split('|')[1].trim()).filter(Boolean); } // One part per tracked file in PART_SCOPE, except eval cases: one part per `evals//`. export function listParts(root) { const out = execFileSync('git', ['ls-files', '-z', '--', ...PART_SCOPE], { cwd: root, encoding: 'utf-8' }); const parts = new Set(); for (const f of out.split('\0').filter(Boolean)) { const segs = f.split('/'); parts.add(segs[0] === 'evals' && segs.length > 2 ? `evals/${segs[1]}/` : f); } return [...parts].sort(); } function resolveRoot(value) { if (!nonEmpty(value)) return null; if (value === '~' || value.startsWith('~/')) return join(homedir(), value.slice(2)); return isAbsolute(value) ? value : resolve(REPO_ROOT, value); } function lineCount(file) { const text = readFileSync(file, 'utf-8'); if (text === '') return 0; return text.split('\n').length - (text.endsWith('\n') ? 1 : 0); } // Problems with one evidence item, each prefixed with `where`. function checkEvidence(item, where, roots) { if (!item || typeof item !== 'object') return [`${where}: an evidence item must be an object`]; const problems = []; if (!nonEmpty(item.says)) problems.push(`${where}: evidence needs \`says\``); if ('loc' in item) { const loc = parseLoc(item.loc); if (!loc) return [...problems, `${where}: ${item.loc} is not ROOT:path:N[-M]`]; const root = roots[loc.root]; if (!root) return [...problems, `${where}: ${item.loc} names an unknown root ${loc.root}`]; const file = resolve(root, loc.path); const rel = relative(root, file); if (rel.startsWith('..') || isAbsolute(rel)) return [...problems, `${where}: ${item.loc} points outside its root`]; if (!existsSync(file) || !statSync(file).isFile()) return [...problems, `${where}: ${item.loc} does not exist`]; const n = lineCount(file); if (loc.end > n) problems.push(`${where}: ${item.loc} cites past the end of a file of ${n} lines`); return problems; } if ('url' in item) { if (!/^https:\/\/\S+$/.test(String(item.url))) problems.push(`${where}: ${item.url} is not an https URL`); if (!nonEmpty(item.quote)) problems.push(`${where}: a web citation needs a verbatim \`quote\``); return problems; } if ('absence' in item) { const a = item.absence ?? {}; if (!nonEmpty(a.query)) problems.push(`${where}: an absence needs the \`query\` that was run`); if (a.hits !== 0) problems.push(`${where}: an absence must report hits: 0, got ${JSON.stringify(a.hits)}`); if (!nonEmpty(a.known_positive)) problems.push(`${where}: an absence needs a \`known_positive\` control`); return problems; } return [...problems, `${where}: evidence must be one of loc, url or absence`]; } function checkEvidenceList(list, where, roots) { if (!Array.isArray(list) || list.length === 0) return [`${where}: no evidence`]; return list.flatMap((item, i) => checkEvidence(item, `${where}[${i}]`, roots)); } function pinProblems(references, roots) { const problems = []; for (const ref of references) { if (!nonEmpty(ref.pin)) continue; const root = roots[ref.id]; if (!root || !existsSync(join(root, '.git'))) { problems.push(`reference ${ref.id}: pinned ${ref.pin} but ${root ?? 'no root'} is not a git clone`); continue; } let head = ''; try { head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf-8' }).trim(); } catch { /* reported below */ } if (!head.startsWith(ref.pin)) problems.push(`reference ${ref.id}: pinned ${ref.pin} but the clone's HEAD is ${head.slice(0, 7) || 'unreadable'}`); } return problems; } function evaluateComparison(data, roots) { const problems = []; const references = Array.isArray(data.references) ? data.references : []; if (references.length === 0) problems.push('no references to compare against'); problems.push(...pinProblems(references, roots)); let sourceAreas = null; const src = parseLoc(`${data.source?.loc ?? ''}:1`); const srcFile = src && roots[src.root] ? resolve(roots[src.root], src.path) : null; if (!srcFile || !existsSync(srcFile)) { problems.push(`source ${data.source?.loc ?? '(none)'}: not readable - the denominator cannot be read`); } else { sourceAreas = parseSourceAreas(readFileSync(srcFile, 'utf-8'), data.source.heading); if (!sourceAreas || sourceAreas.length === 0) { problems.push(`source ${data.source.loc}: no table under ${JSON.stringify(data.source.heading)}`); sourceAreas = null; } } const areas = Array.isArray(data.areas) ? data.areas : []; const byTitle = new Map(areas.map((a) => [a.title, a])); const counted = []; const behind = []; const perReference = Object.fromEntries(references.map((r) => [r.id, 0])); const contested = []; for (const title of sourceAreas ?? []) { if (!byTitle.has(title)) problems.push(`area ${title}: in the source but missing from the table`); } for (const area of areas) { const id = area.id ?? area.title; if (sourceAreas && !sourceAreas.includes(area.title)) { problems.push(`area ${area.title}: not in the source - not counted`); continue; } problems.push(...checkEvidenceList(area.own?.evidence, `${id} own`, roots)); const behindRefs = []; for (const ref of references) { const cell = area.cells?.[ref.id]; if (!cell) { problems.push(`${id} ${ref.id}: cell missing`); continue; } if (!VERDICTS.includes(cell.verdict)) problems.push(`${id} ${ref.id}: verdict ${JSON.stringify(cell.verdict)} is not one of ${VERDICTS.join('|')}`); if (!nonEmpty(cell.why)) problems.push(`${id} ${ref.id}: a verdict needs \`why\``); problems.push(...checkEvidenceList(cell.evidence, `${id} ${ref.id}`, roots)); if (cell.contested) contested.push(`${id} (${ref.id})`); if (!LEVEL_OR_AHEAD.has(cell.verdict)) behindRefs.push(ref.id); else perReference[ref.id]++; } if (behindRefs.length) behind.push({ area: id, references: behindRefs }); else counted.push(id); } const total = sourceAreas ? sourceAreas.length : null; return { problems, comparison: { total, level_or_ahead: total === null ? null : counted.length, behind, per_reference: perReference, contested, references: references.map((r) => `${r.id}${r.pin ? `@${r.pin}` : ''} ${r.name ?? ''}`.trim()), }, }; } function evaluateTidy(tidy, parts, roots) { const problems = []; if (!tidy || typeof tidy !== 'object') { return { problems: ['tidy: missing - the tidy denominator is not measured'], tidy: { parts: parts.length, covered: null, resolved: null } }; } const primitives = Array.isArray(tidy.primitives) ? tidy.primitives : []; const known = new Set(); for (const p of primitives) { known.add(p.id); if (!/^https:\/\/\S+$/.test(String(p.doc ?? ''))) problems.push(`primitive ${p.id}: needs a documentation URL`); if (!nonEmpty(p.quote)) problems.push(`primitive ${p.id}: needs a verbatim \`quote\` from its documentation`); } const listed = Array.isArray(tidy.parts) ? tidy.parts : []; const tree = new Set(parts); const seen = new Set(); let covered = 0; let resolved = 0; for (const part of listed) { const where = `part ${part.path}`; if (seen.has(part.path)) problems.push(`${where}: listed twice`); seen.add(part.path); if (!COVERAGE.includes(part.coverage)) { problems.push(`${where}: coverage ${JSON.stringify(part.coverage)} is not one of ${COVERAGE.join('|')}`); continue; } if (part.resolution !== 'fjernet' && !tree.has(part.path)) problems.push(`${where}: not in the tree and not marked fjernet`); if (part.coverage === 'ikke-dekket') continue; covered++; const by = Array.isArray(part.covered_by) ? part.covered_by : []; if (by.length === 0) problems.push(`${where}: ${part.coverage} needs \`covered_by\``); for (const id of by) if (!known.has(id)) problems.push(`${where}: covered by an unknown primitive ${id}`); if (!RESOLUTIONS.includes(part.resolution)) { problems.push(`${where}: resolution ${JSON.stringify(part.resolution)} is not one of ${RESOLUTIONS.join('|')}`); } else if (part.resolution === 'maalt-grunn') { if (!part.reason) problems.push(`${where}: maalt-grunn needs a \`reason\` with evidence`); else { const p = checkEvidence(part.reason, `${where} reason`, roots); problems.push(...p); if (p.length === 0) resolved++; } } else if (part.resolution === 'fjernet') { if (tree.has(part.path)) problems.push(`${where}: marked fjernet but still in the tree`); else if (!nonEmpty(part.commit)) problems.push(`${where}: fjernet needs the \`commit\` that removed it`); else resolved++; } } for (const p of parts) if (!seen.has(p)) problems.push(`part ${p}: in the tree but missing from the tidy list`); return { problems, tidy: { parts: parts.length, covered, resolved, primitives: primitives.length } }; } export function evaluate(data, { parts = [] } = {}) { const roots = {}; for (const [id, value] of Object.entries(data?.roots ?? {})) roots[id] = resolveRoot(value); const cmp = evaluateComparison(data ?? {}, roots); const tdy = evaluateTidy(data?.tidy, parts, roots); const problems = [...cmp.problems, ...tdy.problems]; const reader = data?.reader ?? {}; const reasons = []; if (problems.length) reasons.push(`${problems.length} evidence problem(s)`); const c = cmp.comparison; if (c.total === null) reasons.push('the comparison denominator is not measured'); else if (c.level_or_ahead < c.total) reasons.push(`${c.level_or_ahead} of ${c.total} areas level or ahead`); if (reader.kind !== 'uavhengig') reasons.push(`the verdicts are by ${reader.kind ?? 'nobody'}, not an independent reader`); if (tdy.tidy.covered === null) reasons.push('the tidy list is not measured'); else if (tdy.tidy.resolved < tdy.tidy.covered) reasons.push(`${tdy.tidy.resolved} of ${tdy.tidy.covered} covered parts resolved`); return { measured: true, reader, comparison: c, tidy: tdy.tidy, problems, reasons, green: reasons.length === 0, }; } export function render(r) { if (!r.measured) return `Comparison gate: NOT MEASURED - ${r.reasons.join('; ')}\nRED`; const c = r.comparison; const out = []; out.push(`Comparison gate - areas where no reference is ahead of Voyage + the operator's way of working`); out.push(` references: ${c.references.join(' · ')}`); out.push(` level or ahead: ${c.level_or_ahead ?? 'NOT MEASURED'} of ${c.total ?? '?'}`); for (const b of c.behind) out.push(` behind: ${b.area} (${b.references.join(', ')})`); out.push(` per reference: ${Object.entries(c.per_reference).map(([id, n]) => `${id}: ${n} of ${c.total ?? '?'}`).join(' · ')}`); if (c.contested.length) out.push(` contested cells: ${c.contested.join(', ')}`); out.push(` reader: ${r.reader.kind ?? 'none'}${r.reader.who ? ` - ${r.reader.who}` : ''}${r.reader.date ? ` (${r.reader.date})` : ''}` + `${r.reader.kind === 'uavhengig' ? '' : ' - green requires an independent reader'}`); const t = r.tidy; out.push(`Tidy - Voyage parts a documented Claude Code primitive covers`); out.push(` parts in the tree: ${t.parts} · covered: ${t.covered ?? 'NOT MEASURED'} · resolved: ${t.resolved ?? '-'} of ${t.covered ?? '-'}`); out.push(`Evidence problems: ${r.problems.length}`); for (const p of r.problems) out.push(` - ${p}`); out.push(r.green ? 'GREEN' : `RED - ${r.reasons.join('; ')}`); return out.join('\n'); } export function main(argv) { let json = false; let dataFile = DEFAULT_DATA; let partsFrom = REPO_ROOT; const usage = 'usage: comparison-gate.mjs [--json] [--data ] [--parts-from ]'; for (let i = 0; i < argv.length; i++) { const a = argv[i]; const value = argv[i + 1]; if (a === '--json') { json = true; continue; } if (a === '--data' && value !== undefined) { dataFile = resolve(value); i++; continue; } if (a === '--parts-from' && value !== undefined) { partsFrom = resolve(value); i++; continue; } process.stderr.write(`comparison-gate: unknown argument ${a}\n${usage}\n`); return 2; } let result; if (!existsSync(dataFile)) { result = { measured: false, green: false, reasons: [`${toPosix(relative(REPO_ROOT, dataFile))} does not exist`] }; } else { let data = null; try { data = JSON.parse(readFileSync(dataFile, 'utf-8')); } catch (err) { result = { measured: false, green: false, reasons: [`${dataFile} is not valid JSON: ${err.message}`] }; } if (data) result = evaluate(data, { parts: listParts(partsFrom) }); } process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`); return result.green ? 0 : 1; } if (isMain(import.meta.url)) { process.exitCode = main(process.argv.slice(2)); }