voyage/scripts/comparison-gate.mjs

#!/usr/bin/env node
// scripts/comparison-gate.mjs
// The comparison gate - the lifted target in PLAN.md (operator, 2026-09-27): Voyage + the
// operator's way of working is level with or ahead of the related frameworks in every area,
// and every Voyage part a built-in Claude Code primitive covers is removed or has a measured
// reason to stay. Two counts, each with a denominator read from its SOURCE:
//
//   comparison  areas where no reference is ahead   (the areas: the first column of the table
//                                                     under `source.heading` in `source.loc`;
//                                                     counted = areas where every reference
//                                                     cell's verdict is `likt` or `foran`)
//   tidy        covered parts that are resolved      (the parts: `git ls-files` over PART_SCOPE,
//                                                     one part per eval case; counted = parts a
//                                                     primitive covers that are `fjernet` or
//                                                     carry a `maalt-grunn` with evidence)
//
// It fells the table on MISSING EVIDENCE, never on a verdict it disagrees with: a cited file
// must exist inside its root and hold the cited lines, a pinned clone must sit at its pin, a
// web citation needs a verbatim quote, and an absence needs zero hits plus a known-positive
// control. A verdict is a reader's judgement; green also requires `reader.kind: uavhengig`.
//
// Exit 0 = both counts full, no evidence problem, independent reader; 1 = anything short of
// that, including a table that does not exist (NOT MEASURED, never 0); 2 = usage error.
// It only READS. The table is local-only (docs/comparison.local.json, `*.local.*`).
//
// Usage: node scripts/comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]

import { readFileSync, existsSync, statSync } from 'node:fs';
import { homedir } from 'node:os';
import { join, resolve, dirname, relative, isAbsolute, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { isMain } from '../lib/util/is-main.mjs';

const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');

export const DEFAULT_DATA = join(REPO_ROOT, 'docs', 'comparison.local.json');
export const PART_SCOPE = ['commands', 'agents', 'hooks', 'lib', 'scripts', 'templates', 'evals', 'settings.json', 'verify.sh'];
export const VERDICTS = ['foran', 'likt', 'bak'];
const LEVEL_OR_AHEAD = new Set(['foran', 'likt']);
export const COVERAGE = ['dekket', 'delvis', 'ikke-dekket'];
export const RESOLUTIONS = ['maalt-grunn', 'aapen', 'fjernet'];

const toPosix = (p) => p.split(sep).join('/');
const nonEmpty = (s) => typeof s === 'string' && s.trim() !== '';

// `ROOT:path:N` or `ROOT:path:N-M`, lines counted from 1. Anything else is not a loc.
export function parseLoc(text) {
  const m = /^([A-Z]{2,}):(.+):(\d+)(?:-(\d+))?$/.exec(String(text ?? ''));
  if (!m) return null;
  const start = Number(m[3]);
  const end = m[4] === undefined ? start : Number(m[4]);
  if (start < 1 || end < start) return null;
  return { root: m[1], path: m[2], start, end };
}

// The first column of the first table after `heading`, header and separator rows excluded.
export function parseSourceAreas(markdown, heading) {
  const all = String(markdown).split('\n');
  const at = all.findIndex((l) => l.trim() === heading);
  if (at === -1) return null;
  const rows = [];
  let inTable = false;
  for (const line of all.slice(at + 1)) {
    if (/^#{1,6}\s/.test(line)) break;
    if (line.trim().startsWith('|')) {
      inTable = true;
      rows.push(line);
    } else if (inTable) break;
  }
  return rows.slice(2).map((r) => r.split('|')[1].trim()).filter(Boolean);
}

// One part per tracked file in PART_SCOPE, except eval cases: one part per `evals/<case>/`.
export function listParts(root) {
  const out = execFileSync('git', ['ls-files', '-z', '--', ...PART_SCOPE], { cwd: root, encoding: 'utf-8' });
  const parts = new Set();
  for (const f of out.split('\0').filter(Boolean)) {
    const segs = f.split('/');
    parts.add(segs[0] === 'evals' && segs.length > 2 ? `evals/${segs[1]}/` : f);
  }
  return [...parts].sort();
}

function resolveRoot(value) {
  if (!nonEmpty(value)) return null;
  if (value === '~' || value.startsWith('~/')) return join(homedir(), value.slice(2));
  return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
}

function lineCount(file) {
  const text = readFileSync(file, 'utf-8');
  if (text === '') return 0;
  return text.split('\n').length - (text.endsWith('\n') ? 1 : 0);
}

// Problems with one evidence item, each prefixed with `where`.
function checkEvidence(item, where, roots) {
  if (!item || typeof item !== 'object') return [`${where}: an evidence item must be an object`];
  const problems = [];
  if (!nonEmpty(item.says)) problems.push(`${where}: evidence needs \`says\``);
  if ('loc' in item) {
    const loc = parseLoc(item.loc);
    if (!loc) return [...problems, `${where}: ${item.loc} is not ROOT:path:N[-M]`];
    const root = roots[loc.root];
    if (!root) return [...problems, `${where}: ${item.loc} names an unknown root ${loc.root}`];
    const file = resolve(root, loc.path);
    const rel = relative(root, file);
    if (rel.startsWith('..') || isAbsolute(rel)) return [...problems, `${where}: ${item.loc} points outside its root`];
    if (!existsSync(file) || !statSync(file).isFile()) return [...problems, `${where}: ${item.loc} does not exist`];
    const n = lineCount(file);
    if (loc.end > n) problems.push(`${where}: ${item.loc} cites past the end of a file of ${n} lines`);
    return problems;
  }
  if ('url' in item) {
    if (!/^https:\/\/\S+$/.test(String(item.url))) problems.push(`${where}: ${item.url} is not an https URL`);
    if (!nonEmpty(item.quote)) problems.push(`${where}: a web citation needs a verbatim \`quote\``);
    return problems;
  }
  if ('absence' in item) {
    const a = item.absence ?? {};
    if (!nonEmpty(a.query)) problems.push(`${where}: an absence needs the \`query\` that was run`);
    if (a.hits !== 0) problems.push(`${where}: an absence must report hits: 0, got ${JSON.stringify(a.hits)}`);
    if (!nonEmpty(a.known_positive)) problems.push(`${where}: an absence needs a \`known_positive\` control`);
    return problems;
  }
  return [...problems, `${where}: evidence must be one of loc, url or absence`];
}

function checkEvidenceList(list, where, roots) {
  if (!Array.isArray(list) || list.length === 0) return [`${where}: no evidence`];
  return list.flatMap((item, i) => checkEvidence(item, `${where}[${i}]`, roots));
}

function pinProblems(references, roots) {
  const problems = [];
  for (const ref of references) {
    if (!nonEmpty(ref.pin)) continue;
    const root = roots[ref.id];
    if (!root || !existsSync(join(root, '.git'))) {
      problems.push(`reference ${ref.id}: pinned ${ref.pin} but ${root ?? 'no root'} is not a git clone`);
      continue;
    }
    let head = '';
    try {
      head = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf-8' }).trim();
    } catch { /* reported below */ }
    if (!head.startsWith(ref.pin)) problems.push(`reference ${ref.id}: pinned ${ref.pin} but the clone's HEAD is ${head.slice(0, 7) || 'unreadable'}`);
  }
  return problems;
}

function evaluateComparison(data, roots) {
  const problems = [];
  const references = Array.isArray(data.references) ? data.references : [];
  if (references.length === 0) problems.push('no references to compare against');
  problems.push(...pinProblems(references, roots));

  let sourceAreas = null;
  const src = parseLoc(`${data.source?.loc ?? ''}:1`);
  const srcFile = src && roots[src.root] ? resolve(roots[src.root], src.path) : null;
  if (!srcFile || !existsSync(srcFile)) {
    problems.push(`source ${data.source?.loc ?? '(none)'}: not readable - the denominator cannot be read`);
  } else {
    sourceAreas = parseSourceAreas(readFileSync(srcFile, 'utf-8'), data.source.heading);
    if (!sourceAreas || sourceAreas.length === 0) {
      problems.push(`source ${data.source.loc}: no table under ${JSON.stringify(data.source.heading)}`);
      sourceAreas = null;
    }
  }

  const areas = Array.isArray(data.areas) ? data.areas : [];
  const byTitle = new Map(areas.map((a) => [a.title, a]));
  const counted = [];
  const behind = [];
  const perReference = Object.fromEntries(references.map((r) => [r.id, 0]));
  const contested = [];
  for (const title of sourceAreas ?? []) {
    if (!byTitle.has(title)) problems.push(`area ${title}: in the source but missing from the table`);
  }
  for (const area of areas) {
    const id = area.id ?? area.title;
    if (sourceAreas && !sourceAreas.includes(area.title)) {
      problems.push(`area ${area.title}: not in the source - not counted`);
      continue;
    }
    problems.push(...checkEvidenceList(area.own?.evidence, `${id} own`, roots));
    const behindRefs = [];
    for (const ref of references) {
      const cell = area.cells?.[ref.id];
      if (!cell) { problems.push(`${id} ${ref.id}: cell missing`); continue; }
      if (!VERDICTS.includes(cell.verdict)) problems.push(`${id} ${ref.id}: verdict ${JSON.stringify(cell.verdict)} is not one of ${VERDICTS.join('|')}`);
      if (!nonEmpty(cell.why)) problems.push(`${id} ${ref.id}: a verdict needs \`why\``);
      problems.push(...checkEvidenceList(cell.evidence, `${id} ${ref.id}`, roots));
      if (cell.contested) contested.push(`${id} (${ref.id})`);
      if (!LEVEL_OR_AHEAD.has(cell.verdict)) behindRefs.push(ref.id);
      else perReference[ref.id]++;
    }
    if (behindRefs.length) behind.push({ area: id, references: behindRefs });
    else counted.push(id);
  }
  const total = sourceAreas ? sourceAreas.length : null;
  return {
    problems,
    comparison: {
      total,
      level_or_ahead: total === null ? null : counted.length,
      behind,
      per_reference: perReference,
      contested,
      references: references.map((r) => `${r.id}${r.pin ? `@${r.pin}` : ''} ${r.name ?? ''}`.trim()),
    },
  };
}

function evaluateTidy(tidy, parts, roots) {
  const problems = [];
  if (!tidy || typeof tidy !== 'object') {
    return { problems: ['tidy: missing - the tidy denominator is not measured'], tidy: { parts: parts.length, covered: null, resolved: null } };
  }
  const primitives = Array.isArray(tidy.primitives) ? tidy.primitives : [];
  const known = new Set();
  for (const p of primitives) {
    known.add(p.id);
    if (!/^https:\/\/\S+$/.test(String(p.doc ?? ''))) problems.push(`primitive ${p.id}: needs a documentation URL`);
    if (!nonEmpty(p.quote)) problems.push(`primitive ${p.id}: needs a verbatim \`quote\` from its documentation`);
  }
  const listed = Array.isArray(tidy.parts) ? tidy.parts : [];
  const tree = new Set(parts);
  const seen = new Set();
  let covered = 0;
  let resolved = 0;
  for (const part of listed) {
    const where = `part ${part.path}`;
    if (seen.has(part.path)) problems.push(`${where}: listed twice`);
    seen.add(part.path);
    if (!COVERAGE.includes(part.coverage)) { problems.push(`${where}: coverage ${JSON.stringify(part.coverage)} is not one of ${COVERAGE.join('|')}`); continue; }
    if (part.resolution !== 'fjernet' && !tree.has(part.path)) problems.push(`${where}: not in the tree and not marked fjernet`);
    if (part.coverage === 'ikke-dekket') continue;
    covered++;
    const by = Array.isArray(part.covered_by) ? part.covered_by : [];
    if (by.length === 0) problems.push(`${where}: ${part.coverage} needs \`covered_by\``);
    for (const id of by) if (!known.has(id)) problems.push(`${where}: covered by an unknown primitive ${id}`);
    if (!RESOLUTIONS.includes(part.resolution)) {
      problems.push(`${where}: resolution ${JSON.stringify(part.resolution)} is not one of ${RESOLUTIONS.join('|')}`);
    } else if (part.resolution === 'maalt-grunn') {
      if (!part.reason) problems.push(`${where}: maalt-grunn needs a \`reason\` with evidence`);
      else {
        const p = checkEvidence(part.reason, `${where} reason`, roots);
        problems.push(...p);
        if (p.length === 0) resolved++;
      }
    } else if (part.resolution === 'fjernet') {
      if (tree.has(part.path)) problems.push(`${where}: marked fjernet but still in the tree`);
      else if (!nonEmpty(part.commit)) problems.push(`${where}: fjernet needs the \`commit\` that removed it`);
      else resolved++;
    }
  }
  for (const p of parts) if (!seen.has(p)) problems.push(`part ${p}: in the tree but missing from the tidy list`);
  return { problems, tidy: { parts: parts.length, covered, resolved, primitives: primitives.length } };
}

export function evaluate(data, { parts = [] } = {}) {
  const roots = {};
  for (const [id, value] of Object.entries(data?.roots ?? {})) roots[id] = resolveRoot(value);
  const cmp = evaluateComparison(data ?? {}, roots);
  const tdy = evaluateTidy(data?.tidy, parts, roots);
  const problems = [...cmp.problems, ...tdy.problems];
  const reader = data?.reader ?? {};
  const reasons = [];
  if (problems.length) reasons.push(`${problems.length} evidence problem(s)`);
  const c = cmp.comparison;
  if (c.total === null) reasons.push('the comparison denominator is not measured');
  else if (c.level_or_ahead < c.total) reasons.push(`${c.level_or_ahead} of ${c.total} areas level or ahead`);
  if (reader.kind !== 'uavhengig') reasons.push(`the verdicts are by ${reader.kind ?? 'nobody'}, not an independent reader`);
  if (tdy.tidy.covered === null) reasons.push('the tidy list is not measured');
  else if (tdy.tidy.resolved < tdy.tidy.covered) reasons.push(`${tdy.tidy.resolved} of ${tdy.tidy.covered} covered parts resolved`);
  return {
    measured: true,
    reader,
    comparison: c,
    tidy: tdy.tidy,
    problems,
    reasons,
    green: reasons.length === 0,
  };
}

export function render(r) {
  if (!r.measured) return `Comparison gate: NOT MEASURED - ${r.reasons.join('; ')}\nRED`;
  const c = r.comparison;
  const out = [];
  out.push(`Comparison gate - areas where no reference is ahead of Voyage + the operator's way of working`);
  out.push(`  references: ${c.references.join(' · ')}`);
  out.push(`  level or ahead: ${c.level_or_ahead ?? 'NOT MEASURED'} of ${c.total ?? '?'}`);
  for (const b of c.behind) out.push(`    behind: ${b.area} (${b.references.join(', ')})`);
  out.push(`  per reference: ${Object.entries(c.per_reference).map(([id, n]) => `${id}: ${n} of ${c.total ?? '?'}`).join(' · ')}`);
  if (c.contested.length) out.push(`  contested cells: ${c.contested.join(', ')}`);
  out.push(`  reader: ${r.reader.kind ?? 'none'}${r.reader.who ? ` - ${r.reader.who}` : ''}${r.reader.date ? ` (${r.reader.date})` : ''}`
    + `${r.reader.kind === 'uavhengig' ? '' : ' - green requires an independent reader'}`);
  const t = r.tidy;
  out.push(`Tidy - Voyage parts a documented Claude Code primitive covers`);
  out.push(`  parts in the tree: ${t.parts} · covered: ${t.covered ?? 'NOT MEASURED'} · resolved: ${t.resolved ?? '-'} of ${t.covered ?? '-'}`);
  out.push(`Evidence problems: ${r.problems.length}`);
  for (const p of r.problems) out.push(`  - ${p}`);
  out.push(r.green ? 'GREEN' : `RED - ${r.reasons.join('; ')}`);
  return out.join('\n');
}

export function main(argv) {
  let json = false;
  let dataFile = DEFAULT_DATA;
  let partsFrom = REPO_ROOT;
  const usage = 'usage: comparison-gate.mjs [--json] [--data <file>] [--parts-from <repo>]';
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    const value = argv[i + 1];
    if (a === '--json') { json = true; continue; }
    if (a === '--data' && value !== undefined) { dataFile = resolve(value); i++; continue; }
    if (a === '--parts-from' && value !== undefined) { partsFrom = resolve(value); i++; continue; }
    process.stderr.write(`comparison-gate: unknown argument ${a}\n${usage}\n`);
    return 2;
  }
  let result;
  if (!existsSync(dataFile)) {
    result = { measured: false, green: false, reasons: [`${toPosix(relative(REPO_ROOT, dataFile))} does not exist`] };
  } else {
    let data = null;
    try {
      data = JSON.parse(readFileSync(dataFile, 'utf-8'));
    } catch (err) {
      result = { measured: false, green: false, reasons: [`${dataFile} is not valid JSON: ${err.message}`] };
    }
    if (data) result = evaluate(data, { parts: listParts(partsFrom) });
  }
  process.stdout.write(`${json ? JSON.stringify(result, null, 2) : render(result)}\n`);
  return result.green ? 0 : 1;
}

if (isMain(import.meta.url)) {
  process.exitCode = main(process.argv.slice(2));
}