#!/usr/bin/env node // check-okf-parity.mjs — the running, n-way per-file OKF §3 parity gate. // // WHY THIS EXISTS (docs/okf-second-brain/log.md, STEG 3): the convention's §3 // minimal contract is enforced by several INDEPENDENT checkers, and two of them // have already diverged on the same input — catalog's frozen okf-check.mjs walks // everything and does not normalize bytes; okr's hardened copy skips innboks/+dot-dirs // and strips BOM/CRLF. A one-shot lift-time parity check cannot catch that drift. // This gate makes parity a RUNNING red/green signal over an adversarial corpus, and // it CARRIES A PROOF THAT IT CAN GO RED (the corpus's `expected: "diverge"` fixtures). // // SCOPE (honest): only two implementations expose a runnable per-file conformance // surface today — catalog + okr. The gate is architected n-way; llm-ingestion-okf's // Python checker (fase 2) and its future Node port drop into IMPLS behind an // available() guard when they legitimately exist. It does NOT fake a third impl. // // CONTRACT: comparison unit is the PER-FILE outcome, not the bundle verdict — two // checkers can both say "OK" while disagreeing on what IS a concept file. Each impl's // existing checkBundle(root) return is normalized to { conceptCount, untyped[], okfVersion }; // the gate diverges iff those normalized outcomes are not identical across the // available impls. Defined over DEFAULT read-mode (strictIngest OFF, not files-scoped). // // The runner COMPARES; it does not author the fasit. Expected agree/diverge per fixture // lives in the corpus manifest (catalog owns the axes; propagation is spec -> fixtures -> // implementations, never the reverse). Zero npm dependencies (node: builtins). import { readdirSync, existsSync, readFileSync, statSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; // --- Implementation registry (n-way; guarded by available()) --- // modulePath is resolved relative to THIS file. Sibling-repo impls (okr) resolve via // the polyrepo layout (catalog + okr siblings). available() lets a catalog-only checkout // (e.g. CI without the sibling) degrade gracefully instead of crashing. export const IMPLS = [ { name: 'catalog', modulePath: new URL('./okf-check.mjs', import.meta.url) }, { name: 'okr', modulePath: new URL('../../okr/scripts/okf-check.mjs', import.meta.url) }, ]; export function implAvailable(impl) { try { return existsSync(fileURLToPath(impl.modulePath)); } catch { return false; } } // Normalize one impl's checkBundle(root) return to the per-file parity contract. // Returns null if the impl module is unavailable (guarded skip, reported by caller). export async function normalizedOutcome(impl, root) { if (!implAvailable(impl)) return null; const mod = await import(impl.modulePath); if (typeof mod.checkBundle !== 'function') { throw new Error(`impl "${impl.name}" exports no checkBundle()`); } const r = mod.checkBundle(root); // default read-mode: no options (strictIngest off, no files) return { name: impl.name, conceptCount: r.scanned, untyped: [...(r.missingType ?? [])].sort(), okfVersion: r.okfVersion ?? null, }; } // Signature over the fields that define per-file parity. Distinct signatures => divergence. function signature(o) { return `${o.conceptCount}|${o.untyped.join(',')}|${o.okfVersion}`; } // Run all impls on one bundle root; decide agree vs diverge among the comparable ones. export async function evaluateBundle(root, impls = IMPLS) { const outcomes = []; const skipped = []; for (const impl of impls) { const o = await normalizedOutcome(impl, root); if (o === null) skipped.push(impl.name); else outcomes.push(o); } const signatures = new Set(outcomes.map(signature)); return { outcomes, skipped, comparableCount: outcomes.length, diverges: outcomes.length >= 2 && signatures.size > 1, }; } // Run the whole corpus against its manifest. Pure over (corpusRoot, impls); no process.exit. export async function runCorpus(corpusRoot, impls = IMPLS) { const manifestPath = join(corpusRoot, 'manifest.json'); if (!existsSync(manifestPath)) { throw new Error(`corpus manifest not found: ${manifestPath}`); } const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); const fixtures = manifest.fixtures ?? {}; const results = []; for (const [name, spec] of Object.entries(fixtures)) { const dir = join(corpusRoot, name); if (!existsSync(dir) || !statSync(dir).isDirectory()) { results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'MISSING' }); continue; } const ev = await evaluateBundle(dir, impls); if (ev.comparableCount < 2) { results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'SKIPPED', ...ev }); continue; } const actual = ev.diverges ? 'diverge' : 'agree'; const status = actual === spec.expected ? 'PASS' : 'FAIL'; results.push({ fixture: name, axis: spec.axis, expected: spec.expected, actual, status, ...ev }); } // Orphan directories present on disk but absent from the manifest (no silent coverage gaps). const onDisk = readdirSync(corpusRoot, { withFileTypes: true }) .filter((e) => e.isDirectory()) .map((e) => e.name); const orphans = onDisk.filter((d) => !(d in fixtures)); const failed = results.filter((r) => r.status === 'FAIL' || r.status === 'MISSING'); const availableImpls = impls.filter(implAvailable).map((i) => i.name); return { results, orphans, failed, availableImpls, exitCode: failed.length === 0 ? 0 : 1 }; } // --- CLI --- const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]; if (isMain) { const arg = process.argv[2]; const corpusRoot = arg ? arg : fileURLToPath(new URL('../test/okf-parity-corpus/', import.meta.url)); const { results, orphans, failed, availableImpls, exitCode } = await runCorpus(corpusRoot); const out = []; out.push(`OKF parity gate over: ${corpusRoot}`); out.push(`Implementations available (${availableImpls.length}): ${availableImpls.join(', ') || 'NONE'}`); if (availableImpls.length < 2) { out.push(' ! Fewer than 2 impls available — parity cannot be established; every fixture skipped.'); out.push(' (This is an environment limitation, not a divergence. In a polyrepo checkout,'); out.push(' ensure the sibling checker repos are present.)'); } out.push(''); for (const r of results) { const tag = r.status === 'PASS' && r.expected === 'diverge' ? 'PASS (red-proof)' : r.status; out.push(`${r.fixture.padEnd(22)} [${(r.axis || '').padEnd(16)}] expect=${(r.expected || '').padEnd(8)} ${tag}`); for (const o of r.outcomes ?? []) { out.push(` ${o.name.padEnd(10)} count=${o.conceptCount} untyped=[${o.untyped.join(', ')}] okf=${o.okfVersion}`); } if (r.skipped?.length) out.push(` (skipped, unavailable: ${r.skipped.join(', ')})`); } out.push(''); const pass = results.filter((r) => r.status === 'PASS').length; const skip = results.filter((r) => r.status === 'SKIPPED').length; out.push(`Summary: ${results.length} fixtures — ${pass} pass, ${failed.length} fail, ${skip} skipped.`); if (orphans.length) out.push(` ! Unmanifested corpus directories (add to manifest): ${orphans.join(', ')}`); out.push(exitCode === 0 ? 'OK: parity holds (red fixtures went red, green fixtures agreed).' : `FAIL: ${failed.length} fixture(s) off expectation.`); process.stdout.write(`${out.join('\n')}\n`); process.exit(exitCode); }