test(okf): add running n-way per-file parity gate + adversarial corpus (STEG 3)

Make OKF §3 checker parity a RUNNING per-file red/green signal instead of a stale one-shot claim. The catalog and okr checkers provably diverge on the same input (catalog walks everything + no BOM/CRLF norm; okr skips innboks/dot-dirs + normalizes); this gate turns that divergence into a monitored, tested fact. - scripts/check-okf-parity.mjs: n-way runner, zero-dep. Normalizes each impl's existing checkBundle(root) return to {conceptCount, untyped[], okfVersion} and compares per file over default read-mode. Impl registry is availability-guarded so a catalog-only checkout degrades gracefully; a documented subprocess seam awaits llm-ingestion-okf's Python checker (fase 2) + Node port (fase 4). - test/okf-parity-corpus/: committed byte-exact adversarial corpus + manifest. Red-proof fixtures (expected: diverge) the gate MUST go red on — byte axis (BOM+CRLF) and tree axis (innboks, dot-dir) each carry one; canon = both. .gitattributes -text pins the BOM/CRLF bytes. Green fixtures pin agreement incl. a SHARED gap (both descend into node_modules). - check-okf-parity.test.mjs: 8/8. Corpus assertions + byte-tracking red-proof + runtime-materialized symlink/NFC-NFD meta-tests + graceful-skip seam. Honest scope: only 2 impls expose a runnable per-file checker today, so the gate runs 2-way and is architected n-way — no faked third impl, no cross-repo code. spec §7 + log.md updated (no self-certified parity; fixes stay upstream-first). Verified: parity 8/8, CLI exit 0; regression 33/33; check-versions 10 OK. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NeK9hkxrU9wFPBYGYnSV1V
2026-07-24 01:23:46 +02:00 · 2026-07-24 01:23:46 +02:00 · 09521f8f2d
commit 09521f8f2d
parent 3be4321a79
31 changed files with 531 additions and 10 deletions
--- a/scripts/check-okf-parity.mjs
+++ b/scripts/check-okf-parity.mjs
@ -0,0 +1,161 @@
+#!/usr/bin/env node
+// check-okf-parity.mjs — the running, n-way per-file OKF §3 parity gate.
+//
+// WHY THIS EXISTS (docs/okf-second-brain/log.md, STEG 3): the convention's §3
+// minimal contract is enforced by several INDEPENDENT checkers, and two of them
+// have already diverged on the same input — catalog's frozen okf-check.mjs walks
+// everything and does not normalize bytes; okr's hardened copy skips innboks/+dot-dirs
+// and strips BOM/CRLF. A one-shot lift-time parity check cannot catch that drift.
+// This gate makes parity a RUNNING red/green signal over an adversarial corpus, and
+// it CARRIES A PROOF THAT IT CAN GO RED (the corpus's `expected: "diverge"` fixtures).
+//
+// SCOPE (honest): only two implementations expose a runnable per-file conformance
+// surface today — catalog + okr. The gate is architected n-way; llm-ingestion-okf's
+// Python checker (fase 2) and its future Node port drop into IMPLS behind an
+// available() guard when they legitimately exist. It does NOT fake a third impl.
+//
+// CONTRACT: comparison unit is the PER-FILE outcome, not the bundle verdict — two
+// checkers can both say "OK" while disagreeing on what IS a concept file. Each impl's
+// existing checkBundle(root) return is normalized to { conceptCount, untyped[], okfVersion };
+// the gate diverges iff those normalized outcomes are not identical across the
+// available impls. Defined over DEFAULT read-mode (strictIngest OFF, not files-scoped).
+//
+// The runner COMPARES; it does not author the fasit. Expected agree/diverge per fixture
+// lives in the corpus manifest (catalog owns the axes; propagation is spec -> fixtures ->
+// implementations, never the reverse). Zero npm dependencies (node: builtins).
+
+import { readdirSync, existsSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// --- Implementation registry (n-way; guarded by available()) ---
+// modulePath is resolved relative to THIS file. Sibling-repo impls (okr) resolve via
+// the polyrepo layout (catalog + okr siblings). available() lets a catalog-only checkout
+// (e.g. CI without the sibling) degrade gracefully instead of crashing.
+export const IMPLS = [
+  { name: 'catalog', modulePath: new URL('./okf-check.mjs', import.meta.url) },
+  { name: 'okr', modulePath: new URL('../../okr/scripts/okf-check.mjs', import.meta.url) },
+];
+
+export function implAvailable(impl) {
+  try {
+    return existsSync(fileURLToPath(impl.modulePath));
+  } catch {
+    return false;
+  }
+}
+
+// Normalize one impl's checkBundle(root) return to the per-file parity contract.
+// Returns null if the impl module is unavailable (guarded skip, reported by caller).
+export async function normalizedOutcome(impl, root) {
+  if (!implAvailable(impl)) return null;
+  const mod = await import(impl.modulePath);
+  if (typeof mod.checkBundle !== 'function') {
+    throw new Error(`impl "${impl.name}" exports no checkBundle()`);
+  }
+  const r = mod.checkBundle(root); // default read-mode: no options (strictIngest off, no files)
+  return {
+    name: impl.name,
+    conceptCount: r.scanned,
+    untyped: [...(r.missingType ?? [])].sort(),
+    okfVersion: r.okfVersion ?? null,
+  };
+}
+
+// Signature over the fields that define per-file parity. Distinct signatures => divergence.
+function signature(o) {
+  return `${o.conceptCount}|${o.untyped.join(',')}|${o.okfVersion}`;
+}
+
+// Run all impls on one bundle root; decide agree vs diverge among the comparable ones.
+export async function evaluateBundle(root, impls = IMPLS) {
+  const outcomes = [];
+  const skipped = [];
+  for (const impl of impls) {
+    const o = await normalizedOutcome(impl, root);
+    if (o === null) skipped.push(impl.name);
+    else outcomes.push(o);
+  }
+  const signatures = new Set(outcomes.map(signature));
+  return {
+    outcomes,
+    skipped,
+    comparableCount: outcomes.length,
+    diverges: outcomes.length >= 2 && signatures.size > 1,
+  };
+}
+
+// Run the whole corpus against its manifest. Pure over (corpusRoot, impls); no process.exit.
+export async function runCorpus(corpusRoot, impls = IMPLS) {
+  const manifestPath = join(corpusRoot, 'manifest.json');
+  if (!existsSync(manifestPath)) {
+    throw new Error(`corpus manifest not found: ${manifestPath}`);
+  }
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+  const fixtures = manifest.fixtures ?? {};
+  const results = [];
+
+  for (const [name, spec] of Object.entries(fixtures)) {
+    const dir = join(corpusRoot, name);
+    if (!existsSync(dir) || !statSync(dir).isDirectory()) {
+      results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'MISSING' });
+      continue;
+    }
+    const ev = await evaluateBundle(dir, impls);
+    if (ev.comparableCount < 2) {
+      results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'SKIPPED', ...ev });
+      continue;
+    }
+    const actual = ev.diverges ? 'diverge' : 'agree';
+    const status = actual === spec.expected ? 'PASS' : 'FAIL';
+    results.push({ fixture: name, axis: spec.axis, expected: spec.expected, actual, status, ...ev });
+  }
+
+  // Orphan directories present on disk but absent from the manifest (no silent coverage gaps).
+  const onDisk = readdirSync(corpusRoot, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+  const orphans = onDisk.filter((d) => !(d in fixtures));
+
+  const failed = results.filter((r) => r.status === 'FAIL' || r.status === 'MISSING');
+  const availableImpls = impls.filter(implAvailable).map((i) => i.name);
+  return { results, orphans, failed, availableImpls, exitCode: failed.length === 0 ? 0 : 1 };
+}
+
+// --- CLI ---
+const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+if (isMain) {
+  const arg = process.argv[2];
+  const corpusRoot = arg
+    ? arg
+    : fileURLToPath(new URL('../test/okf-parity-corpus/', import.meta.url));
+  const { results, orphans, failed, availableImpls, exitCode } = await runCorpus(corpusRoot);
+
+  const out = [];
+  out.push(`OKF parity gate over: ${corpusRoot}`);
+  out.push(`Implementations available (${availableImpls.length}): ${availableImpls.join(', ') || 'NONE'}`);
+  if (availableImpls.length < 2) {
+    out.push('  ! Fewer than 2 impls available — parity cannot be established; every fixture skipped.');
+    out.push('    (This is an environment limitation, not a divergence. In a polyrepo checkout,');
+    out.push('     ensure the sibling checker repos are present.)');
+  }
+  out.push('');
+  for (const r of results) {
+    const tag =
+      r.status === 'PASS' && r.expected === 'diverge' ? 'PASS (red-proof)' : r.status;
+    out.push(`${r.fixture.padEnd(22)} [${(r.axis || '').padEnd(16)}] expect=${(r.expected || '').padEnd(8)} ${tag}`);
+    for (const o of r.outcomes ?? []) {
+      out.push(`    ${o.name.padEnd(10)} count=${o.conceptCount}  untyped=[${o.untyped.join(', ')}]  okf=${o.okfVersion}`);
+    }
+    if (r.skipped?.length) out.push(`    (skipped, unavailable: ${r.skipped.join(', ')})`);
+  }
+  out.push('');
+  const pass = results.filter((r) => r.status === 'PASS').length;
+  const skip = results.filter((r) => r.status === 'SKIPPED').length;
+  out.push(`Summary: ${results.length} fixtures — ${pass} pass, ${failed.length} fail, ${skip} skipped.`);
+  if (orphans.length) out.push(`  ! Unmanifested corpus directories (add to manifest): ${orphans.join(', ')}`);
+  out.push(exitCode === 0 ? 'OK: parity holds (red fixtures went red, green fixtures agreed).' : `FAIL: ${failed.length} fixture(s) off expectation.`);
+
+  process.stdout.write(`${out.join('\n')}\n`);
+  process.exit(exitCode);
+}
--- a/scripts/check-okf-parity.test.mjs
+++ b/scripts/check-okf-parity.test.mjs
@ -0,0 +1,152 @@
+// Tests for the OKF parity gate (check-okf-parity.mjs).
+// Style mirrors okf-check.test.mjs: node:test, temp bundles, subprocess for the CLI
+// exit contract, zero npm deps. Two layers:
+//   1. Over the COMMITTED corpus — green fixtures agree, red fixtures diverge (red-proof).
+//   2. Runtime-materialized meta-tests for the git/FS-hostile axes (symlink, NFC/NFD) and
+//      the byte-tracking red-proof, plus the graceful-skip registry seam.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, writeFileSync, mkdirSync, rmSync, symlinkSync, readdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { IMPLS, evaluateBundle, runCorpus, normalizedOutcome } from './check-okf-parity.mjs';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const RUNNER = join(HERE, 'check-okf-parity.mjs');
+const CORPUS = join(HERE, '..', 'test', 'okf-parity-corpus');
+
+const CATALOG = IMPLS.find((i) => i.name === 'catalog');
+const OKR = IMPLS.find((i) => i.name === 'okr');
+const GHOST = { name: 'ghost', modulePath: new URL('./does-not-exist.mjs', import.meta.url) };
+
+function tmpRoot() {
+  return mkdtempSync(join(tmpdir(), 'okf-parity-'));
+}
+const ROOTINDEX = 'okf_version: 0.1\n\n# Bundle\n';
+const TYPED = '---\ntype: Note\ntitle: t\ndescription: d\nresource: r\ntimestamp: 2026-07-23\n---\n# c\n';
+
+// --- A1/A2: both real impls import cross-repo and yield the contract ---
+test('smoke: catalog + okr both import and return the per-file contract (A1/A2)', async () => {
+  assert.ok(OKR, 'okr must be a registered impl');
+  const dir = tmpRoot();
+  try {
+    writeFileSync(join(dir, 'index.md'), ROOTINDEX);
+    writeFileSync(join(dir, 'c.md'), TYPED);
+    const cat = await normalizedOutcome(CATALOG, dir);
+    const okr = await normalizedOutcome(OKR, dir);
+    for (const o of [cat, okr]) {
+      assert.equal(typeof o.conceptCount, 'number');
+      assert.ok(Array.isArray(o.untyped));
+      assert.equal(o.okfVersion, '0.1');
+    }
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// --- Layer 1: the committed corpus ---
+test('committed corpus: green fixtures agree, red fixtures diverge (red-proof)', async () => {
+  const { results, failed, availableImpls } = await runCorpus(CORPUS);
+  assert.ok(availableImpls.length >= 2, `need >=2 impls to test parity, have: ${availableImpls.join(',')}`);
+  assert.deepEqual(failed, [], `no fixture may miss its expectation; failed: ${JSON.stringify(failed.map((f) => f.fixture))}`);
+  const byName = Object.fromEntries(results.map((r) => [r.fixture, r]));
+  // every green-* agrees, every red-* diverges
+  for (const r of results) {
+    if (r.fixture.startsWith('green-')) assert.equal(r.actual, 'agree', `${r.fixture} must agree`);
+    if (r.fixture.startsWith('red-')) assert.equal(r.actual, 'diverge', `${r.fixture} must diverge`);
+    assert.equal(r.status, 'PASS');
+  }
+  // the canon fixture: catalog FAIL(>=1 untyped) vs okr OK(0 untyped)
+  const canon = byName['red-combined'];
+  const cat = canon.outcomes.find((o) => o.name === 'catalog');
+  const okr = canon.outcomes.find((o) => o.name === 'okr');
+  assert.ok(cat.untyped.length >= 1, 'catalog counts BOM+innboks as untyped');
+  assert.equal(okr.untyped.length, 0, 'okr normalizes BOM + skips innboks -> clean');
+});
+
+test('CLI over committed corpus: exit 0 + "parity holds" + labels red-proof', () => {
+  const stdout = execFileSync('node', [RUNNER], { encoding: 'utf8' });
+  assert.match(stdout, /OK: parity holds/);
+  assert.match(stdout, /PASS \(red-proof\)/);
+});
+
+// --- Layer 2a: byte-tracking red-proof (the gate's redness follows the bytes) ---
+test('red-proof: BOM+CRLF typed concept diverges; the same file as UTF-8/LF agrees', async () => {
+  const impls = [CATALOG, OKR];
+  const withBom = tmpRoot();
+  const clean = tmpRoot();
+  try {
+    writeFileSync(join(withBom, 'index.md'), ROOTINDEX);
+    writeFileSync(join(withBom, 'c.md'), '---\r\ntype: Note\r\n---\r\n# c\r\n'); // BOM + CRLF
+    const evBom = await evaluateBundle(withBom, impls);
+    assert.equal(evBom.diverges, true, 'BOM+CRLF must split catalog (fence miss) vs okr (normalized)');
+
+    writeFileSync(join(clean, 'index.md'), ROOTINDEX);
+    writeFileSync(join(clean, 'c.md'), '---\ntype: Note\n---\n# c\n'); // same concept, plain LF
+    const evClean = await evaluateBundle(clean, impls);
+    assert.equal(evClean.diverges, false, 'neutralizing the bytes makes the gate go green — redness tracks bytes');
+  } finally {
+    rmSync(withBom, { recursive: true, force: true });
+    rmSync(clean, { recursive: true, force: true });
+  }
+});
+
+// --- Layer 2b: symlink axis (runtime-materialized; must not hang/crash) ---
+test('symlink axis: symlinked file + dir + loop are skipped by both, no hang, agree', async () => {
+  const dir = tmpRoot();
+  try {
+    writeFileSync(join(dir, 'index.md'), ROOTINDEX);
+    writeFileSync(join(dir, 'real.md'), TYPED);
+    symlinkSync(join(dir, 'real.md'), join(dir, 'link.md')); // symlinked .md
+    mkdirSync(join(dir, 'sub'));
+    symlinkSync(join(dir, 'sub'), join(dir, 'subline')); // symlinked dir
+    symlinkSync('.', join(dir, 'loop')); // self-loop; must not be followed
+    const ev = await evaluateBundle(dir, [CATALOG, OKR]);
+    assert.equal(ev.diverges, false, 'both skip symlinks (Dirent isFile/isDirectory false) -> agree');
+    assert.equal(ev.comparableCount, 2);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// --- Layer 2c: NFC/NFD filename axis (platform-sensitive, soft — log, never hard-fail) ---
+test('NFC/NFD filename axis: runs without crash; logs what the FS preserved (soft)', async () => {
+  const dir = tmpRoot();
+  try {
+    writeFileSync(join(dir, 'index.md'), ROOTINDEX);
+    writeFileSync(join(dir, 'café.md'), TYPED); // NFC é
+    writeFileSync(join(dir, 'café.md'), TYPED); // NFD é
+    const present = readdirSync(dir).filter((n) => n.startsWith('caf'));
+    // Soft: on APFS both may coexist (2) or normalize to one (1). Either is acceptable;
+    // we only require the gate to survive and both impls to see the same FS.
+    const ev = await evaluateBundle(dir, [CATALOG, OKR]);
+    assert.equal(ev.diverges, false, 'both impls read the same FS bytes -> agree (no filename normalization in either)');
+    console.log(`[nfc-nfd] FS preserved ${present.length} of 2 filename variants: ${present.join(', ')}`);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// --- Layer 2d: graceful-skip registry seam (polyrepo / catalog-only CI) ---
+test('graceful skip: an unavailable impl is skipped with a notice, no crash', async () => {
+  const dir = tmpRoot();
+  try {
+    writeFileSync(join(dir, 'index.md'), ROOTINDEX);
+    writeFileSync(join(dir, 'c.md'), TYPED);
+    const ev = await evaluateBundle(dir, [CATALOG, GHOST]);
+    assert.deepEqual(ev.skipped, ['ghost']);
+    assert.equal(ev.comparableCount, 1);
+    assert.equal(ev.diverges, false, '<2 comparable impls -> cannot diverge');
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test('graceful skip: runCorpus with <2 available impls -> all SKIPPED, exit 0', async () => {
+  const { results, exitCode, availableImpls } = await runCorpus(CORPUS, [CATALOG, GHOST]);
+  assert.deepEqual(availableImpls, ['catalog']);
+  assert.ok(results.every((r) => r.status === 'SKIPPED'), 'every fixture skipped when only 1 impl is present');
+  assert.equal(exitCode, 0, 'a missing sibling is an environment limitation, not a parity failure');
+});