test(okf): add running n-way per-file parity gate + adversarial corpus (STEG 3)

Make OKF §3 checker parity a RUNNING per-file red/green signal instead of a
stale one-shot claim. The catalog and okr checkers provably diverge on the same
input (catalog walks everything + no BOM/CRLF norm; okr skips innboks/dot-dirs +
normalizes); this gate turns that divergence into a monitored, tested fact.

- scripts/check-okf-parity.mjs: n-way runner, zero-dep. Normalizes each impl's
  existing checkBundle(root) return to {conceptCount, untyped[], okfVersion} and
  compares per file over default read-mode. Impl registry is availability-guarded
  so a catalog-only checkout degrades gracefully; a documented subprocess seam
  awaits llm-ingestion-okf's Python checker (fase 2) + Node port (fase 4).
- test/okf-parity-corpus/: committed byte-exact adversarial corpus + manifest.
  Red-proof fixtures (expected: diverge) the gate MUST go red on — byte axis
  (BOM+CRLF) and tree axis (innboks, dot-dir) each carry one; canon = both.
  .gitattributes -text pins the BOM/CRLF bytes. Green fixtures pin agreement
  incl. a SHARED gap (both descend into node_modules).
- check-okf-parity.test.mjs: 8/8. Corpus assertions + byte-tracking red-proof +
  runtime-materialized symlink/NFC-NFD meta-tests + graceful-skip seam.

Honest scope: only 2 impls expose a runnable per-file checker today, so the gate
runs 2-way and is architected n-way — no faked third impl, no cross-repo code.
spec §7 + log.md updated (no self-certified parity; fixes stay upstream-first).

Verified: parity 8/8, CLI exit 0; regression 33/33; check-versions 10 OK.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NeK9hkxrU9wFPBYGYnSV1V
This commit is contained in:
Kjell Tore Guttormsen 2026-07-24 01:23:46 +02:00
commit 09521f8f2d
31 changed files with 531 additions and 10 deletions

View file

@ -0,0 +1,152 @@
// Tests for the OKF parity gate (check-okf-parity.mjs).
// Style mirrors okf-check.test.mjs: node:test, temp bundles, subprocess for the CLI
// exit contract, zero npm deps. Two layers:
// 1. Over the COMMITTED corpus — green fixtures agree, red fixtures diverge (red-proof).
// 2. Runtime-materialized meta-tests for the git/FS-hostile axes (symlink, NFC/NFD) and
// the byte-tracking red-proof, plus the graceful-skip registry seam.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, symlinkSync, readdirSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { IMPLS, evaluateBundle, runCorpus, normalizedOutcome } from './check-okf-parity.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const RUNNER = join(HERE, 'check-okf-parity.mjs');
const CORPUS = join(HERE, '..', 'test', 'okf-parity-corpus');
const CATALOG = IMPLS.find((i) => i.name === 'catalog');
const OKR = IMPLS.find((i) => i.name === 'okr');
const GHOST = { name: 'ghost', modulePath: new URL('./does-not-exist.mjs', import.meta.url) };
function tmpRoot() {
return mkdtempSync(join(tmpdir(), 'okf-parity-'));
}
const ROOTINDEX = 'okf_version: 0.1\n\n# Bundle\n';
const TYPED = '---\ntype: Note\ntitle: t\ndescription: d\nresource: r\ntimestamp: 2026-07-23\n---\n# c\n';
// --- A1/A2: both real impls import cross-repo and yield the contract ---
test('smoke: catalog + okr both import and return the per-file contract (A1/A2)', async () => {
assert.ok(OKR, 'okr must be a registered impl');
const dir = tmpRoot();
try {
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
writeFileSync(join(dir, 'c.md'), TYPED);
const cat = await normalizedOutcome(CATALOG, dir);
const okr = await normalizedOutcome(OKR, dir);
for (const o of [cat, okr]) {
assert.equal(typeof o.conceptCount, 'number');
assert.ok(Array.isArray(o.untyped));
assert.equal(o.okfVersion, '0.1');
}
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// --- Layer 1: the committed corpus ---
test('committed corpus: green fixtures agree, red fixtures diverge (red-proof)', async () => {
const { results, failed, availableImpls } = await runCorpus(CORPUS);
assert.ok(availableImpls.length >= 2, `need >=2 impls to test parity, have: ${availableImpls.join(',')}`);
assert.deepEqual(failed, [], `no fixture may miss its expectation; failed: ${JSON.stringify(failed.map((f) => f.fixture))}`);
const byName = Object.fromEntries(results.map((r) => [r.fixture, r]));
// every green-* agrees, every red-* diverges
for (const r of results) {
if (r.fixture.startsWith('green-')) assert.equal(r.actual, 'agree', `${r.fixture} must agree`);
if (r.fixture.startsWith('red-')) assert.equal(r.actual, 'diverge', `${r.fixture} must diverge`);
assert.equal(r.status, 'PASS');
}
// the canon fixture: catalog FAIL(>=1 untyped) vs okr OK(0 untyped)
const canon = byName['red-combined'];
const cat = canon.outcomes.find((o) => o.name === 'catalog');
const okr = canon.outcomes.find((o) => o.name === 'okr');
assert.ok(cat.untyped.length >= 1, 'catalog counts BOM+innboks as untyped');
assert.equal(okr.untyped.length, 0, 'okr normalizes BOM + skips innboks -> clean');
});
test('CLI over committed corpus: exit 0 + "parity holds" + labels red-proof', () => {
const stdout = execFileSync('node', [RUNNER], { encoding: 'utf8' });
assert.match(stdout, /OK: parity holds/);
assert.match(stdout, /PASS \(red-proof\)/);
});
// --- Layer 2a: byte-tracking red-proof (the gate's redness follows the bytes) ---
test('red-proof: BOM+CRLF typed concept diverges; the same file as UTF-8/LF agrees', async () => {
const impls = [CATALOG, OKR];
const withBom = tmpRoot();
const clean = tmpRoot();
try {
writeFileSync(join(withBom, 'index.md'), ROOTINDEX);
writeFileSync(join(withBom, 'c.md'), '---\r\ntype: Note\r\n---\r\n# c\r\n'); // BOM + CRLF
const evBom = await evaluateBundle(withBom, impls);
assert.equal(evBom.diverges, true, 'BOM+CRLF must split catalog (fence miss) vs okr (normalized)');
writeFileSync(join(clean, 'index.md'), ROOTINDEX);
writeFileSync(join(clean, 'c.md'), '---\ntype: Note\n---\n# c\n'); // same concept, plain LF
const evClean = await evaluateBundle(clean, impls);
assert.equal(evClean.diverges, false, 'neutralizing the bytes makes the gate go green — redness tracks bytes');
} finally {
rmSync(withBom, { recursive: true, force: true });
rmSync(clean, { recursive: true, force: true });
}
});
// --- Layer 2b: symlink axis (runtime-materialized; must not hang/crash) ---
test('symlink axis: symlinked file + dir + loop are skipped by both, no hang, agree', async () => {
const dir = tmpRoot();
try {
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
writeFileSync(join(dir, 'real.md'), TYPED);
symlinkSync(join(dir, 'real.md'), join(dir, 'link.md')); // symlinked .md
mkdirSync(join(dir, 'sub'));
symlinkSync(join(dir, 'sub'), join(dir, 'subline')); // symlinked dir
symlinkSync('.', join(dir, 'loop')); // self-loop; must not be followed
const ev = await evaluateBundle(dir, [CATALOG, OKR]);
assert.equal(ev.diverges, false, 'both skip symlinks (Dirent isFile/isDirectory false) -> agree');
assert.equal(ev.comparableCount, 2);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// --- Layer 2c: NFC/NFD filename axis (platform-sensitive, soft — log, never hard-fail) ---
test('NFC/NFD filename axis: runs without crash; logs what the FS preserved (soft)', async () => {
const dir = tmpRoot();
try {
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
writeFileSync(join(dir, 'café.md'), TYPED); // NFC é
writeFileSync(join(dir, 'café.md'), TYPED); // NFD é
const present = readdirSync(dir).filter((n) => n.startsWith('caf'));
// Soft: on APFS both may coexist (2) or normalize to one (1). Either is acceptable;
// we only require the gate to survive and both impls to see the same FS.
const ev = await evaluateBundle(dir, [CATALOG, OKR]);
assert.equal(ev.diverges, false, 'both impls read the same FS bytes -> agree (no filename normalization in either)');
console.log(`[nfc-nfd] FS preserved ${present.length} of 2 filename variants: ${present.join(', ')}`);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
// --- Layer 2d: graceful-skip registry seam (polyrepo / catalog-only CI) ---
test('graceful skip: an unavailable impl is skipped with a notice, no crash', async () => {
const dir = tmpRoot();
try {
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
writeFileSync(join(dir, 'c.md'), TYPED);
const ev = await evaluateBundle(dir, [CATALOG, GHOST]);
assert.deepEqual(ev.skipped, ['ghost']);
assert.equal(ev.comparableCount, 1);
assert.equal(ev.diverges, false, '<2 comparable impls -> cannot diverge');
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test('graceful skip: runCorpus with <2 available impls -> all SKIPPED, exit 0', async () => {
const { results, exitCode, availableImpls } = await runCorpus(CORPUS, [CATALOG, GHOST]);
assert.deepEqual(availableImpls, ['catalog']);
assert.ok(results.every((r) => r.status === 'SKIPPED'), 'every fixture skipped when only 1 impl is present');
assert.equal(exitCode, 0, 'a missing sibling is an environment limitation, not a parity failure');
});