test(okf): add running n-way per-file parity gate + adversarial corpus (STEG 3)
Make OKF §3 checker parity a RUNNING per-file red/green signal instead of a
stale one-shot claim. The catalog and okr checkers provably diverge on the same
input (catalog walks everything + no BOM/CRLF norm; okr skips innboks/dot-dirs +
normalizes); this gate turns that divergence into a monitored, tested fact.
- scripts/check-okf-parity.mjs: n-way runner, zero-dep. Normalizes each impl's
existing checkBundle(root) return to {conceptCount, untyped[], okfVersion} and
compares per file over default read-mode. Impl registry is availability-guarded
so a catalog-only checkout degrades gracefully; a documented subprocess seam
awaits llm-ingestion-okf's Python checker (fase 2) + Node port (fase 4).
- test/okf-parity-corpus/: committed byte-exact adversarial corpus + manifest.
Red-proof fixtures (expected: diverge) the gate MUST go red on — byte axis
(BOM+CRLF) and tree axis (innboks, dot-dir) each carry one; canon = both.
.gitattributes -text pins the BOM/CRLF bytes. Green fixtures pin agreement
incl. a SHARED gap (both descend into node_modules).
- check-okf-parity.test.mjs: 8/8. Corpus assertions + byte-tracking red-proof +
runtime-materialized symlink/NFC-NFD meta-tests + graceful-skip seam.
Honest scope: only 2 impls expose a runnable per-file checker today, so the gate
runs 2-way and is architected n-way — no faked third impl, no cross-repo code.
spec §7 + log.md updated (no self-certified parity; fixes stay upstream-first).
Verified: parity 8/8, CLI exit 0; regression 33/33; check-versions 10 OK.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NeK9hkxrU9wFPBYGYnSV1V
This commit is contained in:
parent
3be4321a79
commit
09521f8f2d
31 changed files with 531 additions and 10 deletions
152
scripts/check-okf-parity.test.mjs
Normal file
152
scripts/check-okf-parity.test.mjs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
// Tests for the OKF parity gate (check-okf-parity.mjs).
|
||||
// Style mirrors okf-check.test.mjs: node:test, temp bundles, subprocess for the CLI
|
||||
// exit contract, zero npm deps. Two layers:
|
||||
// 1. Over the COMMITTED corpus — green fixtures agree, red fixtures diverge (red-proof).
|
||||
// 2. Runtime-materialized meta-tests for the git/FS-hostile axes (symlink, NFC/NFD) and
|
||||
// the byte-tracking red-proof, plus the graceful-skip registry seam.
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, symlinkSync, readdirSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { IMPLS, evaluateBundle, runCorpus, normalizedOutcome } from './check-okf-parity.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const RUNNER = join(HERE, 'check-okf-parity.mjs');
|
||||
const CORPUS = join(HERE, '..', 'test', 'okf-parity-corpus');
|
||||
|
||||
const CATALOG = IMPLS.find((i) => i.name === 'catalog');
|
||||
const OKR = IMPLS.find((i) => i.name === 'okr');
|
||||
const GHOST = { name: 'ghost', modulePath: new URL('./does-not-exist.mjs', import.meta.url) };
|
||||
|
||||
function tmpRoot() {
|
||||
return mkdtempSync(join(tmpdir(), 'okf-parity-'));
|
||||
}
|
||||
const ROOTINDEX = 'okf_version: 0.1\n\n# Bundle\n';
|
||||
const TYPED = '---\ntype: Note\ntitle: t\ndescription: d\nresource: r\ntimestamp: 2026-07-23\n---\n# c\n';
|
||||
|
||||
// --- A1/A2: both real impls import cross-repo and yield the contract ---
|
||||
test('smoke: catalog + okr both import and return the per-file contract (A1/A2)', async () => {
|
||||
assert.ok(OKR, 'okr must be a registered impl');
|
||||
const dir = tmpRoot();
|
||||
try {
|
||||
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(dir, 'c.md'), TYPED);
|
||||
const cat = await normalizedOutcome(CATALOG, dir);
|
||||
const okr = await normalizedOutcome(OKR, dir);
|
||||
for (const o of [cat, okr]) {
|
||||
assert.equal(typeof o.conceptCount, 'number');
|
||||
assert.ok(Array.isArray(o.untyped));
|
||||
assert.equal(o.okfVersion, '0.1');
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Layer 1: the committed corpus ---
|
||||
test('committed corpus: green fixtures agree, red fixtures diverge (red-proof)', async () => {
|
||||
const { results, failed, availableImpls } = await runCorpus(CORPUS);
|
||||
assert.ok(availableImpls.length >= 2, `need >=2 impls to test parity, have: ${availableImpls.join(',')}`);
|
||||
assert.deepEqual(failed, [], `no fixture may miss its expectation; failed: ${JSON.stringify(failed.map((f) => f.fixture))}`);
|
||||
const byName = Object.fromEntries(results.map((r) => [r.fixture, r]));
|
||||
// every green-* agrees, every red-* diverges
|
||||
for (const r of results) {
|
||||
if (r.fixture.startsWith('green-')) assert.equal(r.actual, 'agree', `${r.fixture} must agree`);
|
||||
if (r.fixture.startsWith('red-')) assert.equal(r.actual, 'diverge', `${r.fixture} must diverge`);
|
||||
assert.equal(r.status, 'PASS');
|
||||
}
|
||||
// the canon fixture: catalog FAIL(>=1 untyped) vs okr OK(0 untyped)
|
||||
const canon = byName['red-combined'];
|
||||
const cat = canon.outcomes.find((o) => o.name === 'catalog');
|
||||
const okr = canon.outcomes.find((o) => o.name === 'okr');
|
||||
assert.ok(cat.untyped.length >= 1, 'catalog counts BOM+innboks as untyped');
|
||||
assert.equal(okr.untyped.length, 0, 'okr normalizes BOM + skips innboks -> clean');
|
||||
});
|
||||
|
||||
test('CLI over committed corpus: exit 0 + "parity holds" + labels red-proof', () => {
|
||||
const stdout = execFileSync('node', [RUNNER], { encoding: 'utf8' });
|
||||
assert.match(stdout, /OK: parity holds/);
|
||||
assert.match(stdout, /PASS \(red-proof\)/);
|
||||
});
|
||||
|
||||
// --- Layer 2a: byte-tracking red-proof (the gate's redness follows the bytes) ---
|
||||
test('red-proof: BOM+CRLF typed concept diverges; the same file as UTF-8/LF agrees', async () => {
|
||||
const impls = [CATALOG, OKR];
|
||||
const withBom = tmpRoot();
|
||||
const clean = tmpRoot();
|
||||
try {
|
||||
writeFileSync(join(withBom, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(withBom, 'c.md'), '---\r\ntype: Note\r\n---\r\n# c\r\n'); // BOM + CRLF
|
||||
const evBom = await evaluateBundle(withBom, impls);
|
||||
assert.equal(evBom.diverges, true, 'BOM+CRLF must split catalog (fence miss) vs okr (normalized)');
|
||||
|
||||
writeFileSync(join(clean, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(clean, 'c.md'), '---\ntype: Note\n---\n# c\n'); // same concept, plain LF
|
||||
const evClean = await evaluateBundle(clean, impls);
|
||||
assert.equal(evClean.diverges, false, 'neutralizing the bytes makes the gate go green — redness tracks bytes');
|
||||
} finally {
|
||||
rmSync(withBom, { recursive: true, force: true });
|
||||
rmSync(clean, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Layer 2b: symlink axis (runtime-materialized; must not hang/crash) ---
|
||||
test('symlink axis: symlinked file + dir + loop are skipped by both, no hang, agree', async () => {
|
||||
const dir = tmpRoot();
|
||||
try {
|
||||
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(dir, 'real.md'), TYPED);
|
||||
symlinkSync(join(dir, 'real.md'), join(dir, 'link.md')); // symlinked .md
|
||||
mkdirSync(join(dir, 'sub'));
|
||||
symlinkSync(join(dir, 'sub'), join(dir, 'subline')); // symlinked dir
|
||||
symlinkSync('.', join(dir, 'loop')); // self-loop; must not be followed
|
||||
const ev = await evaluateBundle(dir, [CATALOG, OKR]);
|
||||
assert.equal(ev.diverges, false, 'both skip symlinks (Dirent isFile/isDirectory false) -> agree');
|
||||
assert.equal(ev.comparableCount, 2);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Layer 2c: NFC/NFD filename axis (platform-sensitive, soft — log, never hard-fail) ---
|
||||
test('NFC/NFD filename axis: runs without crash; logs what the FS preserved (soft)', async () => {
|
||||
const dir = tmpRoot();
|
||||
try {
|
||||
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(dir, 'café.md'), TYPED); // NFC é
|
||||
writeFileSync(join(dir, 'café.md'), TYPED); // NFD é
|
||||
const present = readdirSync(dir).filter((n) => n.startsWith('caf'));
|
||||
// Soft: on APFS both may coexist (2) or normalize to one (1). Either is acceptable;
|
||||
// we only require the gate to survive and both impls to see the same FS.
|
||||
const ev = await evaluateBundle(dir, [CATALOG, OKR]);
|
||||
assert.equal(ev.diverges, false, 'both impls read the same FS bytes -> agree (no filename normalization in either)');
|
||||
console.log(`[nfc-nfd] FS preserved ${present.length} of 2 filename variants: ${present.join(', ')}`);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// --- Layer 2d: graceful-skip registry seam (polyrepo / catalog-only CI) ---
|
||||
test('graceful skip: an unavailable impl is skipped with a notice, no crash', async () => {
|
||||
const dir = tmpRoot();
|
||||
try {
|
||||
writeFileSync(join(dir, 'index.md'), ROOTINDEX);
|
||||
writeFileSync(join(dir, 'c.md'), TYPED);
|
||||
const ev = await evaluateBundle(dir, [CATALOG, GHOST]);
|
||||
assert.deepEqual(ev.skipped, ['ghost']);
|
||||
assert.equal(ev.comparableCount, 1);
|
||||
assert.equal(ev.diverges, false, '<2 comparable impls -> cannot diverge');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('graceful skip: runCorpus with <2 available impls -> all SKIPPED, exit 0', async () => {
|
||||
const { results, exitCode, availableImpls } = await runCorpus(CORPUS, [CATALOG, GHOST]);
|
||||
assert.deepEqual(availableImpls, ['catalog']);
|
||||
assert.ok(results.every((r) => r.status === 'SKIPPED'), 'every fixture skipped when only 1 impl is present');
|
||||
assert.equal(exitCode, 0, 'a missing sibling is an environment limitation, not a parity failure');
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue