test(okf): add running n-way per-file parity gate + adversarial corpus (STEG 3)
Make OKF §3 checker parity a RUNNING per-file red/green signal instead of a
stale one-shot claim. The catalog and okr checkers provably diverge on the same
input (catalog walks everything + no BOM/CRLF norm; okr skips innboks/dot-dirs +
normalizes); this gate turns that divergence into a monitored, tested fact.
- scripts/check-okf-parity.mjs: n-way runner, zero-dep. Normalizes each impl's
existing checkBundle(root) return to {conceptCount, untyped[], okfVersion} and
compares per file over default read-mode. Impl registry is availability-guarded
so a catalog-only checkout degrades gracefully; a documented subprocess seam
awaits llm-ingestion-okf's Python checker (fase 2) + Node port (fase 4).
- test/okf-parity-corpus/: committed byte-exact adversarial corpus + manifest.
Red-proof fixtures (expected: diverge) the gate MUST go red on — byte axis
(BOM+CRLF) and tree axis (innboks, dot-dir) each carry one; canon = both.
.gitattributes -text pins the BOM/CRLF bytes. Green fixtures pin agreement
incl. a SHARED gap (both descend into node_modules).
- check-okf-parity.test.mjs: 8/8. Corpus assertions + byte-tracking red-proof +
runtime-materialized symlink/NFC-NFD meta-tests + graceful-skip seam.
Honest scope: only 2 impls expose a runnable per-file checker today, so the gate
runs 2-way and is architected n-way — no faked third impl, no cross-repo code.
spec §7 + log.md updated (no self-certified parity; fixes stay upstream-first).
Verified: parity 8/8, CLI exit 0; regression 33/33; check-versions 10 OK.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NeK9hkxrU9wFPBYGYnSV1V
This commit is contained in:
parent
3be4321a79
commit
09521f8f2d
31 changed files with 531 additions and 10 deletions
161
scripts/check-okf-parity.mjs
Normal file
161
scripts/check-okf-parity.mjs
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
#!/usr/bin/env node
|
||||
// check-okf-parity.mjs — the running, n-way per-file OKF §3 parity gate.
|
||||
//
|
||||
// WHY THIS EXISTS (docs/okf-second-brain/log.md, STEG 3): the convention's §3
|
||||
// minimal contract is enforced by several INDEPENDENT checkers, and two of them
|
||||
// have already diverged on the same input — catalog's frozen okf-check.mjs walks
|
||||
// everything and does not normalize bytes; okr's hardened copy skips innboks/+dot-dirs
|
||||
// and strips BOM/CRLF. A one-shot lift-time parity check cannot catch that drift.
|
||||
// This gate makes parity a RUNNING red/green signal over an adversarial corpus, and
|
||||
// it CARRIES A PROOF THAT IT CAN GO RED (the corpus's `expected: "diverge"` fixtures).
|
||||
//
|
||||
// SCOPE (honest): only two implementations expose a runnable per-file conformance
|
||||
// surface today — catalog + okr. The gate is architected n-way; llm-ingestion-okf's
|
||||
// Python checker (fase 2) and its future Node port drop into IMPLS behind an
|
||||
// available() guard when they legitimately exist. It does NOT fake a third impl.
|
||||
//
|
||||
// CONTRACT: comparison unit is the PER-FILE outcome, not the bundle verdict — two
|
||||
// checkers can both say "OK" while disagreeing on what IS a concept file. Each impl's
|
||||
// existing checkBundle(root) return is normalized to { conceptCount, untyped[], okfVersion };
|
||||
// the gate diverges iff those normalized outcomes are not identical across the
|
||||
// available impls. Defined over DEFAULT read-mode (strictIngest OFF, not files-scoped).
|
||||
//
|
||||
// The runner COMPARES; it does not author the fasit. Expected agree/diverge per fixture
|
||||
// lives in the corpus manifest (catalog owns the axes; propagation is spec -> fixtures ->
|
||||
// implementations, never the reverse). Zero npm dependencies (node: builtins).
|
||||
|
||||
import { readdirSync, existsSync, readFileSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// --- Implementation registry (n-way; guarded by available()) ---
|
||||
// modulePath is resolved relative to THIS file. Sibling-repo impls (okr) resolve via
|
||||
// the polyrepo layout (catalog + okr siblings). available() lets a catalog-only checkout
|
||||
// (e.g. CI without the sibling) degrade gracefully instead of crashing.
|
||||
export const IMPLS = [
|
||||
{ name: 'catalog', modulePath: new URL('./okf-check.mjs', import.meta.url) },
|
||||
{ name: 'okr', modulePath: new URL('../../okr/scripts/okf-check.mjs', import.meta.url) },
|
||||
];
|
||||
|
||||
export function implAvailable(impl) {
|
||||
try {
|
||||
return existsSync(fileURLToPath(impl.modulePath));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize one impl's checkBundle(root) return to the per-file parity contract.
|
||||
// Returns null if the impl module is unavailable (guarded skip, reported by caller).
|
||||
export async function normalizedOutcome(impl, root) {
|
||||
if (!implAvailable(impl)) return null;
|
||||
const mod = await import(impl.modulePath);
|
||||
if (typeof mod.checkBundle !== 'function') {
|
||||
throw new Error(`impl "${impl.name}" exports no checkBundle()`);
|
||||
}
|
||||
const r = mod.checkBundle(root); // default read-mode: no options (strictIngest off, no files)
|
||||
return {
|
||||
name: impl.name,
|
||||
conceptCount: r.scanned,
|
||||
untyped: [...(r.missingType ?? [])].sort(),
|
||||
okfVersion: r.okfVersion ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
// Signature over the fields that define per-file parity. Distinct signatures => divergence.
|
||||
function signature(o) {
|
||||
return `${o.conceptCount}|${o.untyped.join(',')}|${o.okfVersion}`;
|
||||
}
|
||||
|
||||
// Run all impls on one bundle root; decide agree vs diverge among the comparable ones.
|
||||
export async function evaluateBundle(root, impls = IMPLS) {
|
||||
const outcomes = [];
|
||||
const skipped = [];
|
||||
for (const impl of impls) {
|
||||
const o = await normalizedOutcome(impl, root);
|
||||
if (o === null) skipped.push(impl.name);
|
||||
else outcomes.push(o);
|
||||
}
|
||||
const signatures = new Set(outcomes.map(signature));
|
||||
return {
|
||||
outcomes,
|
||||
skipped,
|
||||
comparableCount: outcomes.length,
|
||||
diverges: outcomes.length >= 2 && signatures.size > 1,
|
||||
};
|
||||
}
|
||||
|
||||
// Run the whole corpus against its manifest. Pure over (corpusRoot, impls); no process.exit.
|
||||
export async function runCorpus(corpusRoot, impls = IMPLS) {
|
||||
const manifestPath = join(corpusRoot, 'manifest.json');
|
||||
if (!existsSync(manifestPath)) {
|
||||
throw new Error(`corpus manifest not found: ${manifestPath}`);
|
||||
}
|
||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||
const fixtures = manifest.fixtures ?? {};
|
||||
const results = [];
|
||||
|
||||
for (const [name, spec] of Object.entries(fixtures)) {
|
||||
const dir = join(corpusRoot, name);
|
||||
if (!existsSync(dir) || !statSync(dir).isDirectory()) {
|
||||
results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'MISSING' });
|
||||
continue;
|
||||
}
|
||||
const ev = await evaluateBundle(dir, impls);
|
||||
if (ev.comparableCount < 2) {
|
||||
results.push({ fixture: name, axis: spec.axis, expected: spec.expected, status: 'SKIPPED', ...ev });
|
||||
continue;
|
||||
}
|
||||
const actual = ev.diverges ? 'diverge' : 'agree';
|
||||
const status = actual === spec.expected ? 'PASS' : 'FAIL';
|
||||
results.push({ fixture: name, axis: spec.axis, expected: spec.expected, actual, status, ...ev });
|
||||
}
|
||||
|
||||
// Orphan directories present on disk but absent from the manifest (no silent coverage gaps).
|
||||
const onDisk = readdirSync(corpusRoot, { withFileTypes: true })
|
||||
.filter((e) => e.isDirectory())
|
||||
.map((e) => e.name);
|
||||
const orphans = onDisk.filter((d) => !(d in fixtures));
|
||||
|
||||
const failed = results.filter((r) => r.status === 'FAIL' || r.status === 'MISSING');
|
||||
const availableImpls = impls.filter(implAvailable).map((i) => i.name);
|
||||
return { results, orphans, failed, availableImpls, exitCode: failed.length === 0 ? 0 : 1 };
|
||||
}
|
||||
|
||||
// --- CLI ---
|
||||
const isMain = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
|
||||
if (isMain) {
|
||||
const arg = process.argv[2];
|
||||
const corpusRoot = arg
|
||||
? arg
|
||||
: fileURLToPath(new URL('../test/okf-parity-corpus/', import.meta.url));
|
||||
const { results, orphans, failed, availableImpls, exitCode } = await runCorpus(corpusRoot);
|
||||
|
||||
const out = [];
|
||||
out.push(`OKF parity gate over: ${corpusRoot}`);
|
||||
out.push(`Implementations available (${availableImpls.length}): ${availableImpls.join(', ') || 'NONE'}`);
|
||||
if (availableImpls.length < 2) {
|
||||
out.push(' ! Fewer than 2 impls available — parity cannot be established; every fixture skipped.');
|
||||
out.push(' (This is an environment limitation, not a divergence. In a polyrepo checkout,');
|
||||
out.push(' ensure the sibling checker repos are present.)');
|
||||
}
|
||||
out.push('');
|
||||
for (const r of results) {
|
||||
const tag =
|
||||
r.status === 'PASS' && r.expected === 'diverge' ? 'PASS (red-proof)' : r.status;
|
||||
out.push(`${r.fixture.padEnd(22)} [${(r.axis || '').padEnd(16)}] expect=${(r.expected || '').padEnd(8)} ${tag}`);
|
||||
for (const o of r.outcomes ?? []) {
|
||||
out.push(` ${o.name.padEnd(10)} count=${o.conceptCount} untyped=[${o.untyped.join(', ')}] okf=${o.okfVersion}`);
|
||||
}
|
||||
if (r.skipped?.length) out.push(` (skipped, unavailable: ${r.skipped.join(', ')})`);
|
||||
}
|
||||
out.push('');
|
||||
const pass = results.filter((r) => r.status === 'PASS').length;
|
||||
const skip = results.filter((r) => r.status === 'SKIPPED').length;
|
||||
out.push(`Summary: ${results.length} fixtures — ${pass} pass, ${failed.length} fail, ${skip} skipped.`);
|
||||
if (orphans.length) out.push(` ! Unmanifested corpus directories (add to manifest): ${orphans.join(', ')}`);
|
||||
out.push(exitCode === 0 ? 'OK: parity holds (red fixtures went red, green fixtures agreed).' : `FAIL: ${failed.length} fixture(s) off expectation.`);
|
||||
|
||||
process.stdout.write(`${out.join('\n')}\n`);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue