Kjell Tore Guttormsen
4a36fd1853
Måleresultatets sentrale påstand — at forslagene er tekstlig ærlige — hvilte på et sesjons-lokalt skript ingen kunne etterprøve. Flyttet inn som bibliotek + CLI med tester, så tallet kan reproduseres fra fersk klon: node scripts/kb-eval/check-o2-returns.mjs Sjekkene avgjør IKKE O2 — betingelse 2 og 3 er fortsatt menneskelige. De avgrenser de to feilmodusene et menneske ikke fanger billig over 46 forslag: - V1: sitert filtekst må finnes ordrett i fila (fanger oppdiktet tekst og stille æøå-transliterering). Gjelder HVER rad, også O3 — en O3 basert på oppdiktet tekst er like feil, bare feil i trygg retning. - V2: forslaget må kunne oppnås ved kun å slette tegn. - V2b: V2 alene er for svak — 'Automatically add' -> 'Add' passerer fordi den store A-en fantes inne i det slettede ordet. Ordnivå-sjekk, case-sensitiv. - V3: skjema- og verdikt-koherens. Suite 996 -> 1021.
74 lines
3.5 KiB
JavaScript
74 lines
3.5 KiB
JavaScript
#!/usr/bin/env node
|
|
// check-o2-returns.mjs — R11 §10 measurement #2: verify and tally the O2/O3
|
|
// classification returns. READ-ONLY; writes nothing.
|
|
//
|
|
// node scripts/kb-eval/check-o2-returns.mjs
|
|
//
|
|
// Runs the V1/V2/V2b/V3 checks (scripts/kb-eval/lib/o2-return-check.mjs) over
|
|
// scripts/kb-eval/data/r11-o2-returns/*.json and prints the measurement: the
|
|
// O2/O3 split, the machine-clean candidate count, and — the actionable part —
|
|
// WHICH of §5's three conditions forecloses each O3. The top-level split alone
|
|
// says nothing; the blocking condition is where the decision lives.
|
|
|
|
import { readFileSync, readdirSync } from 'node:fs';
|
|
import { join, dirname, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
import { checkRow } from './lib/o2-return-check.mjs';
|
|
|
|
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
|
|
const RETURNS = join(REPO, 'scripts/kb-eval/data/r11-o2-returns');
|
|
|
|
const cache = new Map();
|
|
const readRepoFile = (rel) => {
|
|
if (!cache.has(rel)) cache.set(rel, readFileSync(join(REPO, rel), 'utf8'));
|
|
return cache.get(rel);
|
|
};
|
|
|
|
const files = readdirSync(RETURNS).filter((f) => f.endsWith('.json')).sort();
|
|
const rows = files.flatMap((f) =>
|
|
JSON.parse(readFileSync(join(RETURNS, f), 'utf8')).map((r) => ({ ...r, _batch: f })));
|
|
|
|
const findings = rows.flatMap((r) =>
|
|
checkRow(r, readRepoFile).map((f) => ({ ...f, idx: r.idx, file: r.file, line: r.line, batch: r._batch })));
|
|
|
|
const flaggedIdx = new Set(findings.map((f) => f.idx));
|
|
const o2 = rows.filter((r) => r.verdict === 'O2_CANDIDATE');
|
|
const o3 = rows.filter((r) => r.verdict === 'O3');
|
|
const clean = o2.filter((r) => !flaggedIdx.has(r.idx));
|
|
|
|
// Which condition forecloses O2. "human_must_confirm" counts as NOT held: the
|
|
// point of the tri-state is that an unresolved condition is not a satisfied one.
|
|
const held = (v) => v === true || v === 'yes';
|
|
const blockTally = {};
|
|
for (const r of o3) {
|
|
const failed = [
|
|
!held(r.cond1_strictly_less?.holds) && 'cond1',
|
|
!held(r.cond2_remainder_not_misleading?.holds) && 'cond2',
|
|
!held(r.cond3_nothing_confirmed_removed?.holds) && 'cond3',
|
|
].filter(Boolean);
|
|
const key = failed.length ? failed.join('+') : 'none-stated';
|
|
blockTally[key] = (blockTally[key] || 0) + 1;
|
|
}
|
|
const cond3Blocked = o3.filter((r) => !held(r.cond3_nothing_confirmed_removed?.holds)).length;
|
|
|
|
const tally = (xs, key) => xs.reduce((a, x) => ({ ...a, [x[key]]: (a[x[key]] || 0) + 1 }), {});
|
|
const pct = (n) => `${((n / rows.length) * 100).toFixed(1)} %`;
|
|
|
|
console.log(`R11 §10 #2 — R8 multi-part claims (pilot): ${rows.length} items from ${files.length} batches`);
|
|
console.log(` O2_CANDIDATE ${o2.length} (${pct(o2.length)}) · O3 ${o3.length} (${pct(o3.length)})`);
|
|
console.log(` locator_failed: ${rows.filter((r) => r.locator_failed).length}`);
|
|
console.log(` machine-clean O2 candidates: ${clean.length}/${o2.length}`);
|
|
console.log(` O2 confidence: ${JSON.stringify(tally(o2, 'confidence'))}`);
|
|
console.log(`\nO3 blocking conditions: ${JSON.stringify(blockTally)}`);
|
|
console.log(`O3 where condition 3 fails (source supplies a corrected value → swap/rewrite): ${cond3Blocked}/${o3.length}`);
|
|
|
|
console.log(`\nmachine findings: ${findings.length}`);
|
|
for (const f of findings) console.log(` [${f.check}] idx=${f.idx} ${f.file}:${f.line} — ${f.detail}`);
|
|
|
|
console.log('\nO2 candidates (review-grade — conditions 2 and 3 are human-confirmed):');
|
|
for (const r of o2) {
|
|
console.log(` ${r.idx}. ${r.file}:${r.real_line ?? r.line}${flaggedIdx.has(r.idx) ? ' [MACHINE-FLAGGED]' : ''}`);
|
|
}
|
|
|
|
process.exitCode = 0;
|