feat(ms-ai-architect): reproduserbar V1/V2/V2b-sjekk av O2-returene (25 tester) [skip-docs]

Måleresultatets sentrale påstand — at forslagene er tekstlig ærlige — hvilte på
et sesjons-lokalt skript ingen kunne etterprøve. Flyttet inn som bibliotek + CLI
med tester, så tallet kan reproduseres fra fersk klon:

  node scripts/kb-eval/check-o2-returns.mjs

Sjekkene avgjør IKKE O2 — betingelse 2 og 3 er fortsatt menneskelige. De
avgrenser de to feilmodusene et menneske ikke fanger billig over 46 forslag:
- V1: sitert filtekst må finnes ordrett i fila (fanger oppdiktet tekst og stille
  æøå-transliterering). Gjelder HVER rad, også O3 — en O3 basert på oppdiktet
  tekst er like feil, bare feil i trygg retning.
- V2: forslaget må kunne oppnås ved kun å slette tegn.
- V2b: V2 alene er for svak — 'Automatically add' -> 'Add' passerer fordi den
  store A-en fantes inne i det slettede ordet. Ordnivå-sjekk, case-sensitiv.
- V3: skjema- og verdikt-koherens.

Suite 996 -> 1021.

scripts/kb-eval/check-o2-returns.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+// check-o2-returns.mjs — R11 §10 measurement #2: verify and tally the O2/O3
+// classification returns. READ-ONLY; writes nothing.
+//
+//   node scripts/kb-eval/check-o2-returns.mjs
+//
+// Runs the V1/V2/V2b/V3 checks (scripts/kb-eval/lib/o2-return-check.mjs) over
+// scripts/kb-eval/data/r11-o2-returns/*.json and prints the measurement: the
+// O2/O3 split, the machine-clean candidate count, and — the actionable part —
+// WHICH of §5's three conditions forecloses each O3. The top-level split alone
+// says nothing; the blocking condition is where the decision lives.
+
+import { readFileSync, readdirSync } from 'node:fs';
+import { join, dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { checkRow } from './lib/o2-return-check.mjs';
+
+const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const RETURNS = join(REPO, 'scripts/kb-eval/data/r11-o2-returns');
+
+const cache = new Map();
+const readRepoFile = (rel) => {
+  if (!cache.has(rel)) cache.set(rel, readFileSync(join(REPO, rel), 'utf8'));
+  return cache.get(rel);
+};
+
+const files = readdirSync(RETURNS).filter((f) => f.endsWith('.json')).sort();
+const rows = files.flatMap((f) =>
+  JSON.parse(readFileSync(join(RETURNS, f), 'utf8')).map((r) => ({ ...r, _batch: f })));
+
+const findings = rows.flatMap((r) =>
+  checkRow(r, readRepoFile).map((f) => ({ ...f, idx: r.idx, file: r.file, line: r.line, batch: r._batch })));
+
+const flaggedIdx = new Set(findings.map((f) => f.idx));
+const o2 = rows.filter((r) => r.verdict === 'O2_CANDIDATE');
+const o3 = rows.filter((r) => r.verdict === 'O3');
+const clean = o2.filter((r) => !flaggedIdx.has(r.idx));
+
+// Which condition forecloses O2. "human_must_confirm" counts as NOT held: the
+// point of the tri-state is that an unresolved condition is not a satisfied one.
+const held = (v) => v === true || v === 'yes';
+const blockTally = {};
+for (const r of o3) {
+  const failed = [
+    !held(r.cond1_strictly_less?.holds) && 'cond1',
+    !held(r.cond2_remainder_not_misleading?.holds) && 'cond2',
+    !held(r.cond3_nothing_confirmed_removed?.holds) && 'cond3',
+  ].filter(Boolean);
+  const key = failed.length ? failed.join('+') : 'none-stated';
+  blockTally[key] = (blockTally[key] || 0) + 1;
+}
+const cond3Blocked = o3.filter((r) => !held(r.cond3_nothing_confirmed_removed?.holds)).length;
+
+const tally = (xs, key) => xs.reduce((a, x) => ({ ...a, [x[key]]: (a[x[key]] || 0) + 1 }), {});
+const pct = (n) => `${((n / rows.length) * 100).toFixed(1)} %`;
+
+console.log(`R11 §10 #2 — R8 multi-part claims (pilot): ${rows.length} items from ${files.length} batches`);
+console.log(`  O2_CANDIDATE ${o2.length} (${pct(o2.length)}) · O3 ${o3.length} (${pct(o3.length)})`);
+console.log(`  locator_failed: ${rows.filter((r) => r.locator_failed).length}`);
+console.log(`  machine-clean O2 candidates: ${clean.length}/${o2.length}`);
+console.log(`  O2 confidence: ${JSON.stringify(tally(o2, 'confidence'))}`);
+console.log(`\nO3 blocking conditions: ${JSON.stringify(blockTally)}`);
+console.log(`O3 where condition 3 fails (source supplies a corrected value → swap/rewrite): ${cond3Blocked}/${o3.length}`);
+
+console.log(`\nmachine findings: ${findings.length}`);
+for (const f of findings) console.log(`  [${f.check}] idx=${f.idx} ${f.file}:${f.line} — ${f.detail}`);
+
+console.log('\nO2 candidates (review-grade — conditions 2 and 3 are human-confirmed):');
+for (const r of o2) {
+  console.log(`  ${r.idx}. ${r.file}:${r.real_line ?? r.line}${flaggedIdx.has(r.idx) ? '  [MACHINE-FLAGGED]' : ''}`);
+}
+
+process.exitCode = 0;

scripts/kb-eval/lib/o2-return-check.mjs

@@ -0,0 +1,116 @@
+// o2-return-check.mjs — the machine half of R11 §10 measurement #2.
+//
+// O2 candidacy is a PROSE judgement (docs/r11-tiered-fix-design.md §5): conditions
+// 2 and 3 require a human to read the remainder. This module does not attempt
+// that. It bounds the two failure modes that a human reviewing 46 proposals
+// cannot cheaply catch, and that would otherwise waste the human's attention:
+//
+//   V1  the quoted file text must occur VERBATIM in the named file. An agent that
+//       invented the text, or transliterated æ/ø/å, did not read the file — and
+//       its remainder is then a proposal about a file that does not exist.
+//   V2  the proposed remainder must be obtainable from that text by DELETING
+//       characters only. This is the machine-checkable half of condition 1
+//       ("asserts strictly less"); the semantic half stays human.
+//   V2b V2 alone is too weak: deleting a leading word and recapitalising the next
+//       ("Automatically add" -> "Add") still passes, because the capital already
+//       existed inside the deleted word. A word-level, case-sensitive check
+//       catches it. Recapitalising after a subtraction is defensible — but it is
+//       a text change, and the human must SEE it rather than have it pass as
+//       "pure deletion".
+//   V3  schema completeness and verdict/condition coherence.
+//
+// A proposal that fails V1 or V2 is not an O2 candidate whatever its prose says.
+
+const REQUIRED_FIELDS = [
+  'idx', 'file', 'line', 'real_line', 'locator_failed',
+  'file_text_verbatim', 'failing_part', 'proposed_remainder',
+  'cond1_strictly_less', 'cond2_remainder_not_misleading',
+  'cond3_nothing_confirmed_removed', 'verdict', 'o3_reason', 'confidence',
+];
+
+const VERDICTS = new Set(['O2_CANDIDATE', 'O3']);
+
+/** Collapse runs of whitespace — a subtraction legitimately closes the gap it leaves. */
+const normalise = (s) => String(s).replace(/\s+/g, ' ').trim();
+
+/**
+ * Is `sub` obtainable from `full` by deleting characters only?
+ * Case- and diacritic-sensitive by design: a transliterated or recased remainder
+ * is a rewrite, not a subtraction.
+ */
+export function isDeletionOnly(full, sub) {
+  const f = normalise(full);
+  const s = normalise(sub);
+  if (s.length >= f.length) return false;
+  let cursor = 0;
+  for (const ch of s) {
+    cursor = f.indexOf(ch, cursor);
+    if (cursor === -1) return false;
+    cursor += 1;
+  }
+  return true;
+}
+
+const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu;
+
+/** Word forms present in `sub` but absent from `full`, case-sensitively. */
+export function novelWordForms(full, sub) {
+  const before = new Set(String(full).match(WORD_RE) || []);
+  const after = new Set(String(sub).match(WORD_RE) || []);
+  return [...after].filter((w) => !before.has(w));
+}
+
+/**
+ * Check one classification row.
+ * @param {object} row      a return record (see docs/r11-pilot-results.md §9)
+ * @param {(rel: string) => string} readFile reads a repo-relative file, throws if absent
+ * @returns {Array<{check: string, detail: string}>} empty when the row is clean
+ */
+export function checkRow(row, readFile) {
+  const findings = [];
+  const add = (check, detail) => findings.push({ check, detail });
+
+  for (const field of REQUIRED_FIELDS) {
+    if (!(field in row)) add('V3', `missing field: ${field}`);
+  }
+  if (!VERDICTS.has(row.verdict)) add('V3', `unknown verdict: ${row.verdict}`);
+
+  // V1 — applies to EVERY row, not just the O2 candidates. An O3 verdict resting
+  // on invented file text is just as wrong, it is merely wrong in the safe
+  // direction, and the corpus-wide numbers count both.
+  if (row.file_text_verbatim) {
+    let text;
+    try {
+      text = readFile(row.file);
+    } catch (err) {
+      add('V1', `unreadable file: ${err.message}`);
+      return findings;
+    }
+    if (!text.includes(row.file_text_verbatim)) {
+      const loose = normalise(text).includes(normalise(row.file_text_verbatim));
+      add('V1', loose
+        ? 'file_text_verbatim matches only after whitespace normalisation'
+        : 'file_text_verbatim NOT FOUND in the file');
+    }
+  } else if (!row.locator_failed) {
+    add('V1', 'no file_text_verbatim and locator_failed is false');
+  }
+
+  if (row.verdict !== 'O2_CANDIDATE') return findings;
+
+  if (row.locator_failed) add('V3', 'O2_CANDIDATE with locator_failed');
+  if (row.cond1_strictly_less?.holds !== true) add('V3', 'O2_CANDIDATE but condition 1 does not hold');
+
+  if (!row.proposed_remainder) {
+    add('V2', 'O2_CANDIDATE with no proposed_remainder');
+  } else if (!isDeletionOnly(row.file_text_verbatim || '', row.proposed_remainder)) {
+    add('V2', 'proposed_remainder is not deletion-only (adds or reorders characters)');
+  } else {
+    const novel = novelWordForms(row.file_text_verbatim || '', row.proposed_remainder);
+    if (novel.length) {
+      add('V2b', `proposed_remainder introduces word forms absent from the original: ${novel.join(', ')}`);
+    }
+  }
+
+  return findings;
+}