feat(ms-ai-architect): reproduserbar V1/V2/V2b-sjekk av O2-returene (25 tester) [skip-docs]

Måleresultatets sentrale påstand — at forslagene er tekstlig ærlige — hvilte på
et sesjons-lokalt skript ingen kunne etterprøve. Flyttet inn som bibliotek + CLI
med tester, så tallet kan reproduseres fra fersk klon:

  node scripts/kb-eval/check-o2-returns.mjs

Sjekkene avgjør IKKE O2 — betingelse 2 og 3 er fortsatt menneskelige. De
avgrenser de to feilmodusene et menneske ikke fanger billig over 46 forslag:
- V1: sitert filtekst må finnes ordrett i fila (fanger oppdiktet tekst og stille
  æøå-transliterering). Gjelder HVER rad, også O3 — en O3 basert på oppdiktet
  tekst er like feil, bare feil i trygg retning.
- V2: forslaget må kunne oppnås ved kun å slette tegn.
- V2b: V2 alene er for svak — 'Automatically add' -> 'Add' passerer fordi den
  store A-en fantes inne i det slettede ordet. Ordnivå-sjekk, case-sensitiv.
- V3: skjema- og verdikt-koherens.

Suite 996 -> 1021.

scripts/kb-eval/check-o2-returns.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+// check-o2-returns.mjs — R11 §10 measurement #2: verify and tally the O2/O3
+// classification returns. READ-ONLY; writes nothing.
+//
+//   node scripts/kb-eval/check-o2-returns.mjs
+//
+// Runs the V1/V2/V2b/V3 checks (scripts/kb-eval/lib/o2-return-check.mjs) over
+// scripts/kb-eval/data/r11-o2-returns/*.json and prints the measurement: the
+// O2/O3 split, the machine-clean candidate count, and — the actionable part —
+// WHICH of §5's three conditions forecloses each O3. The top-level split alone
+// says nothing; the blocking condition is where the decision lives.
+
+import { readFileSync, readdirSync } from 'node:fs';
+import { join, dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { checkRow } from './lib/o2-return-check.mjs';
+
+const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const RETURNS = join(REPO, 'scripts/kb-eval/data/r11-o2-returns');
+
+const cache = new Map();
+const readRepoFile = (rel) => {
+  if (!cache.has(rel)) cache.set(rel, readFileSync(join(REPO, rel), 'utf8'));
+  return cache.get(rel);
+};
+
+const files = readdirSync(RETURNS).filter((f) => f.endsWith('.json')).sort();
+const rows = files.flatMap((f) =>
+  JSON.parse(readFileSync(join(RETURNS, f), 'utf8')).map((r) => ({ ...r, _batch: f })));
+
+const findings = rows.flatMap((r) =>
+  checkRow(r, readRepoFile).map((f) => ({ ...f, idx: r.idx, file: r.file, line: r.line, batch: r._batch })));
+
+const flaggedIdx = new Set(findings.map((f) => f.idx));
+const o2 = rows.filter((r) => r.verdict === 'O2_CANDIDATE');
+const o3 = rows.filter((r) => r.verdict === 'O3');
+const clean = o2.filter((r) => !flaggedIdx.has(r.idx));
+
+// Which condition forecloses O2. "human_must_confirm" counts as NOT held: the
+// point of the tri-state is that an unresolved condition is not a satisfied one.
+const held = (v) => v === true || v === 'yes';
+const blockTally = {};
+for (const r of o3) {
+  const failed = [
+    !held(r.cond1_strictly_less?.holds) && 'cond1',
+    !held(r.cond2_remainder_not_misleading?.holds) && 'cond2',
+    !held(r.cond3_nothing_confirmed_removed?.holds) && 'cond3',
+  ].filter(Boolean);
+  const key = failed.length ? failed.join('+') : 'none-stated';
+  blockTally[key] = (blockTally[key] || 0) + 1;
+}
+const cond3Blocked = o3.filter((r) => !held(r.cond3_nothing_confirmed_removed?.holds)).length;
+
+const tally = (xs, key) => xs.reduce((a, x) => ({ ...a, [x[key]]: (a[x[key]] || 0) + 1 }), {});
+const pct = (n) => `${((n / rows.length) * 100).toFixed(1)} %`;
+
+console.log(`R11 §10 #2 — R8 multi-part claims (pilot): ${rows.length} items from ${files.length} batches`);
+console.log(`  O2_CANDIDATE ${o2.length} (${pct(o2.length)}) · O3 ${o3.length} (${pct(o3.length)})`);
+console.log(`  locator_failed: ${rows.filter((r) => r.locator_failed).length}`);
+console.log(`  machine-clean O2 candidates: ${clean.length}/${o2.length}`);
+console.log(`  O2 confidence: ${JSON.stringify(tally(o2, 'confidence'))}`);
+console.log(`\nO3 blocking conditions: ${JSON.stringify(blockTally)}`);
+console.log(`O3 where condition 3 fails (source supplies a corrected value → swap/rewrite): ${cond3Blocked}/${o3.length}`);
+
+console.log(`\nmachine findings: ${findings.length}`);
+for (const f of findings) console.log(`  [${f.check}] idx=${f.idx} ${f.file}:${f.line} — ${f.detail}`);
+
+console.log('\nO2 candidates (review-grade — conditions 2 and 3 are human-confirmed):');
+for (const r of o2) {
+  console.log(`  ${r.idx}. ${r.file}:${r.real_line ?? r.line}${flaggedIdx.has(r.idx) ? '  [MACHINE-FLAGGED]' : ''}`);
+}
+
+process.exitCode = 0;

scripts/kb-eval/lib/o2-return-check.mjs

@@ -0,0 +1,116 @@
+// o2-return-check.mjs — the machine half of R11 §10 measurement #2.
+//
+// O2 candidacy is a PROSE judgement (docs/r11-tiered-fix-design.md §5): conditions
+// 2 and 3 require a human to read the remainder. This module does not attempt
+// that. It bounds the two failure modes that a human reviewing 46 proposals
+// cannot cheaply catch, and that would otherwise waste the human's attention:
+//
+//   V1  the quoted file text must occur VERBATIM in the named file. An agent that
+//       invented the text, or transliterated æ/ø/å, did not read the file — and
+//       its remainder is then a proposal about a file that does not exist.
+//   V2  the proposed remainder must be obtainable from that text by DELETING
+//       characters only. This is the machine-checkable half of condition 1
+//       ("asserts strictly less"); the semantic half stays human.
+//   V2b V2 alone is too weak: deleting a leading word and recapitalising the next
+//       ("Automatically add" -> "Add") still passes, because the capital already
+//       existed inside the deleted word. A word-level, case-sensitive check
+//       catches it. Recapitalising after a subtraction is defensible — but it is
+//       a text change, and the human must SEE it rather than have it pass as
+//       "pure deletion".
+//   V3  schema completeness and verdict/condition coherence.
+//
+// A proposal that fails V1 or V2 is not an O2 candidate whatever its prose says.
+
+const REQUIRED_FIELDS = [
+  'idx', 'file', 'line', 'real_line', 'locator_failed',
+  'file_text_verbatim', 'failing_part', 'proposed_remainder',
+  'cond1_strictly_less', 'cond2_remainder_not_misleading',
+  'cond3_nothing_confirmed_removed', 'verdict', 'o3_reason', 'confidence',
+];
+
+const VERDICTS = new Set(['O2_CANDIDATE', 'O3']);
+
+/** Collapse runs of whitespace — a subtraction legitimately closes the gap it leaves. */
+const normalise = (s) => String(s).replace(/\s+/g, ' ').trim();
+
+/**
+ * Is `sub` obtainable from `full` by deleting characters only?
+ * Case- and diacritic-sensitive by design: a transliterated or recased remainder
+ * is a rewrite, not a subtraction.
+ */
+export function isDeletionOnly(full, sub) {
+  const f = normalise(full);
+  const s = normalise(sub);
+  if (s.length >= f.length) return false;
+  let cursor = 0;
+  for (const ch of s) {
+    cursor = f.indexOf(ch, cursor);
+    if (cursor === -1) return false;
+    cursor += 1;
+  }
+  return true;
+}
+
+const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu;
+
+/** Word forms present in `sub` but absent from `full`, case-sensitively. */
+export function novelWordForms(full, sub) {
+  const before = new Set(String(full).match(WORD_RE) || []);
+  const after = new Set(String(sub).match(WORD_RE) || []);
+  return [...after].filter((w) => !before.has(w));
+}
+
+/**
+ * Check one classification row.
+ * @param {object} row      a return record (see docs/r11-pilot-results.md §9)
+ * @param {(rel: string) => string} readFile reads a repo-relative file, throws if absent
+ * @returns {Array<{check: string, detail: string}>} empty when the row is clean
+ */
+export function checkRow(row, readFile) {
+  const findings = [];
+  const add = (check, detail) => findings.push({ check, detail });
+
+  for (const field of REQUIRED_FIELDS) {
+    if (!(field in row)) add('V3', `missing field: ${field}`);
+  }
+  if (!VERDICTS.has(row.verdict)) add('V3', `unknown verdict: ${row.verdict}`);
+
+  // V1 — applies to EVERY row, not just the O2 candidates. An O3 verdict resting
+  // on invented file text is just as wrong, it is merely wrong in the safe
+  // direction, and the corpus-wide numbers count both.
+  if (row.file_text_verbatim) {
+    let text;
+    try {
+      text = readFile(row.file);
+    } catch (err) {
+      add('V1', `unreadable file: ${err.message}`);
+      return findings;
+    }
+    if (!text.includes(row.file_text_verbatim)) {
+      const loose = normalise(text).includes(normalise(row.file_text_verbatim));
+      add('V1', loose
+        ? 'file_text_verbatim matches only after whitespace normalisation'
+        : 'file_text_verbatim NOT FOUND in the file');
+    }
+  } else if (!row.locator_failed) {
+    add('V1', 'no file_text_verbatim and locator_failed is false');
+  }
+
+  if (row.verdict !== 'O2_CANDIDATE') return findings;
+
+  if (row.locator_failed) add('V3', 'O2_CANDIDATE with locator_failed');
+  if (row.cond1_strictly_less?.holds !== true) add('V3', 'O2_CANDIDATE but condition 1 does not hold');
+
+  if (!row.proposed_remainder) {
+    add('V2', 'O2_CANDIDATE with no proposed_remainder');
+  } else if (!isDeletionOnly(row.file_text_verbatim || '', row.proposed_remainder)) {
+    add('V2', 'proposed_remainder is not deletion-only (adds or reorders characters)');
+  } else {
+    const novel = novelWordForms(row.file_text_verbatim || '', row.proposed_remainder);
+    if (novel.length) {
+      add('V2b', `proposed_remainder introduces word forms absent from the original: ${novel.join(', ')}`);
+    }
+  }
+
+  return findings;
+}

tests/kb-eval/test-o2-return-check.test.mjs

@@ -0,0 +1,196 @@
+// test-o2-return-check.test.mjs — R11 §10 measurement #2, the machine half.
+//
+// O2 candidacy is decided by prose (docs/r11-tiered-fix-design.md §5), and prose
+// classification was done by subagents. That leaves two failure modes a human
+// reviewer cannot cheaply catch across 46 items: a proposal quoting file text
+// that is not actually in the file, and a "subtraction" that quietly rewrites.
+// These checks bound both. They do NOT decide O2 — conditions 2 and 3 stay
+// human. They only establish that a proposal is textually honest before a human
+// spends attention on it.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { isDeletionOnly, novelWordForms, checkRow } from '../../scripts/kb-eval/lib/o2-return-check.mjs';
+
+// ------------------------------------------------------------- isDeletionOnly
+
+test('isDeletionOnly accepts a removed clause', () => {
+  assert.equal(isDeletionOnly('Bruk Norway East/West for Redis.', 'Bruk Norway East for Redis.'), true);
+});
+
+test('isDeletionOnly accepts a removed whole line', () => {
+  const before = '- Read\n- Layout\n- General Document';
+  assert.equal(isDeletionOnly(before, '- Read\n- Layout'), true);
+});
+
+test('isDeletionOnly rejects added text', () => {
+  assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway East og West.'), false);
+});
+
+test('isDeletionOnly rejects an equal-length string (nothing was removed)', () => {
+  assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway West.'), false);
+});
+
+test('isDeletionOnly rejects reordering', () => {
+  assert.equal(isDeletionOnly('alpha beta gamma', 'gamma alpha'), false);
+});
+
+test('isDeletionOnly normalises whitespace, because a subtraction collapses the spaces around the removed span', () => {
+  assert.equal(isDeletionOnly('a  b   c', 'a c'), true);
+});
+
+test('isDeletionOnly preserves Norwegian characters rather than folding them', () => {
+  // "høyere" must not be obtainable from a source that only carries "hoyere":
+  // a transliterating agent would otherwise pass the check.
+  assert.equal(isDeletionOnly('krever hoyere semantisk likhet', 'krever høyere likhet'), false);
+  assert.equal(isDeletionOnly('krever høyere semantisk likhet', 'krever høyere likhet'), true);
+});
+
+// -------------------------------------------------------------- novelWordForms
+
+test('novelWordForms is empty for a pure deletion', () => {
+  assert.deepEqual(novelWordForms('Dataverse / SharePoint lagrer data', 'Dataverse lagrer data'), []);
+});
+
+test('novelWordForms catches recapitalisation after a deleted leading word', () => {
+  // The character-subsequence test alone passes this, because the capital A
+  // already exists inside "Automatically". Measured on a real return (idx 14).
+  const before = 'Automatically add reviewed samples';
+  const after = 'Add reviewed samples';
+  assert.equal(isDeletionOnly(before, after), true);
+  assert.deepEqual(novelWordForms(before, after), ['Add']);
+});
+
+test('novelWordForms is case-sensitive but ignores punctuation', () => {
+  assert.deepEqual(novelWordForms('en to tre, fire', 'en to tre'), []);
+});
+
+// -------------------------------------------------------------------- checkRow
+
+const FILE_TEXT = [
+  '# Tittel',
+  '',
+  '- Alpha',
+  '- Beta',
+  '- Gamma',
+  '',
+  '| Model improvement | Automatically add reviewed samples |',
+  '',
+  'Etterord med æ, ø og å.',
+  '',
+].join('\n');
+const readFile = (rel) => {
+  if (rel !== 'skills/x/references/y.md') throw new Error(`ENOENT: ${rel}`);
+  return FILE_TEXT;
+};
+
+function row(over = {}) {
+  return {
+    idx: 1,
+    file: 'skills/x/references/y.md',
+    line: 3,
+    real_line: 3,
+    locator_failed: false,
+    file_text_verbatim: '- Alpha\n- Beta\n- Gamma',
+    failing_part: 'Gamma',
+    proposed_remainder: '- Alpha\n- Beta',
+    cond1_strictly_less: { holds: true, evidence: 'one bullet removed' },
+    cond2_remainder_not_misleading: { holds: 'yes', evidence: 'ok' },
+    cond3_nothing_confirmed_removed: { holds: 'yes', evidence: 'ok' },
+    verdict: 'O2_CANDIDATE',
+    o3_reason: null,
+    confidence: 'high',
+    ...over,
+  };
+}
+
+test('checkRow passes a well-formed O2 candidate', () => {
+  assert.deepEqual(checkRow(row(), readFile), []);
+});
+
+test('checkRow flags quoted file text that is not in the file (V1)', () => {
+  const found = checkRow(row({ file_text_verbatim: '- Alpha\n- Delta' }), readFile);
+  assert.ok(found.some((f) => f.check === 'V1'));
+});
+
+test('checkRow flags a silently transliterated quote as not found (V1)', () => {
+  // The file says "æ, ø og å"; an agent that writes "ae, oe og aa" has not
+  // quoted the file, and the whole point of V1 is to see that.
+  const found = checkRow(row({ file_text_verbatim: 'Etterord med ae, oe og aa.' }), readFile);
+  assert.ok(found.some((f) => f.check === 'V1'));
+});
+
+test('checkRow flags a proposal that adds text (V2)', () => {
+  const found = checkRow(row({ proposed_remainder: '- Alpha\n- Beta\n- Delta' }), readFile);
+  assert.ok(found.some((f) => f.check === 'V2'));
+});
+
+test('checkRow flags recapitalisation separately from outright addition (V2b)', () => {
+  const found = checkRow(
+    row({
+      file_text_verbatim: '| Model improvement | Automatically add reviewed samples |',
+      proposed_remainder: '| Model improvement | Add reviewed samples |',
+    }),
+    readFile,
+  );
+  assert.deepEqual(found.map((f) => f.check), ['V2b']);
+});
+
+test('checkRow flags an O2 candidate with no proposed remainder (V2)', () => {
+  const found = checkRow(row({ proposed_remainder: null }), readFile);
+  assert.ok(found.some((f) => f.check === 'V2'));
+});
+
+test('checkRow flags an O2 candidate whose condition 1 does not hold (V3)', () => {
+  const found = checkRow(row({ cond1_strictly_less: { holds: false, evidence: 'no' } }), readFile);
+  assert.ok(found.some((f) => f.check === 'V3'));
+});
+
+test('checkRow flags an O2 candidate that failed to locate the text (V3)', () => {
+  const found = checkRow(row({ locator_failed: true }), readFile);
+  assert.ok(found.some((f) => f.check === 'V3'));
+});
+
+test('checkRow flags a missing required field (V3)', () => {
+  const r = row();
+  delete r.confidence;
+  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V3'));
+});
+
+test('checkRow flags an unknown verdict (V3)', () => {
+  assert.ok(checkRow(row({ verdict: 'O2' }), readFile).some((f) => f.check === 'V3'));
+});
+
+test('checkRow does NOT require a proposed remainder on an O3 row', () => {
+  const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'condition 3 fails' });
+  assert.deepEqual(checkRow(r, readFile), []);
+});
+
+test('checkRow still verifies the quoted text of an O3 row (V1 applies to every row)', () => {
+  const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'x', file_text_verbatim: '- Delta' });
+  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
+});
+
+test('checkRow reports an unreadable file rather than throwing', () => {
+  const found = checkRow(row({ file: 'skills/x/references/missing.md' }), readFile);
+  assert.equal(found.length, 1);
+  assert.equal(found[0].check, 'V1');
+});
+
+test('checkRow allows a row that failed to locate the text and fell back to O3', () => {
+  const r = {
+    ...row(),
+    locator_failed: true,
+    file_text_verbatim: null,
+    proposed_remainder: null,
+    verdict: 'O3',
+    o3_reason: 'locator failed',
+  };
+  assert.deepEqual(checkRow(r, readFile), []);
+});
+
+test('checkRow flags a row that claims no locator failure but quotes nothing (V1)', () => {
+  const r = row({ file_text_verbatim: null, proposed_remainder: null, verdict: 'O3', o3_reason: 'x' });
+  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
+});