ms-ai-architect/tests/kb-eval/test-o2-return-check.test.mjs

// test-o2-return-check.test.mjs — R11 §10 measurement #2, the machine half.
//
// O2 candidacy is decided by prose (docs/r11-tiered-fix-design.md §5), and prose
// classification was done by subagents. That leaves two failure modes a human
// reviewer cannot cheaply catch across 46 items: a proposal quoting file text
// that is not actually in the file, and a "subtraction" that quietly rewrites.
// These checks bound both. They do NOT decide O2 — conditions 2 and 3 stay
// human. They only establish that a proposal is textually honest before a human
// spends attention on it.

import { test } from 'node:test';
import assert from 'node:assert/strict';

import { isDeletionOnly, novelWordForms, checkRow } from '../../scripts/kb-eval/lib/o2-return-check.mjs';

// ------------------------------------------------------------- isDeletionOnly

test('isDeletionOnly accepts a removed clause', () => {
  assert.equal(isDeletionOnly('Bruk Norway East/West for Redis.', 'Bruk Norway East for Redis.'), true);
});

test('isDeletionOnly accepts a removed whole line', () => {
  const before = '- Read\n- Layout\n- General Document';
  assert.equal(isDeletionOnly(before, '- Read\n- Layout'), true);
});

test('isDeletionOnly rejects added text', () => {
  assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway East og West.'), false);
});

test('isDeletionOnly rejects an equal-length string (nothing was removed)', () => {
  assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway West.'), false);
});

test('isDeletionOnly rejects reordering', () => {
  assert.equal(isDeletionOnly('alpha beta gamma', 'gamma alpha'), false);
});

test('isDeletionOnly normalises whitespace, because a subtraction collapses the spaces around the removed span', () => {
  assert.equal(isDeletionOnly('a  b   c', 'a c'), true);
});

test('isDeletionOnly preserves Norwegian characters rather than folding them', () => {
  // "høyere" must not be obtainable from a source that only carries "hoyere":
  // a transliterating agent would otherwise pass the check.
  assert.equal(isDeletionOnly('krever hoyere semantisk likhet', 'krever høyere likhet'), false);
  assert.equal(isDeletionOnly('krever høyere semantisk likhet', 'krever høyere likhet'), true);
});

// -------------------------------------------------------------- novelWordForms

test('novelWordForms is empty for a pure deletion', () => {
  assert.deepEqual(novelWordForms('Dataverse / SharePoint lagrer data', 'Dataverse lagrer data'), []);
});

test('novelWordForms catches recapitalisation after a deleted leading word', () => {
  // The character-subsequence test alone passes this, because the capital A
  // already exists inside "Automatically". Measured on a real return (idx 14).
  const before = 'Automatically add reviewed samples';
  const after = 'Add reviewed samples';
  assert.equal(isDeletionOnly(before, after), true);
  assert.deepEqual(novelWordForms(before, after), ['Add']);
});

test('novelWordForms is case-sensitive but ignores punctuation', () => {
  assert.deepEqual(novelWordForms('en to tre, fire', 'en to tre'), []);
});

// -------------------------------------------------------------------- checkRow

const FILE_TEXT = [
  '# Tittel',
  '',
  '- Alpha',
  '- Beta',
  '- Gamma',
  '',
  '| Model improvement | Automatically add reviewed samples |',
  '',
  'Etterord med æ, ø og å.',
  '',
].join('\n');
const readFile = (rel) => {
  if (rel !== 'skills/x/references/y.md') throw new Error(`ENOENT: ${rel}`);
  return FILE_TEXT;
};

function row(over = {}) {
  return {
    idx: 1,
    file: 'skills/x/references/y.md',
    line: 3,
    real_line: 3,
    locator_failed: false,
    file_text_verbatim: '- Alpha\n- Beta\n- Gamma',
    failing_part: 'Gamma',
    proposed_remainder: '- Alpha\n- Beta',
    cond1_strictly_less: { holds: true, evidence: 'one bullet removed' },
    cond2_remainder_not_misleading: { holds: 'yes', evidence: 'ok' },
    cond3_nothing_confirmed_removed: { holds: 'yes', evidence: 'ok' },
    verdict: 'O2_CANDIDATE',
    o3_reason: null,
    confidence: 'high',
    ...over,
  };
}

test('checkRow passes a well-formed O2 candidate', () => {
  assert.deepEqual(checkRow(row(), readFile), []);
});

test('checkRow flags quoted file text that is not in the file (V1)', () => {
  const found = checkRow(row({ file_text_verbatim: '- Alpha\n- Delta' }), readFile);
  assert.ok(found.some((f) => f.check === 'V1'));
});

test('checkRow flags a silently transliterated quote as not found (V1)', () => {
  // The file says "æ, ø og å"; an agent that writes "ae, oe og aa" has not
  // quoted the file, and the whole point of V1 is to see that.
  const found = checkRow(row({ file_text_verbatim: 'Etterord med ae, oe og aa.' }), readFile);
  assert.ok(found.some((f) => f.check === 'V1'));
});

test('checkRow flags a proposal that adds text (V2)', () => {
  const found = checkRow(row({ proposed_remainder: '- Alpha\n- Beta\n- Delta' }), readFile);
  assert.ok(found.some((f) => f.check === 'V2'));
});

test('checkRow flags recapitalisation separately from outright addition (V2b)', () => {
  const found = checkRow(
    row({
      file_text_verbatim: '| Model improvement | Automatically add reviewed samples |',
      proposed_remainder: '| Model improvement | Add reviewed samples |',
    }),
    readFile,
  );
  assert.deepEqual(found.map((f) => f.check), ['V2b']);
});

test('checkRow flags an O2 candidate with no proposed remainder (V2)', () => {
  const found = checkRow(row({ proposed_remainder: null }), readFile);
  assert.ok(found.some((f) => f.check === 'V2'));
});

test('checkRow flags an O2 candidate whose condition 1 does not hold (V3)', () => {
  const found = checkRow(row({ cond1_strictly_less: { holds: false, evidence: 'no' } }), readFile);
  assert.ok(found.some((f) => f.check === 'V3'));
});

test('checkRow flags an O2 candidate that failed to locate the text (V3)', () => {
  const found = checkRow(row({ locator_failed: true }), readFile);
  assert.ok(found.some((f) => f.check === 'V3'));
});

test('checkRow flags a missing required field (V3)', () => {
  const r = row();
  delete r.confidence;
  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V3'));
});

test('checkRow flags an unknown verdict (V3)', () => {
  assert.ok(checkRow(row({ verdict: 'O2' }), readFile).some((f) => f.check === 'V3'));
});

test('checkRow does NOT require a proposed remainder on an O3 row', () => {
  const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'condition 3 fails' });
  assert.deepEqual(checkRow(r, readFile), []);
});

test('checkRow still verifies the quoted text of an O3 row (V1 applies to every row)', () => {
  const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'x', file_text_verbatim: '- Delta' });
  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
});

test('checkRow reports an unreadable file rather than throwing', () => {
  const found = checkRow(row({ file: 'skills/x/references/missing.md' }), readFile);
  assert.equal(found.length, 1);
  assert.equal(found[0].check, 'V1');
});

test('checkRow allows a row that failed to locate the text and fell back to O3', () => {
  const r = {
    ...row(),
    locator_failed: true,
    file_text_verbatim: null,
    proposed_remainder: null,
    verdict: 'O3',
    o3_reason: 'locator failed',
  };
  assert.deepEqual(checkRow(r, readFile), []);
});

test('checkRow flags a row that claims no locator failure but quotes nothing (V1)', () => {
  const r = row({ file_text_verbatim: null, proposed_remainder: null, verdict: 'O3', o3_reason: 'x' });
  assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
});