// test-o2-return-check.test.mjs — R11 §10 measurement #2, the machine half. // // O2 candidacy is decided by prose (docs/r11-tiered-fix-design.md §5), and prose // classification was done by subagents. That leaves two failure modes a human // reviewer cannot cheaply catch across 46 items: a proposal quoting file text // that is not actually in the file, and a "subtraction" that quietly rewrites. // These checks bound both. They do NOT decide O2 — conditions 2 and 3 stay // human. They only establish that a proposal is textually honest before a human // spends attention on it. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { isDeletionOnly, novelWordForms, checkRow } from '../../scripts/kb-eval/lib/o2-return-check.mjs'; // ------------------------------------------------------------- isDeletionOnly test('isDeletionOnly accepts a removed clause', () => { assert.equal(isDeletionOnly('Bruk Norway East/West for Redis.', 'Bruk Norway East for Redis.'), true); }); test('isDeletionOnly accepts a removed whole line', () => { const before = '- Read\n- Layout\n- General Document'; assert.equal(isDeletionOnly(before, '- Read\n- Layout'), true); }); test('isDeletionOnly rejects added text', () => { assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway East og West.'), false); }); test('isDeletionOnly rejects an equal-length string (nothing was removed)', () => { assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway West.'), false); }); test('isDeletionOnly rejects reordering', () => { assert.equal(isDeletionOnly('alpha beta gamma', 'gamma alpha'), false); }); test('isDeletionOnly normalises whitespace, because a subtraction collapses the spaces around the removed span', () => { assert.equal(isDeletionOnly('a b c', 'a c'), true); }); test('isDeletionOnly preserves Norwegian characters rather than folding them', () => { // "høyere" must not be obtainable from a source that only carries "hoyere": // a transliterating agent would otherwise pass the check. assert.equal(isDeletionOnly('krever hoyere semantisk likhet', 'krever høyere likhet'), false); assert.equal(isDeletionOnly('krever høyere semantisk likhet', 'krever høyere likhet'), true); }); // -------------------------------------------------------------- novelWordForms test('novelWordForms is empty for a pure deletion', () => { assert.deepEqual(novelWordForms('Dataverse / SharePoint lagrer data', 'Dataverse lagrer data'), []); }); test('novelWordForms catches recapitalisation after a deleted leading word', () => { // The character-subsequence test alone passes this, because the capital A // already exists inside "Automatically". Measured on a real return (idx 14). const before = 'Automatically add reviewed samples'; const after = 'Add reviewed samples'; assert.equal(isDeletionOnly(before, after), true); assert.deepEqual(novelWordForms(before, after), ['Add']); }); test('novelWordForms is case-sensitive but ignores punctuation', () => { assert.deepEqual(novelWordForms('en to tre, fire', 'en to tre'), []); }); // -------------------------------------------------------------------- checkRow const FILE_TEXT = [ '# Tittel', '', '- Alpha', '- Beta', '- Gamma', '', '| Model improvement | Automatically add reviewed samples |', '', 'Etterord med æ, ø og å.', '', ].join('\n'); const readFile = (rel) => { if (rel !== 'skills/x/references/y.md') throw new Error(`ENOENT: ${rel}`); return FILE_TEXT; }; function row(over = {}) { return { idx: 1, file: 'skills/x/references/y.md', line: 3, real_line: 3, locator_failed: false, file_text_verbatim: '- Alpha\n- Beta\n- Gamma', failing_part: 'Gamma', proposed_remainder: '- Alpha\n- Beta', cond1_strictly_less: { holds: true, evidence: 'one bullet removed' }, cond2_remainder_not_misleading: { holds: 'yes', evidence: 'ok' }, cond3_nothing_confirmed_removed: { holds: 'yes', evidence: 'ok' }, verdict: 'O2_CANDIDATE', o3_reason: null, confidence: 'high', ...over, }; } test('checkRow passes a well-formed O2 candidate', () => { assert.deepEqual(checkRow(row(), readFile), []); }); test('checkRow flags quoted file text that is not in the file (V1)', () => { const found = checkRow(row({ file_text_verbatim: '- Alpha\n- Delta' }), readFile); assert.ok(found.some((f) => f.check === 'V1')); }); test('checkRow flags a silently transliterated quote as not found (V1)', () => { // The file says "æ, ø og å"; an agent that writes "ae, oe og aa" has not // quoted the file, and the whole point of V1 is to see that. const found = checkRow(row({ file_text_verbatim: 'Etterord med ae, oe og aa.' }), readFile); assert.ok(found.some((f) => f.check === 'V1')); }); test('checkRow flags a proposal that adds text (V2)', () => { const found = checkRow(row({ proposed_remainder: '- Alpha\n- Beta\n- Delta' }), readFile); assert.ok(found.some((f) => f.check === 'V2')); }); test('checkRow flags recapitalisation separately from outright addition (V2b)', () => { const found = checkRow( row({ file_text_verbatim: '| Model improvement | Automatically add reviewed samples |', proposed_remainder: '| Model improvement | Add reviewed samples |', }), readFile, ); assert.deepEqual(found.map((f) => f.check), ['V2b']); }); test('checkRow flags an O2 candidate with no proposed remainder (V2)', () => { const found = checkRow(row({ proposed_remainder: null }), readFile); assert.ok(found.some((f) => f.check === 'V2')); }); test('checkRow flags an O2 candidate whose condition 1 does not hold (V3)', () => { const found = checkRow(row({ cond1_strictly_less: { holds: false, evidence: 'no' } }), readFile); assert.ok(found.some((f) => f.check === 'V3')); }); test('checkRow flags an O2 candidate that failed to locate the text (V3)', () => { const found = checkRow(row({ locator_failed: true }), readFile); assert.ok(found.some((f) => f.check === 'V3')); }); test('checkRow flags a missing required field (V3)', () => { const r = row(); delete r.confidence; assert.ok(checkRow(r, readFile).some((f) => f.check === 'V3')); }); test('checkRow flags an unknown verdict (V3)', () => { assert.ok(checkRow(row({ verdict: 'O2' }), readFile).some((f) => f.check === 'V3')); }); test('checkRow does NOT require a proposed remainder on an O3 row', () => { const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'condition 3 fails' }); assert.deepEqual(checkRow(r, readFile), []); }); test('checkRow still verifies the quoted text of an O3 row (V1 applies to every row)', () => { const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'x', file_text_verbatim: '- Delta' }); assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1')); }); test('checkRow reports an unreadable file rather than throwing', () => { const found = checkRow(row({ file: 'skills/x/references/missing.md' }), readFile); assert.equal(found.length, 1); assert.equal(found[0].check, 'V1'); }); test('checkRow allows a row that failed to locate the text and fell back to O3', () => { const r = { ...row(), locator_failed: true, file_text_verbatim: null, proposed_remainder: null, verdict: 'O3', o3_reason: 'locator failed', }; assert.deepEqual(checkRow(r, readFile), []); }); test('checkRow flags a row that claims no locator failure but quotes nothing (V1)', () => { const r = row({ file_text_verbatim: null, proposed_remainder: null, verdict: 'O3', o3_reason: 'x' }); assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1')); });