feat(ms-ai-architect): reproduserbar V1/V2/V2b-sjekk av O2-returene (25 tester) [skip-docs]
Måleresultatets sentrale påstand — at forslagene er tekstlig ærlige — hvilte på et sesjons-lokalt skript ingen kunne etterprøve. Flyttet inn som bibliotek + CLI med tester, så tallet kan reproduseres fra fersk klon: node scripts/kb-eval/check-o2-returns.mjs Sjekkene avgjør IKKE O2 — betingelse 2 og 3 er fortsatt menneskelige. De avgrenser de to feilmodusene et menneske ikke fanger billig over 46 forslag: - V1: sitert filtekst må finnes ordrett i fila (fanger oppdiktet tekst og stille æøå-transliterering). Gjelder HVER rad, også O3 — en O3 basert på oppdiktet tekst er like feil, bare feil i trygg retning. - V2: forslaget må kunne oppnås ved kun å slette tegn. - V2b: V2 alene er for svak — 'Automatically add' -> 'Add' passerer fordi den store A-en fantes inne i det slettede ordet. Ordnivå-sjekk, case-sensitiv. - V3: skjema- og verdikt-koherens. Suite 996 -> 1021.
This commit is contained in:
parent
94c99c46dd
commit
4a36fd1853
3 changed files with 386 additions and 0 deletions
196
tests/kb-eval/test-o2-return-check.test.mjs
Normal file
196
tests/kb-eval/test-o2-return-check.test.mjs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
// test-o2-return-check.test.mjs — R11 §10 measurement #2, the machine half.
|
||||
//
|
||||
// O2 candidacy is decided by prose (docs/r11-tiered-fix-design.md §5), and prose
|
||||
// classification was done by subagents. That leaves two failure modes a human
|
||||
// reviewer cannot cheaply catch across 46 items: a proposal quoting file text
|
||||
// that is not actually in the file, and a "subtraction" that quietly rewrites.
|
||||
// These checks bound both. They do NOT decide O2 — conditions 2 and 3 stay
|
||||
// human. They only establish that a proposal is textually honest before a human
|
||||
// spends attention on it.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { isDeletionOnly, novelWordForms, checkRow } from '../../scripts/kb-eval/lib/o2-return-check.mjs';
|
||||
|
||||
// ------------------------------------------------------------- isDeletionOnly
|
||||
|
||||
test('isDeletionOnly accepts a removed clause', () => {
|
||||
assert.equal(isDeletionOnly('Bruk Norway East/West for Redis.', 'Bruk Norway East for Redis.'), true);
|
||||
});
|
||||
|
||||
test('isDeletionOnly accepts a removed whole line', () => {
|
||||
const before = '- Read\n- Layout\n- General Document';
|
||||
assert.equal(isDeletionOnly(before, '- Read\n- Layout'), true);
|
||||
});
|
||||
|
||||
test('isDeletionOnly rejects added text', () => {
|
||||
assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway East og West.'), false);
|
||||
});
|
||||
|
||||
test('isDeletionOnly rejects an equal-length string (nothing was removed)', () => {
|
||||
assert.equal(isDeletionOnly('Bruk Norway East.', 'Bruk Norway West.'), false);
|
||||
});
|
||||
|
||||
test('isDeletionOnly rejects reordering', () => {
|
||||
assert.equal(isDeletionOnly('alpha beta gamma', 'gamma alpha'), false);
|
||||
});
|
||||
|
||||
test('isDeletionOnly normalises whitespace, because a subtraction collapses the spaces around the removed span', () => {
|
||||
assert.equal(isDeletionOnly('a b c', 'a c'), true);
|
||||
});
|
||||
|
||||
test('isDeletionOnly preserves Norwegian characters rather than folding them', () => {
|
||||
// "høyere" must not be obtainable from a source that only carries "hoyere":
|
||||
// a transliterating agent would otherwise pass the check.
|
||||
assert.equal(isDeletionOnly('krever hoyere semantisk likhet', 'krever høyere likhet'), false);
|
||||
assert.equal(isDeletionOnly('krever høyere semantisk likhet', 'krever høyere likhet'), true);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------- novelWordForms
|
||||
|
||||
test('novelWordForms is empty for a pure deletion', () => {
|
||||
assert.deepEqual(novelWordForms('Dataverse / SharePoint lagrer data', 'Dataverse lagrer data'), []);
|
||||
});
|
||||
|
||||
test('novelWordForms catches recapitalisation after a deleted leading word', () => {
|
||||
// The character-subsequence test alone passes this, because the capital A
|
||||
// already exists inside "Automatically". Measured on a real return (idx 14).
|
||||
const before = 'Automatically add reviewed samples';
|
||||
const after = 'Add reviewed samples';
|
||||
assert.equal(isDeletionOnly(before, after), true);
|
||||
assert.deepEqual(novelWordForms(before, after), ['Add']);
|
||||
});
|
||||
|
||||
test('novelWordForms is case-sensitive but ignores punctuation', () => {
|
||||
assert.deepEqual(novelWordForms('en to tre, fire', 'en to tre'), []);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------- checkRow
|
||||
|
||||
const FILE_TEXT = [
|
||||
'# Tittel',
|
||||
'',
|
||||
'- Alpha',
|
||||
'- Beta',
|
||||
'- Gamma',
|
||||
'',
|
||||
'| Model improvement | Automatically add reviewed samples |',
|
||||
'',
|
||||
'Etterord med æ, ø og å.',
|
||||
'',
|
||||
].join('\n');
|
||||
const readFile = (rel) => {
|
||||
if (rel !== 'skills/x/references/y.md') throw new Error(`ENOENT: ${rel}`);
|
||||
return FILE_TEXT;
|
||||
};
|
||||
|
||||
function row(over = {}) {
|
||||
return {
|
||||
idx: 1,
|
||||
file: 'skills/x/references/y.md',
|
||||
line: 3,
|
||||
real_line: 3,
|
||||
locator_failed: false,
|
||||
file_text_verbatim: '- Alpha\n- Beta\n- Gamma',
|
||||
failing_part: 'Gamma',
|
||||
proposed_remainder: '- Alpha\n- Beta',
|
||||
cond1_strictly_less: { holds: true, evidence: 'one bullet removed' },
|
||||
cond2_remainder_not_misleading: { holds: 'yes', evidence: 'ok' },
|
||||
cond3_nothing_confirmed_removed: { holds: 'yes', evidence: 'ok' },
|
||||
verdict: 'O2_CANDIDATE',
|
||||
o3_reason: null,
|
||||
confidence: 'high',
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
test('checkRow passes a well-formed O2 candidate', () => {
|
||||
assert.deepEqual(checkRow(row(), readFile), []);
|
||||
});
|
||||
|
||||
test('checkRow flags quoted file text that is not in the file (V1)', () => {
|
||||
const found = checkRow(row({ file_text_verbatim: '- Alpha\n- Delta' }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V1'));
|
||||
});
|
||||
|
||||
test('checkRow flags a silently transliterated quote as not found (V1)', () => {
|
||||
// The file says "æ, ø og å"; an agent that writes "ae, oe og aa" has not
|
||||
// quoted the file, and the whole point of V1 is to see that.
|
||||
const found = checkRow(row({ file_text_verbatim: 'Etterord med ae, oe og aa.' }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V1'));
|
||||
});
|
||||
|
||||
test('checkRow flags a proposal that adds text (V2)', () => {
|
||||
const found = checkRow(row({ proposed_remainder: '- Alpha\n- Beta\n- Delta' }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V2'));
|
||||
});
|
||||
|
||||
test('checkRow flags recapitalisation separately from outright addition (V2b)', () => {
|
||||
const found = checkRow(
|
||||
row({
|
||||
file_text_verbatim: '| Model improvement | Automatically add reviewed samples |',
|
||||
proposed_remainder: '| Model improvement | Add reviewed samples |',
|
||||
}),
|
||||
readFile,
|
||||
);
|
||||
assert.deepEqual(found.map((f) => f.check), ['V2b']);
|
||||
});
|
||||
|
||||
test('checkRow flags an O2 candidate with no proposed remainder (V2)', () => {
|
||||
const found = checkRow(row({ proposed_remainder: null }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V2'));
|
||||
});
|
||||
|
||||
test('checkRow flags an O2 candidate whose condition 1 does not hold (V3)', () => {
|
||||
const found = checkRow(row({ cond1_strictly_less: { holds: false, evidence: 'no' } }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V3'));
|
||||
});
|
||||
|
||||
test('checkRow flags an O2 candidate that failed to locate the text (V3)', () => {
|
||||
const found = checkRow(row({ locator_failed: true }), readFile);
|
||||
assert.ok(found.some((f) => f.check === 'V3'));
|
||||
});
|
||||
|
||||
test('checkRow flags a missing required field (V3)', () => {
|
||||
const r = row();
|
||||
delete r.confidence;
|
||||
assert.ok(checkRow(r, readFile).some((f) => f.check === 'V3'));
|
||||
});
|
||||
|
||||
test('checkRow flags an unknown verdict (V3)', () => {
|
||||
assert.ok(checkRow(row({ verdict: 'O2' }), readFile).some((f) => f.check === 'V3'));
|
||||
});
|
||||
|
||||
test('checkRow does NOT require a proposed remainder on an O3 row', () => {
|
||||
const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'condition 3 fails' });
|
||||
assert.deepEqual(checkRow(r, readFile), []);
|
||||
});
|
||||
|
||||
test('checkRow still verifies the quoted text of an O3 row (V1 applies to every row)', () => {
|
||||
const r = row({ verdict: 'O3', proposed_remainder: null, o3_reason: 'x', file_text_verbatim: '- Delta' });
|
||||
assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
|
||||
});
|
||||
|
||||
test('checkRow reports an unreadable file rather than throwing', () => {
|
||||
const found = checkRow(row({ file: 'skills/x/references/missing.md' }), readFile);
|
||||
assert.equal(found.length, 1);
|
||||
assert.equal(found[0].check, 'V1');
|
||||
});
|
||||
|
||||
test('checkRow allows a row that failed to locate the text and fell back to O3', () => {
|
||||
const r = {
|
||||
...row(),
|
||||
locator_failed: true,
|
||||
file_text_verbatim: null,
|
||||
proposed_remainder: null,
|
||||
verdict: 'O3',
|
||||
o3_reason: 'locator failed',
|
||||
};
|
||||
assert.deepEqual(checkRow(r, readFile), []);
|
||||
});
|
||||
|
||||
test('checkRow flags a row that claims no locator failure but quotes nothing (V1)', () => {
|
||||
const r = row({ file_text_verbatim: null, proposed_remainder: null, verdict: 'O3', o3_reason: 'x' });
|
||||
assert.ok(checkRow(r, readFile).some((f) => f.check === 'V1'));
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue