// test-fix-op-classify.test.mjs — R11 fix-operation classifier (lib/fix-op.mjs). // // The classifier IS the O1 driver with writes disabled: it attempts the value // swap and checks the §4 invariant (docs/r11-tiered-fix-design.md). Anything it // cannot prove is O3, with a TYPED abort code — the abort taxonomy is the // pilot's measurement #3, so the codes are part of the contract, not diagnostics. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { ABORT_CODES, blockWindow, classifyFlag, extractValueTokens } from '../../scripts/kb-eval/lib/fix-op.mjs'; /** Minimal flag record per docs/r11-flag-format-2026-07.md. */ function flag(over = {}) { return { id: 'x/y.md#1', judge_verdict: 'not_grounded', rule: '', evidence_url: 'https://learn.microsoft.com/x', evidence_quote: 'The service supports max_docs=250 per batch.', reason: 'Source states 250, claim states 100.', file: 'skills/x/references/y.md', line: 3, claim: 'Parameteren max_docs er 100.', disposition: 'outdated', ...over, }; } // The happy-path fixture is deliberately LANGUAGE-NEUTRAL (a key=value pair that // reads the same in the file and in the English quote). That is not fixture // convenience — it is the pilot's measured result: the corpus is Norwegian and // the quotes are English, so a swap is only provable where the surrounding // context is code, a URL, or a parameter key. See the CONTEXT_MISMATCH tests. const FILE = ['# Tittel', '', '| konfig | max_docs=100 |', '| Annet | tekst |', '', 'Etterord.'].join('\n'); // ---------------------------------------------------------------- token types test('extractValueTokens types a plain integer as number', () => { const t = extractValueTokens('Grensen er 100 dokumenter.'); assert.deepEqual( t.map((x) => [x.type, x.value]), [['number', '100']], ); }); test('extractValueTokens types percent, ISO date and version distinctly', () => { const types = (s) => extractValueTokens(s).map((x) => x.type); assert.deepEqual(types('Treffraten er 20 %.'), ['percent']); assert.deepEqual(types('Gjelder fra 2026-08-02.'), ['iso_date']); assert.deepEqual(types('Krever versjon 2.3.0 av pakken.'), ['version']); }); test('extractValueTokens does not emit a status word as a swappable value', () => { assert.deepEqual(extractValueTokens('Hosted agents er i Public Preview.'), []); }); // --------------------------------------------------------------- block window test('blockWindow spans the contiguous non-blank block containing the line', () => { const lines = FILE.split('\n'); assert.deepEqual(blockWindow(lines, 3), { start: 3, end: 4 }); }); test('blockWindow anchored on a table header reaches the rows below it', () => { const lines = ['| A | B |', '| --- | --- |', '| rad | 7 |', '', 'annet']; assert.deepEqual(blockWindow(lines, 1), { start: 1, end: 3 }); }); test('blockWindow on a blank line degenerates to that line only', () => { const lines = FILE.split('\n'); assert.deepEqual(blockWindow(lines, 2), { start: 2, end: 2 }); }); // ------------------------------------------------------------- the O1 happy path test('classifyFlag proves a clean value swap as O1', () => { const r = classifyFlag(flag(), FILE); assert.equal(r.op, 'O1'); assert.equal(r.code, 'PROVEN'); assert.equal(r.proposal.line, 3); assert.equal(r.proposal.token, '100'); assert.equal(r.proposal.replacement, '250'); assert.equal(r.proposal.before, '| konfig | max_docs=100 |'); assert.equal(r.proposal.after, '| konfig | max_docs=250 |'); }); test('the O1 proposal changes exactly one line and leaves the rest byte-identical', () => { const r = classifyFlag(flag(), FILE); const before = FILE.split('\n'); const after = before.slice(); after[r.proposal.line - 1] = r.proposal.after; const changed = after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null); assert.deepEqual(changed, [2]); // §4(2): the rest of the line survives the swap byte-for-byte. assert.equal(r.proposal.before.replace('100', ''), r.proposal.after.replace('250', '')); }); test('the replacement value occurs verbatim in the cited evidence_quote (§4.1)', () => { const f = flag(); const r = classifyFlag(f, FILE); assert.ok(f.evidence_quote.includes(r.proposal.replacement)); }); // ------------------------------------------------------------- abort taxonomy test('an enumerated multi-part claim aborts as MULTI_PART_CLAIM', () => { const r = classifyFlag(flag({ claim: 'Verktøy: Code Interpreter | File Search | 3 andre.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_PART_CLAIM'); }); test('a status-only claim aborts as STATUS_SYNONYM rather than swapping prose', () => { const r = classifyFlag( flag({ claim: 'Hosted agents er i Public Preview.', evidence_quote: 'hosted agents are generally available', }), ['| Hosted agents | **Preview** |'].join('\n'), ); assert.equal(r.op, 'O3'); assert.equal(r.code, 'STATUS_SYNONYM'); }); test('a claim with no swappable value aborts as NO_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Tjenesten anbefales for produksjon.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NO_VALUE_TOKEN'); }); test('two distinct values in one claim abort as MULTI_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Grensen er 100 dokumenter og 50 sider.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_VALUE_TOKEN'); }); test('the same value repeated in a claim is one token, not MULTI_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 100, altså 100 per batch.' }), FILE); assert.equal(r.op, 'O1'); }); test('a value absent from the file block aborts as LOCATOR_MISS', () => { const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 999.', evidence_quote: 'max_docs=250' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_MISS); }); test('a value occurring twice in the block aborts as LOCATOR_AMBIGUOUS', () => { const twice = ['| konfig | max_docs=100 |', '| Tak | 100 kall |'].join('\n'); const r = classifyFlag(flag({ line: 1 }), twice); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_AMBIGUOUS); }); test('a quote with no same-type replacement aborts as NOT_VERBATIM', () => { const r = classifyFlag(flag({ evidence_quote: 'The service supports batching of docs.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); test('type is not crossed: a percent claim cannot be swapped from a bare integer quote', () => { const pct = '| Treffrate | 20 % |'; const r = classifyFlag(flag({ claim: 'Treffraten er 20 %.', evidence_quote: 'a cache holding 35 entries', line: 1 }), pct); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); test('two candidate replacements in the quote abort as MULTI_REPLACEMENT', () => { const r = classifyFlag(flag({ evidence_quote: 'max_docs=250, or 500 in premium' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_REPLACEMENT'); }); test('a quote restating the claim value is not a swap — NOT_VERBATIM, never a no-op edit', () => { const r = classifyFlag(flag({ evidence_quote: 'the limit is max_docs=100' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); // ------------------------------------------------- §4 is not sufficient alone // // Measured on the pilot: §4 as written (new value verbatim in the quote, rest of // the line byte-identical, one line changed) admitted 6 swaps of which 4 were // wrong — it constrains the PROVENANCE of the value and the SHAPE of the edit, // and nothing about whether the two tokens denote the same quantity. The context // condition adds that: the token must sit under a matching label or unit on both // sides. These four tests are the four real failures, reduced to fixtures. test('a unit-crossing swap is refused: 30 days in the file, 24 hours in the quote', () => { const r = classifyFlag( flag({ claim: 'Standard 30 dagers oppbevaring av dokumenter.', evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.', line: 1, }), '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('a metric-crossing swap is refused: indexing rate in the file, query throttle in the quote', () => { const r = classifyFlag( flag({ claim: 'Azure AI Search har rate limits på 3000 requests/sekund per replika.', evidence_quote: '| Search queries | Varies by SU count | 50 queries/sec (aggregate read throttle per index) |', line: 1, }), '**Viktig:** Azure AI Search har rate limits (3000 requests/sekund per replika).', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('a digit inside an identifier is never harvested as a replacement (Agent 365 -> E7)', () => { const r = classifyFlag( flag({ claim: 'Microsoft Agent 365 er inkludert i Copilot Studio-lisens.', evidence_quote: 'Microsoft Agent 365 is available as a stand-alone subscription and is also included with Microsoft 365 E7.', line: 1, }), '- Agent observability (Microsoft Agent 365): Inkludert i Copilot Studio-lisens', ); assert.equal(r.op, 'O3'); assert.notEqual(r.op, 'O1'); }); test('a model identifier is not mutilated by a bare digit from a table cell', () => { const r = classifyFlag( flag({ claim: 'Modellen text-embedding-ada-002 har 1536 dimensjoner.', evidence_quote: '| text-embedding-ada-002 | 2 | GA | 2028-02-09 |', line: 1, }), '| `text-embedding-ada-002` | 1536 | ~80 NOK | Legacy |', ); assert.notEqual(r.op, 'O1'); }); test('a matching key on both sides admits the swap: api-version', () => { const r = classifyFlag( flag({ claim: 'API-versjonen er 2025-09-01.', evidence_quote: 'POST https://[servicename].search.windows.net/indexes?api-version=2026-04-01', line: 1, }), 'POST https://[service].search.windows.net/indexes?api-version=2025-09-01', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.replacement, '2026-04-01'); }); test('key matching normalises punctuation: api_version="X" matches api-version=Y', () => { const r = classifyFlag( flag({ claim: 'Bruker api_version 2024-02-01.', evidence_quote: 'POST https://{endpoint}/openai/deployments/{id}/embeddings?api-version=2024-10-21', line: 1, }), ' api_version="2024-02-01"', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, ' api_version="2024-10-21"'); }); test('the corpus-dominant shape — Norwegian file, English quote, no shared context — is refused', () => { const r = classifyFlag( flag({ claim: 'Grensen er 100 dokumenter.', evidence_quote: 'The service supports up to 250 documents per batch.', line: 1, }), '| Grense | 100 dokumenter |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('the context condition is what refuses the unit crossing — disabling it restores the §4-only verdict', () => { const f = flag({ claim: 'Standard 30 dagers oppbevaring av dokumenter.', evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.', line: 1, }); const text = '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter'; assert.equal(classifyFlag(f, text, { contextCheck: false }).op, 'O1'); assert.equal(classifyFlag(f, text).op, 'O3'); }); // --------------------------------------------------------------- fail-closed test('every outcome is either a proven O1 or an O3 carrying a known abort code', () => { const cases = [ flag(), flag({ claim: 'Ingen tall her.' }), flag({ claim: 'A | B | C med 3 ting.' }), flag({ claim: 'Parameteren max_docs er 999.' }), flag({ evidence_quote: 'ingen tall' }), ]; const known = new Set(['PROVEN', ...Object.values(ABORT_CODES)]); for (const c of cases) { const r = classifyFlag(c, FILE); assert.ok(r.op === 'O1' || r.op === 'O3', `op must be O1|O3, got ${r.op}`); assert.ok(known.has(r.code), `unknown code ${r.code}`); assert.equal(r.op === 'O1', r.code === 'PROVEN'); } }); test('an out-of-range line fails closed instead of throwing', () => { const r = classifyFlag(flag({ line: 9999 }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_MISS); });