// test-fix-op-classify.test.mjs — R11 fix-operation classifier (lib/fix-op.mjs). // // The classifier IS the O1 driver with writes disabled: it attempts the value // swap and checks the §4 invariant (docs/r11-tiered-fix-design.md). Anything it // cannot prove is O3, with a TYPED abort code — the abort taxonomy is the // pilot's measurement #3, so the codes are part of the contract, not diagnostics. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { ABORT_CODES, blockWindow, classifyFlag, extractValueTokens } from '../../scripts/kb-eval/lib/fix-op.mjs'; /** Minimal flag record per docs/r11-flag-format-2026-07.md. */ function flag(over = {}) { return { id: 'x/y.md#1', judge_verdict: 'not_grounded', rule: '', evidence_url: 'https://learn.microsoft.com/x', evidence_quote: 'The service supports max_docs=250 per batch.', reason: 'Source states 250, claim states 100.', file: 'skills/x/references/y.md', line: 3, claim: 'Parameteren max_docs er 100.', disposition: 'outdated', ...over, }; } // The happy-path fixture is deliberately LANGUAGE-NEUTRAL (a key=value pair that // reads the same in the file and in the English quote). That is not fixture // convenience — it is the pilot's measured result: the corpus is Norwegian and // the quotes are English, so a swap is only provable where the surrounding // context is code, a URL, or a parameter key. See the CONTEXT_MISMATCH tests. const FILE = ['# Tittel', '', '| konfig | max_docs=100 |', '| Annet | tekst |', '', 'Etterord.'].join('\n'); // ---------------------------------------------------------------- token types test('extractValueTokens types a plain integer as number', () => { const t = extractValueTokens('Grensen er 100 dokumenter.'); assert.deepEqual( t.map((x) => [x.type, x.value]), [['number', '100']], ); }); test('extractValueTokens types percent, ISO date and version distinctly', () => { const types = (s) => extractValueTokens(s).map((x) => x.type); assert.deepEqual(types('Treffraten er 20 %.'), ['percent']); assert.deepEqual(types('Gjelder fra 2026-08-02.'), ['iso_date']); assert.deepEqual(types('Krever versjon 2.3.0 av pakken.'), ['version']); }); test('extractValueTokens does not emit a status word as a swappable value', () => { assert.deepEqual(extractValueTokens('Hosted agents er i Public Preview.'), []); }); // --------------------------------------------------------------- block window test('blockWindow spans the contiguous non-blank block containing the line', () => { const lines = FILE.split('\n'); assert.deepEqual(blockWindow(lines, 3), { start: 3, end: 4 }); }); test('blockWindow anchored on a table header reaches the rows below it', () => { const lines = ['| A | B |', '| --- | --- |', '| rad | 7 |', '', 'annet']; assert.deepEqual(blockWindow(lines, 1), { start: 1, end: 3 }); }); test('blockWindow on a blank line degenerates to that line only', () => { const lines = FILE.split('\n'); assert.deepEqual(blockWindow(lines, 2), { start: 2, end: 2 }); }); // ------------------------------------------------------------- the O1 happy path test('classifyFlag proves a clean value swap as O1', () => { const r = classifyFlag(flag(), FILE); assert.equal(r.op, 'O1'); assert.equal(r.code, 'PROVEN'); assert.equal(r.proposal.line, 3); assert.equal(r.proposal.token, '100'); assert.equal(r.proposal.replacement, '250'); assert.equal(r.proposal.before, '| konfig | max_docs=100 |'); assert.equal(r.proposal.after, '| konfig | max_docs=250 |'); }); test('the O1 proposal changes exactly one line and leaves the rest byte-identical', () => { const r = classifyFlag(flag(), FILE); const before = FILE.split('\n'); const after = before.slice(); after[r.proposal.line - 1] = r.proposal.after; const changed = after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null); assert.deepEqual(changed, [2]); // §4(2): the rest of the line survives the swap byte-for-byte. assert.equal(r.proposal.before.replace('100', '§'), r.proposal.after.replace('250', '§')); }); test('the replacement value occurs verbatim in the cited evidence_quote (§4.1)', () => { const f = flag(); const r = classifyFlag(f, FILE); assert.ok(f.evidence_quote.includes(r.proposal.replacement)); }); // ------------------------------------------------------------- abort taxonomy test('an enumerated multi-part claim aborts as MULTI_PART_CLAIM', () => { const r = classifyFlag(flag({ claim: 'Verktøy: Code Interpreter | File Search | 3 andre.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_PART_CLAIM'); }); test('a status claim whose pair is outside the ratified table aborts as STATUS_SYNONYM', () => { const r = classifyFlag( flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the service is in early access for selected customers', line: 1, }), ['| Tjenesten | **Preview** |'].join('\n'), ); assert.equal(r.op, 'O3'); assert.equal(r.code, 'STATUS_SYNONYM'); }); test('a claim with no swappable value aborts as NO_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Tjenesten anbefales for produksjon.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NO_VALUE_TOKEN'); }); test('two distinct values in one claim abort as MULTI_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Grensen er 100 dokumenter og 50 sider.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_VALUE_TOKEN'); }); test('the same value repeated in a claim is one token, not MULTI_VALUE_TOKEN', () => { const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 100, altså 100 per batch.' }), FILE); assert.equal(r.op, 'O1'); }); test('a value absent from the file block aborts as LOCATOR_MISS', () => { const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 999.', evidence_quote: 'max_docs=250' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_MISS); }); test('a value occurring twice in the block aborts as LOCATOR_AMBIGUOUS', () => { const twice = ['| konfig | max_docs=100 |', '| Tak | 100 kall |'].join('\n'); const r = classifyFlag(flag({ line: 1 }), twice); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_AMBIGUOUS); }); test('a quote with no same-type replacement aborts as NOT_VERBATIM', () => { const r = classifyFlag(flag({ evidence_quote: 'The service supports batching of docs.' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); test('type is not crossed: a percent claim cannot be swapped from a bare integer quote', () => { const pct = '| Treffrate | 20 % |'; const r = classifyFlag(flag({ claim: 'Treffraten er 20 %.', evidence_quote: 'a cache holding 35 entries', line: 1 }), pct); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); test('two candidate replacements in the quote abort as MULTI_REPLACEMENT', () => { const r = classifyFlag(flag({ evidence_quote: 'max_docs=250, or 500 in premium' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'MULTI_REPLACEMENT'); }); test('a quote restating the claim value is not a swap — NOT_VERBATIM, never a no-op edit', () => { const r = classifyFlag(flag({ evidence_quote: 'the limit is max_docs=100' }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, 'NOT_VERBATIM'); }); // ------------------------------------------------- §4 is not sufficient alone // // Measured on the pilot: §4 as written (new value verbatim in the quote, rest of // the line byte-identical, one line changed) admitted 6 swaps of which 4 were // wrong — it constrains the PROVENANCE of the value and the SHAPE of the edit, // and nothing about whether the two tokens denote the same quantity. The context // condition adds that: the token must sit under a matching label or unit on both // sides. These four tests are the four real failures, reduced to fixtures. test('a unit-crossing swap is refused: 30 days in the file, 24 hours in the quote', () => { const r = classifyFlag( flag({ claim: 'Standard 30 dagers oppbevaring av dokumenter.', evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.', line: 1, }), '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('a metric-crossing swap is refused: indexing rate in the file, query throttle in the quote', () => { const r = classifyFlag( flag({ claim: 'Azure AI Search har rate limits på 3000 requests/sekund per replika.', evidence_quote: '| Search queries | Varies by SU count | 50 queries/sec (aggregate read throttle per index) |', line: 1, }), '**Viktig:** Azure AI Search har rate limits (3000 requests/sekund per replika).', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('a digit inside an identifier is never harvested as a replacement (Agent 365 -> E7)', () => { const r = classifyFlag( flag({ claim: 'Microsoft Agent 365 er inkludert i Copilot Studio-lisens.', evidence_quote: 'Microsoft Agent 365 is available as a stand-alone subscription and is also included with Microsoft 365 E7.', line: 1, }), '- Agent observability (Microsoft Agent 365): Inkludert i Copilot Studio-lisens', ); assert.equal(r.op, 'O3'); assert.notEqual(r.op, 'O1'); }); test('a model identifier is not mutilated by a bare digit from a table cell', () => { const r = classifyFlag( flag({ claim: 'Modellen text-embedding-ada-002 har 1536 dimensjoner.', evidence_quote: '| text-embedding-ada-002 | 2 | GA | 2028-02-09 |', line: 1, }), '| `text-embedding-ada-002` | 1536 | ~80 NOK | Legacy |', ); assert.notEqual(r.op, 'O1'); }); test('a matching key on both sides admits the swap: api-version', () => { const r = classifyFlag( flag({ claim: 'API-versjonen er 2025-09-01.', evidence_quote: 'POST https://[servicename].search.windows.net/indexes?api-version=2026-04-01', line: 1, }), 'POST https://[service].search.windows.net/indexes?api-version=2025-09-01', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.replacement, '2026-04-01'); }); test('key matching normalises punctuation: api_version="X" matches api-version=Y', () => { const r = classifyFlag( flag({ claim: 'Bruker api_version 2024-02-01.', evidence_quote: 'POST https://{endpoint}/openai/deployments/{id}/embeddings?api-version=2024-10-21', line: 1, }), ' api_version="2024-02-01"', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, ' api_version="2024-10-21"'); }); test('the corpus-dominant shape — Norwegian file, English quote, no shared context — is refused', () => { const r = classifyFlag( flag({ claim: 'Grensen er 100 dokumenter.', evidence_quote: 'The service supports up to 250 documents per batch.', line: 1, }), '| Grense | 100 dokumenter |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH); }); test('the context condition is what refuses the unit crossing — disabling it restores the §4-only verdict', () => { const f = flag({ claim: 'Standard 30 dagers oppbevaring av dokumenter.', evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.', line: 1, }); const text = '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter'; assert.equal(classifyFlag(f, text, { contextCheck: false }).op, 'O1'); assert.equal(classifyFlag(f, text).op, 'O3'); }); // ------------------------------------------- §4b the ratified synonym table // // docs/r11-tiered-fix-design.md §4b (operator decision, 2026-08-03). This is the // ONE place where the value written into the file does not itself appear verbatim // in the quote, so the three ratified constraints each carry their own tests: // the table is CLOSED, the file-side token must be a COMPLETE lifecycle label, // and the written value is the CORPUS-side equivalent with the file's own markup // preserved. The fixtures are the real pilot rows, not invented shapes. test('§4b proves a status swap: file **Preview**, source says generally available', () => { const r = classifyFlag( flag({ claim: 'Hosted agents er i Public Preview.', evidence_quote: 'hosted agents are generally available and this header is no longer required.', line: 1, }), '| Hosted agents (din egen kode/container) | **Preview** |', ); assert.equal(r.op, 'O1'); assert.equal(r.code, 'PROVEN'); assert.equal(r.proposal.type, 'status'); assert.equal(r.proposal.after, '| Hosted agents (din egen kode/container) | **GA** |'); }); test('§4b proves the opposite direction: file **GA**, source says (preview)', () => { const r = classifyFlag( flag({ claim: 'Logic Apps-triggerintegrasjon er GA.', evidence_quote: 'Trigger an agent by using Logic Apps (preview) (classic)', line: 1, }), '| Logic Apps-triggerintegrasjon | **GA** |', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, '| Logic Apps-triggerintegrasjon | **Preview** |'); }); test('§4b handles an unmarked table cell: | Public Preview | -> | GA |', () => { const r = classifyFlag( flag({ claim: 'Phishing Triage Agent i Defender XDR er i Public Preview.', evidence_quote: 'Email and collaboration alert triage capabilities are already generally available (GA).', line: 1, }), '| **Phishing Triage Agent** | Defender XDR | Autonomt triage av phishing-hendelser. | Public Preview |', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, '| **Phishing Triage Agent** | Defender XDR | Autonomt triage av phishing-hendelser. | GA |'); }); // -- constraint 1: the table is CLOSED, never extended by inference at run time test('§4b constraint 1: an unlisted corpus-side label is never mapped', () => { const r = classifyFlag( flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the feature is generally available', line: 1 }), '| Tjenesten | **Beta** |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 1: a source phrasing that merely resembles a listed one is not mapped', () => { const r = classifyFlag( flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the feature is generally accessible to all tenants', line: 1 }), '| Tjenesten | **Preview** |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 1: an api-version suffix is not a lifecycle statement', () => { const r = classifyFlag( flag({ claim: 'Workflows er GA.', evidence_quote: 'Call the endpoint with api-version=2025-11-15-preview.', line: 1 }), '| Workflows | **GA** |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 1: private preview is its own row, not the preview row', () => { const r = classifyFlag( flag({ claim: 'Funksjonen er GA.', evidence_quote: 'This capability is in private preview for selected customers.', line: 1 }), '| Funksjonen | **GA** |', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, '| Funksjonen | **Private Preview** |'); }); // -- constraint 2: a COMPLETE lifecycle label, never a substring of a sentence test('§4b constraint 2: a parenthesised (preview) inside a list item is not a complete label', () => { const r = classifyFlag( flag({ claim: 'Microsoft Entra ID-autentisering for Redis er i preview.', evidence_quote: 'Microsoft Entra ID authentication for Redis is generally available.', line: 1, }), '- Microsoft Entra ID authentication for Redis (preview)', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 2: an emphasised run that is more than the label is not a complete label', () => { const r = classifyFlag( flag({ claim: 'DSPM er i preview.', evidence_quote: 'The new version of Data Security Posture Management is now generally available.', line: 1, }), '- **DSPM (preview):** New version med enhanced AI activities tab', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 2: a label inside a JSON string value is never touched', () => { const r = classifyFlag( flag({ claim: 'Azure Policy-definisjonen er i preview.', evidence_quote: 'The policy definition is generally available.', line: 1, }), ' "policyDefinitionName": "[Preview]: Azure Machine Learning Deployments should only use approved Registry Models",', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 2: a bare label in prose (no emphasis, no cell) is not swappable', () => { const r = classifyFlag( flag({ claim: 'Hybrid search er GA.', evidence_quote: 'Vector optimization techniques are generally available.', line: 1 }), '**Status:** GA (Hybrid search), Preview (Scalar quantization)', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b constraint 2: an emphasised label inside a sentence IS complete and swappable', () => { const r = classifyFlag( flag({ claim: 'Hosted agents er i Preview.', evidence_quote: 'hosted agents are generally available', line: 1 }), 'Hosted agents er **Preview** i norsk sky.', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, 'Hosted agents er **GA** i norsk sky.'); }); test('§4b constraint 2: two labels on the same line abort — which one is a judgement', () => { const r = classifyFlag( flag({ claim: 'Verktøyet er i Preview.', evidence_quote: 'the tool is generally available', line: 1 }), '| Verktøyet | **Preview** | **GA** |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b the status locator is line-scoped: a label on a neighbouring row is not used', () => { const table = ['| Verktøy | Status |', '| SharePoint | Beskrivelse uten status |', '| Fabric | **Preview** |'].join('\n'); const r = classifyFlag( flag({ claim: 'SharePoint-verktøyet er i Preview.', evidence_quote: 'SharePoint tool is generally available', line: 2 }), table, ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); // -- constraint 3: write the CORPUS-side equivalent, with the file's own markup test('§4b constraint 3: the English source phrasing is never pasted into the file', () => { const r = classifyFlag( flag({ claim: 'Vulnerability Remediation Agent er GA.', evidence_quote: 'This feature is in public preview. For more information, see Public preview in Microsoft Intune.', line: 1, }), '| **Vulnerability Remediation Agent** | Microsoft Intune | Trinnvis remediering via Intune | GA |', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.replacement, 'Preview'); assert.ok(!/public preview/i.test(r.proposal.after), 'the quote phrasing must not reach the file'); assert.equal(r.proposal.after.endsWith('| Preview |'), true); }); test('§4b constraint 3: backtick markup is preserved exactly as the file wrote it', () => { const r = classifyFlag( flag({ claim: 'Verktøyet er i Preview.', evidence_quote: 'the tool is now generally available', line: 1 }), '| Verktøyet | `Preview` |', ); assert.equal(r.op, 'O1'); assert.equal(r.proposal.after, '| Verktøyet | `GA` |'); }); test('§4b a file already agreeing with the source is not a swap — never a no-op edit', () => { const r = classifyFlag( flag({ claim: 'SharePoint-verktøyet er i Preview (de øvrige er GA).', evidence_quote: 'Microsoft Fabric (preview) ... SharePoint (preview)', line: 1, }), '| **SharePoint** | Knowledge | Tilgang til interne dokumenter | Preview |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b a quote carrying two different lifecycle rows aborts — the source is ambiguous', () => { const r = classifyFlag( flag({ claim: 'DSPM er GA.', evidence_quote: 'General availability (GA): The new version of Data Security Posture Management is now generally available. Partner solutions remain in preview.', line: 1, }), '| DSPM | **GA** |', ); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM); }); test('§4b a proven status swap changes exactly one line and keeps the rest byte-identical', () => { const text = ['| Verktøy | Status |', '|---|---|', '| Hosted agents | **Preview** |', '', 'Etterord.'].join('\n'); const r = classifyFlag( flag({ claim: 'Hosted agents er i Preview.', evidence_quote: 'hosted agents are generally available', line: 3 }), text, ); assert.equal(r.op, 'O1'); const before = text.split('\n'); const after = before.slice(); after[r.proposal.line - 1] = r.proposal.after; assert.deepEqual( after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null), [2], ); assert.equal(r.proposal.before.replace('**Preview**', '§'), r.proposal.after.replace('**GA**', '§')); }); // --------------------------------------------------------------- fail-closed test('every outcome is either a proven O1 or an O3 carrying a known abort code', () => { const cases = [ flag(), flag({ claim: 'Ingen tall her.' }), flag({ claim: 'A | B | C med 3 ting.' }), flag({ claim: 'Parameteren max_docs er 999.' }), flag({ evidence_quote: 'ingen tall' }), flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the service is generally available', line: 1 }), flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'early access', line: 1 }), ]; const known = new Set(['PROVEN', ...Object.values(ABORT_CODES)]); for (const c of cases) { const r = classifyFlag(c, FILE); assert.ok(r.op === 'O1' || r.op === 'O3', `op must be O1|O3, got ${r.op}`); assert.ok(known.has(r.code), `unknown code ${r.code}`); assert.equal(r.op === 'O1', r.code === 'PROVEN'); } }); test('an out-of-range line fails closed instead of throwing', () => { const r = classifyFlag(flag({ line: 9999 }), FILE); assert.equal(r.op, 'O3'); assert.equal(r.code, ABORT_CODES.LOCATOR_MISS); });