ms-ai-architect/tests/kb-eval/test-fix-op-classify.test.mjs

// test-fix-op-classify.test.mjs — R11 fix-operation classifier (lib/fix-op.mjs).
//
// The classifier IS the O1 driver with writes disabled: it attempts the value
// swap and checks the §4 invariant (docs/r11-tiered-fix-design.md). Anything it
// cannot prove is O3, with a TYPED abort code — the abort taxonomy is the
// pilot's measurement #3, so the codes are part of the contract, not diagnostics.

import { test } from 'node:test';
import assert from 'node:assert/strict';

import { ABORT_CODES, blockWindow, classifyFlag, extractValueTokens } from '../../scripts/kb-eval/lib/fix-op.mjs';

/** Minimal flag record per docs/r11-flag-format-2026-07.md. */
function flag(over = {}) {
  return {
    id: 'x/y.md#1',
    judge_verdict: 'not_grounded',
    rule: '',
    evidence_url: 'https://learn.microsoft.com/x',
    evidence_quote: 'The service supports max_docs=250 per batch.',
    reason: 'Source states 250, claim states 100.',
    file: 'skills/x/references/y.md',
    line: 3,
    claim: 'Parameteren max_docs er 100.',
    disposition: 'outdated',
    ...over,
  };
}

// The happy-path fixture is deliberately LANGUAGE-NEUTRAL (a key=value pair that
// reads the same in the file and in the English quote). That is not fixture
// convenience — it is the pilot's measured result: the corpus is Norwegian and
// the quotes are English, so a swap is only provable where the surrounding
// context is code, a URL, or a parameter key. See the CONTEXT_MISMATCH tests.
const FILE = ['# Tittel', '', '| konfig | max_docs=100 |', '| Annet | tekst |', '', 'Etterord.'].join('\n');

// ---------------------------------------------------------------- token types

test('extractValueTokens types a plain integer as number', () => {
  const t = extractValueTokens('Grensen er 100 dokumenter.');
  assert.deepEqual(
    t.map((x) => [x.type, x.value]),
    [['number', '100']],
  );
});

test('extractValueTokens types percent, ISO date and version distinctly', () => {
  const types = (s) => extractValueTokens(s).map((x) => x.type);
  assert.deepEqual(types('Treffraten er 20 %.'), ['percent']);
  assert.deepEqual(types('Gjelder fra 2026-08-02.'), ['iso_date']);
  assert.deepEqual(types('Krever versjon 2.3.0 av pakken.'), ['version']);
});

test('extractValueTokens does not emit a status word as a swappable value', () => {
  assert.deepEqual(extractValueTokens('Hosted agents er i Public Preview.'), []);
});

// --------------------------------------------------------------- block window

test('blockWindow spans the contiguous non-blank block containing the line', () => {
  const lines = FILE.split('\n');
  assert.deepEqual(blockWindow(lines, 3), { start: 3, end: 4 });
});

test('blockWindow anchored on a table header reaches the rows below it', () => {
  const lines = ['| A | B |', '| --- | --- |', '| rad | 7 |', '', 'annet'];
  assert.deepEqual(blockWindow(lines, 1), { start: 1, end: 3 });
});

test('blockWindow on a blank line degenerates to that line only', () => {
  const lines = FILE.split('\n');
  assert.deepEqual(blockWindow(lines, 2), { start: 2, end: 2 });
});

// ------------------------------------------------------------- the O1 happy path

test('classifyFlag proves a clean value swap as O1', () => {
  const r = classifyFlag(flag(), FILE);
  assert.equal(r.op, 'O1');
  assert.equal(r.code, 'PROVEN');
  assert.equal(r.proposal.line, 3);
  assert.equal(r.proposal.token, '100');
  assert.equal(r.proposal.replacement, '250');
  assert.equal(r.proposal.before, '| konfig | max_docs=100 |');
  assert.equal(r.proposal.after, '| konfig | max_docs=250 |');
});

test('the O1 proposal changes exactly one line and leaves the rest byte-identical', () => {
  const r = classifyFlag(flag(), FILE);
  const before = FILE.split('\n');
  const after = before.slice();
  after[r.proposal.line - 1] = r.proposal.after;
  const changed = after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null);
  assert.deepEqual(changed, [2]);
  // §4(2): the rest of the line survives the swap byte-for-byte.
  assert.equal(r.proposal.before.replace('100', '§'), r.proposal.after.replace('250', '§'));
});

test('the replacement value occurs verbatim in the cited evidence_quote (§4.1)', () => {
  const f = flag();
  const r = classifyFlag(f, FILE);
  assert.ok(f.evidence_quote.includes(r.proposal.replacement));
});

// ------------------------------------------------------------- abort taxonomy

test('an enumerated multi-part claim aborts as MULTI_PART_CLAIM', () => {
  const r = classifyFlag(flag({ claim: 'Verktøy: Code Interpreter | File Search | 3 andre.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_PART_CLAIM');
});

test('a status claim whose pair is outside the ratified table aborts as STATUS_SYNONYM', () => {
  const r = classifyFlag(
    flag({
      claim: 'Tjenesten er i Preview.',
      evidence_quote: 'the service is in early access for selected customers',
      line: 1,
    }),
    ['| Tjenesten | **Preview** |'].join('\n'),
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'STATUS_SYNONYM');
});

test('a claim with no swappable value aborts as NO_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Tjenesten anbefales for produksjon.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NO_VALUE_TOKEN');
});

test('two distinct values in one claim abort as MULTI_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Grensen er 100 dokumenter og 50 sider.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_VALUE_TOKEN');
});

test('the same value repeated in a claim is one token, not MULTI_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 100, altså 100 per batch.' }), FILE);
  assert.equal(r.op, 'O1');
});

test('a value absent from the file block aborts as LOCATOR_MISS', () => {
  const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 999.', evidence_quote: 'max_docs=250' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_MISS);
});

test('a value occurring twice in the block aborts as LOCATOR_AMBIGUOUS', () => {
  const twice = ['| konfig | max_docs=100 |', '| Tak | 100 kall |'].join('\n');
  const r = classifyFlag(flag({ line: 1 }), twice);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_AMBIGUOUS);
});

test('a quote with no same-type replacement aborts as NOT_VERBATIM', () => {
  const r = classifyFlag(flag({ evidence_quote: 'The service supports batching of docs.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

test('type is not crossed: a percent claim cannot be swapped from a bare integer quote', () => {
  const pct = '| Treffrate | 20 % |';
  const r = classifyFlag(flag({ claim: 'Treffraten er 20 %.', evidence_quote: 'a cache holding 35 entries', line: 1 }), pct);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

test('two candidate replacements in the quote abort as MULTI_REPLACEMENT', () => {
  const r = classifyFlag(flag({ evidence_quote: 'max_docs=250, or 500 in premium' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_REPLACEMENT');
});

test('a quote restating the claim value is not a swap — NOT_VERBATIM, never a no-op edit', () => {
  const r = classifyFlag(flag({ evidence_quote: 'the limit is max_docs=100' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

// ------------------------------------------------- §4 is not sufficient alone
//
// Measured on the pilot: §4 as written (new value verbatim in the quote, rest of
// the line byte-identical, one line changed) admitted 6 swaps of which 4 were
// wrong — it constrains the PROVENANCE of the value and the SHAPE of the edit,
// and nothing about whether the two tokens denote the same quantity. The context
// condition adds that: the token must sit under a matching label or unit on both
// sides. These four tests are the four real failures, reduced to fixtures.

test('a unit-crossing swap is refused: 30 days in the file, 24 hours in the quote', () => {
  const r = classifyFlag(
    flag({
      claim: 'Standard 30 dagers oppbevaring av dokumenter.',
      evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.',
      line: 1,
    }),
    '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('a metric-crossing swap is refused: indexing rate in the file, query throttle in the quote', () => {
  const r = classifyFlag(
    flag({
      claim: 'Azure AI Search har rate limits på 3000 requests/sekund per replika.',
      evidence_quote: '| Search queries | Varies by SU count | 50 queries/sec (aggregate read throttle per index) |',
      line: 1,
    }),
    '**Viktig:** Azure AI Search har rate limits (3000 requests/sekund per replika).',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('a digit inside an identifier is never harvested as a replacement (Agent 365 -> E7)', () => {
  const r = classifyFlag(
    flag({
      claim: 'Microsoft Agent 365 er inkludert i Copilot Studio-lisens.',
      evidence_quote: 'Microsoft Agent 365 is available as a stand-alone subscription and is also included with Microsoft 365 E7.',
      line: 1,
    }),
    '- Agent observability (Microsoft Agent 365): Inkludert i Copilot Studio-lisens',
  );
  assert.equal(r.op, 'O3');
  assert.notEqual(r.op, 'O1');
});

test('a model identifier is not mutilated by a bare digit from a table cell', () => {
  const r = classifyFlag(
    flag({
      claim: 'Modellen text-embedding-ada-002 har 1536 dimensjoner.',
      evidence_quote: '| text-embedding-ada-002 | 2 | GA | 2028-02-09 |',
      line: 1,
    }),
    '| `text-embedding-ada-002` | 1536 | ~80 NOK | Legacy |',
  );
  assert.notEqual(r.op, 'O1');
});

test('a matching key on both sides admits the swap: api-version', () => {
  const r = classifyFlag(
    flag({
      claim: 'API-versjonen er 2025-09-01.',
      evidence_quote: 'POST https://[servicename].search.windows.net/indexes?api-version=2026-04-01',
      line: 1,
    }),
    'POST https://[service].search.windows.net/indexes?api-version=2025-09-01',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.replacement, '2026-04-01');
});

test('key matching normalises punctuation: api_version="X" matches api-version=Y', () => {
  const r = classifyFlag(
    flag({
      claim: 'Bruker api_version 2024-02-01.',
      evidence_quote: 'POST https://{endpoint}/openai/deployments/{id}/embeddings?api-version=2024-10-21',
      line: 1,
    }),
    '    api_version="2024-02-01"',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '    api_version="2024-10-21"');
});

test('the corpus-dominant shape — Norwegian file, English quote, no shared context — is refused', () => {
  const r = classifyFlag(
    flag({
      claim: 'Grensen er 100 dokumenter.',
      evidence_quote: 'The service supports up to 250 documents per batch.',
      line: 1,
    }),
    '| Grense | 100 dokumenter |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('the context condition is what refuses the unit crossing — disabling it restores the §4-only verdict', () => {
  const f = flag({
    claim: 'Standard 30 dagers oppbevaring av dokumenter.',
    evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.',
    line: 1,
  });
  const text = '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter';
  assert.equal(classifyFlag(f, text, { contextCheck: false }).op, 'O1');
  assert.equal(classifyFlag(f, text).op, 'O3');
});

// ------------------------------------------- §4b the ratified synonym table
//
// docs/r11-tiered-fix-design.md §4b (operator decision, 2026-08-03). This is the
// ONE place where the value written into the file does not itself appear verbatim
// in the quote, so the three ratified constraints each carry their own tests:
// the table is CLOSED, the file-side token must be a COMPLETE lifecycle label,
// and the written value is the CORPUS-side equivalent with the file's own markup
// preserved. The fixtures are the real pilot rows, not invented shapes.

test('§4b proves a status swap: file **Preview**, source says generally available', () => {
  const r = classifyFlag(
    flag({
      claim: 'Hosted agents er i Public Preview.',
      evidence_quote: 'hosted agents are generally available and this header is no longer required.',
      line: 1,
    }),
    '| Hosted agents (din egen kode/container) | **Preview** |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.code, 'PROVEN');
  assert.equal(r.proposal.type, 'status');
  assert.equal(r.proposal.after, '| Hosted agents (din egen kode/container) | **GA** |');
});

test('§4b proves the opposite direction: file **GA**, source says (preview)', () => {
  const r = classifyFlag(
    flag({
      claim: 'Logic Apps-triggerintegrasjon er GA.',
      evidence_quote: 'Trigger an agent by using Logic Apps (preview) (classic)',
      line: 1,
    }),
    '| Logic Apps-triggerintegrasjon | **GA** |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '| Logic Apps-triggerintegrasjon | **Preview** |');
});

test('§4b handles an unmarked table cell: | Public Preview | -> | GA |', () => {
  const r = classifyFlag(
    flag({
      claim: 'Phishing Triage Agent i Defender XDR er i Public Preview.',
      evidence_quote: 'Email and collaboration alert triage capabilities are already generally available (GA).',
      line: 1,
    }),
    '| **Phishing Triage Agent** | Defender XDR | Autonomt triage av phishing-hendelser. | Public Preview |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '| **Phishing Triage Agent** | Defender XDR | Autonomt triage av phishing-hendelser. | GA |');
});

// -- constraint 1: the table is CLOSED, never extended by inference at run time

test('§4b constraint 1: an unlisted corpus-side label is never mapped', () => {
  const r = classifyFlag(
    flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the feature is generally available', line: 1 }),
    '| Tjenesten | **Beta** |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 1: a source phrasing that merely resembles a listed one is not mapped', () => {
  const r = classifyFlag(
    flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the feature is generally accessible to all tenants', line: 1 }),
    '| Tjenesten | **Preview** |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 1: an api-version suffix is not a lifecycle statement', () => {
  const r = classifyFlag(
    flag({ claim: 'Workflows er GA.', evidence_quote: 'Call the endpoint with api-version=2025-11-15-preview.', line: 1 }),
    '| Workflows | **GA** |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 1: private preview is its own row, not the preview row', () => {
  const r = classifyFlag(
    flag({ claim: 'Funksjonen er GA.', evidence_quote: 'This capability is in private preview for selected customers.', line: 1 }),
    '| Funksjonen | **GA** |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '| Funksjonen | **Private Preview** |');
});

// -- constraint 2: a COMPLETE lifecycle label, never a substring of a sentence

test('§4b constraint 2: a parenthesised (preview) inside a list item is not a complete label', () => {
  const r = classifyFlag(
    flag({
      claim: 'Microsoft Entra ID-autentisering for Redis er i preview.',
      evidence_quote: 'Microsoft Entra ID authentication for Redis is generally available.',
      line: 1,
    }),
    '- Microsoft Entra ID authentication for Redis (preview)',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 2: an emphasised run that is more than the label is not a complete label', () => {
  const r = classifyFlag(
    flag({
      claim: 'DSPM er i preview.',
      evidence_quote: 'The new version of Data Security Posture Management is now generally available.',
      line: 1,
    }),
    '- **DSPM (preview):** New version med enhanced AI activities tab',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 2: a label inside a JSON string value is never touched', () => {
  const r = classifyFlag(
    flag({
      claim: 'Azure Policy-definisjonen er i preview.',
      evidence_quote: 'The policy definition is generally available.',
      line: 1,
    }),
    '  "policyDefinitionName": "[Preview]: Azure Machine Learning Deployments should only use approved Registry Models",',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 2: a bare label in prose (no emphasis, no cell) is not swappable', () => {
  const r = classifyFlag(
    flag({ claim: 'Hybrid search er GA.', evidence_quote: 'Vector optimization techniques are generally available.', line: 1 }),
    '**Status:** GA (Hybrid search), Preview (Scalar quantization)',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b constraint 2: an emphasised label inside a sentence IS complete and swappable', () => {
  const r = classifyFlag(
    flag({ claim: 'Hosted agents er i Preview.', evidence_quote: 'hosted agents are generally available', line: 1 }),
    'Hosted agents er **Preview** i norsk sky.',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, 'Hosted agents er **GA** i norsk sky.');
});

test('§4b constraint 2: two labels on the same line abort — which one is a judgement', () => {
  const r = classifyFlag(
    flag({ claim: 'Verktøyet er i Preview.', evidence_quote: 'the tool is generally available', line: 1 }),
    '| Verktøyet | **Preview** | **GA** |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b the status locator is line-scoped: a label on a neighbouring row is not used', () => {
  const table = ['| Verktøy | Status |', '| SharePoint | Beskrivelse uten status |', '| Fabric | **Preview** |'].join('\n');
  const r = classifyFlag(
    flag({ claim: 'SharePoint-verktøyet er i Preview.', evidence_quote: 'SharePoint tool is generally available', line: 2 }),
    table,
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

// -- constraint 3: write the CORPUS-side equivalent, with the file's own markup

test('§4b constraint 3: the English source phrasing is never pasted into the file', () => {
  const r = classifyFlag(
    flag({
      claim: 'Vulnerability Remediation Agent er GA.',
      evidence_quote: 'This feature is in public preview. For more information, see Public preview in Microsoft Intune.',
      line: 1,
    }),
    '| **Vulnerability Remediation Agent** | Microsoft Intune | Trinnvis remediering via Intune | GA |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.replacement, 'Preview');
  assert.ok(!/public preview/i.test(r.proposal.after), 'the quote phrasing must not reach the file');
  assert.equal(r.proposal.after.endsWith('| Preview |'), true);
});

test('§4b constraint 3: backtick markup is preserved exactly as the file wrote it', () => {
  const r = classifyFlag(
    flag({ claim: 'Verktøyet er i Preview.', evidence_quote: 'the tool is now generally available', line: 1 }),
    '| Verktøyet | `Preview` |',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '| Verktøyet | `GA` |');
});

test('§4b a file already agreeing with the source is not a swap — never a no-op edit', () => {
  const r = classifyFlag(
    flag({
      claim: 'SharePoint-verktøyet er i Preview (de øvrige er GA).',
      evidence_quote: 'Microsoft Fabric (preview) ... SharePoint (preview)',
      line: 1,
    }),
    '| **SharePoint** | Knowledge | Tilgang til interne dokumenter | Preview |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b a quote carrying two different lifecycle rows aborts — the source is ambiguous', () => {
  const r = classifyFlag(
    flag({
      claim: 'DSPM er GA.',
      evidence_quote:
        'General availability (GA): The new version of Data Security Posture Management is now generally available. Partner solutions remain in preview.',
      line: 1,
    }),
    '| DSPM | **GA** |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.STATUS_SYNONYM);
});

test('§4b a proven status swap changes exactly one line and keeps the rest byte-identical', () => {
  const text = ['| Verktøy | Status |', '|---|---|', '| Hosted agents | **Preview** |', '', 'Etterord.'].join('\n');
  const r = classifyFlag(
    flag({ claim: 'Hosted agents er i Preview.', evidence_quote: 'hosted agents are generally available', line: 3 }),
    text,
  );
  assert.equal(r.op, 'O1');
  const before = text.split('\n');
  const after = before.slice();
  after[r.proposal.line - 1] = r.proposal.after;
  assert.deepEqual(
    after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null),
    [2],
  );
  assert.equal(r.proposal.before.replace('**Preview**', '§'), r.proposal.after.replace('**GA**', '§'));
});

// --------------------------------------------------------------- fail-closed

test('every outcome is either a proven O1 or an O3 carrying a known abort code', () => {
  const cases = [
    flag(),
    flag({ claim: 'Ingen tall her.' }),
    flag({ claim: 'A | B | C med 3 ting.' }),
    flag({ claim: 'Parameteren max_docs er 999.' }),
    flag({ evidence_quote: 'ingen tall' }),
    flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'the service is generally available', line: 1 }),
    flag({ claim: 'Tjenesten er i Preview.', evidence_quote: 'early access', line: 1 }),
  ];
  const known = new Set(['PROVEN', ...Object.values(ABORT_CODES)]);
  for (const c of cases) {
    const r = classifyFlag(c, FILE);
    assert.ok(r.op === 'O1' || r.op === 'O3', `op must be O1|O3, got ${r.op}`);
    assert.ok(known.has(r.code), `unknown code ${r.code}`);
    assert.equal(r.op === 'O1', r.code === 'PROVEN');
  }
});

test('an out-of-range line fails closed instead of throwing', () => {
  const r = classifyFlag(flag({ line: 9999 }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_MISS);
});