ms-ai-architect/tests/kb-eval/test-fix-op-classify.test.mjs

// test-fix-op-classify.test.mjs — R11 fix-operation classifier (lib/fix-op.mjs).
//
// The classifier IS the O1 driver with writes disabled: it attempts the value
// swap and checks the §4 invariant (docs/r11-tiered-fix-design.md). Anything it
// cannot prove is O3, with a TYPED abort code — the abort taxonomy is the
// pilot's measurement #3, so the codes are part of the contract, not diagnostics.

import { test } from 'node:test';
import assert from 'node:assert/strict';

import { ABORT_CODES, blockWindow, classifyFlag, extractValueTokens } from '../../scripts/kb-eval/lib/fix-op.mjs';

/** Minimal flag record per docs/r11-flag-format-2026-07.md. */
function flag(over = {}) {
  return {
    id: 'x/y.md#1',
    judge_verdict: 'not_grounded',
    rule: '',
    evidence_url: 'https://learn.microsoft.com/x',
    evidence_quote: 'The service supports max_docs=250 per batch.',
    reason: 'Source states 250, claim states 100.',
    file: 'skills/x/references/y.md',
    line: 3,
    claim: 'Parameteren max_docs er 100.',
    disposition: 'outdated',
    ...over,
  };
}

// The happy-path fixture is deliberately LANGUAGE-NEUTRAL (a key=value pair that
// reads the same in the file and in the English quote). That is not fixture
// convenience — it is the pilot's measured result: the corpus is Norwegian and
// the quotes are English, so a swap is only provable where the surrounding
// context is code, a URL, or a parameter key. See the CONTEXT_MISMATCH tests.
const FILE = ['# Tittel', '', '| konfig | max_docs=100 |', '| Annet | tekst |', '', 'Etterord.'].join('\n');

// ---------------------------------------------------------------- token types

test('extractValueTokens types a plain integer as number', () => {
  const t = extractValueTokens('Grensen er 100 dokumenter.');
  assert.deepEqual(
    t.map((x) => [x.type, x.value]),
    [['number', '100']],
  );
});

test('extractValueTokens types percent, ISO date and version distinctly', () => {
  const types = (s) => extractValueTokens(s).map((x) => x.type);
  assert.deepEqual(types('Treffraten er 20 %.'), ['percent']);
  assert.deepEqual(types('Gjelder fra 2026-08-02.'), ['iso_date']);
  assert.deepEqual(types('Krever versjon 2.3.0 av pakken.'), ['version']);
});

test('extractValueTokens does not emit a status word as a swappable value', () => {
  assert.deepEqual(extractValueTokens('Hosted agents er i Public Preview.'), []);
});

// --------------------------------------------------------------- block window

test('blockWindow spans the contiguous non-blank block containing the line', () => {
  const lines = FILE.split('\n');
  assert.deepEqual(blockWindow(lines, 3), { start: 3, end: 4 });
});

test('blockWindow anchored on a table header reaches the rows below it', () => {
  const lines = ['| A | B |', '| --- | --- |', '| rad | 7 |', '', 'annet'];
  assert.deepEqual(blockWindow(lines, 1), { start: 1, end: 3 });
});

test('blockWindow on a blank line degenerates to that line only', () => {
  const lines = FILE.split('\n');
  assert.deepEqual(blockWindow(lines, 2), { start: 2, end: 2 });
});

// ------------------------------------------------------------- the O1 happy path

test('classifyFlag proves a clean value swap as O1', () => {
  const r = classifyFlag(flag(), FILE);
  assert.equal(r.op, 'O1');
  assert.equal(r.code, 'PROVEN');
  assert.equal(r.proposal.line, 3);
  assert.equal(r.proposal.token, '100');
  assert.equal(r.proposal.replacement, '250');
  assert.equal(r.proposal.before, '| konfig | max_docs=100 |');
  assert.equal(r.proposal.after, '| konfig | max_docs=250 |');
});

test('the O1 proposal changes exactly one line and leaves the rest byte-identical', () => {
  const r = classifyFlag(flag(), FILE);
  const before = FILE.split('\n');
  const after = before.slice();
  after[r.proposal.line - 1] = r.proposal.after;
  const changed = after.map((l, i) => (l === before[i] ? null : i)).filter((i) => i !== null);
  assert.deepEqual(changed, [2]);
  // §4(2): the rest of the line survives the swap byte-for-byte.
  assert.equal(r.proposal.before.replace('100', ''), r.proposal.after.replace('250', ''));
});

test('the replacement value occurs verbatim in the cited evidence_quote (§4.1)', () => {
  const f = flag();
  const r = classifyFlag(f, FILE);
  assert.ok(f.evidence_quote.includes(r.proposal.replacement));
});

// ------------------------------------------------------------- abort taxonomy

test('an enumerated multi-part claim aborts as MULTI_PART_CLAIM', () => {
  const r = classifyFlag(flag({ claim: 'Verktøy: Code Interpreter | File Search | 3 andre.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_PART_CLAIM');
});

test('a status-only claim aborts as STATUS_SYNONYM rather than swapping prose', () => {
  const r = classifyFlag(
    flag({
      claim: 'Hosted agents er i Public Preview.',
      evidence_quote: 'hosted agents are generally available',
    }),
    ['| Hosted agents | **Preview** |'].join('\n'),
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'STATUS_SYNONYM');
});

test('a claim with no swappable value aborts as NO_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Tjenesten anbefales for produksjon.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NO_VALUE_TOKEN');
});

test('two distinct values in one claim abort as MULTI_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Grensen er 100 dokumenter og 50 sider.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_VALUE_TOKEN');
});

test('the same value repeated in a claim is one token, not MULTI_VALUE_TOKEN', () => {
  const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 100, altså 100 per batch.' }), FILE);
  assert.equal(r.op, 'O1');
});

test('a value absent from the file block aborts as LOCATOR_MISS', () => {
  const r = classifyFlag(flag({ claim: 'Parameteren max_docs er 999.', evidence_quote: 'max_docs=250' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_MISS);
});

test('a value occurring twice in the block aborts as LOCATOR_AMBIGUOUS', () => {
  const twice = ['| konfig | max_docs=100 |', '| Tak | 100 kall |'].join('\n');
  const r = classifyFlag(flag({ line: 1 }), twice);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_AMBIGUOUS);
});

test('a quote with no same-type replacement aborts as NOT_VERBATIM', () => {
  const r = classifyFlag(flag({ evidence_quote: 'The service supports batching of docs.' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

test('type is not crossed: a percent claim cannot be swapped from a bare integer quote', () => {
  const pct = '| Treffrate | 20 % |';
  const r = classifyFlag(flag({ claim: 'Treffraten er 20 %.', evidence_quote: 'a cache holding 35 entries', line: 1 }), pct);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

test('two candidate replacements in the quote abort as MULTI_REPLACEMENT', () => {
  const r = classifyFlag(flag({ evidence_quote: 'max_docs=250, or 500 in premium' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'MULTI_REPLACEMENT');
});

test('a quote restating the claim value is not a swap — NOT_VERBATIM, never a no-op edit', () => {
  const r = classifyFlag(flag({ evidence_quote: 'the limit is max_docs=100' }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, 'NOT_VERBATIM');
});

// ------------------------------------------------- §4 is not sufficient alone
//
// Measured on the pilot: §4 as written (new value verbatim in the quote, rest of
// the line byte-identical, one line changed) admitted 6 swaps of which 4 were
// wrong — it constrains the PROVENANCE of the value and the SHAPE of the edit,
// and nothing about whether the two tokens denote the same quantity. The context
// condition adds that: the token must sit under a matching label or unit on both
// sides. These four tests are the four real failures, reduced to fixtures.

test('a unit-crossing swap is refused: 30 days in the file, 24 hours in the quote', () => {
  const r = classifyFlag(
    flag({
      claim: 'Standard 30 dagers oppbevaring av dokumenter.',
      evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.',
      line: 1,
    }),
    '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('a metric-crossing swap is refused: indexing rate in the file, query throttle in the quote', () => {
  const r = classifyFlag(
    flag({
      claim: 'Azure AI Search har rate limits på 3000 requests/sekund per replika.',
      evidence_quote: '| Search queries | Varies by SU count | 50 queries/sec (aggregate read throttle per index) |',
      line: 1,
    }),
    '**Viktig:** Azure AI Search har rate limits (3000 requests/sekund per replika).',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('a digit inside an identifier is never harvested as a replacement (Agent 365 -> E7)', () => {
  const r = classifyFlag(
    flag({
      claim: 'Microsoft Agent 365 er inkludert i Copilot Studio-lisens.',
      evidence_quote: 'Microsoft Agent 365 is available as a stand-alone subscription and is also included with Microsoft 365 E7.',
      line: 1,
    }),
    '- Agent observability (Microsoft Agent 365): Inkludert i Copilot Studio-lisens',
  );
  assert.equal(r.op, 'O3');
  assert.notEqual(r.op, 'O1');
});

test('a model identifier is not mutilated by a bare digit from a table cell', () => {
  const r = classifyFlag(
    flag({
      claim: 'Modellen text-embedding-ada-002 har 1536 dimensjoner.',
      evidence_quote: '| text-embedding-ada-002 | 2 | GA | 2028-02-09 |',
      line: 1,
    }),
    '| `text-embedding-ada-002` | 1536 | ~80 NOK | Legacy |',
  );
  assert.notEqual(r.op, 'O1');
});

test('a matching key on both sides admits the swap: api-version', () => {
  const r = classifyFlag(
    flag({
      claim: 'API-versjonen er 2025-09-01.',
      evidence_quote: 'POST https://[servicename].search.windows.net/indexes?api-version=2026-04-01',
      line: 1,
    }),
    'POST https://[service].search.windows.net/indexes?api-version=2025-09-01',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.replacement, '2026-04-01');
});

test('key matching normalises punctuation: api_version="X" matches api-version=Y', () => {
  const r = classifyFlag(
    flag({
      claim: 'Bruker api_version 2024-02-01.',
      evidence_quote: 'POST https://{endpoint}/openai/deployments/{id}/embeddings?api-version=2024-10-21',
      line: 1,
    }),
    '    api_version="2024-02-01"',
  );
  assert.equal(r.op, 'O1');
  assert.equal(r.proposal.after, '    api_version="2024-10-21"');
});

test('the corpus-dominant shape — Norwegian file, English quote, no shared context — is refused', () => {
  const r = classifyFlag(
    flag({
      claim: 'Grensen er 100 dokumenter.',
      evidence_quote: 'The service supports up to 250 documents per batch.',
      line: 1,
    }),
    '| Grense | 100 dokumenter |',
  );
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.CONTEXT_MISMATCH);
});

test('the context condition is what refuses the unit crossing — disabling it restores the §4-only verdict', () => {
  const f = flag({
    claim: 'Standard 30 dagers oppbevaring av dokumenter.',
    evidence_quote: 'Your data is then deleted 24 hours from the time that you submit an analyze request.',
    line: 1,
  });
  const text = '4. **Data retention:** Standard 30-dagers oppbevaring av dokumenter';
  assert.equal(classifyFlag(f, text, { contextCheck: false }).op, 'O1');
  assert.equal(classifyFlag(f, text).op, 'O3');
});

// --------------------------------------------------------------- fail-closed

test('every outcome is either a proven O1 or an O3 carrying a known abort code', () => {
  const cases = [
    flag(),
    flag({ claim: 'Ingen tall her.' }),
    flag({ claim: 'A | B | C med 3 ting.' }),
    flag({ claim: 'Parameteren max_docs er 999.' }),
    flag({ evidence_quote: 'ingen tall' }),
  ];
  const known = new Set(['PROVEN', ...Object.values(ABORT_CODES)]);
  for (const c of cases) {
    const r = classifyFlag(c, FILE);
    assert.ok(r.op === 'O1' || r.op === 'O3', `op must be O1|O3, got ${r.op}`);
    assert.ok(known.has(r.code), `unknown code ${r.code}`);
    assert.equal(r.op === 'O1', r.code === 'PROVEN');
  }
});

test('an out-of-range line fails closed instead of throwing', () => {
  const r = classifyFlag(flag({ line: 9999 }), FILE);
  assert.equal(r.op, 'O3');
  assert.equal(r.code, ABORT_CODES.LOCATOR_MISS);
});