ms-ai-architect/tests/kb-update/test-verify-out.test.mjs

// tests/kb-update/test-verify-out.test.mjs
// Unit tests for scripts/kb-update/lib/verify-out.mjs — lag 5 (verifisering-UT):
// the adversarial "try-to-refute" + status-gate that runs AFTER transformation
// (lag 4) and BEFORE a candidate change is written to a KB file.
//
// Kjernekriterium (roadmap §48): den KJENTE agentic-retrieval-regresjonen som
// fixtur → flagged:true, IKKE auto-applied. Lag 5 skal aldri auto-applye en
// status-paastand (GA/preview/versjon/pris) — den flagges ALLTID for operatoer
// (spec §21). verify-out er en REN klassifiserer: den beregner en verdict og
// skriver aldri (invariant nederst).

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { detectStatusClaim, classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const FIXTURE = JSON.parse(
  readFileSync(join(__dirname, '..', 'fixtures', 'kb-update', 'agentic-retrieval-regression.json'), 'utf8'),
);

// --- detectStatusClaim — release stage / version / price tokens ---

test('detectStatusClaim — recognizes GA', () => {
  const r = detectStatusClaim('Tjenesten er nå generally available (GA).');
  assert.equal(r.isStatus, true);
  assert.ok(r.kinds.includes('release-stage'));
});

test('detectStatusClaim — recognizes public preview', () => {
  assert.equal(detectStatusClaim('Available in public preview').isStatus, true);
});

test('detectStatusClaim — recognizes a preview API-version date (2026-05-01-preview)', () => {
  const r = detectStatusClaim('Krever REST 2026-05-01-preview for multi-turn.');
  assert.equal(r.isStatus, true);
  assert.ok(r.kinds.includes('version'));
});

test('detectStatusClaim — recognizes deprecation / retirement', () => {
  assert.equal(detectStatusClaim('This API is deprecated and will be retired.').isStatus, true);
});

test('detectStatusClaim — recognizes price / SLA', () => {
  assert.equal(detectStatusClaim('Priset til $30 per million tokens.').isStatus, true);
  assert.equal(detectStatusClaim('99.9% uptime SLA').isStatus, true);
});

test('detectStatusClaim — benign prose is NOT a status claim', () => {
  const r = detectStatusClaim(
    'RAG kombinerer henting med generering for å gi kontekst-forankrede svar.',
  );
  assert.equal(r.isStatus, false);
  assert.deepEqual(r.kinds, []);
});

test('detectStatusClaim — a bare calendar date alone is NOT a status claim', () => {
  // Unngaa aa flagge enhver datert setning; bare -preview-versjoner / release-ord teller.
  assert.equal(detectStatusClaim('Sist verifisert 2026-06-19 av teamet.').isStatus, false);
});

// --- classifyChange — ACCEPTANCE CRITERION (roadmap §48) ---

test('agentic-retrieval regression fixture → flagged, NOT auto-applied', () => {
  const v = classifyChange(FIXTURE.change);
  assert.equal(v.verdict, 'flagged', 'the known regression must be flagged for the operator');
  assert.notEqual(v.verdict, 'auto-applied');
  assert.equal(v.status_claim, true, 'GA/preview transition is a status claim');
  assert.ok(v.reasons.length >= 1, 'a flag must carry at least one human-readable reason');
  assert.ok(
    v.reasons.some((r) => /status/i.test(r)),
    'the status-gate reason must be present',
  );
});

// --- classifyChange — benign change auto-applies ---

test('benign non-status change with matching authority → auto-applied', () => {
  const v = classifyChange({
    field: 'RAG-introduksjon — formulering',
    old_value: 'RAG henter dokumenter og genererer svar.',
    new_value: 'RAG henter relevante dokumenter og genererer kontekst-forankrede svar.',
    source_url: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview',
    authority_source: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview',
    refutations: [],
  });
  assert.equal(v.verdict, 'auto-applied');
  assert.equal(v.status_claim, false);
  assert.deepEqual(v.reasons, []);
});

test('benign change with no authority info (null) still auto-applies', () => {
  const v = classifyChange({
    field: 'beskrivelse',
    old_value: 'Et verktoey for henting.',
    new_value: 'Et verktoey for henting og rangering.',
    source_url: 'https://learn.microsoft.com/azure/search/x',
    authority_source: null,
    refutations: [],
  });
  assert.equal(v.verdict, 'auto-applied');
});

// --- classifyChange — adversarial refutation flags ---

test('any refutation with refuted:true → flagged (adversarial panel found counter-evidence)', () => {
  const v = classifyChange({
    field: 'beskrivelse',
    old_value: 'Stoetter X.',
    new_value: 'Stoetter X og Y.',
    source_url: 'https://learn.microsoft.com/azure/search/x',
    authority_source: 'https://learn.microsoft.com/azure/search/x',
    refutations: [
      { refuted: false, reason: 'kunne ikke motbevise X' },
      { refuted: true, reason: 'autoritetskilden nevner ikke Y' },
    ],
  });
  assert.equal(v.verdict, 'flagged');
  assert.ok(v.reasons.some((r) => /refut|motbevis/i.test(r)));
});

// --- classifyChange — authority mismatch flags (regression root cause) ---

test('source_url differs from designated authority_source → flagged', () => {
  const v = classifyChange({
    field: 'beskrivelse uten status-ord',
    old_value: 'Stoetter henting.',
    new_value: 'Stoetter henting og caching.',
    source_url: 'https://learn.microsoft.com/azure/search/whats-new',
    authority_source: 'https://learn.microsoft.com/azure/search/search-agentic-retrieval-how-to-retrieve',
    refutations: [],
  });
  assert.equal(v.verdict, 'flagged');
  assert.ok(v.reasons.some((r) => /authority|autoritet/i.test(r)));
});

// --- INVARIANT (spec §83 / STATE.md): verify-out is a pure classifier; it
// NEVER imports a write path. It computes a verdict — applying the change is the
// gate's job, not lag 5's. Import-specific (a comment can't create a false hit).

test('verify-out imports NO write-utils (pure classifier)', () => {
  const src = readFileSync(
    join(__dirname, '..', '..', 'scripts', 'kb-update', 'lib', 'verify-out.mjs'),
    'utf8',
  );
  const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n');
  assert.doesNotMatch(importLines, /\bsaveDecisions\b/);
  assert.doesNotMatch(importLines, /\bsaveRegistry\b/);
  assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/);
  assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/);
});