// tests/kb-update/test-verify-out.test.mjs // Unit tests for scripts/kb-update/lib/verify-out.mjs — lag 5 (verifisering-UT): // the adversarial "try-to-refute" + status-gate that runs AFTER transformation // (lag 4) and BEFORE a candidate change is written to a KB file. // // Kjernekriterium (roadmap §48): den KJENTE agentic-retrieval-regresjonen som // fixtur → flagged:true, IKKE auto-applied. Lag 5 skal aldri auto-applye en // status-paastand (GA/preview/versjon/pris) — den flagges ALLTID for operatoer // (spec §21). verify-out er en REN klassifiserer: den beregner en verdict og // skriver aldri (invariant nederst). import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { detectStatusClaim, classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const FIXTURE = JSON.parse( readFileSync(join(__dirname, '..', 'fixtures', 'kb-update', 'agentic-retrieval-regression.json'), 'utf8'), ); // --- detectStatusClaim — release stage / version / price tokens --- test('detectStatusClaim — recognizes GA', () => { const r = detectStatusClaim('Tjenesten er nå generally available (GA).'); assert.equal(r.isStatus, true); assert.ok(r.kinds.includes('release-stage')); }); test('detectStatusClaim — recognizes public preview', () => { assert.equal(detectStatusClaim('Available in public preview').isStatus, true); }); test('detectStatusClaim — recognizes a preview API-version date (2026-05-01-preview)', () => { const r = detectStatusClaim('Krever REST 2026-05-01-preview for multi-turn.'); assert.equal(r.isStatus, true); assert.ok(r.kinds.includes('version')); }); test('detectStatusClaim — recognizes deprecation / retirement', () => { assert.equal(detectStatusClaim('This API is deprecated and will be retired.').isStatus, true); }); test('detectStatusClaim — recognizes price / SLA', () => { assert.equal(detectStatusClaim('Priset til $30 per million tokens.').isStatus, true); assert.equal(detectStatusClaim('99.9% uptime SLA').isStatus, true); }); test('detectStatusClaim — benign prose is NOT a status claim', () => { const r = detectStatusClaim( 'RAG kombinerer henting med generering for å gi kontekst-forankrede svar.', ); assert.equal(r.isStatus, false); assert.deepEqual(r.kinds, []); }); test('detectStatusClaim — a bare calendar date alone is NOT a status claim', () => { // Unngaa aa flagge enhver datert setning; bare -preview-versjoner / release-ord teller. assert.equal(detectStatusClaim('Sist verifisert 2026-06-19 av teamet.').isStatus, false); }); // --- classifyChange — ACCEPTANCE CRITERION (roadmap §48) --- test('agentic-retrieval regression fixture → flagged, NOT auto-applied', () => { const v = classifyChange(FIXTURE.change); assert.equal(v.verdict, 'flagged', 'the known regression must be flagged for the operator'); assert.notEqual(v.verdict, 'auto-applied'); assert.equal(v.status_claim, true, 'GA/preview transition is a status claim'); assert.ok(v.reasons.length >= 1, 'a flag must carry at least one human-readable reason'); assert.ok( v.reasons.some((r) => /status/i.test(r)), 'the status-gate reason must be present', ); }); // --- classifyChange — benign change auto-applies --- test('benign non-status change with matching authority → auto-applied', () => { const v = classifyChange({ field: 'RAG-introduksjon — formulering', old_value: 'RAG henter dokumenter og genererer svar.', new_value: 'RAG henter relevante dokumenter og genererer kontekst-forankrede svar.', source_url: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview', authority_source: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview', refutations: [], }); assert.equal(v.verdict, 'auto-applied'); assert.equal(v.status_claim, false); assert.deepEqual(v.reasons, []); }); test('benign change with no authority info (null) still auto-applies', () => { const v = classifyChange({ field: 'beskrivelse', old_value: 'Et verktoey for henting.', new_value: 'Et verktoey for henting og rangering.', source_url: 'https://learn.microsoft.com/azure/search/x', authority_source: null, refutations: [], }); assert.equal(v.verdict, 'auto-applied'); }); // --- classifyChange — adversarial refutation flags --- test('any refutation with refuted:true → flagged (adversarial panel found counter-evidence)', () => { const v = classifyChange({ field: 'beskrivelse', old_value: 'Stoetter X.', new_value: 'Stoetter X og Y.', source_url: 'https://learn.microsoft.com/azure/search/x', authority_source: 'https://learn.microsoft.com/azure/search/x', refutations: [ { refuted: false, reason: 'kunne ikke motbevise X' }, { refuted: true, reason: 'autoritetskilden nevner ikke Y' }, ], }); assert.equal(v.verdict, 'flagged'); assert.ok(v.reasons.some((r) => /refut|motbevis/i.test(r))); }); // --- classifyChange — authority mismatch flags (regression root cause) --- test('source_url differs from designated authority_source → flagged', () => { const v = classifyChange({ field: 'beskrivelse uten status-ord', old_value: 'Stoetter henting.', new_value: 'Stoetter henting og caching.', source_url: 'https://learn.microsoft.com/azure/search/whats-new', authority_source: 'https://learn.microsoft.com/azure/search/search-agentic-retrieval-how-to-retrieve', refutations: [], }); assert.equal(v.verdict, 'flagged'); assert.ok(v.reasons.some((r) => /authority|autoritet/i.test(r))); }); // --- INVARIANT (spec §83 / STATE.md): verify-out is a pure classifier; it // NEVER imports a write path. It computes a verdict — applying the change is the // gate's job, not lag 5's. Import-specific (a comment can't create a false hit). test('verify-out imports NO write-utils (pure classifier)', () => { const src = readFileSync( join(__dirname, '..', '..', 'scripts', 'kb-update', 'lib', 'verify-out.mjs'), 'utf8', ); const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n'); assert.doesNotMatch(importLines, /\bsaveDecisions\b/); assert.doesNotMatch(importLines, /\bsaveRegistry\b/); assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/); assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/); });