From 6af3624b1700aa8ddd00c1991abf606561c47bec Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 19 Jun 2026 22:00:36 +0200 Subject: [PATCH] feat(ms-ai-architect): Sesjon 4 - verifisering-ut (lag 5) + adversarial regresjonstest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lag 5 kjører ETTER transformasjon (lag 4) og FØR en kandidat-endring skrives til en KB-fil. Den fanger regresjons-klassen: en status-påstand «korrigert» mot en tilfeldig sitert side og stille auto-applyet (den kjente agentic-retrieval- regresjonen — hovedkontekst måtte rette manuelt). Regel (spec §21): status-påstander (GA/preview/versjon/pris) flagges ALLTID for operatør, aldri auto-applyet — uansett hvor sikker evidensen ser ut. - lib/verify-out.mjs: ren klassifiserer, null deps (speiler decisions-io). detectStatusClaim(text) → {isStatus, kinds}; classifyChange(change) → {verdict: flagged|auto-applied, status_claim, reasons}. Tre flag-regler: status-gate (§21) · adversarial refutering · autoritets-mismatch (regresjonens rotårsak). Konservativ med vilje: ved tvil flagges. SKRIVER ALDRI. - Fixtur tests/fixtures/kb-update/agentic-retrieval-regression.json: den kjente regresjonen (flat «GA» mot nyansert «delvis GA … resten preview»). - TDD: 13 tester før kode, inkl. import-invariant (ingen write-utils). - Wiret i kb-update.md §4 c2 (lag 5 mellom identifiser-endring og skriv). Kriterium møtt: fixtur → flagged:true, ikke auto-applied. Tester: validate 239 · kb-update 95 (+13) · kb-eval 13 · kb-integrity 115/115. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ --- commands/kb-update.md | 7 +- scripts/kb-update/lib/verify-out.mjs | 107 ++++++++++++ .../agentic-retrieval-regression.json | 11 ++ tests/kb-update/test-verify-out.test.mjs | 153 ++++++++++++++++++ 4 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 scripts/kb-update/lib/verify-out.mjs create mode 100644 tests/fixtures/kb-update/agentic-retrieval-regression.json create mode 100644 tests/kb-update/test-verify-out.test.mjs diff --git a/commands/kb-update.md b/commands/kb-update.md index cc57faa..163cec4 100644 --- a/commands/kb-update.md +++ b/commands/kb-update.md @@ -94,7 +94,12 @@ For hver fil i valgte prioriteter: a. **Les nåværende fil:** `Read` på filstien b. **Hent oppdaterte kilder:** for hver URL i `change-report.json[file].changed_urls`, kjør `microsoft_docs_fetch` på URLen c. **Identifiser endringer:** sammenlign hentet markdown mot eksisterende seksjoner i fila. Fokuser på faktuelle endringer (ny info, oppdaterte features, deprecation-varsler) — ikke små formuleringsendringer -d. **Oppdater fila:** `Edit` med relevante endringer. Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato +c2. **Verifisering-ut (lag 5) — FØR du skriver.** Hver kandidat-endring fra (c) går gjennom `lib/verify-out.mjs` `classifyChange({field, old_value, new_value, source_url, authority_source, refutations})`. Dette fanger regresjons-klassen (en status-påstand «korrigert» mot en tilfeldig sitert side og stille auto-applyet — agentic-retrieval-regresjonen): + - **`flagged`** → IKKE auto-skriv. Vis endringen til operatør med `reasons[]`; operatør avgjør om den tas inn. **Status-påstander (GA/preview/versjon/pris) er ALLTID `flagged`** (spec §21) — uansett hvor sikker kilden ser ut. + - **`auto-applied`** → trygt å ta inn i (d) (benign, ikke-status, ingen motbevis, autoritets-match). + - **Adversarial motbevis-panel (LLM-runtime):** for status-/load-bearing-påstander, kjør et lite panel som *prøver å motbevise* `new_value` mot den utpekte `authority_source`, og mat resultatene inn som `refutations[]` (`[{refuted, reason}]`). Multi-agent foreslås når lag 4/5 kjøres i produksjon (roadmap §71). + - **Invariant:** `verify-out.mjs` skriver aldri — den returnerer kun en verdict. Selve skrivingen skjer i (d), gated. Verifisert av `tests/kb-update/test-verify-out.test.mjs`. +d. **Oppdater fila:** `Edit` med endringer som er `auto-applied` eller eksplisitt godkjent av operatør i (c2). Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato e. **Committ:** `git add ` + `git commit -m "chore(ms-ai-architect): refresh KB $(basename ) [skip-docs]"` med mindre `--single-commit` ble gitt ### 5. Single-commit modus diff --git a/scripts/kb-update/lib/verify-out.mjs b/scripts/kb-update/lib/verify-out.mjs new file mode 100644 index 0000000..543a0c3 --- /dev/null +++ b/scripts/kb-update/lib/verify-out.mjs @@ -0,0 +1,107 @@ +// verify-out.mjs — Lag 5: verifisering-UT. Den adversarielle "proev-aa-motbevis" +// + status-gaten som kjoerer ETTER transformasjon (lag 4) og FOER en kandidat- +// endring skrives til en KB-fil. Den fanger regresjons-klassen: en status-paastand +// (GA/preview/versjon/pris) som ble "korrigert" mot en tilfeldig sitert side og +// stille auto-applyet (agentic-retrieval-regresjonen — roadmap §7, §48). +// +// Lagets regel (spec §21): status-paastander flagges ALLTID for operatoer — aldri +// auto-applyet, uansett hvor sikker evidensen ser ut. verify-out er en REN +// klassifiserer: den beregner en verdict og SKRIVER ALDRI (ingen write-util- +// import). Aa applye endringen er gatens jobb, ikke lag 5s. +// +// Konservativ med vilje: ved tvil flagges. Aa over-flagge koster operatoeren et +// blikk; aa under-flagge ga regresjonen. Null avhengigheter (speiler decisions-io). + +// --- Status-claim-detektorer ------------------------------------------------- + +// GA matches case-SENSITIVE for aa unngaa norsk "ga" (preteritum av "gi"). +const RE_GA = /\bGA\b/; +const RE_RELEASE = /general(ly)?\s+available|generelt\s+tilgjengelig|\bpublic\s+preview\b|\bprivate\s+preview\b|\bpreview\b|\bbeta\b|deprecat\w*|retir\w*|\bsunset\b|end[-\s]of[-\s]life|\bEOL\b/i; +const RE_VERSION = /\b\d{4}-\d{2}-\d{2}-preview\b|\bapi-version\b/i; +const RE_PRICE = /\$\s?\d|\bNOK\b|\bUSD\b|\bEUR\b|per\s+(million|thousand|1[MK])\s+tokens?|\bpris\w*|\bpricing\b|\bSLA\b|\d+(\.\d+)?\s?%\s*(uptime|oppetid)|\buptime\b/i; + +/** + * Is this text a status/lifecycle/version/price claim? + * Release-stage, API-version (preview-datoer), and price/SLA are the signals the + * naive auto-correct mechanism mangled. A bare calendar date is NOT a status + * claim (only -preview-suffixed versions count) — so dated prose still auto-applies. + * @param {string} text + * @returns {{isStatus: boolean, kinds: string[]}} + */ +export function detectStatusClaim(text) { + const s = String(text ?? ''); + const kinds = []; + if (RE_GA.test(s) || RE_RELEASE.test(s)) kinds.push('release-stage'); + if (RE_VERSION.test(s)) kinds.push('version'); + if (RE_PRICE.test(s)) kinds.push('price'); + return { isStatus: kinds.length > 0, kinds }; +} + +function normUrl(u) { + return String(u).trim().replace(/\/+$/, '').toLowerCase(); +} + +/** + * Classify a candidate KB change: may it be auto-applied, or must the operator + * see it first? Flagged wins — every triggering rule contributes a reason. + * + * A change object: + * field — what is changing (label) + * old_value — current KB claim + * new_value — proposed replacement + * source_url — where new_value was fetched from (optional) + * authority_source — the designated authority for this claim (optional; null + * today until lag 3 verifisering-INN backfills it) + * refutations — adversarial panel output: [{refuted: boolean, reason?}] + * (LLM runtime; empty here is fine) + * + * @param {object} change + * @returns {{verdict: 'flagged'|'auto-applied', status_claim: boolean, reasons: string[]}} + */ +export function classifyChange(change) { + const reasons = []; + const kinds = new Set(); + let statusClaim = false; + + // Rule 1 — STATUS-GATE (spec §21): any status claim in field/old/new → flag, always. + for (const t of [change.field, change.old_value, change.new_value]) { + const d = detectStatusClaim(t); + if (d.isStatus) { + statusClaim = true; + d.kinds.forEach((k) => kinds.add(k)); + } + } + if (statusClaim) { + reasons.push( + `status claim (${[...kinds].join(', ')}) — always operator-gated (spec §21), never auto-applied`, + ); + } + + // Rule 2 — ADVERSARIAL: the try-to-refute panel found counter-evidence. + const refs = Array.isArray(change.refutations) ? change.refutations : []; + const refuted = refs.filter((r) => r && r.refuted === true); + if (refuted.length > 0) { + reasons.push( + `adversarial panel refuted the claim (${refuted.length}) — e.g. ${refuted[0].reason ?? 'no reason given'}`, + ); + } + + // Rule 3 — AUTHORITY MISMATCH (regression root cause): the new value was + // anchored on something other than the designated authority. Inert while + // authority_source is null (pre-lag-3), active once it is backfilled. + if ( + change.authority_source && + change.source_url && + normUrl(change.authority_source) !== normUrl(change.source_url) + ) { + reasons.push( + `source is not the designated authority_source — anchored on ${change.source_url} instead of ${change.authority_source}`, + ); + } + + return { + verdict: reasons.length ? 'flagged' : 'auto-applied', + status_claim: statusClaim, + reasons, + }; +} diff --git a/tests/fixtures/kb-update/agentic-retrieval-regression.json b/tests/fixtures/kb-update/agentic-retrieval-regression.json new file mode 100644 index 0000000..fe45a23 --- /dev/null +++ b/tests/fixtures/kb-update/agentic-retrieval-regression.json @@ -0,0 +1,11 @@ +{ + "_fixture": "Den KJENTE regresjonen (roadmap §7, §48). kb-update flatet agentic-retrievals nyanserte status ('delvis GA via REST 2026-04-01; answer synthesis + multi-turn forblir preview') til en flat 'generally available (GA)', forankret mot en tilfeldig sitert 'what's new'-side i stedet for den utpekte autoritetskilden. Hovedkonteksten måtte rette manuelt. Verdiene er ILLUSTRATIVE for regresjonens FORM — dette er en test-input, ikke en currency-paastand (verifiseringsplikt). Lag 5 skal FLAGGE denne (status-paastand), aldri auto-applye.", + "change": { + "field": "Azure AI Search agentic retrieval — release status", + "old_value": "Azure AI Search agentic retrieval er delvis GA via REST 2026-04-01 (ekstraktiv grounding). Answer synthesis, ikke-minimal reasoning effort og multi-turn messages forblir preview (2026-05-01-preview). Azure- og Foundry-portalen gir kun preview-tilgang.", + "new_value": "Azure AI Search agentic retrieval er nå generally available (GA).", + "source_url": "https://learn.microsoft.com/azure/search/whats-new", + "authority_source": "https://learn.microsoft.com/azure/search/search-agentic-retrieval-how-to-retrieve", + "refutations": [] + } +} diff --git a/tests/kb-update/test-verify-out.test.mjs b/tests/kb-update/test-verify-out.test.mjs new file mode 100644 index 0000000..3ffa411 --- /dev/null +++ b/tests/kb-update/test-verify-out.test.mjs @@ -0,0 +1,153 @@ +// tests/kb-update/test-verify-out.test.mjs +// Unit tests for scripts/kb-update/lib/verify-out.mjs — lag 5 (verifisering-UT): +// the adversarial "try-to-refute" + status-gate that runs AFTER transformation +// (lag 4) and BEFORE a candidate change is written to a KB file. +// +// Kjernekriterium (roadmap §48): den KJENTE agentic-retrieval-regresjonen som +// fixtur → flagged:true, IKKE auto-applied. Lag 5 skal aldri auto-applye en +// status-paastand (GA/preview/versjon/pris) — den flagges ALLTID for operatoer +// (spec §21). verify-out er en REN klassifiserer: den beregner en verdict og +// skriver aldri (invariant nederst). + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { detectStatusClaim, classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = JSON.parse( + readFileSync(join(__dirname, '..', 'fixtures', 'kb-update', 'agentic-retrieval-regression.json'), 'utf8'), +); + +// --- detectStatusClaim — release stage / version / price tokens --- + +test('detectStatusClaim — recognizes GA', () => { + const r = detectStatusClaim('Tjenesten er nå generally available (GA).'); + assert.equal(r.isStatus, true); + assert.ok(r.kinds.includes('release-stage')); +}); + +test('detectStatusClaim — recognizes public preview', () => { + assert.equal(detectStatusClaim('Available in public preview').isStatus, true); +}); + +test('detectStatusClaim — recognizes a preview API-version date (2026-05-01-preview)', () => { + const r = detectStatusClaim('Krever REST 2026-05-01-preview for multi-turn.'); + assert.equal(r.isStatus, true); + assert.ok(r.kinds.includes('version')); +}); + +test('detectStatusClaim — recognizes deprecation / retirement', () => { + assert.equal(detectStatusClaim('This API is deprecated and will be retired.').isStatus, true); +}); + +test('detectStatusClaim — recognizes price / SLA', () => { + assert.equal(detectStatusClaim('Priset til $30 per million tokens.').isStatus, true); + assert.equal(detectStatusClaim('99.9% uptime SLA').isStatus, true); +}); + +test('detectStatusClaim — benign prose is NOT a status claim', () => { + const r = detectStatusClaim( + 'RAG kombinerer henting med generering for å gi kontekst-forankrede svar.', + ); + assert.equal(r.isStatus, false); + assert.deepEqual(r.kinds, []); +}); + +test('detectStatusClaim — a bare calendar date alone is NOT a status claim', () => { + // Unngaa aa flagge enhver datert setning; bare -preview-versjoner / release-ord teller. + assert.equal(detectStatusClaim('Sist verifisert 2026-06-19 av teamet.').isStatus, false); +}); + +// --- classifyChange — ACCEPTANCE CRITERION (roadmap §48) --- + +test('agentic-retrieval regression fixture → flagged, NOT auto-applied', () => { + const v = classifyChange(FIXTURE.change); + assert.equal(v.verdict, 'flagged', 'the known regression must be flagged for the operator'); + assert.notEqual(v.verdict, 'auto-applied'); + assert.equal(v.status_claim, true, 'GA/preview transition is a status claim'); + assert.ok(v.reasons.length >= 1, 'a flag must carry at least one human-readable reason'); + assert.ok( + v.reasons.some((r) => /status/i.test(r)), + 'the status-gate reason must be present', + ); +}); + +// --- classifyChange — benign change auto-applies --- + +test('benign non-status change with matching authority → auto-applied', () => { + const v = classifyChange({ + field: 'RAG-introduksjon — formulering', + old_value: 'RAG henter dokumenter og genererer svar.', + new_value: 'RAG henter relevante dokumenter og genererer kontekst-forankrede svar.', + source_url: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview', + authority_source: 'https://learn.microsoft.com/azure/search/retrieval-augmented-generation-overview', + refutations: [], + }); + assert.equal(v.verdict, 'auto-applied'); + assert.equal(v.status_claim, false); + assert.deepEqual(v.reasons, []); +}); + +test('benign change with no authority info (null) still auto-applies', () => { + const v = classifyChange({ + field: 'beskrivelse', + old_value: 'Et verktoey for henting.', + new_value: 'Et verktoey for henting og rangering.', + source_url: 'https://learn.microsoft.com/azure/search/x', + authority_source: null, + refutations: [], + }); + assert.equal(v.verdict, 'auto-applied'); +}); + +// --- classifyChange — adversarial refutation flags --- + +test('any refutation with refuted:true → flagged (adversarial panel found counter-evidence)', () => { + const v = classifyChange({ + field: 'beskrivelse', + old_value: 'Stoetter X.', + new_value: 'Stoetter X og Y.', + source_url: 'https://learn.microsoft.com/azure/search/x', + authority_source: 'https://learn.microsoft.com/azure/search/x', + refutations: [ + { refuted: false, reason: 'kunne ikke motbevise X' }, + { refuted: true, reason: 'autoritetskilden nevner ikke Y' }, + ], + }); + assert.equal(v.verdict, 'flagged'); + assert.ok(v.reasons.some((r) => /refut|motbevis/i.test(r))); +}); + +// --- classifyChange — authority mismatch flags (regression root cause) --- + +test('source_url differs from designated authority_source → flagged', () => { + const v = classifyChange({ + field: 'beskrivelse uten status-ord', + old_value: 'Stoetter henting.', + new_value: 'Stoetter henting og caching.', + source_url: 'https://learn.microsoft.com/azure/search/whats-new', + authority_source: 'https://learn.microsoft.com/azure/search/search-agentic-retrieval-how-to-retrieve', + refutations: [], + }); + assert.equal(v.verdict, 'flagged'); + assert.ok(v.reasons.some((r) => /authority|autoritet/i.test(r))); +}); + +// --- INVARIANT (spec §83 / STATE.md): verify-out is a pure classifier; it +// NEVER imports a write path. It computes a verdict — applying the change is the +// gate's job, not lag 5's. Import-specific (a comment can't create a false hit). + +test('verify-out imports NO write-utils (pure classifier)', () => { + const src = readFileSync( + join(__dirname, '..', '..', 'scripts', 'kb-update', 'lib', 'verify-out.mjs'), + 'utf8', + ); + const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n'); + assert.doesNotMatch(importLines, /\bsaveDecisions\b/); + assert.doesNotMatch(importLines, /\bsaveRegistry\b/); + assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/); + assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/); +});