ms-ai-architect/scripts/kb-update/lib/verify-out.mjs
Kjell Tore Guttormsen 6af3624b17 feat(ms-ai-architect): Sesjon 4 - verifisering-ut (lag 5) + adversarial regresjonstest
Lag 5 kjører ETTER transformasjon (lag 4) og FØR en kandidat-endring skrives til
en KB-fil. Den fanger regresjons-klassen: en status-påstand «korrigert» mot en
tilfeldig sitert side og stille auto-applyet (den kjente agentic-retrieval-
regresjonen — hovedkontekst måtte rette manuelt).

Regel (spec §21): status-påstander (GA/preview/versjon/pris) flagges ALLTID for
operatør, aldri auto-applyet — uansett hvor sikker evidensen ser ut.

- lib/verify-out.mjs: ren klassifiserer, null deps (speiler decisions-io).
  detectStatusClaim(text) → {isStatus, kinds}; classifyChange(change) →
  {verdict: flagged|auto-applied, status_claim, reasons}. Tre flag-regler:
  status-gate (§21) · adversarial refutering · autoritets-mismatch (regresjonens
  rotårsak). Konservativ med vilje: ved tvil flagges. SKRIVER ALDRI.
- Fixtur tests/fixtures/kb-update/agentic-retrieval-regression.json: den kjente
  regresjonen (flat «GA» mot nyansert «delvis GA … resten preview»).
- TDD: 13 tester før kode, inkl. import-invariant (ingen write-utils).
- Wiret i kb-update.md §4 c2 (lag 5 mellom identifiser-endring og skriv).

Kriterium møtt: fixtur → flagged:true, ikke auto-applied.
Tester: validate 239 · kb-update 95 (+13) · kb-eval 13 · kb-integrity 115/115.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
2026-06-19 22:00:36 +02:00

107 lines
4.6 KiB
JavaScript

// verify-out.mjs — Lag 5: verifisering-UT. Den adversarielle "proev-aa-motbevis"
// + status-gaten som kjoerer ETTER transformasjon (lag 4) og FOER en kandidat-
// endring skrives til en KB-fil. Den fanger regresjons-klassen: en status-paastand
// (GA/preview/versjon/pris) som ble "korrigert" mot en tilfeldig sitert side og
// stille auto-applyet (agentic-retrieval-regresjonen — roadmap §7, §48).
//
// Lagets regel (spec §21): status-paastander flagges ALLTID for operatoer — aldri
// auto-applyet, uansett hvor sikker evidensen ser ut. verify-out er en REN
// klassifiserer: den beregner en verdict og SKRIVER ALDRI (ingen write-util-
// import). Aa applye endringen er gatens jobb, ikke lag 5s.
//
// Konservativ med vilje: ved tvil flagges. Aa over-flagge koster operatoeren et
// blikk; aa under-flagge ga regresjonen. Null avhengigheter (speiler decisions-io).
// --- Status-claim-detektorer -------------------------------------------------
// GA matches case-SENSITIVE for aa unngaa norsk "ga" (preteritum av "gi").
const RE_GA = /\bGA\b/;
const RE_RELEASE = /general(ly)?\s+available|generelt\s+tilgjengelig|\bpublic\s+preview\b|\bprivate\s+preview\b|\bpreview\b|\bbeta\b|deprecat\w*|retir\w*|\bsunset\b|end[-\s]of[-\s]life|\bEOL\b/i;
const RE_VERSION = /\b\d{4}-\d{2}-\d{2}-preview\b|\bapi-version\b/i;
const RE_PRICE = /\$\s?\d|\bNOK\b|\bUSD\b|\bEUR\b|per\s+(million|thousand|1[MK])\s+tokens?|\bpris\w*|\bpricing\b|\bSLA\b|\d+(\.\d+)?\s?%\s*(uptime|oppetid)|\buptime\b/i;
/**
* Is this text a status/lifecycle/version/price claim?
* Release-stage, API-version (preview-datoer), and price/SLA are the signals the
* naive auto-correct mechanism mangled. A bare calendar date is NOT a status
* claim (only -preview-suffixed versions count) — so dated prose still auto-applies.
* @param {string} text
* @returns {{isStatus: boolean, kinds: string[]}}
*/
export function detectStatusClaim(text) {
const s = String(text ?? '');
const kinds = [];
if (RE_GA.test(s) || RE_RELEASE.test(s)) kinds.push('release-stage');
if (RE_VERSION.test(s)) kinds.push('version');
if (RE_PRICE.test(s)) kinds.push('price');
return { isStatus: kinds.length > 0, kinds };
}
function normUrl(u) {
return String(u).trim().replace(/\/+$/, '').toLowerCase();
}
/**
* Classify a candidate KB change: may it be auto-applied, or must the operator
* see it first? Flagged wins — every triggering rule contributes a reason.
*
* A change object:
* field — what is changing (label)
* old_value — current KB claim
* new_value — proposed replacement
* source_url — where new_value was fetched from (optional)
* authority_source — the designated authority for this claim (optional; null
* today until lag 3 verifisering-INN backfills it)
* refutations — adversarial panel output: [{refuted: boolean, reason?}]
* (LLM runtime; empty here is fine)
*
* @param {object} change
* @returns {{verdict: 'flagged'|'auto-applied', status_claim: boolean, reasons: string[]}}
*/
export function classifyChange(change) {
const reasons = [];
const kinds = new Set();
let statusClaim = false;
// Rule 1 — STATUS-GATE (spec §21): any status claim in field/old/new → flag, always.
for (const t of [change.field, change.old_value, change.new_value]) {
const d = detectStatusClaim(t);
if (d.isStatus) {
statusClaim = true;
d.kinds.forEach((k) => kinds.add(k));
}
}
if (statusClaim) {
reasons.push(
`status claim (${[...kinds].join(', ')}) — always operator-gated (spec §21), never auto-applied`,
);
}
// Rule 2 — ADVERSARIAL: the try-to-refute panel found counter-evidence.
const refs = Array.isArray(change.refutations) ? change.refutations : [];
const refuted = refs.filter((r) => r && r.refuted === true);
if (refuted.length > 0) {
reasons.push(
`adversarial panel refuted the claim (${refuted.length}) — e.g. ${refuted[0].reason ?? 'no reason given'}`,
);
}
// Rule 3 — AUTHORITY MISMATCH (regression root cause): the new value was
// anchored on something other than the designated authority. Inert while
// authority_source is null (pre-lag-3), active once it is backfilled.
if (
change.authority_source &&
change.source_url &&
normUrl(change.authority_source) !== normUrl(change.source_url)
) {
reasons.push(
`source is not the designated authority_source — anchored on ${change.source_url} instead of ${change.authority_source}`,
);
}
return {
verdict: reasons.length ? 'flagged' : 'auto-applied',
status_claim: statusClaim,
reasons,
};
}