feat(ms-ai-architect): Sesjon 4 - verifisering-ut (lag 5) + adversarial regresjonstest
Lag 5 kjører ETTER transformasjon (lag 4) og FØR en kandidat-endring skrives til
en KB-fil. Den fanger regresjons-klassen: en status-påstand «korrigert» mot en
tilfeldig sitert side og stille auto-applyet (den kjente agentic-retrieval-
regresjonen — hovedkontekst måtte rette manuelt).
Regel (spec §21): status-påstander (GA/preview/versjon/pris) flagges ALLTID for
operatør, aldri auto-applyet — uansett hvor sikker evidensen ser ut.
- lib/verify-out.mjs: ren klassifiserer, null deps (speiler decisions-io).
detectStatusClaim(text) → {isStatus, kinds}; classifyChange(change) →
{verdict: flagged|auto-applied, status_claim, reasons}. Tre flag-regler:
status-gate (§21) · adversarial refutering · autoritets-mismatch (regresjonens
rotårsak). Konservativ med vilje: ved tvil flagges. SKRIVER ALDRI.
- Fixtur tests/fixtures/kb-update/agentic-retrieval-regression.json: den kjente
regresjonen (flat «GA» mot nyansert «delvis GA … resten preview»).
- TDD: 13 tester før kode, inkl. import-invariant (ingen write-utils).
- Wiret i kb-update.md §4 c2 (lag 5 mellom identifiser-endring og skriv).
Kriterium møtt: fixtur → flagged:true, ikke auto-applied.
Tester: validate 239 · kb-update 95 (+13) · kb-eval 13 · kb-integrity 115/115.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01REiKFhP4w6xGXXqWKpPCJJ
This commit is contained in:
parent
57166e82d6
commit
6af3624b17
4 changed files with 277 additions and 1 deletions
107
scripts/kb-update/lib/verify-out.mjs
Normal file
107
scripts/kb-update/lib/verify-out.mjs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
// verify-out.mjs — Lag 5: verifisering-UT. Den adversarielle "proev-aa-motbevis"
|
||||
// + status-gaten som kjoerer ETTER transformasjon (lag 4) og FOER en kandidat-
|
||||
// endring skrives til en KB-fil. Den fanger regresjons-klassen: en status-paastand
|
||||
// (GA/preview/versjon/pris) som ble "korrigert" mot en tilfeldig sitert side og
|
||||
// stille auto-applyet (agentic-retrieval-regresjonen — roadmap §7, §48).
|
||||
//
|
||||
// Lagets regel (spec §21): status-paastander flagges ALLTID for operatoer — aldri
|
||||
// auto-applyet, uansett hvor sikker evidensen ser ut. verify-out er en REN
|
||||
// klassifiserer: den beregner en verdict og SKRIVER ALDRI (ingen write-util-
|
||||
// import). Aa applye endringen er gatens jobb, ikke lag 5s.
|
||||
//
|
||||
// Konservativ med vilje: ved tvil flagges. Aa over-flagge koster operatoeren et
|
||||
// blikk; aa under-flagge ga regresjonen. Null avhengigheter (speiler decisions-io).
|
||||
|
||||
// --- Status-claim-detektorer -------------------------------------------------
|
||||
|
||||
// GA matches case-SENSITIVE for aa unngaa norsk "ga" (preteritum av "gi").
|
||||
const RE_GA = /\bGA\b/;
|
||||
const RE_RELEASE = /general(ly)?\s+available|generelt\s+tilgjengelig|\bpublic\s+preview\b|\bprivate\s+preview\b|\bpreview\b|\bbeta\b|deprecat\w*|retir\w*|\bsunset\b|end[-\s]of[-\s]life|\bEOL\b/i;
|
||||
const RE_VERSION = /\b\d{4}-\d{2}-\d{2}-preview\b|\bapi-version\b/i;
|
||||
const RE_PRICE = /\$\s?\d|\bNOK\b|\bUSD\b|\bEUR\b|per\s+(million|thousand|1[MK])\s+tokens?|\bpris\w*|\bpricing\b|\bSLA\b|\d+(\.\d+)?\s?%\s*(uptime|oppetid)|\buptime\b/i;
|
||||
|
||||
/**
|
||||
* Is this text a status/lifecycle/version/price claim?
|
||||
* Release-stage, API-version (preview-datoer), and price/SLA are the signals the
|
||||
* naive auto-correct mechanism mangled. A bare calendar date is NOT a status
|
||||
* claim (only -preview-suffixed versions count) — so dated prose still auto-applies.
|
||||
* @param {string} text
|
||||
* @returns {{isStatus: boolean, kinds: string[]}}
|
||||
*/
|
||||
export function detectStatusClaim(text) {
|
||||
const s = String(text ?? '');
|
||||
const kinds = [];
|
||||
if (RE_GA.test(s) || RE_RELEASE.test(s)) kinds.push('release-stage');
|
||||
if (RE_VERSION.test(s)) kinds.push('version');
|
||||
if (RE_PRICE.test(s)) kinds.push('price');
|
||||
return { isStatus: kinds.length > 0, kinds };
|
||||
}
|
||||
|
||||
function normUrl(u) {
|
||||
return String(u).trim().replace(/\/+$/, '').toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify a candidate KB change: may it be auto-applied, or must the operator
|
||||
* see it first? Flagged wins — every triggering rule contributes a reason.
|
||||
*
|
||||
* A change object:
|
||||
* field — what is changing (label)
|
||||
* old_value — current KB claim
|
||||
* new_value — proposed replacement
|
||||
* source_url — where new_value was fetched from (optional)
|
||||
* authority_source — the designated authority for this claim (optional; null
|
||||
* today until lag 3 verifisering-INN backfills it)
|
||||
* refutations — adversarial panel output: [{refuted: boolean, reason?}]
|
||||
* (LLM runtime; empty here is fine)
|
||||
*
|
||||
* @param {object} change
|
||||
* @returns {{verdict: 'flagged'|'auto-applied', status_claim: boolean, reasons: string[]}}
|
||||
*/
|
||||
export function classifyChange(change) {
|
||||
const reasons = [];
|
||||
const kinds = new Set();
|
||||
let statusClaim = false;
|
||||
|
||||
// Rule 1 — STATUS-GATE (spec §21): any status claim in field/old/new → flag, always.
|
||||
for (const t of [change.field, change.old_value, change.new_value]) {
|
||||
const d = detectStatusClaim(t);
|
||||
if (d.isStatus) {
|
||||
statusClaim = true;
|
||||
d.kinds.forEach((k) => kinds.add(k));
|
||||
}
|
||||
}
|
||||
if (statusClaim) {
|
||||
reasons.push(
|
||||
`status claim (${[...kinds].join(', ')}) — always operator-gated (spec §21), never auto-applied`,
|
||||
);
|
||||
}
|
||||
|
||||
// Rule 2 — ADVERSARIAL: the try-to-refute panel found counter-evidence.
|
||||
const refs = Array.isArray(change.refutations) ? change.refutations : [];
|
||||
const refuted = refs.filter((r) => r && r.refuted === true);
|
||||
if (refuted.length > 0) {
|
||||
reasons.push(
|
||||
`adversarial panel refuted the claim (${refuted.length}) — e.g. ${refuted[0].reason ?? 'no reason given'}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Rule 3 — AUTHORITY MISMATCH (regression root cause): the new value was
|
||||
// anchored on something other than the designated authority. Inert while
|
||||
// authority_source is null (pre-lag-3), active once it is backfilled.
|
||||
if (
|
||||
change.authority_source &&
|
||||
change.source_url &&
|
||||
normUrl(change.authority_source) !== normUrl(change.source_url)
|
||||
) {
|
||||
reasons.push(
|
||||
`source is not the designated authority_source — anchored on ${change.source_url} instead of ${change.authority_source}`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
verdict: reasons.length ? 'flagged' : 'auto-applied',
|
||||
status_claim: statusClaim,
|
||||
reasons,
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue