diff --git a/commands/kb-update.md b/commands/kb-update.md index 679fcd1..447c66f 100644 --- a/commands/kb-update.md +++ b/commands/kb-update.md @@ -101,12 +101,13 @@ For hver fil i valgte prioriteter: a. **Les nåværende fil:** `Read` på filstien b. **Hent oppdaterte kilder:** for hver URL i `change-report.json[file].changed_urls`, kjør `microsoft_docs_fetch` på URLen c. **Identifiser endringer:** sammenlign hentet markdown mot eksisterende seksjoner i fila. Fokuser på faktuelle endringer (ny info, oppdaterte features, deprecation-varsler) — ikke små formuleringsendringer -c2. **Verifisering-ut (lag 5) — FØR du skriver.** Hver kandidat-endring fra (c) går gjennom `lib/verify-out.mjs` `classifyChange({field, old_value, new_value, source_url, authority_source, refutations})`. Dette fanger regresjons-klassen (en status-påstand «korrigert» mot en tilfeldig sitert side og stille auto-applyet — agentic-retrieval-regresjonen): +c1. **Autoritetskilde-binding (lag 3) — header-as-truth.** `authority_source = resolveAuthority()` (`lib/authority.mjs`) — fila sin `**Source:**`-header ER den utpekte autoriteten for dens påstander. `null` hvis fila mangler headeren (vokser etter hvert som lag 4 regenererer filer). Dette er inngangsdataen til lag-5 regel 3 (autoritets-mismatch): er en status-/load-bearing-endrings `source_url` ≠ `authority_source`, flagges den. Aldri gjett en autoritet fra fila sine siterte URLer (251/303 filer siterer 4–10 → ville vært gjetning). +c2. **Verifisering-ut (lag 5) — FØR du skriver.** Hver kandidat-endring fra (c) går gjennom `lib/verify-out.mjs` `classifyChange({field, old_value, new_value, source_url, authority_source, refutations})` med `authority_source` fra (c1). Dette fanger regresjons-klassen (en status-påstand «korrigert» mot en tilfeldig sitert side og stille auto-applyet — agentic-retrieval-regresjonen): - **`flagged`** → IKKE auto-skriv. Vis endringen til operatør med `reasons[]`; operatør avgjør om den tas inn. **Status-påstander (GA/preview/versjon/pris) er ALLTID `flagged`** (spec §21) — uansett hvor sikker kilden ser ut. - **`auto-applied`** → trygt å ta inn i (d) (benign, ikke-status, ingen motbevis, autoritets-match). - **Adversarial motbevis-panel (LLM-runtime):** for status-/load-bearing-påstander, kjør et lite panel som *prøver å motbevise* `new_value` mot den utpekte `authority_source`, og mat resultatene inn som `refutations[]` (`[{refuted, reason}]`). Multi-agent foreslås når lag 4/5 kjøres i produksjon (roadmap §71). - **Invariant:** `verify-out.mjs` skriver aldri — den returnerer kun en verdict. Selve skrivingen skjer i (d), gated. Verifisert av `tests/kb-update/test-verify-out.test.mjs`. -d. **Oppdater fila:** `Edit` med endringer som er `auto-applied` eller eksplisitt godkjent av operatør i (c2). Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato. **Lag-4-kontrakt:** kjør `validateKbFile()` (`lib/transform.mjs`) før skriving — den skal være `valid: true`. Mangler fila et `**Source:**`-header (pre-lag-4-filer gjør det; dekning er 0 %), legg det til i header-blokka med den utpekte autoritets-URLen for hovedkilden, slik at lag 3 senere kan backfille `authority_source`. +d. **Oppdater fila:** `Edit` med endringer som er `auto-applied` eller eksplisitt godkjent av operatør i (c2). Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato. **Lag-4-kontrakt:** kjør `validateKbFile()` (`lib/transform.mjs`) før skriving — den skal være `valid: true`. Mangler fila et `**Source:**`-header, legg det til i header-blokka med den utpekte autoritets-URLen for hovedkilden — da fanger lag 3 (`resolveAuthority` + `build-registry`) den som `authority_source`, og lag-5 regel 3 blir virksom for fila. e. **Committ:** `git add ` + `git commit -m "chore(ms-ai-architect): refresh KB $(basename ) [skip-docs]"` med mindre `--single-commit` ble gitt ### 5. Single-commit modus diff --git a/scripts/kb-update/build-registry.mjs b/scripts/kb-update/build-registry.mjs index 1420cd5..06c5569 100644 --- a/scripts/kb-update/build-registry.mjs +++ b/scripts/kb-update/build-registry.mjs @@ -9,7 +9,7 @@ import { join, relative, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { normalizeUrl, extractUrls } from './lib/url-normalize.mjs'; import { loadRegistry, saveRegistry } from './lib/registry-io.mjs'; -import { parseSourceHeader } from './lib/kb-headers.mjs'; +import { resolveAuthority, markUrlAuthority } from './lib/authority.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = join(__dirname, '..', '..'); @@ -36,6 +36,7 @@ const existing = merge ? loadRegistry() : null; const urlToFiles = new Map(); // normalizedUrl → Set let totalFiles = 0; let filesWithSourceHeader = 0; // coverage of the optional **Source:** authority header +const declaredSources = new Set(); // normalized authority URLs declared by files (lag 3) const skillDirs = readdirSync(SKILLS_DIR, { withFileTypes: true }) .filter(d => d.isDirectory()) @@ -51,10 +52,15 @@ for (const skill of skillDirs) { const urls = extractUrls(content); const relPath = relative(PLUGIN_ROOT, file); - // Read the optional **Source:**/**Primary source:** authority header. - // 0% coverage today; tracked here so we can populate authority_source as - // files adopt it (population semantics designed in the verification layer). - if (parseSourceHeader(content)) filesWithSourceHeader++; + // Lag 3 (header-as-truth): the **Source:** header is the file's declared + // authority. Collect declared sources (normalized) so each URL entry can be + // marked as a designated authority below. + const authority = resolveAuthority(content); + if (authority) { + filesWithSourceHeader++; + const n = normalizeUrl(authority); + if (n) declaredSources.add(n); + } for (const url of urls) { if (!urlToFiles.has(url)) urlToFiles.set(url, new Set()); @@ -79,9 +85,10 @@ for (const [url, files] of urlToFiles) { sitemap_lastmod: prev?.sitemap_lastmod || null, reference_files: [...files].sort(), status: prev?.status || 'unpolled', - // Reserved schema (lag 0): authority_source = declared verification source - // (**Source:** header, 0% coverage today); course = Learn course-track ref. - authority_source: prev?.authority_source ?? null, + // Lag 3: authority_source = this URL iff a file declares it as its **Source:** + // (a designated authority), else null. Recomputed from headers (truth), not + // carried over. course = Learn course-track ref (reserved). + authority_source: markUrlAuthority(url, declaredSources), course: prev?.course ?? null, }; } @@ -93,6 +100,8 @@ const multiRef = [...urlToFiles.values()].filter(s => s.size > 1).length; console.log(`Registry built: ${urlToFiles.size} unique URLs from ${totalFiles} files`); console.log(` URLs referenced by multiple files: ${multiRef}`); console.log(` Files declaring **Source:** header: ${filesWithSourceHeader}/${totalFiles}`); +const authMarked = Object.values(registry.urls).filter(e => e.authority_source).length; +console.log(` URL entries marked as authority_source: ${authMarked}/${urlToFiles.size}`); if (merge && existing?.urls) { const newUrls = [...urlToFiles.keys()].filter(u => !existing.urls[u]).length; console.log(` New URLs added (merge): ${newUrls}`); diff --git a/scripts/kb-update/lib/authority.mjs b/scripts/kb-update/lib/authority.mjs new file mode 100644 index 0000000..ac6b3d2 --- /dev/null +++ b/scripts/kb-update/lib/authority.mjs @@ -0,0 +1,80 @@ +// authority.mjs — Lag 3: VERIFISERING-INN. Establishes the per-file authority +// binding that feeds verify-out rule 3 (authority mismatch — the regression's +// root cause) and build-registry's per-URL authority_source field. +// +// Design (operatør-valg 2026-06-19): HEADER-AS-TRUTH. A file's **Source:** header +// IS the designated authority for that file's claims — full stop. We never GUESS +// an authority from a file's many cited URLs (251/303 files cite 4–10 URLs, so a +// derivation would be a guess, and a wrong authority makes rule 3 misfire). Coverage +// grows as files adopt the header (lag 4 writes it for every regenerated file). +// +// Pure lib (mirrors decisions-io / verify-out / transform): the only imports are +// read-only sibling parsers (kb-headers, url-normalize). It NEVER writes. + +import { parseSourceHeader } from './kb-headers.mjs'; +import { normalizeUrl } from './url-normalize.mjs'; + +const RE_HTTP_URL = /^https?:\/\/\S+$/i; + +/** + * The designated authority for a file's claims = its declared **Source:** header, + * IF that value is a real http(s) URL. A free-text "source" is not an authority + * binding (verifiseringsplikt: no guessed authority). The URL is returned as + * declared (not normalized) — it may legitimately be a non-Microsoft authority + * (e.g. an EU regulation for a governance file); verify-out normalizes when it + * compares. Returns null when absent or not a URL. + * @param {string} content — full reference-file content (header scanned) + * @returns {string|null} + */ +export function resolveAuthority(content) { + const src = parseSourceHeader(content); + if (!src) return null; + return RE_HTTP_URL.test(src.trim()) ? src.trim() : null; +} + +/** + * Collect the set of declared authority sources across many files, normalized to + * match registry keys. Only learn.microsoft.com authorities survive normalization + * (normalizeUrl returns null otherwise) — which is correct, since the registry + * only tracks Microsoft Learn URLs. A non-MS authority is a valid per-file binding + * but is simply not a tracked registry entry to mark. + * @param {string[]} contents — reference-file contents + * @returns {Set} normalized declared-source URLs + */ +export function collectDeclaredSources(contents) { + const set = new Set(); + for (const c of contents || []) { + const a = resolveAuthority(c); + if (!a) continue; + const n = normalizeUrl(a); + if (n) set.add(n); + } + return set; +} + +/** + * The authority_source value for a registry URL entry: the URL itself when some + * file declares it as its **Source:** (i.e. this URL is a designated authority), + * else null. The registry is URL-keyed, so storing the URL is type-consistent with + * the change.authority_source field rule 3 consumes; null keeps the field honest + * where no file vouches for the URL. + * @param {string} normUrl — a normalized registry URL key + * @param {Set} declaredSet — output of collectDeclaredSources + * @returns {string|null} + */ +export function markUrlAuthority(normUrl, declaredSet) { + return declaredSet && declaredSet.has(normUrl) ? normUrl : null; +} + +/** + * Coverage metric for the criterion ("authority_source-dekning > 0%"). + * @param {{urls: object}} registry + * @returns {{withAuthority: number, total: number, ratio: number}} + */ +export function authorityCoverage(registry) { + const urls = (registry && registry.urls) || {}; + const entries = Object.values(urls); + const withAuthority = entries.filter((e) => e && e.authority_source).length; + const total = entries.length; + return { withAuthority, total, ratio: total ? withAuthority / total : 0 }; +} diff --git a/tests/kb-update/test-authority.test.mjs b/tests/kb-update/test-authority.test.mjs new file mode 100644 index 0000000..431cc3a --- /dev/null +++ b/tests/kb-update/test-authority.test.mjs @@ -0,0 +1,143 @@ +// tests/kb-update/test-authority.test.mjs +// TDD for lag 3 — VERIFISERING-INN. lib/authority.mjs establishes the per-file +// authority binding: the **Source:** header IS the designated authority for a +// file's claims (operatør-valg 2026-06-19: header-as-truth). This is what feeds +// verify-out rule 3 (authority mismatch) — the regression's root cause — and what +// build-registry uses to populate the per-URL authority_source field. +// +// Pure lib (mirrors decisions-io / verify-out / transform): no write-util imports. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + resolveAuthority, + collectDeclaredSources, + markUrlAuthority, + authorityCoverage, +} from '../../scripts/kb-update/lib/authority.mjs'; +import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const FILE_WITH_SOURCE = + '# Tittel\n\n**Last updated:** 2026-06\n**Status:** GA\n' + + '**Source:** https://learn.microsoft.com/azure/search/hybrid-search-overview\n\n---\n\n## Innhold\n'; + +// --- resolveAuthority: the **Source:** header is the per-file authority --------- + +test('resolveAuthority returns the declared **Source:** URL', () => { + assert.equal( + resolveAuthority(FILE_WITH_SOURCE), + 'https://learn.microsoft.com/azure/search/hybrid-search-overview', + ); +}); + +test('resolveAuthority returns null when no Source header is present', () => { + assert.equal(resolveAuthority('# Tittel\n\n**Status:** GA\n\n---\n'), null); +}); + +test('resolveAuthority rejects a non-URL Source value (no guessed authority)', () => { + // Verifiseringsplikt: a free-text "source" is not an authority binding. + assert.equal(resolveAuthority('# T\n\n**Source:** se Microsoft-dokumentasjonen\n\n---\n'), null); +}); + +test('resolveAuthority accepts a non-Microsoft authority (e.g. EU regulation)', () => { + // Governance files may anchor on the regulation itself, not a learn.microsoft.com page. + const c = '# T\n\n**Source:** https://eur-lex.europa.eu/eli/reg/2024/1689/oj\n\n---\n'; + assert.equal(resolveAuthority(c), 'https://eur-lex.europa.eu/eli/reg/2024/1689/oj'); +}); + +// --- collectDeclaredSources: normalized set for registry matching -------------- + +test('collectDeclaredSources returns the normalized MS-Learn sources declared across files', () => { + const set = collectDeclaredSources([ + FILE_WITH_SOURCE, + '# B\n\n**Source:** https://learn.microsoft.com/azure/foundry/openai/how-to/batch-blob-storage\n\n---\n', + '# C — no source\n\n**Status:** GA\n\n---\n', + '# D — non-MS authority\n\n**Source:** https://eur-lex.europa.eu/x\n\n---\n', + ]); + assert.ok(set.has('https://learn.microsoft.com/azure/search/hybrid-search-overview')); + assert.ok(set.has('https://learn.microsoft.com/azure/foundry/openai/how-to/batch-blob-storage')); + // non-MS authority is not a tracked registry URL → excluded from the matching set + assert.equal(set.size, 2); +}); + +test('collectDeclaredSources normalizes (trailing slash / locale) to match registry keys', () => { + const set = collectDeclaredSources([ + '# T\n\n**Source:** https://learn.microsoft.com/en-us/azure/search/hybrid-search-overview/\n\n---\n', + ]); + assert.ok(set.has('https://learn.microsoft.com/azure/search/hybrid-search-overview')); +}); + +// --- markUrlAuthority: a URL entry is an authority iff a file declares it ------- + +test('markUrlAuthority returns the URL when it is a declared authority, else null', () => { + const declared = new Set(['https://learn.microsoft.com/azure/search/hybrid-search-overview']); + assert.equal( + markUrlAuthority('https://learn.microsoft.com/azure/search/hybrid-search-overview', declared), + 'https://learn.microsoft.com/azure/search/hybrid-search-overview', + ); + assert.equal(markUrlAuthority('https://learn.microsoft.com/azure/other/page', declared), null); +}); + +// --- authorityCoverage: the coverage metric (criterion: > 0%) ------------------ + +test('authorityCoverage counts entries with a non-null authority_source', () => { + const registry = { + urls: { + 'https://learn.microsoft.com/a': { authority_source: 'https://learn.microsoft.com/a' }, + 'https://learn.microsoft.com/b': { authority_source: null }, + 'https://learn.microsoft.com/c': { authority_source: 'https://learn.microsoft.com/c' }, + }, + }; + const cov = authorityCoverage(registry); + assert.equal(cov.withAuthority, 2); + assert.equal(cov.total, 3); + assert.ok(cov.ratio > 0); +}); + +// --- END-TO-END: header → authority → verify-out rule 3 (the criterion) -------- + +test('rule 3 fires end-to-end on a real authority: source_url ≠ header authority → flagged', () => { + // A NON-status change so rules 1 (status-gate) and 2 (refutation) stay silent — + // this isolates rule 3 and proves the header→authority→mismatch path works. + const authority = resolveAuthority(FILE_WITH_SOURCE); + const verdict = classifyChange({ + field: 'description', + old_value: 'En setning.', + new_value: 'En annen setning.', + source_url: 'https://learn.microsoft.com/azure/search/some-other-tangential-page', + authority_source: authority, + refutations: [], + }); + assert.equal(verdict.verdict, 'flagged'); + assert.equal(verdict.status_claim, false); // not a status claim — rule 3 alone flagged it + assert.ok(verdict.reasons.some((r) => /authority/i.test(r)), `expected an authority reason: ${verdict.reasons}`); +}); + +test('rule 3 stays silent when source_url matches the header authority', () => { + const authority = resolveAuthority(FILE_WITH_SOURCE); + const verdict = classifyChange({ + field: 'description', + old_value: 'En setning.', + new_value: 'En annen setning.', + source_url: authority, // same as authority → no mismatch + authority_source: authority, + refutations: [], + }); + assert.equal(verdict.verdict, 'auto-applied'); +}); + +// --- ARCHITECTURE INVARIANT: authority.mjs imports NO write-utils -------------- + +test('authority.mjs imports NO write-utils (pure, never writes)', () => { + const src = readFileSync(join(__dirname, '../../scripts/kb-update/lib/authority.mjs'), 'utf8'); + const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n'); + assert.doesNotMatch(importLines, /\bsaveRegistry\b/); + assert.doesNotMatch(importLines, /\bsaveDecisions\b/); + assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/); + assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/); +});