diff --git a/commands/kb-update.md b/commands/kb-update.md index f526820..5e83278 100644 --- a/commands/kb-update.md +++ b/commands/kb-update.md @@ -127,8 +127,8 @@ saveDecisions(led); d. **For `approved`:** registrér URLen i `url-registry` (gated) så den fanges av polling heretter — via `lib/registry-io.mjs` `saveRegistry`. Bruk `suggested_skill`/`suggested_category` fra kandidaten. Deretter kjør **transformasjonslaget (lag 4)** for å lage KB-fila: 1. `microsoft_docs_fetch` på den godkjente URLen → kildedokument. 2. Destillér via `scripts/kb-update/transform-prompt.md` (doc→KB-fil; status-påstander holdes eksplisitte). Multi-agent parallell fan-out foreslås i produksjon (roadmap §71), speilet på `generate-skills`-mønsteret. - 3. `content = buildKbHeader({title, status, category, source: , lastUpdated: }) + body` (`lib/transform.mjs`). **`Status` + `Source` er obligatoriske** — `Source` i header-blokka (øverste 500 bytes) er det som lar lag 3 backfille `authority_source`. - 4. `validateKbFile(content)` MÅ være `valid: true` før noe gates videre (ellers be modellen fylle manglende felt). + 3. `content = composeKbFile({title, status, category, source: , lastUpdated: }, body)` (`lib/transform.mjs`) — bygger header + (store filer >100 linjer) en deterministisk `## Innhold`-TOC + brødtekst. **`Status` + `Source` er obligatoriske** — `Source` i header-blokka (øverste 500 bytes) er det som lar lag 3 backfille `authority_source`. TOC-en fødes inn så regenerering ikke stripper den (Fase 1c). + 4. `validateKbFile(content)` MÅ være `valid: true` (title + Last updated + Status + Source; **store filer også TOC**) før noe gates videre (ellers be modellen fylle manglende felt). 5. For hver status-/load-bearing-påstand: `buildChange({...})` → **lag 5** `classifyChange(...)` (samme gate som §4 c2). Status-påstander er alltid `flagged`. 6. `resolveTargetPath(tax, category, filename)` → eierskill-sti via taksonomien (`null` = ukjent kategori → flagg for operatør, ikke skriv). 7. Først etter operatør-gate: atomisk skriving (`lib/atomic-write.mjs` + `lib/backup.mjs`). **`transform.mjs` skriver aldri selv** — verifisert av `tests/kb-update/test-transform.test.mjs` (import-invariant). Kriterium verifisert av `tests/kb-eval/test-transform-criterion.test.mjs` (regenerer 1 fil → eval ≥ baseline). @@ -185,7 +185,7 @@ c2. **Verifisering-ut (lag 5) — FØR du skriver.** Hver kandidat-endring fra ( - **`auto-applied`** → trygt å ta inn i (d) (benign, ikke-status, ingen motbevis, autoritets-match). - **Adversarial motbevis-panel (LLM-runtime):** for status-/load-bearing-påstander, kjør et lite panel som *prøver å motbevise* `new_value` mot den utpekte `authority_source`, og mat resultatene inn som `refutations[]` (`[{refuted, reason}]`). Multi-agent foreslås når lag 4/5 kjøres i produksjon (roadmap §71). - **Invariant:** `verify-out.mjs` skriver aldri — den returnerer kun en verdict. Selve skrivingen skjer i (d), gated. Verifisert av `tests/kb-update/test-verify-out.test.mjs`. -d. **Oppdater fila:** `Edit` med endringer som er `auto-applied` eller eksplisitt godkjent av operatør i (c2). Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato. **Lag-4-kontrakt:** kjør `validateKbFile()` (`lib/transform.mjs`) før skriving — den skal være `valid: true`. Mangler fila et `**Source:**`-header, legg det til i header-blokka med den utpekte autoritets-URLen for hovedkilden — da fanger lag 3 (`resolveAuthority` + `build-registry`) den som `authority_source`, og lag-5 regel 3 blir virksom for fila. +d. **Oppdater fila:** `Edit` med endringer som er `auto-applied` eller eksplisitt godkjent av operatør i (c2). Behold "For Cosmo"-seksjonen og overordnet struktur. Oppdater `Last updated: YYYY-MM-DD`-header til dagens dato. **Lag-4-kontrakt:** kjør `validateKbFile()` (`lib/transform.mjs`) før skriving — den skal være `valid: true`. Mangler fila et `**Source:**`-header, legg det til i header-blokka med den utpekte autoritets-URLen for hovedkilden — da fanger lag 3 (`resolveAuthority` + `build-registry`) den som `authority_source`, og lag-5 regel 3 blir virksom for fila. Mangler en **stor fil (>100 linjer)** en `## Innhold`-TOC, generer den med `buildToc()` og legg den inn rett etter header-`---` — `validateKbFile` krever den nå for store filer, så in-place-oppdateringer backfiller TOC inkrementelt (Fase 1c). e. **Committ:** `git add ` + `git commit -m "chore(ms-ai-architect): refresh KB $(basename ) [skip-docs]"` med mindre `--single-commit` ble gitt ### 5. Single-commit modus diff --git a/scripts/kb-update/lib/transform.mjs b/scripts/kb-update/lib/transform.mjs index 62cc49e..8ddfba3 100644 --- a/scripts/kb-update/lib/transform.mjs +++ b/scripts/kb-update/lib/transform.mjs @@ -22,6 +22,43 @@ import { getCategorySkill } from './taxonomy.mjs'; // Source is what lets lag 3 backfill authority_source and lag-5 rule 3 eventually fire. const REQUIRED_META = ['title', 'status', 'category', 'source', 'lastUpdated']; +// A reference file longer than this many lines should carry a table of contents +// (best-practices §Structure). MIRRORS eval.mjs N4_LARGE_FILE_LINES — kept in sync +// behaviourally by test-transform's checkN4 parity assertion, NOT by importing across +// the kb-update ↔ kb-eval boundary (transform.mjs's only dependency stays kb-headers). +const TOC_MIN_LINES = 100; + +/** Line count counted exactly as checkN4 does (trailing blank lines ignored). */ +function lineCount(content) { + return String(content ?? '').replace(/\n+$/, '').split('\n').length; +} + +/** Is this file large enough to require a TOC? (mirror of checkN4's "large" filter) */ +function isLargeContent(content) { + return lineCount(content) > TOC_MIN_LINES; +} + +/** + * Does the content already carry a TOC? Mirror of eval.mjs hasToc (internal there): + * an explicit TOC heading, OR a cluster (≥3) of in-page anchor links near the top. + */ +function hasToc(content) { + const s = String(content ?? ''); + if (/^#{1,3}\s*(table of contents|contents|innhold(?:sfortegnelse)?|toc)\b/im.test(s)) return true; + const head = s.split('\n').slice(0, 40).join('\n'); + return (head.match(/\]\(#/g) || []).length >= 3; +} + +/** GitHub-style heading slug: lowercase, drop punctuation (keep letters/digits/_/-), + * spaces → hyphens. Unicode-aware so Norwegian æ/ø/å survive in anchors. */ +function slugify(heading) { + return String(heading ?? '') + .toLowerCase() + .replace(/[^\p{L}\p{N}\s_-]/gu, '') + .trim() + .replace(/\s/g, '-'); +} + /** * Build the deterministic KB-file header block. Status and Source live IN the * header (top 500 bytes) so kb-headers.parseSourceHeader and the "Last updated:" @@ -52,6 +89,56 @@ export function buildKbHeader(meta) { ); } +/** + * Build a deterministic `## Innhold` table of contents from a body's level-2 (`##`) + * section headings — the plugin house style (matches the existing TOC'd files and the + * heading eval.mjs hasToc recognises). The LLM should NOT hand-author anchor slugs; + * this generates GitHub-compatible ones. Headings inside ``` code fences are ignored, + * and the `## Innhold` heading never lists itself (so re-running is idempotent). + * + * @param {string} body — the file body (under the header `---`) + * @returns {string} a `## Innhold` block ending in a newline, or '' if there are no + * `##` sections to list + */ +export function buildToc(body) { + const lines = String(body ?? '').split('\n'); + const titles = []; + let inFence = false; + for (const line of lines) { + if (/^\s*```/.test(line)) { inFence = !inFence; continue; } + if (inFence) continue; + const m = /^##\s+(.+?)\s*$/.exec(line); // level-2 only (not # title, not ### sub) + if (!m) continue; + const title = m[1].trim(); + if (/^innhold(?:sfortegnelse)?$/i.test(title)) continue; // never list the TOC itself + titles.push(title); + } + if (titles.length === 0) return ''; + const items = titles.map((t) => `- [${t}](#${slugify(t)})`).join('\n'); + return `## Innhold\n\n${items}\n`; +} + +/** + * Assemble a complete KB file from header meta + body: the deterministic generator the + * write-path uses (replaces the bare `buildKbHeader(meta) + body` concatenation). A + * large file (>100 lines, per N4) is born with a `## Innhold` TOC inserted after the + * header so regeneration never strips the TOC back out (the Fase 1c regression guard). + * Idempotent: a body that already carries a TOC is not double-TOC'd. + * + * @param {object} meta — same shape as buildKbHeader + * @param {string} body — the distilled body (under the header `---`) + * @returns {string} the full file content + */ +export function composeKbFile(meta, body) { + const header = buildKbHeader(meta); + const b = String(body ?? ''); + const full = header + b; + if (!isLargeContent(full) || hasToc(full)) return full; + const toc = buildToc(b); + if (!toc) return full; + return `${header}\n${toc}\n${b}`; +} + const RE_TITLE = /^#\s+\S/m; const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i; const RE_STATUS = /\*\*Status:\*\*\s*\S/i; @@ -59,8 +146,10 @@ const RE_STATUS = /\*\*Status:\*\*\s*\S/i; /** * Validate that a candidate KB file carries the mandatory contract: a title, a * "Last updated:" header (so it is never dateless → never the 'unverified' bucket), - * a Status field, and a Source header in the scannable region. Used by the gate - * before a transformed file may be written. + * a Status field, a Source header in the scannable region, and — for a large file + * (>100 lines, per N4) — a table of contents. The TOC clause is the write-path + * regression gate: a regenerated large file that lost its TOC fails here and the + * caller re-prompts, so KB-update can never silently strip a TOC back out. * * @param {string} content — full file content * @returns {{valid: boolean, missing: string[]}} @@ -72,6 +161,7 @@ export function validateKbFile(content) { if (!RE_LAST_UPDATED.test(s)) missing.push('last_updated'); if (!RE_STATUS.test(s)) missing.push('status'); if (parseSourceHeader(s) === null) missing.push('source'); + if (isLargeContent(s) && !hasToc(s)) missing.push('toc'); return { valid: missing.length === 0, missing }; } diff --git a/scripts/kb-update/transform-prompt.md b/scripts/kb-update/transform-prompt.md index 9c96530..a705175 100644 --- a/scripts/kb-update/transform-prompt.md +++ b/scripts/kb-update/transform-prompt.md @@ -42,9 +42,12 @@ ikke av modellen. Den ser slik ut og MÅ stå først: --- ``` -Modellen skriver **kun brødteksten under `---`**. Etterpå kjører -`validateKbFile(header + body)` og MÅ returnere `valid: true` (title + Last updated + -Status + Source til stede) før noe gates videre. +Modellen skriver **kun brødteksten under `---`**. `composeKbFile({...}, body)` setter så +sammen hele fila — header + (for **store filer >100 linjer**) en deterministisk +`## Innhold`-TOC fra brødtekstens `##`-seksjoner + brødteksten. Etterpå kjører +`validateKbFile(content)` og MÅ returnere `valid: true` (title + Last updated + Status + +Source til stede; **for store filer også TOC**) før noe gates videre. TOC-en fødes inn så +en senere regenerering ikke stripper den (write-path-regresjon — Fase 1c). ## Prompt (brødtekst) @@ -87,8 +90,8 @@ Ingen innledning eller avslutningskommentar — kun markdown-brødteksten. ## Etter destillasjon (caller, gated) -1. `content = buildKbHeader({...}) + body` -2. `validateKbFile(content)` → må være `valid: true` (ellers: be modellen fikse manglende felt). +1. `content = composeKbFile({title, status, category, source, lastUpdated}, body)` — header + (store filer) `## Innhold`-TOC + brødtekst. Erstatter den bare `buildKbHeader(...) + body`-konkateneringen. +2. `validateKbFile(content)` → må være `valid: true` (title + Last updated + Status + Source; **store filer også TOC**) (ellers: be modellen fikse manglende felt). 3. For hver status-/load-bearing-påstand: `buildChange({field, oldValue, newValue, sourceUrl, authoritySource})` → `classifyChange(...)` (lag 5). `flagged` → operatør avgjør; status-påstander er alltid `flagged`. 4. `resolveTargetPath(tax, category, filename)` → eierskill-sti (null = ukjent kategori → gate, ikke skriv). diff --git a/tests/kb-eval/test-transform-criterion.test.mjs b/tests/kb-eval/test-transform-criterion.test.mjs index 03400a3..1e09a85 100644 --- a/tests/kb-eval/test-transform-criterion.test.mjs +++ b/tests/kb-eval/test-transform-criterion.test.mjs @@ -4,12 +4,14 @@ // Two halves, both runnable, no LLM: // (1) The live deterministic eval for a target skill is ≥ the recorded baseline // — the literal criterion, a regression guard over eval-baseline.json. -// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body + -// a fresh buildKbHeader) yields a file that is valid, dated, source-anchored, +// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body +// composed via composeKbFile) yields a file that is valid, dated, source-anchored, // routed back to the same skill dir, and preserves the body verbatim — so // writing it in place is score-preserving (same path → same ref count → same -// deterministic eval). It is in fact strictly BETTER: the pre-lag-4 file has -// no **Source:** header (the 0%-coverage failure mode), the regenerated one does. +// deterministic eval). It is in fact strictly BETTER on two axes: the pre-lag-4 +// file has no **Source:** header (the 0%-coverage failure mode) and no TOC (N4) — +// the regenerated one carries both (composeKbFile births a TOC for large files, +// so `valid: true` here transitively proves the large file is TOC'd: Fase 1c). import { test } from 'node:test'; import assert from 'node:assert/strict'; @@ -17,7 +19,7 @@ import { readFileSync } from 'node:fs'; import { dirname, join, basename } from 'node:path'; import { fileURLToPath } from 'node:url'; import { evalSkill } from '../../scripts/kb-eval/eval.mjs'; -import { buildKbHeader, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs'; +import { composeKbFile, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs'; import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -69,21 +71,25 @@ test('regenerate 1 real file → valid, dated, source-anchored, body preserved, const category = parts[3]; const filename = basename(refRel); - const regenerated = buildKbHeader({ + const regenerated = composeKbFile({ title: 'AI Foundry Disaster Recovery Planning', status: 'GA', category, source: 'https://learn.microsoft.com/azure/ai-foundry/concepts/disaster-recovery', lastUpdated: '2026-06', - }) + body; + }, body); - // Strictly better than the original: now valid (Source present), still dated. + // Strictly better than the original: now valid (Source present + TOC for the large + // body), still dated. validateKbFile requires a TOC on large files, so valid:true + // here is also the Fase 1c anti-regression proof. const v = validateKbFile(regenerated); assert.equal(v.valid, true, `regenerated file invalid: ${v.missing}`); // Body preserved verbatim — transformation does not drop content. assert.ok(regenerated.includes(body), 'body not preserved'); - assert.ok(regenerated.includes('Azure AI Foundry'), 'expected body content missing'); + // Spot-check a stable body marker. ("Azure AI Foundry" was the old product name; + // the file now uses "Microsoft Foundry"/"AI Foundry" after the rename.) + assert.ok(regenerated.includes('AI Foundry'), 'expected body content missing'); // Routes back to the SAME skill/category dir → in-place → count-preserving → eval-score preserved. assert.equal(resolveTargetPath(tax, category, filename), refRel); diff --git a/tests/kb-update/test-transform.test.mjs b/tests/kb-update/test-transform.test.mjs index 1f2a25b..3e0c166 100644 --- a/tests/kb-update/test-transform.test.mjs +++ b/tests/kb-update/test-transform.test.mjs @@ -21,10 +21,25 @@ import { validateKbFile, buildChange, resolveTargetPath, + buildToc, + composeKbFile, } from '../../scripts/kb-update/lib/transform.mjs'; import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs'; import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs'; import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs'; +// The N4 (TOC-in-large-files) check is the cross-subsystem best-practice contract. +// Fase 1c births files that satisfy it; we assert parity against the REAL check so +// a future drift in N4's definition fails this test rather than silently diverging. +import { checkN4 } from '../../scripts/kb-eval/eval.mjs'; + +// A body long enough that header+body exceeds N4's 100-line "large file" threshold, +// carrying real ## section headings the TOC must pick up. +const LARGE_BODY = + '## Introduksjon\n\nNoe innledning.\n\n' + + '## Kjernekomponenter\n\n' + + 'Mye innhold ...\n'.repeat(110) + + '\n## Beslutningsveiledning\n\nVelg X når ...\n'; +const SHORT_BODY = '## Introduksjon\n\nKort.\n'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -143,6 +158,88 @@ test('resolveTargetPath returns null for an unknown category (caller must gate)' assert.equal(resolveTargetPath(tax, 'does-not-exist', 'x.md'), null); }); +// --- buildToc: deterministic TOC from the body's ## section headings ----------- + +test('buildToc emits a ## Innhold heading with anchor links for each ## section', () => { + const toc = buildToc('## Introduksjon\n\ntekst\n\n## Kostnad og lisensiering\n\ntekst\n'); + assert.match(toc, /^##\s+Innhold/m); + assert.match(toc, /- \[Introduksjon\]\(#introduksjon\)/); + assert.match(toc, /- \[Kostnad og lisensiering\]\(#kostnad-og-lisensiering\)/); +}); + +test('buildToc slugifies GitHub-style: lowercase, punctuation stripped, spaces→hyphens', () => { + const toc = buildToc('## 1. Introduksjon & Oversikt\n\ntekst\n'); + assert.match(toc, /- \[1\. Introduksjon & Oversikt\]\(#1-introduksjon--oversikt\)/); +}); + +test('buildToc returns empty string when the body has no ## sections', () => { + assert.equal(buildToc('Bare brødtekst, ingen overskrifter.\n'), ''); + assert.equal(buildToc('# Bare en H1\n\ntekst\n'), ''); // H1 is the header title, not a section +}); + +test('buildToc never lists the Innhold heading itself (idempotent re-runs)', () => { + const body = '## Innhold\n\n- [Introduksjon](#introduksjon)\n\n## Introduksjon\n\ntekst\n'; + const toc = buildToc(body); + // exactly one anchor — to Introduksjon — never an anchor back to Innhold + assert.doesNotMatch(toc, /\(#innhold\)/); + assert.match(toc, /\(#introduksjon\)/); +}); + +// --- composeKbFile: the deterministic generator (header + TOC-if-large + body) -- + +test('composeKbFile births a large file with canonical header AND a TOC', () => { + const file = composeKbFile(META, LARGE_BODY); + assert.match(file, /^# Azure AI Search - Hybrid Retrieval/m); // canonical header first + assert.equal(parseSourceHeader(file), META.source); // Source in top-500-byte region + assert.match(file, /^##\s+Innhold/m); // TOC present + assert.equal(validateKbFile(file).valid, true); +}); + +test('composeKbFile output for a large file passes the REAL N4 check (anti-regression oracle)', () => { + const file = composeKbFile(META, LARGE_BODY); + const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: file }]); + assert.equal(n4.largeFiles, 1); + assert.equal(n4.withToc, 1); + assert.equal(n4.pass, true); +}); + +test('composeKbFile does NOT add a TOC to a small file (TOC is for large files only)', () => { + const file = composeKbFile(META, SHORT_BODY); + assert.doesNotMatch(file, /^##\s+Innhold/m); + assert.equal(validateKbFile(file).valid, true); +}); + +test('composeKbFile is idempotent: a body that already carries a TOC is not double-TOCd', () => { + const once = composeKbFile(META, LARGE_BODY); + const body = once.slice(once.indexOf('---\n') + 4); // strip header, keep TOC+body + const twice = composeKbFile(META, body); + assert.equal((twice.match(/^##\s+Innhold/gm) || []).length, 1); + assert.equal(validateKbFile(twice).valid, true); +}); + +// --- validateKbFile: large files must carry a TOC (the write-path regression gate) + +test('validateKbFile flags a large file missing a TOC', () => { + const file = buildKbHeader(META) + '\n' + 'En linje med innhold.\n'.repeat(120); + const r = validateKbFile(file); + assert.equal(r.valid, false); + assert.ok(r.missing.includes('toc'), `expected 'toc' in missing: ${r.missing}`); +}); + +test('validateKbFile accepts a large file that carries a TOC', () => { + const file = composeKbFile(META, LARGE_BODY); + const r = validateKbFile(file); + assert.equal(r.valid, true); + assert.ok(!r.missing.includes('toc')); +}); + +test('validateKbFile does NOT require a TOC for a small file', () => { + const file = buildKbHeader(META) + '\n## Introduksjon\n\nKort innhold.\n'; + const r = validateKbFile(file); + assert.equal(r.valid, true); + assert.ok(!r.missing.includes('toc')); +}); + // --- ARCHITECTURE INVARIANT: transform.mjs is a pure lib, imports NO write-utils test('transform.mjs imports NO write-utils (pure transformer, never writes)', () => {