feat(ms-ai-architect): generator-anti-regresjon Fase 1c — composeKbFile + TOC-gate

Nye ref-filer fodes best-practice-konforme sa neste KB-update ikke
reverserer TOC-arbeidet (write-path-regresjon). MA fore 1b.

- buildToc(body): deterministisk ## Innhold fra ##-seksjoner, GitHub-slugs,
  fence-aware, idempotent (lister aldri seg selv)
- composeKbFile(meta, body): generatoren — header + (store filer >100 linjer) TOC
  + brodtekst; erstatter bar buildKbHeader+body-konkatenering
- validateKbFile: stor fil uten TOC -> missing:['toc'] (write-path-gaten)
- Terskel 100 speiler eval.mjs N4_LARGE_FILE_LINES; bundet atferdsmessig via
  test som importerer ekte checkN4 (anti-regresjons-orakel), ikke cross-import
- Wiret composeKbFile inn i transform-prompt.md + kb-update.md (create + in-place)

TDD: +11 transform-tester. kb-update 349/349, kb-eval 154/154 (503 totalt).
Fikset pre-eksisterende skjor assertion i criterion-testen (stale
'Azure AI Foundry' -> 'AI Foundry'; produktet omdopt) — var rod for 1c.
This commit is contained in:
Kjell Tore Guttormsen 2026-06-26 10:59:40 +02:00
commit 2240f1efdd
5 changed files with 215 additions and 19 deletions

View file

@ -4,12 +4,14 @@
// Two halves, both runnable, no LLM:
// (1) The live deterministic eval for a target skill is ≥ the recorded baseline
// — the literal criterion, a regression guard over eval-baseline.json.
// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body +
// a fresh buildKbHeader) yields a file that is valid, dated, source-anchored,
// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body
// composed via composeKbFile) yields a file that is valid, dated, source-anchored,
// routed back to the same skill dir, and preserves the body verbatim — so
// writing it in place is score-preserving (same path → same ref count → same
// deterministic eval). It is in fact strictly BETTER: the pre-lag-4 file has
// no **Source:** header (the 0%-coverage failure mode), the regenerated one does.
// deterministic eval). It is in fact strictly BETTER on two axes: the pre-lag-4
// file has no **Source:** header (the 0%-coverage failure mode) and no TOC (N4) —
// the regenerated one carries both (composeKbFile births a TOC for large files,
// so `valid: true` here transitively proves the large file is TOC'd: Fase 1c).
import { test } from 'node:test';
import assert from 'node:assert/strict';
@ -17,7 +19,7 @@ import { readFileSync } from 'node:fs';
import { dirname, join, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { evalSkill } from '../../scripts/kb-eval/eval.mjs';
import { buildKbHeader, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
import { composeKbFile, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
@ -69,21 +71,25 @@ test('regenerate 1 real file → valid, dated, source-anchored, body preserved,
const category = parts[3];
const filename = basename(refRel);
const regenerated = buildKbHeader({
const regenerated = composeKbFile({
title: 'AI Foundry Disaster Recovery Planning',
status: 'GA',
category,
source: 'https://learn.microsoft.com/azure/ai-foundry/concepts/disaster-recovery',
lastUpdated: '2026-06',
}) + body;
}, body);
// Strictly better than the original: now valid (Source present), still dated.
// Strictly better than the original: now valid (Source present + TOC for the large
// body), still dated. validateKbFile requires a TOC on large files, so valid:true
// here is also the Fase 1c anti-regression proof.
const v = validateKbFile(regenerated);
assert.equal(v.valid, true, `regenerated file invalid: ${v.missing}`);
// Body preserved verbatim — transformation does not drop content.
assert.ok(regenerated.includes(body), 'body not preserved');
assert.ok(regenerated.includes('Azure AI Foundry'), 'expected body content missing');
// Spot-check a stable body marker. ("Azure AI Foundry" was the old product name;
// the file now uses "Microsoft Foundry"/"AI Foundry" after the rename.)
assert.ok(regenerated.includes('AI Foundry'), 'expected body content missing');
// Routes back to the SAME skill/category dir → in-place → count-preserving → eval-score preserved.
assert.equal(resolveTargetPath(tax, category, filename), refRel);

View file

@ -21,10 +21,25 @@ import {
validateKbFile,
buildChange,
resolveTargetPath,
buildToc,
composeKbFile,
} from '../../scripts/kb-update/lib/transform.mjs';
import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs';
import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs';
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
// The N4 (TOC-in-large-files) check is the cross-subsystem best-practice contract.
// Fase 1c births files that satisfy it; we assert parity against the REAL check so
// a future drift in N4's definition fails this test rather than silently diverging.
import { checkN4 } from '../../scripts/kb-eval/eval.mjs';
// A body long enough that header+body exceeds N4's 100-line "large file" threshold,
// carrying real ## section headings the TOC must pick up.
const LARGE_BODY =
'## Introduksjon\n\nNoe innledning.\n\n' +
'## Kjernekomponenter\n\n' +
'Mye innhold ...\n'.repeat(110) +
'\n## Beslutningsveiledning\n\nVelg X når ...\n';
const SHORT_BODY = '## Introduksjon\n\nKort.\n';
const __dirname = dirname(fileURLToPath(import.meta.url));
@ -143,6 +158,88 @@ test('resolveTargetPath returns null for an unknown category (caller must gate)'
assert.equal(resolveTargetPath(tax, 'does-not-exist', 'x.md'), null);
});
// --- buildToc: deterministic TOC from the body's ## section headings -----------
test('buildToc emits a ## Innhold heading with anchor links for each ## section', () => {
const toc = buildToc('## Introduksjon\n\ntekst\n\n## Kostnad og lisensiering\n\ntekst\n');
assert.match(toc, /^##\s+Innhold/m);
assert.match(toc, /- \[Introduksjon\]\(#introduksjon\)/);
assert.match(toc, /- \[Kostnad og lisensiering\]\(#kostnad-og-lisensiering\)/);
});
test('buildToc slugifies GitHub-style: lowercase, punctuation stripped, spaces→hyphens', () => {
const toc = buildToc('## 1. Introduksjon & Oversikt\n\ntekst\n');
assert.match(toc, /- \[1\. Introduksjon & Oversikt\]\(#1-introduksjon--oversikt\)/);
});
test('buildToc returns empty string when the body has no ## sections', () => {
assert.equal(buildToc('Bare brødtekst, ingen overskrifter.\n'), '');
assert.equal(buildToc('# Bare en H1\n\ntekst\n'), ''); // H1 is the header title, not a section
});
test('buildToc never lists the Innhold heading itself (idempotent re-runs)', () => {
const body = '## Innhold\n\n- [Introduksjon](#introduksjon)\n\n## Introduksjon\n\ntekst\n';
const toc = buildToc(body);
// exactly one anchor — to Introduksjon — never an anchor back to Innhold
assert.doesNotMatch(toc, /\(#innhold\)/);
assert.match(toc, /\(#introduksjon\)/);
});
// --- composeKbFile: the deterministic generator (header + TOC-if-large + body) --
test('composeKbFile births a large file with canonical header AND a TOC', () => {
const file = composeKbFile(META, LARGE_BODY);
assert.match(file, /^# Azure AI Search - Hybrid Retrieval/m); // canonical header first
assert.equal(parseSourceHeader(file), META.source); // Source in top-500-byte region
assert.match(file, /^##\s+Innhold/m); // TOC present
assert.equal(validateKbFile(file).valid, true);
});
test('composeKbFile output for a large file passes the REAL N4 check (anti-regression oracle)', () => {
const file = composeKbFile(META, LARGE_BODY);
const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: file }]);
assert.equal(n4.largeFiles, 1);
assert.equal(n4.withToc, 1);
assert.equal(n4.pass, true);
});
test('composeKbFile does NOT add a TOC to a small file (TOC is for large files only)', () => {
const file = composeKbFile(META, SHORT_BODY);
assert.doesNotMatch(file, /^##\s+Innhold/m);
assert.equal(validateKbFile(file).valid, true);
});
test('composeKbFile is idempotent: a body that already carries a TOC is not double-TOCd', () => {
const once = composeKbFile(META, LARGE_BODY);
const body = once.slice(once.indexOf('---\n') + 4); // strip header, keep TOC+body
const twice = composeKbFile(META, body);
assert.equal((twice.match(/^##\s+Innhold/gm) || []).length, 1);
assert.equal(validateKbFile(twice).valid, true);
});
// --- validateKbFile: large files must carry a TOC (the write-path regression gate)
test('validateKbFile flags a large file missing a TOC', () => {
const file = buildKbHeader(META) + '\n' + 'En linje med innhold.\n'.repeat(120);
const r = validateKbFile(file);
assert.equal(r.valid, false);
assert.ok(r.missing.includes('toc'), `expected 'toc' in missing: ${r.missing}`);
});
test('validateKbFile accepts a large file that carries a TOC', () => {
const file = composeKbFile(META, LARGE_BODY);
const r = validateKbFile(file);
assert.equal(r.valid, true);
assert.ok(!r.missing.includes('toc'));
});
test('validateKbFile does NOT require a TOC for a small file', () => {
const file = buildKbHeader(META) + '\n## Introduksjon\n\nKort innhold.\n';
const r = validateKbFile(file);
assert.equal(r.valid, true);
assert.ok(!r.missing.includes('toc'));
});
// --- ARCHITECTURE INVARIANT: transform.mjs is a pure lib, imports NO write-utils
test('transform.mjs imports NO write-utils (pure transformer, never writes)', () => {