feat(ms-ai-architect): generator-anti-regresjon Fase 1c — composeKbFile + TOC-gate
Nye ref-filer fodes best-practice-konforme sa neste KB-update ikke reverserer TOC-arbeidet (write-path-regresjon). MA fore 1b. - buildToc(body): deterministisk ## Innhold fra ##-seksjoner, GitHub-slugs, fence-aware, idempotent (lister aldri seg selv) - composeKbFile(meta, body): generatoren — header + (store filer >100 linjer) TOC + brodtekst; erstatter bar buildKbHeader+body-konkatenering - validateKbFile: stor fil uten TOC -> missing:['toc'] (write-path-gaten) - Terskel 100 speiler eval.mjs N4_LARGE_FILE_LINES; bundet atferdsmessig via test som importerer ekte checkN4 (anti-regresjons-orakel), ikke cross-import - Wiret composeKbFile inn i transform-prompt.md + kb-update.md (create + in-place) TDD: +11 transform-tester. kb-update 349/349, kb-eval 154/154 (503 totalt). Fikset pre-eksisterende skjor assertion i criterion-testen (stale 'Azure AI Foundry' -> 'AI Foundry'; produktet omdopt) — var rod for 1c.
This commit is contained in:
parent
89eccf8881
commit
2240f1efdd
5 changed files with 215 additions and 19 deletions
|
|
@ -4,12 +4,14 @@
|
|||
// Two halves, both runnable, no LLM:
|
||||
// (1) The live deterministic eval for a target skill is ≥ the recorded baseline
|
||||
// — the literal criterion, a regression guard over eval-baseline.json.
|
||||
// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body +
|
||||
// a fresh buildKbHeader) yields a file that is valid, dated, source-anchored,
|
||||
// (2) Regenerating one REAL reference file through the lag-4 pipeline (real body
|
||||
// composed via composeKbFile) yields a file that is valid, dated, source-anchored,
|
||||
// routed back to the same skill dir, and preserves the body verbatim — so
|
||||
// writing it in place is score-preserving (same path → same ref count → same
|
||||
// deterministic eval). It is in fact strictly BETTER: the pre-lag-4 file has
|
||||
// no **Source:** header (the 0%-coverage failure mode), the regenerated one does.
|
||||
// deterministic eval). It is in fact strictly BETTER on two axes: the pre-lag-4
|
||||
// file has no **Source:** header (the 0%-coverage failure mode) and no TOC (N4) —
|
||||
// the regenerated one carries both (composeKbFile births a TOC for large files,
|
||||
// so `valid: true` here transitively proves the large file is TOC'd: Fase 1c).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
|
@ -17,7 +19,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { dirname, join, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { evalSkill } from '../../scripts/kb-eval/eval.mjs';
|
||||
import { buildKbHeader, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { composeKbFile, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -69,21 +71,25 @@ test('regenerate 1 real file → valid, dated, source-anchored, body preserved,
|
|||
const category = parts[3];
|
||||
const filename = basename(refRel);
|
||||
|
||||
const regenerated = buildKbHeader({
|
||||
const regenerated = composeKbFile({
|
||||
title: 'AI Foundry Disaster Recovery Planning',
|
||||
status: 'GA',
|
||||
category,
|
||||
source: 'https://learn.microsoft.com/azure/ai-foundry/concepts/disaster-recovery',
|
||||
lastUpdated: '2026-06',
|
||||
}) + body;
|
||||
}, body);
|
||||
|
||||
// Strictly better than the original: now valid (Source present), still dated.
|
||||
// Strictly better than the original: now valid (Source present + TOC for the large
|
||||
// body), still dated. validateKbFile requires a TOC on large files, so valid:true
|
||||
// here is also the Fase 1c anti-regression proof.
|
||||
const v = validateKbFile(regenerated);
|
||||
assert.equal(v.valid, true, `regenerated file invalid: ${v.missing}`);
|
||||
|
||||
// Body preserved verbatim — transformation does not drop content.
|
||||
assert.ok(regenerated.includes(body), 'body not preserved');
|
||||
assert.ok(regenerated.includes('Azure AI Foundry'), 'expected body content missing');
|
||||
// Spot-check a stable body marker. ("Azure AI Foundry" was the old product name;
|
||||
// the file now uses "Microsoft Foundry"/"AI Foundry" after the rename.)
|
||||
assert.ok(regenerated.includes('AI Foundry'), 'expected body content missing');
|
||||
|
||||
// Routes back to the SAME skill/category dir → in-place → count-preserving → eval-score preserved.
|
||||
assert.equal(resolveTargetPath(tax, category, filename), refRel);
|
||||
|
|
|
|||
|
|
@ -21,10 +21,25 @@ import {
|
|||
validateKbFile,
|
||||
buildChange,
|
||||
resolveTargetPath,
|
||||
buildToc,
|
||||
composeKbFile,
|
||||
} from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs';
|
||||
import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs';
|
||||
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
|
||||
// The N4 (TOC-in-large-files) check is the cross-subsystem best-practice contract.
|
||||
// Fase 1c births files that satisfy it; we assert parity against the REAL check so
|
||||
// a future drift in N4's definition fails this test rather than silently diverging.
|
||||
import { checkN4 } from '../../scripts/kb-eval/eval.mjs';
|
||||
|
||||
// A body long enough that header+body exceeds N4's 100-line "large file" threshold,
|
||||
// carrying real ## section headings the TOC must pick up.
|
||||
const LARGE_BODY =
|
||||
'## Introduksjon\n\nNoe innledning.\n\n' +
|
||||
'## Kjernekomponenter\n\n' +
|
||||
'Mye innhold ...\n'.repeat(110) +
|
||||
'\n## Beslutningsveiledning\n\nVelg X når ...\n';
|
||||
const SHORT_BODY = '## Introduksjon\n\nKort.\n';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
|
|
@ -143,6 +158,88 @@ test('resolveTargetPath returns null for an unknown category (caller must gate)'
|
|||
assert.equal(resolveTargetPath(tax, 'does-not-exist', 'x.md'), null);
|
||||
});
|
||||
|
||||
// --- buildToc: deterministic TOC from the body's ## section headings -----------
|
||||
|
||||
test('buildToc emits a ## Innhold heading with anchor links for each ## section', () => {
|
||||
const toc = buildToc('## Introduksjon\n\ntekst\n\n## Kostnad og lisensiering\n\ntekst\n');
|
||||
assert.match(toc, /^##\s+Innhold/m);
|
||||
assert.match(toc, /- \[Introduksjon\]\(#introduksjon\)/);
|
||||
assert.match(toc, /- \[Kostnad og lisensiering\]\(#kostnad-og-lisensiering\)/);
|
||||
});
|
||||
|
||||
test('buildToc slugifies GitHub-style: lowercase, punctuation stripped, spaces→hyphens', () => {
|
||||
const toc = buildToc('## 1. Introduksjon & Oversikt\n\ntekst\n');
|
||||
assert.match(toc, /- \[1\. Introduksjon & Oversikt\]\(#1-introduksjon--oversikt\)/);
|
||||
});
|
||||
|
||||
test('buildToc returns empty string when the body has no ## sections', () => {
|
||||
assert.equal(buildToc('Bare brødtekst, ingen overskrifter.\n'), '');
|
||||
assert.equal(buildToc('# Bare en H1\n\ntekst\n'), ''); // H1 is the header title, not a section
|
||||
});
|
||||
|
||||
test('buildToc never lists the Innhold heading itself (idempotent re-runs)', () => {
|
||||
const body = '## Innhold\n\n- [Introduksjon](#introduksjon)\n\n## Introduksjon\n\ntekst\n';
|
||||
const toc = buildToc(body);
|
||||
// exactly one anchor — to Introduksjon — never an anchor back to Innhold
|
||||
assert.doesNotMatch(toc, /\(#innhold\)/);
|
||||
assert.match(toc, /\(#introduksjon\)/);
|
||||
});
|
||||
|
||||
// --- composeKbFile: the deterministic generator (header + TOC-if-large + body) --
|
||||
|
||||
test('composeKbFile births a large file with canonical header AND a TOC', () => {
|
||||
const file = composeKbFile(META, LARGE_BODY);
|
||||
assert.match(file, /^# Azure AI Search - Hybrid Retrieval/m); // canonical header first
|
||||
assert.equal(parseSourceHeader(file), META.source); // Source in top-500-byte region
|
||||
assert.match(file, /^##\s+Innhold/m); // TOC present
|
||||
assert.equal(validateKbFile(file).valid, true);
|
||||
});
|
||||
|
||||
test('composeKbFile output for a large file passes the REAL N4 check (anti-regression oracle)', () => {
|
||||
const file = composeKbFile(META, LARGE_BODY);
|
||||
const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: file }]);
|
||||
assert.equal(n4.largeFiles, 1);
|
||||
assert.equal(n4.withToc, 1);
|
||||
assert.equal(n4.pass, true);
|
||||
});
|
||||
|
||||
test('composeKbFile does NOT add a TOC to a small file (TOC is for large files only)', () => {
|
||||
const file = composeKbFile(META, SHORT_BODY);
|
||||
assert.doesNotMatch(file, /^##\s+Innhold/m);
|
||||
assert.equal(validateKbFile(file).valid, true);
|
||||
});
|
||||
|
||||
test('composeKbFile is idempotent: a body that already carries a TOC is not double-TOCd', () => {
|
||||
const once = composeKbFile(META, LARGE_BODY);
|
||||
const body = once.slice(once.indexOf('---\n') + 4); // strip header, keep TOC+body
|
||||
const twice = composeKbFile(META, body);
|
||||
assert.equal((twice.match(/^##\s+Innhold/gm) || []).length, 1);
|
||||
assert.equal(validateKbFile(twice).valid, true);
|
||||
});
|
||||
|
||||
// --- validateKbFile: large files must carry a TOC (the write-path regression gate)
|
||||
|
||||
test('validateKbFile flags a large file missing a TOC', () => {
|
||||
const file = buildKbHeader(META) + '\n' + 'En linje med innhold.\n'.repeat(120);
|
||||
const r = validateKbFile(file);
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.missing.includes('toc'), `expected 'toc' in missing: ${r.missing}`);
|
||||
});
|
||||
|
||||
test('validateKbFile accepts a large file that carries a TOC', () => {
|
||||
const file = composeKbFile(META, LARGE_BODY);
|
||||
const r = validateKbFile(file);
|
||||
assert.equal(r.valid, true);
|
||||
assert.ok(!r.missing.includes('toc'));
|
||||
});
|
||||
|
||||
test('validateKbFile does NOT require a TOC for a small file', () => {
|
||||
const file = buildKbHeader(META) + '\n## Introduksjon\n\nKort innhold.\n';
|
||||
const r = validateKbFile(file);
|
||||
assert.equal(r.valid, true);
|
||||
assert.ok(!r.missing.includes('toc'));
|
||||
});
|
||||
|
||||
// --- ARCHITECTURE INVARIANT: transform.mjs is a pure lib, imports NO write-utils
|
||||
|
||||
test('transform.mjs imports NO write-utils (pure transformer, never writes)', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue