ms-ai-architect/tests/kb-update/test-transform.test.mjs

496 lines
22 KiB
JavaScript

// tests/kb-update/test-transform.test.mjs
// TDD for lag 4 — TRANSFORMASJON (doc → KB-fil). lib/transform.mjs is a pure
// helper library (mirrors decisions-io / verify-out): it builds the deterministic
// KB-file header, validates the mandatory-field contract, routes the file to its
// owning skill via the taxonomy (lag 0), and constructs the lag-5 `change` object.
// It NEVER writes — applying the file is the operator gate's job (commands/kb-update.md).
//
// Why Source lives in the HEADER block: build-registry / kb-headers.parseSourceHeader
// scan only the top 500 bytes, so a source buried in a bottom section (as the old
// prompt-template did) is invisible → the 0% authority_source coverage. Putting
// **Source:** in the header is what lets lag 3 backfill authority_source and lag-5
// rule 3 (authority mismatch) eventually fire.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
buildKbHeader,
validateKbFile,
buildChange,
resolveTargetPath,
buildToc,
composeKbFile,
insertToc,
stampVerifiedMeta,
} from '../../scripts/kb-update/lib/transform.mjs';
import {
parseSourceHeader,
parseTypeHeader,
parseVerifiedHeader,
parseVerifiedByHeader,
} from '../../scripts/kb-update/lib/kb-headers.mjs';
import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs';
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
// The N4 (TOC-in-large-files) check is the cross-subsystem best-practice contract.
// Fase 1c births files that satisfy it; we assert parity against the REAL check so
// a future drift in N4's definition fails this test rather than silently diverging.
import { checkN4 } from '../../scripts/kb-eval/eval.mjs';
// A body long enough that header+body exceeds N4's 100-line "large file" threshold,
// carrying real ## section headings the TOC must pick up.
const LARGE_BODY =
'## Introduksjon\n\nNoe innledning.\n\n' +
'## Kjernekomponenter\n\n' +
'Mye innhold ...\n'.repeat(110) +
'\n## Beslutningsveiledning\n\nVelg X når ...\n';
const SHORT_BODY = '## Introduksjon\n\nKort.\n';
const __dirname = dirname(fileURLToPath(import.meta.url));
// A born-verified reference meta: Spor 3 Port 1 contract adds type/verified/verified_by
// (reference files are "born verified" — stamped only after the v2 judge clears them).
const META = {
title: 'Azure AI Search - Hybrid Retrieval',
status: 'GA',
category: 'rag-architecture',
type: 'reference',
source: 'https://learn.microsoft.com/azure/search/hybrid-search-overview',
lastUpdated: '2026-06',
verified: '2026-06-29',
verified_by: 'judge-v2',
};
// --- buildKbHeader: mandatory fields, parseable by the real header scanners ----
test('buildKbHeader emits title, Last updated, Status, Source, Category in the header block', () => {
const h = buildKbHeader(META);
assert.match(h, /^# Azure AI Search - Hybrid Retrieval/m);
assert.match(h, /\*\*Last updated:\*\*\s*2026-06/);
assert.match(h, /\*\*Status:\*\*\s*GA/);
assert.match(h, /\*\*Category:\*\*\s*rag-architecture/);
assert.match(h, /\*\*Source:\*\*\s*https:\/\/learn\.microsoft\.com/);
});
test('buildKbHeader Source is within the top-500-byte region kb-headers scans', () => {
const h = buildKbHeader(META);
// The whole point: parseSourceHeader (top 500 bytes) must find it.
assert.equal(parseSourceHeader(h), META.source);
});
test('buildKbHeader throws when a mandatory field is missing (status-felt obligatorisk)', () => {
assert.throws(() => buildKbHeader({ ...META, status: undefined }), /status/i);
assert.throws(() => buildKbHeader({ ...META, source: '' }), /source/i);
assert.throws(() => buildKbHeader({ ...META, title: null }), /title/i);
assert.throws(() => buildKbHeader({ ...META, lastUpdated: undefined }), /last.?updated/i);
});
// --- Spor 3 Port 1/Port 2: type-aware create-guard contract --------------------
// Every reference file is "born verified": carries Type + Verified + Verified by in
// the header, all within the top-500-byte region the scanners read.
test('buildKbHeader emits Type, Verified, Verified by in the header (Port 1 contract)', () => {
const h = buildKbHeader(META);
assert.match(h, /\*\*Type:\*\*\s*reference/);
assert.match(h, /\*\*Verified:\*\*\s*2026-06-29/);
assert.match(h, /\*\*Verified by:\*\*\s*judge-v2/);
});
test('buildKbHeader: the new fields are parseable in the top-500-byte region', () => {
const h = buildKbHeader(META);
assert.equal(parseTypeHeader(h), 'reference');
assert.equal(parseVerifiedHeader(h), '2026-06-29');
assert.equal(parseVerifiedByHeader(h), 'judge-v2');
});
test('buildKbHeader: Verified by (the last header line) stays inside the 500-byte scan region for a realistic long title + URL', () => {
// Verified by sits after Source (the long field) — guard that a realistic worst case
// still fits, so a valid file is never wrongly flagged as missing verified_by.
const h = buildKbHeader({
title: 'Azure AI Foundry Model Router and Deployment Strategies for Enterprise',
status: 'Public Preview',
category: 'azure-ai-services',
type: 'reference',
source: 'https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/model-router-deployment-strategies-overview',
lastUpdated: '2026-06',
verified: '2026-06-29',
verified_by: 'judge-v2',
});
assert.equal(parseVerifiedByHeader(h), 'judge-v2', 'verified_by fell outside the 500-byte header region');
});
test('buildKbHeader: a reference file MUST carry verified + verified_by (born-verified guard)', () => {
assert.throws(() => buildKbHeader({ ...META, verified: undefined }), /verified/i);
assert.throws(() => buildKbHeader({ ...META, verified_by: '' }), /verified_by|verified by/i);
});
test('buildKbHeader: type defaults to reference when omitted (back-compat for callers)', () => {
const { type, ...noType } = META;
const h = buildKbHeader(noType);
assert.equal(parseTypeHeader(h), 'reference');
});
test('buildKbHeader: an unknown type is rejected', () => {
assert.throws(() => buildKbHeader({ ...META, type: 'blogpost' }), /type/i);
});
test('buildKbHeader: a non-reference file (methodology) needs NO source/verified', () => {
// Template/methodology/regulatory are out of correctness-scope — they carry no MS source.
const h = buildKbHeader({
title: 'ROS-metodikk (NS 5814)',
status: 'Stable',
category: 'norwegian-public-sector-ai-governance',
type: 'methodology',
lastUpdated: '2026-06',
});
assert.match(h, /\*\*Type:\*\*\s*methodology/);
assert.doesNotMatch(h, /\*\*Source:\*\*/);
assert.doesNotMatch(h, /\*\*Verified:\*\*/);
});
test('buildKbHeader: a non-reference file may NOT carry a Microsoft Learn source (contract)', () => {
assert.throws(
() => buildKbHeader({
title: 'X', status: 'Stable', category: 'c', type: 'template',
lastUpdated: '2026-06', source: 'https://learn.microsoft.com/azure/x',
}),
/source/i,
);
});
// --- validateKbFile: type-aware enforcement of the full contract ---------------
test('validateKbFile: a reference file missing verified/verified_by is invalid', () => {
// Header has Source but no Verified / Verified by → not born-verified → rejected.
const file =
'# T\n\n**Last updated:** 2026-06\n**Status:** GA\n**Type:** reference\n' +
'**Source:** https://learn.microsoft.com/x\n\n---\n\n## A\n\ntekst\n';
const r = validateKbFile(file);
assert.equal(r.valid, false);
assert.ok(r.missing.includes('verified'), `expected 'verified' in ${r.missing}`);
assert.ok(r.missing.includes('verified_by'), `expected 'verified_by' in ${r.missing}`);
});
test('validateKbFile: a complete born-verified reference file is valid', () => {
const file = buildKbHeader(META) + '\n## Introduksjon\n\nNoe innhold.\n';
assert.equal(validateKbFile(file).valid, true);
});
test('validateKbFile: a methodology file without source/verified is valid (out of scope)', () => {
const file =
'# ROS-metodikk\n\n**Last updated:** 2026-06\n**Status:** Stable\n**Type:** methodology\n\n' +
'---\n\n## Metode\n\nNS 5814.\n';
const r = validateKbFile(file);
assert.equal(r.valid, true, `unexpected missing: ${r.missing}`);
});
// --- stampVerifiedMeta: the born-verified gate (judge clears → stamp; else refuse) --
// transform.mjs never calls the judge (pure lib). The command runs the v2 claim judge,
// aggregates per-claim results into {pass}, and calls this. A failing verdict throws —
// "nytt innhold gjennom judgen FØR commit; ellers ingen fil." (§4 Port 2.)
test('stampVerifiedMeta stamps verified=today + verified_by=judge-vN on a passing verdict', () => {
const base = { title: 'T', status: 'GA', category: 'c',
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
const m = stampVerifiedMeta(base, { pass: true }, '2026-06-29');
assert.equal(m.type, 'reference');
assert.equal(m.verified, '2026-06-29');
assert.equal(m.verified_by, 'judge-v3.1'); // current GATE-PASS judge (v3.1 adopted G1)
});
test('stampVerifiedMeta honours an explicit judgeVersion', () => {
const base = { title: 'T', status: 'GA', category: 'c',
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
assert.equal(stampVerifiedMeta(base, { pass: true, judgeVersion: 3 }, '2026-06-29').verified_by, 'judge-v3');
});
test('stampVerifiedMeta supports a human verification path', () => {
const base = { title: 'T', status: 'GA', category: 'c',
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
assert.equal(stampVerifiedMeta(base, { pass: true, by: 'human' }, '2026-06-29').verified_by, 'human');
});
test('stampVerifiedMeta REFUSES to stamp when the judge verdict is not passing', () => {
const base = { title: 'T', status: 'GA', category: 'c',
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
assert.throws(() => stampVerifiedMeta(base, { pass: false }, '2026-06-29'), /judge|verdict|verified/i);
assert.throws(() => stampVerifiedMeta(base, null, '2026-06-29'), /judge|verdict|verified/i);
});
test('stampVerifiedMeta rejects an invalid today date', () => {
const base = { title: 'T', status: 'GA', category: 'c',
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
assert.throws(() => stampVerifiedMeta(base, { pass: true }, 'i går'), /date/i);
});
test('stampVerifiedMeta → composeKbFile is a complete born-verified file (end-to-end)', () => {
const base = { title: 'Azure AI Search', status: 'GA', category: 'rag-architecture',
source: 'https://learn.microsoft.com/azure/search/x', lastUpdated: '2026-06' };
const stamped = stampVerifiedMeta(base, { pass: true }, '2026-06-29');
const file = composeKbFile(stamped, '## Introduksjon\n\nKort.\n');
assert.equal(validateKbFile(file).valid, true);
assert.equal(parseVerifiedHeader(file), '2026-06-29');
assert.equal(parseVerifiedByHeader(file), 'judge-v3.1'); // default stamp parses back (dotted minor survives)
});
// --- validateKbFile: catches files missing the mandatory contract -------------
test('validateKbFile accepts a header-conformant file', () => {
const file = buildKbHeader(META) + '\n## Introduksjon\n\nNoe innhold.\n';
const r = validateKbFile(file);
assert.equal(r.valid, true);
assert.deepEqual(r.missing, []);
});
test('validateKbFile reports each missing mandatory field', () => {
// No Status, no Source, no Last updated, no title.
const r = validateKbFile('Bare brødtekst uten header.\n');
assert.equal(r.valid, false);
for (const f of ['title', 'last_updated', 'status', 'source']) {
assert.ok(r.missing.includes(f), `expected '${f}' in missing: ${r.missing}`);
}
});
test('validateKbFile flags a file with body-only source (the 0%-coverage failure mode)', () => {
// Source in a bottom section, NOT the header → invisible to top-500-byte scan.
const file =
'# Tittel\n\n**Last updated:** 2026-06\n**Status:** Preview\n\n---\n\n' +
'Mye innhold ...\n'.repeat(40) +
'\n## Kilder\n\n**Source:** https://learn.microsoft.com/x\n';
const r = validateKbFile(file);
assert.equal(r.valid, false);
assert.ok(r.missing.includes('source'));
});
// --- buildChange: the lag-4 → lag-5 contract ----------------------------------
test('buildChange produces a classifyChange-compatible object with safe defaults', () => {
const c = buildChange({
field: 'status',
oldValue: 'Preview',
newValue: 'GA',
sourceUrl: META.source,
});
assert.equal(c.field, 'status');
assert.equal(c.old_value, 'Preview');
assert.equal(c.new_value, 'GA');
assert.equal(c.source_url, META.source);
assert.equal(c.authority_source, null); // pre-lag-3 default
assert.deepEqual(c.refutations, []);
});
test('buildChange output flows through lag 5 — a status change is flagged, never auto-applied', () => {
const c = buildChange({ field: 'status', oldValue: 'Preview', newValue: 'GA', sourceUrl: META.source });
const verdict = classifyChange(c);
assert.equal(verdict.verdict, 'flagged');
assert.equal(verdict.status_claim, true);
});
test('buildChange output flows through lag 5 — a non-status change can auto-apply', () => {
const c = buildChange({
field: 'description',
oldValue: 'En kort beskrivelse.',
newValue: 'En litt lengre, oppdatert beskrivelse.',
sourceUrl: META.source,
});
assert.equal(classifyChange(c).verdict, 'auto-applied');
});
// --- resolveTargetPath: taxonomy-driven routing (replaces category-skill-map) --
test('resolveTargetPath routes via the taxonomy category_skill (disk-truth)', () => {
const tax = loadTaxonomy();
assert.equal(
resolveTargetPath(tax, 'rag-architecture', 'hybrid-retrieval.md'),
'skills/ms-ai-engineering/references/rag-architecture/hybrid-retrieval.md',
);
// a former divergence — taxonomy says advisor, not engineering
assert.equal(
resolveTargetPath(tax, 'prompt-engineering', 'few-shot.md'),
'skills/ms-ai-advisor/references/prompt-engineering/few-shot.md',
);
});
test('resolveTargetPath returns null for an unknown category (caller must gate)', () => {
const tax = loadTaxonomy();
assert.equal(resolveTargetPath(tax, 'does-not-exist', 'x.md'), null);
});
// --- buildToc: deterministic TOC from the body's ## section headings -----------
test('buildToc emits a ## Innhold heading with anchor links for each ## section', () => {
const toc = buildToc('## Introduksjon\n\ntekst\n\n## Kostnad og lisensiering\n\ntekst\n');
assert.match(toc, /^##\s+Innhold/m);
assert.match(toc, /- \[Introduksjon\]\(#introduksjon\)/);
assert.match(toc, /- \[Kostnad og lisensiering\]\(#kostnad-og-lisensiering\)/);
});
test('buildToc slugifies GitHub-style: lowercase, punctuation stripped, spaces→hyphens', () => {
const toc = buildToc('## 1. Introduksjon & Oversikt\n\ntekst\n');
assert.match(toc, /- \[1\. Introduksjon & Oversikt\]\(#1-introduksjon--oversikt\)/);
});
test('buildToc returns empty string when the body has no ## sections', () => {
assert.equal(buildToc('Bare brødtekst, ingen overskrifter.\n'), '');
assert.equal(buildToc('# Bare en H1\n\ntekst\n'), ''); // H1 is the header title, not a section
});
test('buildToc never lists the Innhold heading itself (idempotent re-runs)', () => {
const body = '## Innhold\n\n- [Introduksjon](#introduksjon)\n\n## Introduksjon\n\ntekst\n';
const toc = buildToc(body);
// exactly one anchor — to Introduksjon — never an anchor back to Innhold
assert.doesNotMatch(toc, /\(#innhold\)/);
assert.match(toc, /\(#introduksjon\)/);
});
// --- composeKbFile: the deterministic generator (header + TOC-if-large + body) --
test('composeKbFile births a large file with canonical header AND a TOC', () => {
const file = composeKbFile(META, LARGE_BODY);
assert.match(file, /^# Azure AI Search - Hybrid Retrieval/m); // canonical header first
assert.equal(parseSourceHeader(file), META.source); // Source in top-500-byte region
assert.match(file, /^##\s+Innhold/m); // TOC present
assert.equal(validateKbFile(file).valid, true);
});
test('composeKbFile output for a large file passes the REAL N4 check (anti-regression oracle)', () => {
const file = composeKbFile(META, LARGE_BODY);
const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: file }]);
assert.equal(n4.largeFiles, 1);
assert.equal(n4.withToc, 1);
assert.equal(n4.pass, true);
});
test('composeKbFile does NOT add a TOC to a small file (TOC is for large files only)', () => {
const file = composeKbFile(META, SHORT_BODY);
assert.doesNotMatch(file, /^##\s+Innhold/m);
assert.equal(validateKbFile(file).valid, true);
});
test('composeKbFile is idempotent: a body that already carries a TOC is not double-TOCd', () => {
const once = composeKbFile(META, LARGE_BODY);
const body = once.slice(once.indexOf('---\n') + 4); // strip header, keep TOC+body
const twice = composeKbFile(META, body);
assert.equal((twice.match(/^##\s+Innhold/gm) || []).length, 1);
assert.equal(validateKbFile(twice).valid, true);
});
// --- insertToc: backfill a TOC into an EXISTING large file (Fase 1b) -----------
// composeKbFile builds a NEW file from meta+body; insertToc retrofits the ~20 largest
// hand-written files on disk, whose legacy headers vary (some carry a `---` rule, some
// don't), without touching their bodies. It places `## Innhold` immediately before the
// first non-fenced `## ` section — the same relative slot composeKbFile uses.
// Large legacy file WITHOUT a `---` header rule (the secure-model-deployment shape).
const LEGACY_NO_RULE =
'# Secure Model Deployment\n\n' +
'**Kategori:** AI Security Engineering\n' +
'**Dato:** 2026-02-05\n\n' +
'## Introduksjon\n\n' + 'Brødtekst.\n'.repeat(110) +
'\n## Container Scanning\n\nMer tekst.\n' +
'\n## Avslutning\n\nSlutt.\n';
// Large legacy file WITH a `---` header rule (the ros-ai-threat-library shape).
const LEGACY_WITH_RULE =
'# AI-trusselbibliotek\n\n' +
'**Sist oppdatert:** 2026-06-18\n' +
'**Kategori:** Norwegian Public Sector AI Governance\n\n' +
'---\n\n' +
'## Oversikt\n\n' + 'Brødtekst.\n'.repeat(110) +
'\n## Metode\n\nMer tekst.\n';
test('insertToc adds a ## Innhold before the first ## section, body untouched (no `---` header)', () => {
const out = insertToc(LEGACY_NO_RULE);
assert.equal((out.match(/^##\s+Innhold/gm) || []).length, 1);
// TOC sits before the first content section …
assert.ok(out.indexOf('## Innhold') < out.indexOf('## Introduksjon'));
// … and lists every ## section in order
assert.match(out, /- \[Introduksjon\]\(#introduksjon\)/);
assert.match(out, /- \[Container Scanning\]\(#container-scanning\)/);
assert.match(out, /- \[Avslutning\]\(#avslutning\)/);
// body from the first section onward is byte-identical (zero churn)
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
assert.equal(tail(out), tail(LEGACY_NO_RULE));
});
test('insertToc places the TOC after a `---` header rule, before the first section', () => {
const out = insertToc(LEGACY_WITH_RULE);
assert.ok(out.indexOf('---') < out.indexOf('## Innhold'));
assert.ok(out.indexOf('## Innhold') < out.indexOf('## Oversikt'));
const tail = (s) => s.slice(s.indexOf('## Oversikt'));
assert.equal(tail(out), tail(LEGACY_WITH_RULE)); // body preserved
});
test('insertToc is a no-op on a small file (TOC is for large files only)', () => {
const small = '# T\n\n## A\n\ntekst\n\n## B\n\nmer\n';
assert.equal(insertToc(small), small);
});
test('insertToc is idempotent: a file that already carries a TOC is unchanged', () => {
const once = insertToc(LEGACY_NO_RULE);
const twice = insertToc(once);
assert.equal(twice, once);
assert.equal((once.match(/^##\s+Innhold/gm) || []).length, 1);
});
test('insertToc turns an N4-failing large file into an N4-passing one (real-check oracle)', () => {
const before = checkN4([{ path: 'skills/x/references/y/z.md', content: LEGACY_NO_RULE }]);
assert.equal(before.largeFiles, 1);
assert.equal(before.withToc, 0);
assert.equal(before.pass, false);
const after = checkN4([{ path: 'skills/x/references/y/z.md', content: insertToc(LEGACY_NO_RULE) }]);
assert.equal(after.withToc, 1);
assert.equal(after.pass, true);
});
test('insertToc is fence-aware: a ## inside a code fence is neither anchor nor listed', () => {
const fenced =
'# T\n\n**Kategori:** X\n\n' +
'```md\n## Not A Real Section\n```\n\n' +
'## Faktisk Seksjon\n\n' + 'Brødtekst.\n'.repeat(110) +
'\n## Andre Seksjon\n\nx.\n';
const out = insertToc(fenced);
assert.doesNotMatch(out, /\[Not A Real Section\]\(#/); // fenced heading not listed
assert.match(out, /- \[Faktisk Seksjon\]\(#faktisk-seksjon\)/);
assert.match(out, /- \[Andre Seksjon\]\(#andre-seksjon\)/);
// the fenced block stays above the TOC; TOC sits before the first real section
assert.ok(out.indexOf('## Not A Real Section') < out.indexOf('## Innhold'));
assert.ok(out.indexOf('## Innhold') < out.indexOf('## Faktisk Seksjon'));
});
// --- validateKbFile: large files must carry a TOC (the write-path regression gate)
test('validateKbFile flags a large file missing a TOC', () => {
const file = buildKbHeader(META) + '\n' + 'En linje med innhold.\n'.repeat(120);
const r = validateKbFile(file);
assert.equal(r.valid, false);
assert.ok(r.missing.includes('toc'), `expected 'toc' in missing: ${r.missing}`);
});
test('validateKbFile accepts a large file that carries a TOC', () => {
const file = composeKbFile(META, LARGE_BODY);
const r = validateKbFile(file);
assert.equal(r.valid, true);
assert.ok(!r.missing.includes('toc'));
});
test('validateKbFile does NOT require a TOC for a small file', () => {
const file = buildKbHeader(META) + '\n## Introduksjon\n\nKort innhold.\n';
const r = validateKbFile(file);
assert.equal(r.valid, true);
assert.ok(!r.missing.includes('toc'));
});
// --- ARCHITECTURE INVARIANT: transform.mjs is a pure lib, imports NO write-utils
test('transform.mjs imports NO write-utils (pure transformer, never writes)', () => {
const src = readFileSync(join(__dirname, '../../scripts/kb-update/lib/transform.mjs'), 'utf8');
const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n');
assert.doesNotMatch(importLines, /\bsaveDecisions\b/);
assert.doesNotMatch(importLines, /\bsaveRegistry\b/);
assert.doesNotMatch(importLines, /\bsaveReport\b/);
assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/);
assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/);
});