feat(ms-ai-architect): Spor 3 Port 2 — create-time-guard (born-verified contract) [skip-docs]
Type-aware create-guard for reference-filer (Spor 3 Port 1/Port 2): - kb-headers.mjs: parseTypeHeader/parseVerifiedHeader/parseVerifiedByHeader (samme top-500-byte bold-label-skann som parseSourceHeader). - transform.mjs: buildKbHeader er type-aware (reference krever source+verified+verified_by; non-reference kaster pa MS-Learn-source) + emitterer Type/Verified/Verified by; validateKbFile type-aware; stampVerifiedMeta = fodt-verifisert-gate (stempler KUN ved bestatt judge-verdikt, ellers kaster). - validate-kb-file.mjs (ny): kjorbar gate generatorer kaller for commit (exit != 0 ved kontraktbrudd). - generate-skills.md + kb-update.md + transform-prompt.md: wiret til fodt-verifisert kontrakt (header-felt + judge-steg + create-guard-gate). Suite 586/586 (33 nye + 1 invariant). Plugin-validering 239/0/0. Utenfor scope (flagget): generate-skills.sh legacy (sonnet/Cosmo/no-source), Cosmo-persona i generatorene (S-Cosmo), korpus-migrering av 306 filer (Spor 1).
This commit is contained in:
parent
0ded2f9413
commit
5a0ba1fc9e
10 changed files with 621 additions and 54 deletions
|
|
@ -19,7 +19,7 @@ import { readFileSync } from 'node:fs';
|
|||
import { dirname, join, basename } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { evalSkill } from '../../scripts/kb-eval/eval.mjs';
|
||||
import { composeKbFile, validateKbFile, resolveTargetPath } from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { composeKbFile, validateKbFile, resolveTargetPath, stampVerifiedMeta } from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -71,16 +71,20 @@ test('regenerate 1 real file → valid, dated, source-anchored, body preserved,
|
|||
const category = parts[3];
|
||||
const filename = basename(refRel);
|
||||
|
||||
const regenerated = composeKbFile({
|
||||
// Born-verified (Spor 3 Port 2): the regenerated file is stamped only after a passing
|
||||
// judge verdict. Here the verdict is supplied directly (no LLM in this test) — the
|
||||
// point is the regenerated file carries the full Port 1 contract (source + verified).
|
||||
const stamped = stampVerifiedMeta({
|
||||
title: 'AI Foundry Disaster Recovery Planning',
|
||||
status: 'GA',
|
||||
category,
|
||||
source: 'https://learn.microsoft.com/azure/ai-foundry/concepts/disaster-recovery',
|
||||
lastUpdated: '2026-06',
|
||||
}, body);
|
||||
}, { pass: true }, '2026-06-29');
|
||||
const regenerated = composeKbFile(stamped, body);
|
||||
|
||||
// Strictly better than the original: now valid (Source present + TOC for the large
|
||||
// body), still dated. validateKbFile requires a TOC on large files, so valid:true
|
||||
// Strictly better than the original: now valid (Source + Verified present + TOC for the
|
||||
// large body), still dated. validateKbFile requires a TOC on large files, so valid:true
|
||||
// here is also the Fase 1c anti-regression proof.
|
||||
const v = validateKbFile(regenerated);
|
||||
assert.equal(v.valid, true, `regenerated file invalid: ${v.missing}`);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,12 @@
|
|||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs';
|
||||
import {
|
||||
parseSourceHeader,
|
||||
parseTypeHeader,
|
||||
parseVerifiedHeader,
|
||||
parseVerifiedByHeader,
|
||||
} from '../../scripts/kb-update/lib/kb-headers.mjs';
|
||||
|
||||
test('parseSourceHeader — **Source:** header', () => {
|
||||
const content = '# Title\n\n**Source:** https://learn.microsoft.com/azure/foundry/concepts/x\n**Status:** GA\n';
|
||||
|
|
@ -50,3 +55,62 @@ test('parseSourceHeader — trims trailing whitespace from URL', () => {
|
|||
const content = '**Source:** https://learn.microsoft.com/azure/trim \n';
|
||||
assert.equal(parseSourceHeader(content), 'https://learn.microsoft.com/azure/trim');
|
||||
});
|
||||
|
||||
// --- parseTypeHeader: the Port 1 frontmatter-contract `type` field --------------
|
||||
// Spor 3 Port 1: every file carries type (reference|template|methodology|regulatory).
|
||||
// Same top-500-byte bold-label scan as parseSourceHeader (one source of truth).
|
||||
|
||||
test('parseTypeHeader — **Type:** header', () => {
|
||||
const content = '# T\n\n**Status:** GA\n**Type:** reference\n**Source:** https://learn.microsoft.com/x\n';
|
||||
assert.equal(parseTypeHeader(content), 'reference');
|
||||
});
|
||||
|
||||
test('parseTypeHeader — case-insensitive label, lowercased value', () => {
|
||||
assert.equal(parseTypeHeader('**type:** Methodology\n'), 'methodology');
|
||||
});
|
||||
|
||||
test('parseTypeHeader — no header returns null', () => {
|
||||
assert.equal(parseTypeHeader('# T\n\n**Status:** GA\n'), null);
|
||||
assert.equal(parseTypeHeader(''), null);
|
||||
assert.equal(parseTypeHeader(null), null);
|
||||
});
|
||||
|
||||
test('parseTypeHeader — only scans the top of the file (header region)', () => {
|
||||
const buried = '# T\n' + 'x'.repeat(800) + '\n**Type:** template\n';
|
||||
assert.equal(parseTypeHeader(buried), null);
|
||||
});
|
||||
|
||||
// --- parseVerifiedHeader: the Port 1 `verified` date (born-verified stamp) ------
|
||||
|
||||
test('parseVerifiedHeader — full date', () => {
|
||||
assert.equal(parseVerifiedHeader('**Verified:** 2026-06-29\n'), '2026-06-29');
|
||||
});
|
||||
|
||||
test('parseVerifiedHeader — YYYY-MM date', () => {
|
||||
assert.equal(parseVerifiedHeader('**Verified:** 2026-06\n'), '2026-06');
|
||||
});
|
||||
|
||||
test('parseVerifiedHeader — does NOT match the "**Verified by:**" line', () => {
|
||||
// "Verified by:" is a distinct label; the date parser must not swallow it.
|
||||
assert.equal(parseVerifiedHeader('**Verified by:** judge-v2\n'), null);
|
||||
});
|
||||
|
||||
test('parseVerifiedHeader — no header / garbage returns null', () => {
|
||||
assert.equal(parseVerifiedHeader('**Status:** GA\n'), null);
|
||||
assert.equal(parseVerifiedHeader(null), null);
|
||||
});
|
||||
|
||||
// --- parseVerifiedByHeader: the Port 1 `verified_by` field (judge-vN | human) ---
|
||||
|
||||
test('parseVerifiedByHeader — judge version', () => {
|
||||
assert.equal(parseVerifiedByHeader('**Verified by:** judge-v2\n'), 'judge-v2');
|
||||
});
|
||||
|
||||
test('parseVerifiedByHeader — human', () => {
|
||||
assert.equal(parseVerifiedByHeader('**Verified by:** human\n'), 'human');
|
||||
});
|
||||
|
||||
test('parseVerifiedByHeader — no header returns null', () => {
|
||||
assert.equal(parseVerifiedByHeader('**Verified:** 2026-06-29\n'), null);
|
||||
assert.equal(parseVerifiedByHeader(''), null);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -24,8 +24,14 @@ import {
|
|||
buildToc,
|
||||
composeKbFile,
|
||||
insertToc,
|
||||
stampVerifiedMeta,
|
||||
} from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs';
|
||||
import {
|
||||
parseSourceHeader,
|
||||
parseTypeHeader,
|
||||
parseVerifiedHeader,
|
||||
parseVerifiedByHeader,
|
||||
} from '../../scripts/kb-update/lib/kb-headers.mjs';
|
||||
import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs';
|
||||
import { loadTaxonomy } from '../../scripts/kb-update/lib/taxonomy.mjs';
|
||||
// The N4 (TOC-in-large-files) check is the cross-subsystem best-practice contract.
|
||||
|
|
@ -44,12 +50,17 @@ const SHORT_BODY = '## Introduksjon\n\nKort.\n';
|
|||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// A born-verified reference meta: Spor 3 Port 1 contract adds type/verified/verified_by
|
||||
// (reference files are "born verified" — stamped only after the v2 judge clears them).
|
||||
const META = {
|
||||
title: 'Azure AI Search - Hybrid Retrieval',
|
||||
status: 'GA',
|
||||
category: 'rag-architecture',
|
||||
type: 'reference',
|
||||
source: 'https://learn.microsoft.com/azure/search/hybrid-search-overview',
|
||||
lastUpdated: '2026-06',
|
||||
verified: '2026-06-29',
|
||||
verified_by: 'judge-v2',
|
||||
};
|
||||
|
||||
// --- buildKbHeader: mandatory fields, parseable by the real header scanners ----
|
||||
|
|
@ -76,6 +87,154 @@ test('buildKbHeader throws when a mandatory field is missing (status-felt obliga
|
|||
assert.throws(() => buildKbHeader({ ...META, lastUpdated: undefined }), /last.?updated/i);
|
||||
});
|
||||
|
||||
// --- Spor 3 Port 1/Port 2: type-aware create-guard contract --------------------
|
||||
// Every reference file is "born verified": carries Type + Verified + Verified by in
|
||||
// the header, all within the top-500-byte region the scanners read.
|
||||
|
||||
test('buildKbHeader emits Type, Verified, Verified by in the header (Port 1 contract)', () => {
|
||||
const h = buildKbHeader(META);
|
||||
assert.match(h, /\*\*Type:\*\*\s*reference/);
|
||||
assert.match(h, /\*\*Verified:\*\*\s*2026-06-29/);
|
||||
assert.match(h, /\*\*Verified by:\*\*\s*judge-v2/);
|
||||
});
|
||||
|
||||
test('buildKbHeader: the new fields are parseable in the top-500-byte region', () => {
|
||||
const h = buildKbHeader(META);
|
||||
assert.equal(parseTypeHeader(h), 'reference');
|
||||
assert.equal(parseVerifiedHeader(h), '2026-06-29');
|
||||
assert.equal(parseVerifiedByHeader(h), 'judge-v2');
|
||||
});
|
||||
|
||||
test('buildKbHeader: Verified by (the last header line) stays inside the 500-byte scan region for a realistic long title + URL', () => {
|
||||
// Verified by sits after Source (the long field) — guard that a realistic worst case
|
||||
// still fits, so a valid file is never wrongly flagged as missing verified_by.
|
||||
const h = buildKbHeader({
|
||||
title: 'Azure AI Foundry Model Router and Deployment Strategies for Enterprise',
|
||||
status: 'Public Preview',
|
||||
category: 'azure-ai-services',
|
||||
type: 'reference',
|
||||
source: 'https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/model-router-deployment-strategies-overview',
|
||||
lastUpdated: '2026-06',
|
||||
verified: '2026-06-29',
|
||||
verified_by: 'judge-v2',
|
||||
});
|
||||
assert.equal(parseVerifiedByHeader(h), 'judge-v2', 'verified_by fell outside the 500-byte header region');
|
||||
});
|
||||
|
||||
test('buildKbHeader: a reference file MUST carry verified + verified_by (born-verified guard)', () => {
|
||||
assert.throws(() => buildKbHeader({ ...META, verified: undefined }), /verified/i);
|
||||
assert.throws(() => buildKbHeader({ ...META, verified_by: '' }), /verified_by|verified by/i);
|
||||
});
|
||||
|
||||
test('buildKbHeader: type defaults to reference when omitted (back-compat for callers)', () => {
|
||||
const { type, ...noType } = META;
|
||||
const h = buildKbHeader(noType);
|
||||
assert.equal(parseTypeHeader(h), 'reference');
|
||||
});
|
||||
|
||||
test('buildKbHeader: an unknown type is rejected', () => {
|
||||
assert.throws(() => buildKbHeader({ ...META, type: 'blogpost' }), /type/i);
|
||||
});
|
||||
|
||||
test('buildKbHeader: a non-reference file (methodology) needs NO source/verified', () => {
|
||||
// Template/methodology/regulatory are out of correctness-scope — they carry no MS source.
|
||||
const h = buildKbHeader({
|
||||
title: 'ROS-metodikk (NS 5814)',
|
||||
status: 'Stable',
|
||||
category: 'norwegian-public-sector-ai-governance',
|
||||
type: 'methodology',
|
||||
lastUpdated: '2026-06',
|
||||
});
|
||||
assert.match(h, /\*\*Type:\*\*\s*methodology/);
|
||||
assert.doesNotMatch(h, /\*\*Source:\*\*/);
|
||||
assert.doesNotMatch(h, /\*\*Verified:\*\*/);
|
||||
});
|
||||
|
||||
test('buildKbHeader: a non-reference file may NOT carry a Microsoft Learn source (contract)', () => {
|
||||
assert.throws(
|
||||
() => buildKbHeader({
|
||||
title: 'X', status: 'Stable', category: 'c', type: 'template',
|
||||
lastUpdated: '2026-06', source: 'https://learn.microsoft.com/azure/x',
|
||||
}),
|
||||
/source/i,
|
||||
);
|
||||
});
|
||||
|
||||
// --- validateKbFile: type-aware enforcement of the full contract ---------------
|
||||
|
||||
test('validateKbFile: a reference file missing verified/verified_by is invalid', () => {
|
||||
// Header has Source but no Verified / Verified by → not born-verified → rejected.
|
||||
const file =
|
||||
'# T\n\n**Last updated:** 2026-06\n**Status:** GA\n**Type:** reference\n' +
|
||||
'**Source:** https://learn.microsoft.com/x\n\n---\n\n## A\n\ntekst\n';
|
||||
const r = validateKbFile(file);
|
||||
assert.equal(r.valid, false);
|
||||
assert.ok(r.missing.includes('verified'), `expected 'verified' in ${r.missing}`);
|
||||
assert.ok(r.missing.includes('verified_by'), `expected 'verified_by' in ${r.missing}`);
|
||||
});
|
||||
|
||||
test('validateKbFile: a complete born-verified reference file is valid', () => {
|
||||
const file = buildKbHeader(META) + '\n## Introduksjon\n\nNoe innhold.\n';
|
||||
assert.equal(validateKbFile(file).valid, true);
|
||||
});
|
||||
|
||||
test('validateKbFile: a methodology file without source/verified is valid (out of scope)', () => {
|
||||
const file =
|
||||
'# ROS-metodikk\n\n**Last updated:** 2026-06\n**Status:** Stable\n**Type:** methodology\n\n' +
|
||||
'---\n\n## Metode\n\nNS 5814.\n';
|
||||
const r = validateKbFile(file);
|
||||
assert.equal(r.valid, true, `unexpected missing: ${r.missing}`);
|
||||
});
|
||||
|
||||
// --- stampVerifiedMeta: the born-verified gate (judge clears → stamp; else refuse) --
|
||||
// transform.mjs never calls the judge (pure lib). The command runs the v2 claim judge,
|
||||
// aggregates per-claim results into {pass}, and calls this. A failing verdict throws —
|
||||
// "nytt innhold gjennom judgen FØR commit; ellers ingen fil." (§4 Port 2.)
|
||||
|
||||
test('stampVerifiedMeta stamps verified=today + verified_by=judge-vN on a passing verdict', () => {
|
||||
const base = { title: 'T', status: 'GA', category: 'c',
|
||||
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
|
||||
const m = stampVerifiedMeta(base, { pass: true }, '2026-06-29');
|
||||
assert.equal(m.type, 'reference');
|
||||
assert.equal(m.verified, '2026-06-29');
|
||||
assert.equal(m.verified_by, 'judge-v2'); // current GATE-PASS judge
|
||||
});
|
||||
|
||||
test('stampVerifiedMeta honours an explicit judgeVersion', () => {
|
||||
const base = { title: 'T', status: 'GA', category: 'c',
|
||||
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
|
||||
assert.equal(stampVerifiedMeta(base, { pass: true, judgeVersion: 3 }, '2026-06-29').verified_by, 'judge-v3');
|
||||
});
|
||||
|
||||
test('stampVerifiedMeta supports a human verification path', () => {
|
||||
const base = { title: 'T', status: 'GA', category: 'c',
|
||||
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
|
||||
assert.equal(stampVerifiedMeta(base, { pass: true, by: 'human' }, '2026-06-29').verified_by, 'human');
|
||||
});
|
||||
|
||||
test('stampVerifiedMeta REFUSES to stamp when the judge verdict is not passing', () => {
|
||||
const base = { title: 'T', status: 'GA', category: 'c',
|
||||
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
|
||||
assert.throws(() => stampVerifiedMeta(base, { pass: false }, '2026-06-29'), /judge|verdict|verified/i);
|
||||
assert.throws(() => stampVerifiedMeta(base, null, '2026-06-29'), /judge|verdict|verified/i);
|
||||
});
|
||||
|
||||
test('stampVerifiedMeta rejects an invalid today date', () => {
|
||||
const base = { title: 'T', status: 'GA', category: 'c',
|
||||
source: 'https://learn.microsoft.com/x', lastUpdated: '2026-06' };
|
||||
assert.throws(() => stampVerifiedMeta(base, { pass: true }, 'i går'), /date/i);
|
||||
});
|
||||
|
||||
test('stampVerifiedMeta → composeKbFile is a complete born-verified file (end-to-end)', () => {
|
||||
const base = { title: 'Azure AI Search', status: 'GA', category: 'rag-architecture',
|
||||
source: 'https://learn.microsoft.com/azure/search/x', lastUpdated: '2026-06' };
|
||||
const stamped = stampVerifiedMeta(base, { pass: true }, '2026-06-29');
|
||||
const file = composeKbFile(stamped, '## Introduksjon\n\nKort.\n');
|
||||
assert.equal(validateKbFile(file).valid, true);
|
||||
assert.equal(parseVerifiedHeader(file), '2026-06-29');
|
||||
assert.equal(parseVerifiedByHeader(file), 'judge-v2');
|
||||
});
|
||||
|
||||
// --- validateKbFile: catches files missing the mandatory contract -------------
|
||||
|
||||
test('validateKbFile accepts a header-conformant file', () => {
|
||||
|
|
|
|||
80
tests/kb-update/test-validate-kb-file.test.mjs
Normal file
80
tests/kb-update/test-validate-kb-file.test.mjs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
// tests/kb-update/test-validate-kb-file.test.mjs
|
||||
// The executable create-guard (Spor 3 Port 2): scripts/kb-update/validate-kb-file.mjs
|
||||
// is the gate any generator calls AFTER writing a candidate file and BEFORE committing.
|
||||
// It wraps transform.validateKbFile over one or more paths and exits non-zero if any
|
||||
// file violates the contract (no source / not born-verified / large file without a TOC).
|
||||
//
|
||||
// validatePaths is a pure function with an injectable reader, so these tests never
|
||||
// touch disk — they feed synthetic content and assert the verdict + ok flag.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { validatePaths } from '../../scripts/kb-update/validate-kb-file.mjs';
|
||||
import { buildKbHeader } from '../../scripts/kb-update/lib/transform.mjs';
|
||||
|
||||
const GOOD = buildKbHeader({
|
||||
title: 'Azure AI Search - Hybrid Retrieval',
|
||||
status: 'GA',
|
||||
category: 'rag-architecture',
|
||||
type: 'reference',
|
||||
source: 'https://learn.microsoft.com/azure/search/hybrid-search-overview',
|
||||
lastUpdated: '2026-06',
|
||||
verified: '2026-06-29',
|
||||
verified_by: 'judge-v2',
|
||||
}) + '\n## Introduksjon\n\nNoe innhold.\n';
|
||||
|
||||
// A reference file with a Source but NOT born-verified (no Verified / Verified by).
|
||||
const NOT_VERIFIED =
|
||||
'# T\n\n**Last updated:** 2026-06\n**Status:** GA\n**Type:** reference\n' +
|
||||
'**Source:** https://learn.microsoft.com/x\n\n---\n\n## A\n\ntekst\n';
|
||||
|
||||
// A reference file with no Source header at all (the 0%-coverage failure mode).
|
||||
const NO_SOURCE = '# T\n\n**Last updated:** 2026-06\n**Status:** GA\n\n---\n\n## A\n\ntekst\n';
|
||||
|
||||
function reader(map) {
|
||||
return (p) => {
|
||||
if (!(p in map)) throw new Error(`unexpected read: ${p}`);
|
||||
return map[p];
|
||||
};
|
||||
}
|
||||
|
||||
test('validatePaths — all files valid → ok:true', () => {
|
||||
const r = validatePaths(['a.md', 'b.md'], reader({ 'a.md': GOOD, 'b.md': GOOD }));
|
||||
assert.equal(r.ok, true);
|
||||
assert.equal(r.results.length, 2);
|
||||
assert.ok(r.results.every((x) => x.valid));
|
||||
});
|
||||
|
||||
test('validatePaths — a file missing source fails the gate', () => {
|
||||
const r = validatePaths(['bad.md'], reader({ 'bad.md': NO_SOURCE }));
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.results[0].valid, false);
|
||||
assert.ok(r.results[0].missing.includes('source'));
|
||||
});
|
||||
|
||||
test('validatePaths — a reference file that is not born-verified fails the gate', () => {
|
||||
const r = validatePaths(['nv.md'], reader({ 'nv.md': NOT_VERIFIED }));
|
||||
assert.equal(r.ok, false);
|
||||
assert.ok(r.results[0].missing.includes('verified'));
|
||||
assert.ok(r.results[0].missing.includes('verified_by'));
|
||||
});
|
||||
|
||||
test('validatePaths — one bad file in a batch fails the whole gate', () => {
|
||||
const r = validatePaths(['ok.md', 'bad.md'], reader({ 'ok.md': GOOD, 'bad.md': NO_SOURCE }));
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.results.find((x) => x.path === 'ok.md').valid, true);
|
||||
assert.equal(r.results.find((x) => x.path === 'bad.md').valid, false);
|
||||
});
|
||||
|
||||
test('validatePaths — a path that cannot be read is reported as invalid, not thrown', () => {
|
||||
const r = validatePaths(['missing.md'], () => { throw new Error('ENOENT'); });
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.results[0].valid, false);
|
||||
assert.ok(r.results[0].missing.some((m) => /read|enoent/i.test(m)));
|
||||
});
|
||||
|
||||
test('validatePaths — no paths → ok:true (nothing to gate)', () => {
|
||||
const r = validatePaths([], reader({}));
|
||||
assert.equal(r.ok, true);
|
||||
assert.deepEqual(r.results, []);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue