From cec98b5cb5b98c1aa51da689f7f25e20655992a0 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Sat, 4 Jul 2026 09:20:32 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20Spor=201=20=E2=80=94?= =?UTF-8?q?=20unified=20migrate-corpus=20applier=20(backup+atomisk+advisor?= =?UTF-8?q?-fence+per-felt-idempotent,=20TDD)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/kb-update/migrate-corpus.mjs | 231 ++++++++++++++++ tests/kb-update/test-migrate-apply.test.mjs | 281 ++++++++++++++++++++ 2 files changed, 512 insertions(+) create mode 100644 scripts/kb-update/migrate-corpus.mjs create mode 100644 tests/kb-update/test-migrate-apply.test.mjs diff --git a/scripts/kb-update/migrate-corpus.mjs b/scripts/kb-update/migrate-corpus.mjs new file mode 100644 index 0000000..5190ab1 --- /dev/null +++ b/scripts/kb-update/migrate-corpus.mjs @@ -0,0 +1,231 @@ +#!/usr/bin/env node +// migrate-corpus.mjs — the unified Spor 1 migration applier (Step 7). +// +// Stamps the Port-1 header contract (**Type:** + **Source:**) and a `## Innhold` TOC +// onto the ~327 NON-advisor reference files, reading its plan from the enriched +// ref-type manifest (scripts/kb-update/data/ref-type-manifest.json). It is the single +// write-path for the corpus mutation — deterministic, atomic, advisor-fenced, +// per-field idempotent, with a pre-write backup and a post-write parse assertion. +// +// ARCHITECTURE: pure-core / IO-shell split (mirrors validate-kb-file.mjs). The decision +// is `planFileMutation` (no IO); the shell `migrateCorpus` walks the corpus, takes ONE +// backup before the first write, writes atomically, and re-reads each file to assert the +// header survived the 500-byte scan window. The pure primitives it composes +// (normalizeStaleVerified → insertHeaderFields → insertToc) are already idempotent and +// 500B-aware; this file adds no header logic of its own. +// +// Hard rules (plan Step 7): +// - advisor allowlist: only the 4 non-advisor skills are walked; a target path that +// matches /ms-ai-advisor/ is a hard throw (belt-and-suspenders on the pure fn). +// - atomic: atomicWriteSync only, never the raw fs writer. +// - backup: backupDir(root, backupRoot) before the first write; restore() retained. +// - per-field idempotent: delegated to insertHeaderFields (skips set Type/Source) and +// insertToc (skips a file that already hasToc / is small). +// - post-write assertion: re-read each written file, assert parseTypeHeader (and, when a +// Source was intended, parseSourceHeader) non-null — catches 500B truncation. On +// failure the backup is restored and the run throws (no partial corpus is left). +// +// Usage: +// node scripts/kb-update/migrate-corpus.mjs # dry-run report (default) +// node scripts/kb-update/migrate-corpus.mjs --write # apply the migration +// node scripts/kb-update/migrate-corpus.mjs --json # machine-readable report +// Exit code: 0 = ok; 2 = usage error. + +import { readFileSync, readdirSync, existsSync, realpathSync } from 'node:fs'; +import { join, dirname, relative, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + insertHeaderFields, + normalizeStaleVerified, + insertToc, +} from './lib/transform.mjs'; +import { parseTypeHeader, parseSourceHeader } from './lib/kb-headers.mjs'; +import { atomicWriteSync } from './lib/atomic-write.mjs'; +import { backupDir } from './lib/backup.mjs'; + +// The 4 non-advisor skills, enumerated explicitly (NO shell brace-expansion). Advisor +// (ms-ai-advisor) is deliberately absent — it is classified in the manifest but never +// mutated (S-Cosmo). +const NON_ADVISOR_SKILLS = [ + 'ms-ai-engineering', + 'ms-ai-governance', + 'ms-ai-infrastructure', + 'ms-ai-security', +]; + +const ADVISOR_RE = /(^|\/)ms-ai-advisor(\/|$)/; + +const PLUGIN_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const DEFAULT_SKILLS_DIR = join(PLUGIN_ROOT, 'skills'); +const DEFAULT_BACKUP_ROOT = join(PLUGIN_ROOT, '.kb-backup'); +const DEFAULT_MANIFEST_PATH = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'data', 'ref-type-manifest.json'); + +/** + * Decide the mutation for a single corpus file — pure, no IO. Composes the Session-2 + * primitives in the mandated order: strip a stale header `**Verified:** MCP` → + * insert `**Type:**` (+ `**Source:**` for a sourced reference) → insert a `## Innhold` + * TOC on a large file. Each primitive is individually idempotent, so re-running plans + * an unchanged file as a no-op. + * + * The manifest entry drives Type/Source: a `reference` with a `source` (and not + * `deferred`) gets Type+Source; a deferred or non-reference entry gets Type only + * (insertHeaderFields never emits a Source outside the reference-with-source case). + * + * @param {string} relpath — canonical `skills//references/…` path (advisor guard) + * @param {string} content — the file's current bytes + * @param {object|undefined} manifestEntry — the ref-type manifest row for this file + * @returns {{relpath: string, changed: boolean, content: string, actions: string[], + * expectType: boolean, expectSource: boolean, skipReason: string|null}} + * @throws if relpath is under ms-ai-advisor (advisor fence) + */ +export function planFileMutation(relpath, content, manifestEntry) { + if (ADVISOR_RE.test(relpath)) { + throw new Error(`planFileMutation: refusing to mutate an ms-ai-advisor file: ${relpath}`); + } + const original = String(content ?? ''); + if (!manifestEntry) { + return { relpath, changed: false, content: original, actions: [], expectType: false, expectSource: false, skipReason: 'no-manifest-entry' }; + } + + const type = manifestEntry.type ?? 'reference'; + const isReference = type === 'reference'; + // Deferred entries (advisor-scope / no-ms-citation) carry no defensible authority — Type only. + const source = manifestEntry.deferred === true ? undefined : manifestEntry.source; + const hasSource = source !== undefined && source !== null && String(source).trim() !== ''; + + const actions = []; + let out = normalizeStaleVerified(original); + if (out !== original) actions.push('normalize-verified'); + + const afterNormalize = out; + out = insertHeaderFields(out, { type, source }); + if (out !== afterNormalize) actions.push(isReference && hasSource ? 'type+source' : 'type'); + + const afterHeader = out; + out = insertToc(out); + if (out !== afterHeader) actions.push('toc'); + + return { + relpath, + changed: out !== original, + content: out, + actions, + expectType: true, // every migrated file must carry a readable Type + expectSource: isReference && hasSource, // …and a readable Source when one was intended + skipReason: null, + }; +} + +/** Recursively collect absolute paths of `*.md` files under `dir` (missing dir → []). */ +function walkMarkdown(dir) { + if (!existsSync(dir)) return []; + const out = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) out.push(...walkMarkdown(full)); + else if (entry.isFile() && entry.name.endsWith('.md')) out.push(full); + } + return out; +} + +/** Canonical `skills/…`-rooted manifest key for a file under the skills dir. */ +function canonicalRelpath(skillsDir, fullPath) { + return `skills/${relative(skillsDir, fullPath).split(sep).join('/')}`; +} + +/** + * Walk the 4 non-advisor skills under `root`, plan each file against the manifest, and + * (when `write`) apply the changed ones atomically behind a single pre-write backup with + * a post-write parse assertion. Dry-run by default. + * + * @param {{root: string, backupRoot: string, manifest: object, write?: boolean}} opts + * @returns {{root: string, write: boolean, backupPath: string|null, restore: (()=>void)|null, + * results: Array, counts: {total:number, changed:number, written:number, skipped:number}}} + */ +export function migrateCorpus({ root, backupRoot, manifest, write = false }) { + const skillsDir = root; + const worklist = []; + for (const skill of NON_ADVISOR_SKILLS) { + const refDir = join(skillsDir, skill, 'references'); + for (const full of walkMarkdown(refDir)) { + const relpath = canonicalRelpath(skillsDir, full); + worklist.push({ full, relpath, content: readFileSync(full, 'utf8') }); + } + } + + const results = worklist.map((w) => { + const plan = planFileMutation(w.relpath, w.content, manifest[w.relpath]); + return { ...plan, full: w.full, wrote: false }; + }); + + const changed = results.filter((r) => r.changed); + const counts = { + total: results.length, + changed: changed.length, + written: 0, + skipped: results.filter((r) => r.skipReason).length, + }; + + if (!write || changed.length === 0) { + return { root, write, backupPath: null, restore: null, results, counts }; + } + + // ONE backup before the first write — restore() undoes the whole batch. + const { backupPath, restore } = backupDir(skillsDir, backupRoot); + + for (const r of changed) { + atomicWriteSync(r.full, r.content); + const written = readFileSync(r.full, 'utf8'); + const typeOk = !r.expectType || parseTypeHeader(written) !== null; + const sourceOk = !r.expectSource || parseSourceHeader(written) !== null; + if (!typeOk || !sourceOk) { + restore(); // roll the whole batch back — never leave a partial corpus + throw new Error( + `migrateCorpus: post-write header truncated for ${r.relpath} ` + + `(inserted field fell past the ${500}-byte scan window; corpus restored from backup)`, + ); + } + r.wrote = true; + counts.written += 1; + } + + return { root, write, backupPath, restore, results, counts }; +} + +function main(argv) { + const args = argv.slice(2); + const write = args.includes('--write'); + const json = args.includes('--json'); + const unknown = args.filter((a) => !['--write', '--json'].includes(a)); + if (unknown.length) { + process.stderr.write(`usage: migrate-corpus.mjs [--write] [--json]\nunknown arg(s): ${unknown.join(' ')}\n`); + process.exit(2); + } + if (!existsSync(DEFAULT_MANIFEST_PATH)) { + process.stderr.write(`migrate-corpus: manifest not found at ${DEFAULT_MANIFEST_PATH}\nRun classify-ref-type.mjs --write first.\n`); + process.exit(2); + } + const manifest = JSON.parse(readFileSync(DEFAULT_MANIFEST_PATH, 'utf8')); + const report = migrateCorpus({ root: DEFAULT_SKILLS_DIR, backupRoot: DEFAULT_BACKUP_ROOT, manifest, write }); + + if (json) { + const { restore, results, ...rest } = report; + process.stdout.write(JSON.stringify({ ...rest, results: results.map(({ full, content, ...r }) => r) }, null, 2) + '\n'); + } else { + const verb = write ? 'wrote' : 'would change'; + process.stdout.write( + `migrate-corpus (${write ? 'APPLY' : 'dry-run'}): ${verb} ${report.counts.written || report.counts.changed}/${report.counts.total} files` + + `${report.backupPath ? `; backup: ${report.backupPath}` : ''}\n`, + ); + } + process.exit(0); +} + +const isMain = (() => { + try { + return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); + } catch { + return false; + } +})(); +if (isMain) main(process.argv); diff --git a/tests/kb-update/test-migrate-apply.test.mjs b/tests/kb-update/test-migrate-apply.test.mjs new file mode 100644 index 0000000..0147258 --- /dev/null +++ b/tests/kb-update/test-migrate-apply.test.mjs @@ -0,0 +1,281 @@ +// tests/kb-update/test-migrate-apply.test.mjs +// Integration tests for scripts/kb-update/migrate-corpus.mjs — the unified Spor 1 +// applier (Step 7). Hermetic: every case runs against a fresh mkdtemp corpus via the +// injected `--root`/`backupRoot`, so nothing ever writes to the real skills/ tree. +// +// Covers the Step-7 acceptance set: idempotency, advisor-exclusion, throw-guard, +// atomic (no *.tmp.* leftover + import-grep), backup+restore round-trip, dry-run +// default byte-identity, body-after-`---` identity + one `## Innhold`, post-write +// parse assertion (500B truncation), and normalize→insert ordering. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + mkdtempSync, + mkdirSync, + rmSync, + writeFileSync, + readFileSync, + readdirSync, + existsSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + planFileMutation, + migrateCorpus, +} from '../../scripts/kb-update/migrate-corpus.mjs'; +import { + parseTypeHeader, + parseSourceHeader, + parseVerifiedHeader, +} from '../../scripts/kb-update/lib/kb-headers.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SRC = join(HERE, '..', '..', 'scripts', 'kb-update', 'migrate-corpus.mjs'); + +function withTmp(fn) { + const dir = mkdtempSync(join(tmpdir(), 'migrate-test-')); + try { + return fn(dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Materialize a corpus under `root` from a {relpath: content} map. */ +function makeCorpus(root, files) { + for (const [rel, content] of Object.entries(files)) { + const path = join(root, rel); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, 'utf8'); + } +} + +/** Read every file under `root` into a {relpath: content} map (posix keys). */ +function readAll(root) { + const out = {}; + function walk(dir, prefix) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) walk(join(dir, entry.name), rel); + else if (entry.isFile()) out[rel] = readFileSync(join(dir, entry.name), 'utf8'); + } + } + walk(root, ''); + return out; +} + +/** Canonical `skills/…` manifest key for a corpus-relative path. */ +const key = (rel) => `skills/${rel}`; + +const REF_SMALL = `# Azure AI Foundry Overview + +**Category:** Engineering +**Status:** Active + +--- + +## Overview + +Foundry is a platform. + +## Details + +More detail here. +`; + +const LARGE_SECTIONS = Array.from({ length: 8 }, (_, i) => + `## Section ${i + 1}\n\n${'body paragraph line.\n'.repeat(14)}`, +).join('\n'); +const REF_LARGE = `# Big Reference + +**Category:** Engineering +**Status:** Active + +--- + +${LARGE_SECTIONS}`; + +const REF_STALE_VERIFIED = `# Stale Verified File + +**Category:** Engineering +**Status:** Active +**Verified:** MCP 2026-06 + +--- + +## Overview + +Body. +`; + +test('planFileMutation — reference entry stamps Type + Source; body after --- byte-identical', () => { + const entry = { type: 'reference', source: 'https://learn.microsoft.com/azure/ai-foundry' }; + const plan = planFileMutation(key('ms-ai-engineering/references/a.md'), REF_SMALL, entry); + assert.equal(plan.changed, true); + assert.equal(parseTypeHeader(plan.content), 'reference'); + assert.equal(parseSourceHeader(plan.content), entry.source); + // Body from the first `---` rule onward is untouched (header-only insertion; small file → no TOC). + const marker = '\n---\n'; + assert.equal(plan.content.slice(plan.content.indexOf(marker)), REF_SMALL.slice(REF_SMALL.indexOf(marker))); +}); + +test('planFileMutation — deferred reference (no source) gets Type only, never Source', () => { + const entry = { type: 'reference', deferred: true, deferReason: 'no-ms-citation' }; + const plan = planFileMutation(key('ms-ai-governance/references/d.md'), REF_SMALL, entry); + assert.equal(parseTypeHeader(plan.content), 'reference'); + assert.equal(parseSourceHeader(plan.content), null); + assert.equal(plan.expectSource, false); +}); + +test('planFileMutation — throw-guard fires on an advisor target path', () => { + const entry = { type: 'reference', source: 'https://learn.microsoft.com/x' }; + assert.throws( + () => planFileMutation('skills/ms-ai-advisor/references/architecture/cost-models.md', REF_SMALL, entry), + /advisor/i, + ); +}); + +test('planFileMutation — normalize then insert applied in order on a **Verified:** MCP fixture', () => { + const entry = { type: 'reference', source: 'https://learn.microsoft.com/azure/x' }; + const plan = planFileMutation(key('ms-ai-security/references/v.md'), REF_STALE_VERIFIED, entry); + // Stale header Verified stripped (normalize ran) AND Type inserted (insert ran). + assert.equal(parseVerifiedHeader(plan.content), null); + assert.equal(parseTypeHeader(plan.content), 'reference'); + assert.equal(parseSourceHeader(plan.content), entry.source); +}); + +test('planFileMutation — already-migrated content is a no-op (idempotent)', () => { + const entry = { type: 'reference', source: 'https://learn.microsoft.com/azure/ai-foundry' }; + const once = planFileMutation(key('ms-ai-engineering/references/a.md'), REF_SMALL, entry); + const twice = planFileMutation(key('ms-ai-engineering/references/a.md'), once.content, entry); + assert.equal(twice.changed, false); + assert.equal(twice.content, once.content); +}); + +test('migrateCorpus — dry-run (default) leaves the corpus byte-identical on disk', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { 'ms-ai-engineering/references/a.md': REF_SMALL }); + const before = readAll(root); + const manifest = { [key('ms-ai-engineering/references/a.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' } }; + const report = migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: false }); + assert.equal(report.write, false); + assert.ok(report.counts.changed > 0, 'dry-run still reports the pending change'); + assert.equal(report.backupPath, null); + assert.deepEqual(readAll(root), before); + }); +}); + +test('migrateCorpus — --write stamps non-advisor files and leaves advisor byte-identical', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { + 'ms-ai-engineering/references/a.md': REF_SMALL, + 'ms-ai-advisor/references/architecture/cost-models.md': REF_SMALL, + }); + const advisorBefore = readFileSync(join(root, 'ms-ai-advisor/references/architecture/cost-models.md'), 'utf8'); + const manifest = { + [key('ms-ai-engineering/references/a.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' }, + // Advisor entry is present in the real manifest (deferred), but the walker must never reach it. + [key('ms-ai-advisor/references/architecture/cost-models.md')]: { type: 'reference', deferred: true, deferReason: 'advisor-scope' }, + }; + const report = migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }); + assert.equal(report.counts.written, 1); + // Non-advisor mutated. + const eng = readFileSync(join(root, 'ms-ai-engineering/references/a.md'), 'utf8'); + assert.equal(parseTypeHeader(eng), 'reference'); + assert.equal(parseSourceHeader(eng), 'https://learn.microsoft.com/x'); + // Advisor untouched — byte-identical. + assert.equal(readFileSync(join(root, 'ms-ai-advisor/references/architecture/cost-models.md'), 'utf8'), advisorBefore); + }); +}); + +test('migrateCorpus — second --write run writes 0 files (idempotent on disk)', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { 'ms-ai-engineering/references/a.md': REF_SMALL }); + const manifest = { [key('ms-ai-engineering/references/a.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' } }; + const opts = { root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }; + const first = migrateCorpus(opts); + assert.equal(first.counts.written, 1); + const second = migrateCorpus(opts); + assert.equal(second.counts.written, 0); + assert.equal(second.counts.changed, 0); + }); +}); + +test('migrateCorpus — atomic write leaves no *.tmp.* leftover', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { 'ms-ai-engineering/references/a.md': REF_SMALL }); + const manifest = { [key('ms-ai-engineering/references/a.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' } }; + migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }); + const names = Object.keys(readAll(root)); + assert.equal(names.some((n) => /\.tmp\./.test(n)), false, 'no atomic-write temp file survived'); + }); +}); + +test('migrate-corpus source — uses atomicWriteSync, never bare writeFileSync', () => { + const src = readFileSync(SRC, 'utf8'); + assert.match(src, /atomicWriteSync/, 'imports/uses the atomic writer'); + assert.doesNotMatch(src, /\bwriteFileSync\b/, 'never calls writeFileSync directly for corpus writes'); + assert.match(src, /ms-ai-advisor/, 'carries the advisor fence'); +}); + +test('migrateCorpus — backup exists and restore() round-trips byte-identity', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { + 'ms-ai-engineering/references/a.md': REF_SMALL, + 'ms-ai-security/references/b.md': REF_STALE_VERIFIED, + }); + const original = readAll(root); + const manifest = { + [key('ms-ai-engineering/references/a.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' }, + [key('ms-ai-security/references/b.md')]: { type: 'reference', source: 'https://learn.microsoft.com/y' }, + }; + const report = migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }); + assert.ok(report.backupPath && existsSync(report.backupPath), 'backup snapshot exists'); + assert.equal(typeof report.restore, 'function'); + // Corpus is mutated now… + assert.notDeepEqual(readAll(root), original); + // …restore brings it back byte-identical. + report.restore(); + assert.deepEqual(readAll(root), original); + }); +}); + +test('migrateCorpus — large file gains exactly one ## Innhold, sections preserved', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + makeCorpus(root, { 'ms-ai-engineering/references/big.md': REF_LARGE }); + const manifest = { [key('ms-ai-engineering/references/big.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' } }; + migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }); + const out = readFileSync(join(root, 'ms-ai-engineering/references/big.md'), 'utf8'); + assert.equal((out.match(/^## Innhold$/gm) || []).length, 1, 'exactly one TOC heading'); + assert.equal(parseTypeHeader(out), 'reference'); + // All original sections survive. + for (let i = 1; i <= 8; i++) assert.match(out, new RegExp(`^## Section ${i}$`, 'm')); + }); +}); + +test('migrateCorpus — post-write parse assertion catches a deep-preamble (>500B) file', () => { + withTmp((tmp) => { + const root = join(tmp, 'skills'); + // A title long enough that the inserted **Type:** line lands past the 500-byte + // scan window → parseTypeHeader returns null on re-read → the applier must throw. + const deep = `# ${'A'.repeat(490)}\n\n**Category:** Engineering\n\n## Section\n\nBody.\n`; + makeCorpus(root, { 'ms-ai-engineering/references/deep.md': deep }); + const original = readAll(root); + const manifest = { [key('ms-ai-engineering/references/deep.md')]: { type: 'reference', source: 'https://learn.microsoft.com/x' } }; + assert.throws( + () => migrateCorpus({ root, backupRoot: join(tmp, '.kb-backup'), manifest, write: true }), + /truncat|500|header/i, + ); + // The applier restored the corpus on the failed assertion — no partial mutation left behind. + assert.deepEqual(readAll(root), original); + }); +});