feat(ms-ai-architect): Spor 1 — unified migrate-corpus applier (backup+atomisk+advisor-fence+per-felt-idempotent, TDD) [skip-docs]

scripts/kb-update/migrate-corpus.mjs

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+// migrate-corpus.mjs — the unified Spor 1 migration applier (Step 7).
+//
+// Stamps the Port-1 header contract (**Type:** + **Source:**) and a `## Innhold` TOC
+// onto the ~327 NON-advisor reference files, reading its plan from the enriched
+// ref-type manifest (scripts/kb-update/data/ref-type-manifest.json). It is the single
+// write-path for the corpus mutation — deterministic, atomic, advisor-fenced,
+// per-field idempotent, with a pre-write backup and a post-write parse assertion.
+//
+// ARCHITECTURE: pure-core / IO-shell split (mirrors validate-kb-file.mjs). The decision
+// is `planFileMutation` (no IO); the shell `migrateCorpus` walks the corpus, takes ONE
+// backup before the first write, writes atomically, and re-reads each file to assert the
+// header survived the 500-byte scan window. The pure primitives it composes
+// (normalizeStaleVerified → insertHeaderFields → insertToc) are already idempotent and
+// 500B-aware; this file adds no header logic of its own.
+//
+// Hard rules (plan Step 7):
+//   - advisor allowlist: only the 4 non-advisor skills are walked; a target path that
+//     matches /ms-ai-advisor/ is a hard throw (belt-and-suspenders on the pure fn).
+//   - atomic: atomicWriteSync only, never the raw fs writer.
+//   - backup: backupDir(root, backupRoot) before the first write; restore() retained.
+//   - per-field idempotent: delegated to insertHeaderFields (skips set Type/Source) and
+//     insertToc (skips a file that already hasToc / is small).
+//   - post-write assertion: re-read each written file, assert parseTypeHeader (and, when a
+//     Source was intended, parseSourceHeader) non-null — catches 500B truncation. On
+//     failure the backup is restored and the run throws (no partial corpus is left).
+//
+// Usage:
+//   node scripts/kb-update/migrate-corpus.mjs            # dry-run report (default)
+//   node scripts/kb-update/migrate-corpus.mjs --write    # apply the migration
+//   node scripts/kb-update/migrate-corpus.mjs --json     # machine-readable report
+// Exit code: 0 = ok; 2 = usage error.
+
+import { readFileSync, readdirSync, existsSync, realpathSync } from 'node:fs';
+import { join, dirname, relative, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  insertHeaderFields,
+  normalizeStaleVerified,
+  insertToc,
+} from './lib/transform.mjs';
+import { parseTypeHeader, parseSourceHeader } from './lib/kb-headers.mjs';
+import { atomicWriteSync } from './lib/atomic-write.mjs';
+import { backupDir } from './lib/backup.mjs';
+
+// The 4 non-advisor skills, enumerated explicitly (NO shell brace-expansion). Advisor
+// (ms-ai-advisor) is deliberately absent — it is classified in the manifest but never
+// mutated (S-Cosmo).
+const NON_ADVISOR_SKILLS = [
+  'ms-ai-engineering',
+  'ms-ai-governance',
+  'ms-ai-infrastructure',
+  'ms-ai-security',
+];
+
+const ADVISOR_RE = /(^|\/)ms-ai-advisor(\/|$)/;
+
+const PLUGIN_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const DEFAULT_SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
+const DEFAULT_BACKUP_ROOT = join(PLUGIN_ROOT, '.kb-backup');
+const DEFAULT_MANIFEST_PATH = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'data', 'ref-type-manifest.json');
+
+/**
+ * Decide the mutation for a single corpus file — pure, no IO. Composes the Session-2
+ * primitives in the mandated order: strip a stale header `**Verified:** MCP` →
+ * insert `**Type:**` (+ `**Source:**` for a sourced reference) → insert a `## Innhold`
+ * TOC on a large file. Each primitive is individually idempotent, so re-running plans
+ * an unchanged file as a no-op.
+ *
+ * The manifest entry drives Type/Source: a `reference` with a `source` (and not
+ * `deferred`) gets Type+Source; a deferred or non-reference entry gets Type only
+ * (insertHeaderFields never emits a Source outside the reference-with-source case).
+ *
+ * @param {string} relpath — canonical `skills/<skill>/references/…` path (advisor guard)
+ * @param {string} content — the file's current bytes
+ * @param {object|undefined} manifestEntry — the ref-type manifest row for this file
+ * @returns {{relpath: string, changed: boolean, content: string, actions: string[],
+ *            expectType: boolean, expectSource: boolean, skipReason: string|null}}
+ * @throws if relpath is under ms-ai-advisor (advisor fence)
+ */
+export function planFileMutation(relpath, content, manifestEntry) {
+  if (ADVISOR_RE.test(relpath)) {
+    throw new Error(`planFileMutation: refusing to mutate an ms-ai-advisor file: ${relpath}`);
+  }
+  const original = String(content ?? '');
+  if (!manifestEntry) {
+    return { relpath, changed: false, content: original, actions: [], expectType: false, expectSource: false, skipReason: 'no-manifest-entry' };
+  }
+
+  const type = manifestEntry.type ?? 'reference';
+  const isReference = type === 'reference';
+  // Deferred entries (advisor-scope / no-ms-citation) carry no defensible authority — Type only.
+  const source = manifestEntry.deferred === true ? undefined : manifestEntry.source;
+  const hasSource = source !== undefined && source !== null && String(source).trim() !== '';
+
+  const actions = [];
+  let out = normalizeStaleVerified(original);
+  if (out !== original) actions.push('normalize-verified');
+
+  const afterNormalize = out;
+  out = insertHeaderFields(out, { type, source });
+  if (out !== afterNormalize) actions.push(isReference && hasSource ? 'type+source' : 'type');
+
+  const afterHeader = out;
+  out = insertToc(out);
+  if (out !== afterHeader) actions.push('toc');
+
+  return {
+    relpath,
+    changed: out !== original,
+    content: out,
+    actions,
+    expectType: true, // every migrated file must carry a readable Type
+    expectSource: isReference && hasSource, // …and a readable Source when one was intended
+    skipReason: null,
+  };
+}
+
+/** Recursively collect absolute paths of `*.md` files under `dir` (missing dir → []). */
+function walkMarkdown(dir) {
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) out.push(...walkMarkdown(full));
+    else if (entry.isFile() && entry.name.endsWith('.md')) out.push(full);
+  }
+  return out;
+}
+
+/** Canonical `skills/…`-rooted manifest key for a file under the skills dir. */
+function canonicalRelpath(skillsDir, fullPath) {
+  return `skills/${relative(skillsDir, fullPath).split(sep).join('/')}`;
+}
+
+/**
+ * Walk the 4 non-advisor skills under `root`, plan each file against the manifest, and
+ * (when `write`) apply the changed ones atomically behind a single pre-write backup with
+ * a post-write parse assertion. Dry-run by default.
+ *
+ * @param {{root: string, backupRoot: string, manifest: object, write?: boolean}} opts
+ * @returns {{root: string, write: boolean, backupPath: string|null, restore: (()=>void)|null,
+ *            results: Array, counts: {total:number, changed:number, written:number, skipped:number}}}
+ */
+export function migrateCorpus({ root, backupRoot, manifest, write = false }) {
+  const skillsDir = root;
+  const worklist = [];
+  for (const skill of NON_ADVISOR_SKILLS) {
+    const refDir = join(skillsDir, skill, 'references');
+    for (const full of walkMarkdown(refDir)) {
+      const relpath = canonicalRelpath(skillsDir, full);
+      worklist.push({ full, relpath, content: readFileSync(full, 'utf8') });
+    }
+  }
+
+  const results = worklist.map((w) => {
+    const plan = planFileMutation(w.relpath, w.content, manifest[w.relpath]);
+    return { ...plan, full: w.full, wrote: false };
+  });
+
+  const changed = results.filter((r) => r.changed);
+  const counts = {
+    total: results.length,
+    changed: changed.length,
+    written: 0,
+    skipped: results.filter((r) => r.skipReason).length,
+  };
+
+  if (!write || changed.length === 0) {
+    return { root, write, backupPath: null, restore: null, results, counts };
+  }
+
+  // ONE backup before the first write — restore() undoes the whole batch.
+  const { backupPath, restore } = backupDir(skillsDir, backupRoot);
+
+  for (const r of changed) {
+    atomicWriteSync(r.full, r.content);
+    const written = readFileSync(r.full, 'utf8');
+    const typeOk = !r.expectType || parseTypeHeader(written) !== null;
+    const sourceOk = !r.expectSource || parseSourceHeader(written) !== null;
+    if (!typeOk || !sourceOk) {
+      restore(); // roll the whole batch back — never leave a partial corpus
+      throw new Error(
+        `migrateCorpus: post-write header truncated for ${r.relpath} ` +
+          `(inserted field fell past the ${500}-byte scan window; corpus restored from backup)`,
+      );
+    }
+    r.wrote = true;
+    counts.written += 1;
+  }
+
+  return { root, write, backupPath, restore, results, counts };
+}
+
+function main(argv) {
+  const args = argv.slice(2);
+  const write = args.includes('--write');
+  const json = args.includes('--json');
+  const unknown = args.filter((a) => !['--write', '--json'].includes(a));
+  if (unknown.length) {
+    process.stderr.write(`usage: migrate-corpus.mjs [--write] [--json]\nunknown arg(s): ${unknown.join(' ')}\n`);
+    process.exit(2);
+  }
+  if (!existsSync(DEFAULT_MANIFEST_PATH)) {
+    process.stderr.write(`migrate-corpus: manifest not found at ${DEFAULT_MANIFEST_PATH}\nRun classify-ref-type.mjs --write first.\n`);
+    process.exit(2);
+  }
+  const manifest = JSON.parse(readFileSync(DEFAULT_MANIFEST_PATH, 'utf8'));
+  const report = migrateCorpus({ root: DEFAULT_SKILLS_DIR, backupRoot: DEFAULT_BACKUP_ROOT, manifest, write });
+
+  if (json) {
+    const { restore, results, ...rest } = report;
+    process.stdout.write(JSON.stringify({ ...rest, results: results.map(({ full, content, ...r }) => r) }, null, 2) + '\n');
+  } else {
+    const verb = write ? 'wrote' : 'would change';
+    process.stdout.write(
+      `migrate-corpus (${write ? 'APPLY' : 'dry-run'}): ${verb} ${report.counts.written || report.counts.changed}/${report.counts.total} files` +
+        `${report.backupPath ? `; backup: ${report.backupPath}` : ''}\n`,
+    );
+  }
+  process.exit(0);
+}
+
+const isMain = (() => {
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+})();
+if (isMain) main(process.argv);