ms-ai-architect/scripts/kb-update/migrate-corpus.mjs

#!/usr/bin/env node
// migrate-corpus.mjs — the unified Spor 1 migration applier (Step 7).
//
// Stamps the Port-1 header contract (**Type:** + **Source:**) and a `## Innhold` TOC
// onto the ~327 NON-advisor reference files, reading its plan from the enriched
// ref-type manifest (scripts/kb-update/data/ref-type-manifest.json). It is the single
// write-path for the corpus mutation — deterministic, atomic, advisor-fenced,
// per-field idempotent, with a pre-write backup and a post-write parse assertion.
//
// ARCHITECTURE: pure-core / IO-shell split (mirrors validate-kb-file.mjs). The decision
// is `planFileMutation` (no IO); the shell `migrateCorpus` walks the corpus, takes ONE
// backup before the first write, writes atomically, and re-reads each file to assert the
// header survived the 500-byte scan window. The pure primitives it composes
// (normalizeStaleVerified → insertHeaderFields → insertToc) are already idempotent and
// 500B-aware; this file adds no header logic of its own.
//
// Hard rules (plan Step 7):
//   - advisor allowlist: only the 4 non-advisor skills are walked; a target path that
//     matches /ms-ai-advisor/ is a hard throw (belt-and-suspenders on the pure fn).
//   - atomic: atomicWriteSync only, never the raw fs writer.
//   - backup: backupDir(root, backupRoot) before the first write; restore() retained.
//   - per-field idempotent: delegated to insertHeaderFields (skips set Type/Source) and
//     insertToc (skips a file that already hasToc / is small).
//   - post-write assertion: re-read each written file, assert parseTypeHeader (and, when a
//     Source was intended, parseSourceHeader) non-null — catches 500B truncation. On
//     failure the backup is restored and the run throws (no partial corpus is left).
//
// Usage:
//   node scripts/kb-update/migrate-corpus.mjs            # dry-run report (default)
//   node scripts/kb-update/migrate-corpus.mjs --write    # apply the migration
//   node scripts/kb-update/migrate-corpus.mjs --json     # machine-readable report
// Exit code: 0 = ok; 2 = usage error.

import { readFileSync, readdirSync, existsSync, realpathSync } from 'node:fs';
import { join, dirname, relative, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
  insertHeaderFields,
  normalizeStaleVerified,
  insertToc,
} from './lib/transform.mjs';
import { parseTypeHeader, parseSourceHeader } from './lib/kb-headers.mjs';
import { atomicWriteSync } from './lib/atomic-write.mjs';
import { backupDir } from './lib/backup.mjs';

// The 4 non-advisor skills, enumerated explicitly (NO shell brace-expansion). Advisor
// (ms-ai-advisor) is deliberately absent — it is classified in the manifest but never
// mutated (S-Cosmo).
const NON_ADVISOR_SKILLS = [
  'ms-ai-engineering',
  'ms-ai-governance',
  'ms-ai-infrastructure',
  'ms-ai-security',
];

const ADVISOR_RE = /(^|\/)ms-ai-advisor(\/|$)/;

const PLUGIN_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const DEFAULT_SKILLS_DIR = join(PLUGIN_ROOT, 'skills');
const DEFAULT_BACKUP_ROOT = join(PLUGIN_ROOT, '.kb-backup');
const DEFAULT_MANIFEST_PATH = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'data', 'ref-type-manifest.json');

/**
 * Decide the mutation for a single corpus file — pure, no IO. Composes the Session-2
 * primitives in the mandated order: strip a stale header `**Verified:** MCP` →
 * insert `**Type:**` (+ `**Source:**` for a sourced reference) → insert a `## Innhold`
 * TOC on a large file. Each primitive is individually idempotent, so re-running plans
 * an unchanged file as a no-op.
 *
 * The manifest entry drives Type/Source: a `reference` with a `source` (and not
 * `deferred`) gets Type+Source; a deferred or non-reference entry gets Type only
 * (insertHeaderFields never emits a Source outside the reference-with-source case).
 *
 * @param {string} relpath — canonical `skills/<skill>/references/…` path (advisor guard)
 * @param {string} content — the file's current bytes
 * @param {object|undefined} manifestEntry — the ref-type manifest row for this file
 * @returns {{relpath: string, changed: boolean, content: string, actions: string[],
 *            expectType: boolean, expectSource: boolean, skipReason: string|null}}
 * @throws if relpath is under ms-ai-advisor (advisor fence)
 */
export function planFileMutation(relpath, content, manifestEntry) {
  if (ADVISOR_RE.test(relpath)) {
    throw new Error(`planFileMutation: refusing to mutate an ms-ai-advisor file: ${relpath}`);
  }
  const original = String(content ?? '');
  if (!manifestEntry) {
    return { relpath, changed: false, content: original, actions: [], expectType: false, expectSource: false, skipReason: 'no-manifest-entry' };
  }

  const type = manifestEntry.type ?? 'reference';
  const isReference = type === 'reference';
  // Deferred entries (advisor-scope / no-ms-citation) carry no defensible authority — Type only.
  const source = manifestEntry.deferred === true ? undefined : manifestEntry.source;
  const hasSource = source !== undefined && source !== null && String(source).trim() !== '';

  const actions = [];
  let out = normalizeStaleVerified(original);
  if (out !== original) actions.push('normalize-verified');

  const afterNormalize = out;
  out = insertHeaderFields(out, { type, source });
  if (out !== afterNormalize) actions.push(isReference && hasSource ? 'type+source' : 'type');

  const afterHeader = out;
  out = insertToc(out);
  if (out !== afterHeader) actions.push('toc');

  return {
    relpath,
    changed: out !== original,
    content: out,
    actions,
    expectType: true, // every migrated file must carry a readable Type
    expectSource: isReference && hasSource, // …and a readable Source when one was intended
    skipReason: null,
  };
}

/** Recursively collect absolute paths of `*.md` files under `dir` (missing dir → []). */
function walkMarkdown(dir) {
  if (!existsSync(dir)) return [];
  const out = [];
  for (const entry of readdirSync(dir, { withFileTypes: true })) {
    const full = join(dir, entry.name);
    if (entry.isDirectory()) out.push(...walkMarkdown(full));
    else if (entry.isFile() && entry.name.endsWith('.md')) out.push(full);
  }
  return out;
}

/** Canonical `skills/…`-rooted manifest key for a file under the skills dir. */
function canonicalRelpath(skillsDir, fullPath) {
  return `skills/${relative(skillsDir, fullPath).split(sep).join('/')}`;
}

/**
 * Walk the 4 non-advisor skills under `root`, plan each file against the manifest, and
 * (when `write`) apply the changed ones atomically behind a single pre-write backup with
 * a post-write parse assertion. Dry-run by default.
 *
 * @param {{root: string, backupRoot: string, manifest: object, write?: boolean}} opts
 * @returns {{root: string, write: boolean, backupPath: string|null, restore: (()=>void)|null,
 *            results: Array, counts: {total:number, changed:number, written:number, skipped:number}}}
 */
export function migrateCorpus({ root, backupRoot, manifest, write = false }) {
  const skillsDir = root;
  const worklist = [];
  for (const skill of NON_ADVISOR_SKILLS) {
    const refDir = join(skillsDir, skill, 'references');
    for (const full of walkMarkdown(refDir)) {
      const relpath = canonicalRelpath(skillsDir, full);
      worklist.push({ full, relpath, content: readFileSync(full, 'utf8') });
    }
  }

  const results = worklist.map((w) => {
    const plan = planFileMutation(w.relpath, w.content, manifest[w.relpath]);
    return { ...plan, full: w.full, wrote: false };
  });

  const changed = results.filter((r) => r.changed);
  const counts = {
    total: results.length,
    changed: changed.length,
    written: 0,
    skipped: results.filter((r) => r.skipReason).length,
  };

  if (!write || changed.length === 0) {
    return { root, write, backupPath: null, restore: null, results, counts };
  }

  // ONE backup before the first write — restore() undoes the whole batch.
  const { backupPath, restore } = backupDir(skillsDir, backupRoot);

  for (const r of changed) {
    atomicWriteSync(r.full, r.content);
    const written = readFileSync(r.full, 'utf8');
    const typeOk = !r.expectType || parseTypeHeader(written) !== null;
    const sourceOk = !r.expectSource || parseSourceHeader(written) !== null;
    if (!typeOk || !sourceOk) {
      restore(); // roll the whole batch back — never leave a partial corpus
      throw new Error(
        `migrateCorpus: post-write header truncated for ${r.relpath} ` +
          `(inserted field fell past the ${500}-byte scan window; corpus restored from backup)`,
      );
    }
    r.wrote = true;
    counts.written += 1;
  }

  return { root, write, backupPath, restore, results, counts };
}

function main(argv) {
  const args = argv.slice(2);
  const write = args.includes('--write');
  const json = args.includes('--json');
  const unknown = args.filter((a) => !['--write', '--json'].includes(a));
  if (unknown.length) {
    process.stderr.write(`usage: migrate-corpus.mjs [--write] [--json]\nunknown arg(s): ${unknown.join(' ')}\n`);
    process.exit(2);
  }
  if (!existsSync(DEFAULT_MANIFEST_PATH)) {
    process.stderr.write(`migrate-corpus: manifest not found at ${DEFAULT_MANIFEST_PATH}\nRun classify-ref-type.mjs --write first.\n`);
    process.exit(2);
  }
  const manifest = JSON.parse(readFileSync(DEFAULT_MANIFEST_PATH, 'utf8'));
  const report = migrateCorpus({ root: DEFAULT_SKILLS_DIR, backupRoot: DEFAULT_BACKUP_ROOT, manifest, write });

  if (json) {
    const { restore, results, ...rest } = report;
    process.stdout.write(JSON.stringify({ ...rest, results: results.map(({ full, content, ...r }) => r) }, null, 2) + '\n');
  } else {
    const verb = write ? 'wrote' : 'would change';
    process.stdout.write(
      `migrate-corpus (${write ? 'APPLY' : 'dry-run'}): ${verb} ${report.counts.written || report.counts.changed}/${report.counts.total} files` +
        `${report.backupPath ? `; backup: ${report.backupPath}` : ''}\n`,
    );
  }
  process.exit(0);
}

const isMain = (() => {
  try {
    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
  } catch {
    return false;
  }
})();
if (isMain) main(process.argv);