ms-ai-architect/scripts/kb-update/lib/transform.mjs

// transform.mjs — Lag 4: TRANSFORMASJON (doc → KB-fil). Pure helper library that
// turns an approved, fetched Microsoft Learn document into a KB reference file in
// the plugin's house format. It is the deterministic spine the LLM destillation
// (commands/kb-update.md, transform-prompt.md) hangs on:
//
//   buildKbHeader   — the mandatory header block (Status + Source obligatory)
//   validateKbFile  — enforce the mandatory-field contract on any candidate file
//   buildChange     — construct the lag-5 `change` object (the lag-4 → lag-5 seam)
//   resolveTargetPath — route the file to its owning skill via the taxonomy (lag 0)
//
// ARCHITECTURE INVARIANT (spec §83): this lib NEVER writes. It returns content,
// verdicts and paths; applying a file is the operator gate's job, after lag 5
// (verify-out) has cleared it. Mirrors decisions-io / verify-out: pure functions,
// no write-util imports. The only dependency is kb-headers (read-only parser) so
// validateKbFile uses the SAME source detection as build-registry — one source of
// truth for "what counts as a Source header" (the top-500-byte region).

import {
  parseSourceHeader,
  parseTypeHeader,
  parseVerifiedHeader,
  parseVerifiedByHeader,
} from './kb-headers.mjs';
import { getCategorySkill } from './taxonomy.mjs';

// Base mandatory header fields for EVERY file type. Status is obligatory per spec
// ("status-felt obligatorisk").
const REQUIRED_BASE_META = ['title', 'status', 'category', 'lastUpdated'];

// Spor 3 Port 1 — the frontmatter-contract `type`. Only `reference` files are in the
// correctness scope (they carry a Microsoft source we verify against). Template /
// methodology / regulatory are out of scope and carry NO Microsoft source.
const VALID_TYPES = ['reference', 'template', 'methodology', 'regulatory'];
const DEFAULT_TYPE = 'reference';

// Extra fields a `reference` file MUST carry to be "born verified" (Spor 3 Port 2):
// `source` lets lag-5 authority checks fire; `verified`/`verified_by` record that the
// content cleared the judge before it shipped. A reference file without these is the
// drift-introducing failure mode the create-guard exists to stop.
const REQUIRED_REFERENCE_META = ['source', 'verified', 'verified_by'];

// The current GATE-PASS claim judge (judge-claim-prompt-v2). stampVerifiedMeta records
// `verified_by: judge-v<JUDGE_VERSION>` unless the caller overrides or marks `human`.
const JUDGE_VERSION = 2;

// A Microsoft Learn / docs authority URL — a non-reference file may NOT carry one.
const MS_LEARN_RE = /\b(?:learn|docs)\.microsoft\.com/i;

const DATE_RE = /^\d{4}-\d{2}(?:-\d{2})?$/;

// A reference file longer than this many lines should carry a table of contents
// (best-practices §Structure). MIRRORS eval.mjs N4_LARGE_FILE_LINES — kept in sync
// behaviourally by test-transform's checkN4 parity assertion, NOT by importing across
// the kb-update ↔ kb-eval boundary (transform.mjs's only dependency stays kb-headers).
const TOC_MIN_LINES = 100;

/** Line count counted exactly as checkN4 does (trailing blank lines ignored). */
function lineCount(content) {
  return String(content ?? '').replace(/\n+$/, '').split('\n').length;
}

/** Is this file large enough to require a TOC? (mirror of checkN4's "large" filter) */
function isLargeContent(content) {
  return lineCount(content) > TOC_MIN_LINES;
}

/**
 * Does the content already carry a TOC? Mirror of eval.mjs hasToc (internal there):
 * an explicit TOC heading, OR a cluster (≥3) of in-page anchor links near the top.
 */
function hasToc(content) {
  const s = String(content ?? '');
  if (/^#{1,3}\s*(table of contents|contents|innhold(?:sfortegnelse)?|toc)\b/im.test(s)) return true;
  const head = s.split('\n').slice(0, 40).join('\n');
  return (head.match(/\]\(#/g) || []).length >= 3;
}

/** GitHub-style heading slug: lowercase, drop punctuation (keep letters/digits/_/-),
 *  spaces → hyphens. Unicode-aware so Norwegian æ/ø/å survive in anchors. */
function slugify(heading) {
  return String(heading ?? '')
    .toLowerCase()
    .replace(/[^\p{L}\p{N}\s_-]/gu, '')
    .trim()
    .replace(/\s/g, '-');
}

/**
 * Build the deterministic KB-file header block. Status, Source and the Port 1 contract
 * fields (Type / Verified / Verified by) live IN the header (top 500 bytes) so
 * kb-headers.* and the "Last updated:" scanner can see them — the old bottom-of-file
 * source section was invisible to those scanners, which is why authority_source
 * coverage was 0%.
 *
 * Type-aware create-guard (Spor 3 Port 2): a `reference` file MUST carry
 * source + verified + verified_by (it is "born verified"); a non-reference file
 * (template/methodology/regulatory) is out of correctness scope and may NOT carry a
 * Microsoft Learn source. Either violation is a hard throw — the create-time gate.
 *
 * @param {{title: string, status: string, category: string, lastUpdated: string,
 *          type?: string, source?: string, verified?: string, verified_by?: string}} meta
 * @returns {string} the header block, ending with the `---` rule + trailing newline
 * @throws if the type is unknown, a mandatory field is missing, or a non-reference
 *   file carries a Microsoft Learn source
 */
export function buildKbHeader(meta) {
  const m = meta ?? {};
  const type = m.type ?? DEFAULT_TYPE;
  if (!VALID_TYPES.includes(type)) {
    throw new Error(`buildKbHeader: invalid type '${type}' (expected ${VALID_TYPES.join('|')})`);
  }
  const isReference = type === 'reference';
  const empty = (k) => {
    const v = m[k];
    return v === undefined || v === null || String(v).trim() === '';
  };
  const required = isReference
    ? [...REQUIRED_BASE_META, ...REQUIRED_REFERENCE_META]
    : REQUIRED_BASE_META;
  const missing = required.filter(empty);
  if (missing.length) {
    throw new Error(`buildKbHeader: missing mandatory field(s): ${missing.join(', ')}`);
  }
  if (!isReference && !empty('source') && MS_LEARN_RE.test(String(m.source))) {
    throw new Error(
      `buildKbHeader: a ${type} file may not carry a Microsoft Learn source (out of correctness scope)`,
    );
  }
  let header =
    `# ${m.title}\n\n` +
    `**Last updated:** ${m.lastUpdated}\n` +
    `**Status:** ${m.status}\n` +
    `**Category:** ${m.category}\n` +
    `**Type:** ${type}\n`;
  if (!empty('source')) header += `**Source:** ${m.source}\n`;
  if (!empty('verified')) header += `**Verified:** ${m.verified}\n`;
  if (!empty('verified_by')) header += `**Verified by:** ${m.verified_by}\n`;
  return `${header}\n---\n`;
}

/**
 * Born-verified gate (Spor 3 Port 2): stamp `verified`/`verified_by` onto a reference
 * meta IFF the judge verdict passed — otherwise REFUSE (throw). This encodes
 * "nytt/regenerert innhold kjøres gjennom judgen FØR commit → født verifisert; ellers
 * ingen fil". transform.mjs never calls the judge itself (pure lib); the command runs
 * the v2 claim judge, aggregates per-claim results into `{pass}`, and calls this. A
 * failing verdict means no file is born.
 *
 * @param {object} baseMeta — the reference meta (title/status/category/source/lastUpdated)
 * @param {{pass: boolean, judgeVersion?: number, by?: 'human'}} verdict
 * @param {string} today — YYYY-MM or YYYY-MM-DD, supplied by the caller (lib stays pure)
 * @returns {object} baseMeta with type='reference' + verified + verified_by set
 * @throws if the verdict is not passing, or today is not a valid date
 */
export function stampVerifiedMeta(baseMeta, verdict, today) {
  if (!verdict || verdict.pass !== true) {
    throw new Error('stampVerifiedMeta: refusing to stamp verified — judge verdict not passing');
  }
  if (!DATE_RE.test(String(today ?? ''))) {
    throw new Error(`stampVerifiedMeta: invalid today date '${today}' (expected YYYY-MM[-DD])`);
  }
  const verifiedBy =
    verdict.by === 'human'
      ? 'human'
      : `judge-v${Number.isInteger(verdict.judgeVersion) ? verdict.judgeVersion : JUDGE_VERSION}`;
  return { ...baseMeta, type: 'reference', verified: today, verified_by: verifiedBy };
}

/**
 * Build a deterministic `## Innhold` table of contents from a body's level-2 (`##`)
 * section headings — the plugin house style (matches the existing TOC'd files and the
 * heading eval.mjs hasToc recognises). The LLM should NOT hand-author anchor slugs;
 * this generates GitHub-compatible ones. Headings inside ``` code fences are ignored,
 * and the `## Innhold` heading never lists itself (so re-running is idempotent).
 *
 * @param {string} body — the file body (under the header `---`)
 * @returns {string} a `## Innhold` block ending in a newline, or '' if there are no
 *   `##` sections to list
 */
export function buildToc(body) {
  const lines = String(body ?? '').split('\n');
  const titles = [];
  let inFence = false;
  for (const line of lines) {
    if (/^\s*```/.test(line)) { inFence = !inFence; continue; }
    if (inFence) continue;
    const m = /^##\s+(.+?)\s*$/.exec(line); // level-2 only (not # title, not ### sub)
    if (!m) continue;
    const title = m[1].trim();
    if (/^innhold(?:sfortegnelse)?$/i.test(title)) continue; // never list the TOC itself
    titles.push(title);
  }
  if (titles.length === 0) return '';
  const items = titles.map((t) => `- [${t}](#${slugify(t)})`).join('\n');
  return `## Innhold\n\n${items}\n`;
}

/**
 * Assemble a complete KB file from header meta + body: the deterministic generator the
 * write-path uses (replaces the bare `buildKbHeader(meta) + body` concatenation). A
 * large file (>100 lines, per N4) is born with a `## Innhold` TOC inserted after the
 * header so regeneration never strips the TOC back out (the Fase 1c regression guard).
 * Idempotent: a body that already carries a TOC is not double-TOC'd.
 *
 * @param {object} meta — same shape as buildKbHeader
 * @param {string} body — the distilled body (under the header `---`)
 * @returns {string} the full file content
 */
export function composeKbFile(meta, body) {
  const header = buildKbHeader(meta);
  const b = String(body ?? '');
  const full = header + b;
  if (!isLargeContent(full) || hasToc(full)) return full;
  const toc = buildToc(b);
  if (!toc) return full;
  return `${header}\n${toc}\n${b}`;
}

/**
 * Backfill a `## Innhold` TOC into an EXISTING large reference file that lacks one,
 * placing it immediately before the first non-fenced `## ` section heading — the same
 * relative slot composeKbFile gives a freshly-generated file (TOC between the header
 * preamble and the first content section), but robust to the varied legacy headers on
 * disk: some carry a `---` rule after the metadata, some don't. This is the Fase 1b
 * retrofit for the ~20 largest hand-written files; composeKbFile covers NEW files.
 *
 * No-op (returns content unchanged) when the file is small (≤ TOC_MIN_LINES) or already
 * carries a TOC — so it is idempotent and never churns a small file. The body below the
 * insertion point is left byte-identical: only the `## Innhold` block is added.
 *
 * @param {string} content — full existing file content
 * @returns {string} content with a `## Innhold` block inserted, or unchanged
 */
export function insertToc(content) {
  const s = String(content ?? '');
  if (!isLargeContent(s) || hasToc(s)) return s;
  const toc = buildToc(s);
  if (!toc) return s; // no ## sections to list — nothing to anchor a TOC to
  const lines = s.split('\n');
  let inFence = false;
  let idx = -1;
  for (let i = 0; i < lines.length; i++) {
    if (/^\s*```/.test(lines[i])) { inFence = !inFence; continue; }
    if (inFence) continue;
    if (/^##\s+/.test(lines[i])) { idx = i; break; }
  }
  if (idx === -1) return s; // defensive: buildToc found a section we didn't — leave as-is
  const before = lines.slice(0, idx);
  while (before.length && before[before.length - 1].trim() === '') before.pop();
  const block = ['', ...toc.replace(/\n$/, '').split('\n'), ''];
  return [...before, ...block, ...lines.slice(idx)].join('\n');
}

const RE_TITLE = /^#\s+\S/m;
const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i;
const RE_STATUS = /\*\*Status:\*\*\s*\S/i;

/**
 * Validate that a candidate KB file carries the mandatory contract: a title, a
 * "Last updated:" header (so it is never dateless → never the 'unverified' bucket),
 * a Status field, and — for a large file (>100 lines, per N4) — a table of contents.
 * The TOC clause is the write-path regression gate: a regenerated large file that lost
 * its TOC fails here and the caller re-prompts, so KB-update can never silently strip a
 * TOC back out.
 *
 * Type-aware (Spor 3 Port 2): a `reference` file MUST additionally carry source +
 * verified + verified_by (born-verified). A file with no `**Type:**` header is treated
 * as a reference — the create path always stamps Type, and the legacy corpus (no Type)
 * is migrated under Spor 1, where this same contract is the target. Non-reference files
 * (template/methodology/regulatory) are out of correctness scope and need no source.
 *
 * @param {string} content — full file content
 * @returns {{valid: boolean, missing: string[]}}
 */
export function validateKbFile(content) {
  const s = String(content ?? '');
  const missing = [];
  if (!RE_TITLE.test(s)) missing.push('title');
  if (!RE_LAST_UPDATED.test(s)) missing.push('last_updated');
  if (!RE_STATUS.test(s)) missing.push('status');
  const type = parseTypeHeader(s) ?? DEFAULT_TYPE;
  if (type === 'reference') {
    if (parseSourceHeader(s) === null) missing.push('source');
    if (parseVerifiedHeader(s) === null) missing.push('verified');
    if (parseVerifiedByHeader(s) === null) missing.push('verified_by');
  }
  if (isLargeContent(s) && !hasToc(s)) missing.push('toc');
  return { valid: missing.length === 0, missing };
}

/**
 * Construct the lag-5 `change` object from a transformation result — the explicit,
 * documented seam between lag 4 and lag 5. Decouples lag 4's field names from
 * verify-out's, and supplies the safe defaults lag 5 expects (no refutations yet;
 * authority_source null until lag 3 backfills it).
 *
 * @param {{field: string, oldValue?: string, newValue: string, sourceUrl?: string,
 *          authoritySource?: string|null, refutations?: Array}} t
 * @returns {{field: string, old_value: string, new_value: string,
 *            source_url: string|null, authority_source: string|null, refutations: Array}}
 */
export function buildChange(t) {
  const o = t ?? {};
  return {
    field: o.field ?? '',
    old_value: o.oldValue ?? '',
    new_value: o.newValue ?? '',
    source_url: o.sourceUrl ?? null,
    authority_source: o.authoritySource ?? null,
    refutations: Array.isArray(o.refutations) ? o.refutations : [],
  };
}

/**
 * Resolve where a transformed file belongs, routing via the taxonomy's canonical
 * category → skill map (physical-disk truth — this is what replaced
 * category-skill-map.json). Returns null for an unknown category so the caller
 * must gate rather than silently misroute.
 *
 * @param {object} tax — loaded domain taxonomy
 * @param {string} category
 * @param {string} filename — kebab-case .md filename
 * @returns {string|null} repo-relative path, or null if the category is unknown
 */
export function resolveTargetPath(tax, category, filename) {
  const skill = getCategorySkill(tax, category);
  if (!skill) return null;
  return `skills/${skill}/references/${category}/${filename}`;
}