ms-ai-architect/scripts/kb-update/lib/transform.mjs

// transform.mjs — Lag 4: TRANSFORMASJON (doc → KB-fil). Pure helper library that
// turns an approved, fetched Microsoft Learn document into a KB reference file in
// the plugin's house format. It is the deterministic spine the LLM destillation
// (commands/kb-update.md, transform-prompt.md) hangs on:
//
//   buildKbHeader   — the mandatory header block (Status + Source obligatory)
//   validateKbFile  — enforce the mandatory-field contract on any candidate file
//   buildChange     — construct the lag-5 `change` object (the lag-4 → lag-5 seam)
//   resolveTargetPath — route the file to its owning skill via the taxonomy (lag 0)
//
// ARCHITECTURE INVARIANT (spec §83): this lib NEVER writes. It returns content,
// verdicts and paths; applying a file is the operator gate's job, after lag 5
// (verify-out) has cleared it. Mirrors decisions-io / verify-out: pure functions,
// no write-util imports. The only dependency is kb-headers (read-only parser) so
// validateKbFile uses the SAME source detection as build-registry — one source of
// truth for "what counts as a Source header" (the top-500-byte region).

import {
  parseSourceHeader,
  parseTypeHeader,
  parseVerifiedHeader,
  parseVerifiedByHeader,
} from './kb-headers.mjs';
import { getCategorySkill } from './taxonomy.mjs';

// Base mandatory header fields for EVERY file type. Status is obligatory per spec
// ("status-felt obligatorisk").
const REQUIRED_BASE_META = ['title', 'status', 'category', 'lastUpdated'];

// Spor 3 Port 1 — the frontmatter-contract `type`. Only `reference` files are in the
// correctness scope (they carry a Microsoft source we verify against). Template /
// methodology / regulatory are out of scope and carry NO Microsoft source.
const VALID_TYPES = ['reference', 'template', 'methodology', 'regulatory'];
const DEFAULT_TYPE = 'reference';

// Extra fields a `reference` file MUST carry to be "born verified" (Spor 3 Port 2):
// `source` lets lag-5 authority checks fire; `verified`/`verified_by` record that the
// content cleared the judge before it shipped. A reference file without these is the
// drift-introducing failure mode the create-guard exists to stop.
const REQUIRED_REFERENCE_META = ['source', 'verified', 'verified_by'];

// The current GATE-PASS claim judge (judge-claim-prompt-v3.1, adopted G1 2026-06-30).
// A version *label string* — integer major + optional minor — so provenance names the
// adopted judge exactly (v3 was a distinct, rejected version). stampVerifiedMeta records
// `verified_by: judge-v<JUDGE_VERSION>` unless the caller overrides or marks `human`.
const JUDGE_VERSION = '3.1';

// A Microsoft Learn / docs authority URL — a non-reference file may NOT carry one.
const MS_LEARN_RE = /\b(?:learn|docs)\.microsoft\.com/i;

const DATE_RE = /^\d{4}-\d{2}(?:-\d{2})?$/;

// The top-of-file region the kb-headers parsers scan. MIRRORS kb-headers.mjs:6
// (HEADER_REGION_BYTES) — kept local rather than imported so this lib's only
// kb-headers dependency stays the read-only parser functions (same discipline as
// TOC_MIN_LINES mirroring eval.mjs). insertHeaderFields / normalizeStaleVerified use it
// to keep the fields they insert — and the stale Verified they strip — inside the window
// those parsers actually read.
const HEADER_REGION_BYTES = 500;

// A reference file longer than this many lines should carry a table of contents
// (best-practices §Structure). MIRRORS eval.mjs N4_LARGE_FILE_LINES — kept in sync
// behaviourally by test-transform's checkN4 parity assertion, NOT by importing across
// the kb-update ↔ kb-eval boundary (transform.mjs's only dependency stays kb-headers).
const TOC_MIN_LINES = 100;

/** Line count counted exactly as checkN4 does (trailing blank lines ignored). */
function lineCount(content) {
  return String(content ?? '').replace(/\n+$/, '').split('\n').length;
}

/** Is this file large enough to require a TOC? (mirror of checkN4's "large" filter) */
function isLargeContent(content) {
  return lineCount(content) > TOC_MIN_LINES;
}

/**
 * Does the content already carry a TOC? Mirror of eval.mjs hasToc (internal there):
 * an explicit TOC heading, OR a cluster (≥3) of in-page anchor links near the top.
 */
function hasToc(content) {
  const s = String(content ?? '');
  if (/^#{1,3}\s*(table of contents|contents|innhold(?:sfortegnelse)?|toc)\b/im.test(s)) return true;
  const head = s.split('\n').slice(0, 40).join('\n');
  return (head.match(/\]\(#/g) || []).length >= 3;
}

/** GitHub-style heading slug: lowercase, drop punctuation (keep letters/digits/_/-),
 *  spaces → hyphens. Unicode-aware so Norwegian æ/ø/å survive in anchors. */
function slugify(heading) {
  return String(heading ?? '')
    .toLowerCase()
    .replace(/[^\p{L}\p{N}\s_-]/gu, '')
    .trim()
    .replace(/\s/g, '-');
}

/**
 * Build the deterministic KB-file header block. Status, Source and the Port 1 contract
 * fields (Type / Verified / Verified by) live IN the header (top 500 bytes) so
 * kb-headers.* and the "Last updated:" scanner can see them — the old bottom-of-file
 * source section was invisible to those scanners, which is why authority_source
 * coverage was 0%.
 *
 * Type-aware create-guard (Spor 3 Port 2): a `reference` file MUST carry
 * source + verified + verified_by (it is "born verified"); a non-reference file
 * (template/methodology/regulatory) is out of correctness scope and may NOT carry a
 * Microsoft Learn source. Either violation is a hard throw — the create-time gate.
 *
 * @param {{title: string, status: string, category: string, lastUpdated: string,
 *          type?: string, source?: string, verified?: string, verified_by?: string}} meta
 * @returns {string} the header block, ending with the `---` rule + trailing newline
 * @throws if the type is unknown, a mandatory field is missing, or a non-reference
 *   file carries a Microsoft Learn source
 */
export function buildKbHeader(meta) {
  const m = meta ?? {};
  const type = m.type ?? DEFAULT_TYPE;
  if (!VALID_TYPES.includes(type)) {
    throw new Error(`buildKbHeader: invalid type '${type}' (expected ${VALID_TYPES.join('|')})`);
  }
  const isReference = type === 'reference';
  const empty = (k) => {
    const v = m[k];
    return v === undefined || v === null || String(v).trim() === '';
  };
  const required = isReference
    ? [...REQUIRED_BASE_META, ...REQUIRED_REFERENCE_META]
    : REQUIRED_BASE_META;
  const missing = required.filter(empty);
  if (missing.length) {
    throw new Error(`buildKbHeader: missing mandatory field(s): ${missing.join(', ')}`);
  }
  if (!isReference && !empty('source') && MS_LEARN_RE.test(String(m.source))) {
    throw new Error(
      `buildKbHeader: a ${type} file may not carry a Microsoft Learn source (out of correctness scope)`,
    );
  }
  let header =
    `# ${m.title}\n\n` +
    `**Last updated:** ${m.lastUpdated}\n` +
    `**Status:** ${m.status}\n` +
    `**Category:** ${m.category}\n` +
    `**Type:** ${type}\n`;
  if (!empty('source')) header += `**Source:** ${m.source}\n`;
  if (!empty('verified')) header += `**Verified:** ${m.verified}\n`;
  if (!empty('verified_by')) header += `**Verified by:** ${m.verified_by}\n`;
  return `${header}\n---\n`;
}

/**
 * Born-verified gate (Spor 3 Port 2): stamp `verified`/`verified_by` onto a reference
 * meta IFF the judge verdict passed — otherwise REFUSE (throw). This encodes
 * "nytt/regenerert innhold kjøres gjennom judgen FØR commit → født verifisert; ellers
 * ingen fil". transform.mjs never calls the judge itself (pure lib); the command runs
 * the v3.1 claim judge, aggregates per-claim results into `{pass}`, and calls this. A
 * failing verdict means no file is born.
 *
 * @param {object} baseMeta — the reference meta (title/status/category/source/lastUpdated)
 * @param {{pass: boolean, judgeVersion?: number, by?: 'human'}} verdict
 * @param {string} today — YYYY-MM or YYYY-MM-DD, supplied by the caller (lib stays pure)
 * @returns {object} baseMeta with type='reference' + verified + verified_by set
 * @throws if the verdict is not passing, or today is not a valid date
 */
export function stampVerifiedMeta(baseMeta, verdict, today) {
  if (!verdict || verdict.pass !== true) {
    throw new Error('stampVerifiedMeta: refusing to stamp verified — judge verdict not passing');
  }
  if (!DATE_RE.test(String(today ?? ''))) {
    throw new Error(`stampVerifiedMeta: invalid today date '${today}' (expected YYYY-MM[-DD])`);
  }
  // Default stamp = the current JUDGE_VERSION label ('3.1'). An explicit integer override
  // is still honoured (judgeVersion:3 → judge-v3); a minor-bearing string override is not
  // yet (falls back to default) — tracked as a §8 follow-up, harmless since the pipeline
  // always uses the default.
  const verifiedBy =
    verdict.by === 'human'
      ? 'human'
      : `judge-v${Number.isInteger(verdict.judgeVersion) ? verdict.judgeVersion : JUDGE_VERSION}`;
  return { ...baseMeta, type: 'reference', verified: today, verified_by: verifiedBy };
}

/**
 * Build a deterministic `## Innhold` table of contents from a body's level-2 (`##`)
 * section headings — the plugin house style (matches the existing TOC'd files and the
 * heading eval.mjs hasToc recognises). The LLM should NOT hand-author anchor slugs;
 * this generates GitHub-compatible ones. Headings inside ``` code fences are ignored,
 * and the `## Innhold` heading never lists itself (so re-running is idempotent).
 *
 * @param {string} body — the file body (under the header `---`)
 * @returns {string} a `## Innhold` block ending in a newline, or '' if there are no
 *   `##` sections to list
 */
export function buildToc(body) {
  const lines = String(body ?? '').split('\n');
  const titles = [];
  let inFence = false;
  for (const line of lines) {
    if (/^\s*```/.test(line)) { inFence = !inFence; continue; }
    if (inFence) continue;
    const m = /^##\s+(.+?)\s*$/.exec(line); // level-2 only (not # title, not ### sub)
    if (!m) continue;
    const title = m[1].trim();
    if (/^innhold(?:sfortegnelse)?$/i.test(title)) continue; // never list the TOC itself
    titles.push(title);
  }
  if (titles.length === 0) return '';
  const items = titles.map((t) => `- [${t}](#${slugify(t)})`).join('\n');
  return `## Innhold\n\n${items}\n`;
}

/**
 * Assemble a complete KB file from header meta + body: the deterministic generator the
 * write-path uses (replaces the bare `buildKbHeader(meta) + body` concatenation). A
 * large file (>100 lines, per N4) is born with a `## Innhold` TOC inserted after the
 * header so regeneration never strips the TOC back out (the Fase 1c regression guard).
 * Idempotent: a body that already carries a TOC is not double-TOC'd.
 *
 * @param {object} meta — same shape as buildKbHeader
 * @param {string} body — the distilled body (under the header `---`)
 * @returns {string} the full file content
 */
export function composeKbFile(meta, body) {
  const header = buildKbHeader(meta);
  const b = String(body ?? '');
  const full = header + b;
  if (!isLargeContent(full) || hasToc(full)) return full;
  const toc = buildToc(b);
  if (!toc) return full;
  return `${header}\n${toc}\n${b}`;
}

/**
 * Backfill a `## Innhold` TOC into an EXISTING large reference file that lacks one,
 * placing it immediately before the first non-fenced `## ` section heading — the same
 * relative slot composeKbFile gives a freshly-generated file (TOC between the header
 * preamble and the first content section), but robust to the varied legacy headers on
 * disk: some carry a `---` rule after the metadata, some don't. This is the Fase 1b
 * retrofit for the ~20 largest hand-written files; composeKbFile covers NEW files.
 *
 * No-op (returns content unchanged) when the file is small (≤ TOC_MIN_LINES) or already
 * carries a TOC — so it is idempotent and never churns a small file. The body below the
 * insertion point is left byte-identical: only the `## Innhold` block is added.
 *
 * @param {string} content — full existing file content
 * @returns {string} content with a `## Innhold` block inserted, or unchanged
 */
export function insertToc(content) {
  const s = String(content ?? '');
  if (!isLargeContent(s) || hasToc(s)) return s;
  const toc = buildToc(s);
  if (!toc) return s; // no ## sections to list — nothing to anchor a TOC to
  const lines = s.split('\n');
  let inFence = false;
  let idx = -1;
  for (let i = 0; i < lines.length; i++) {
    if (/^\s*```/.test(lines[i])) { inFence = !inFence; continue; }
    if (inFence) continue;
    if (/^##\s+/.test(lines[i])) { idx = i; break; }
  }
  if (idx === -1) return s; // defensive: buildToc found a section we didn't — leave as-is
  const before = lines.slice(0, idx);
  while (before.length && before[before.length - 1].trim() === '') before.pop();
  const block = ['', ...toc.replace(/\n$/, '').split('\n'), ''];
  return [...before, ...block, ...lines.slice(idx)].join('\n');
}

// A bold-label meta line, e.g. **Category:** / **Kategori:** / **Status:** /
// **Sist oppdatert:** / **Confidence:** — and a pipe-delimited meta row, which also
// begins with a bold label. The non-greedy label body excludes `*` so it stops at the
// first `:**`. Language-agnostic by construction: it keys on the bold-label shape, not
// on a fixed label vocabulary.
const RE_META_LINE = /^\s*\*\*[^*\n]+?:\*\*/;

/**
 * Backfill the Port-1 `**Type:**` (and, for a reference, `**Source:**`) header field(s)
 * into an EXISTING legacy KB file that lacks them — the Spor 1 migration primitive for
 * the ~327 non-advisor files. Unlike buildKbHeader (which emits a COMPLETE new header),
 * this inserts ONLY the missing field(s), placing them immediately after the last line of
 * the first contiguous run of bold-label meta lines (RE_META_LINE — covers the
 * **Category:** / **Kategori:** / pipe dialects). It never inserts before the first `## `
 * section or past the 500-byte scan window, so the fields land where kb-headers.* read
 * them; the body is byte-identical.
 *
 * Per-field idempotent: skips `**Type:**` when parseTypeHeader is already set, skips
 * `**Source:**` when parseSourceHeader is already set. A non-reference type NEVER receives
 * a `**Source:**` (mirrors buildKbHeader's out-of-correctness-scope rule) — Type only.
 *
 * @param {string} content — full existing file content
 * @param {{type?: string, source?: string}} meta — resolved type + (reference) authority URL
 * @returns {string} content with the missing header field(s) inserted, or unchanged
 * @throws if the type is not one of VALID_TYPES
 */
export function insertHeaderFields(content, meta) {
  const s = String(content ?? '');
  const m = meta ?? {};
  const type = m.type ?? DEFAULT_TYPE;
  if (!VALID_TYPES.includes(type)) {
    throw new Error(`insertHeaderFields: invalid type '${type}' (expected ${VALID_TYPES.join('|')})`);
  }
  const isReference = type === 'reference';
  const src = m.source;
  const hasSource = src !== undefined && src !== null && String(src).trim() !== '';
  const needType = parseTypeHeader(s) === null;
  const needSource = isReference && hasSource && parseSourceHeader(s) === null;
  if (!needType && !needSource) return s;

  const toInsert = [];
  if (needType) toInsert.push(`**Type:** ${type}`);
  if (needSource) toInsert.push(`**Source:** ${String(src).trim()}`);

  const lines = s.split('\n');
  let anchorIdx = -1; // last line of the first contiguous meta run that fits the window
  let titleIdx = -1; // fallback anchor for the "no meta line" dialect
  let cum = 0;
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];
    if (/^##\s/.test(line) || /^---\s*$/.test(line)) break; // reached body / rule
    if (titleIdx === -1 && /^#\s+\S/.test(line)) titleIdx = i;
    const nextCum = cum + line.length + 1; // byte offset just AFTER this line (= the insertion point)
    if (RE_META_LINE.test(line)) {
      // Anchor after this meta line ONLY if the insertion point still fits the scan window.
      // A meta line whose own length pushes the offset past 500B (a corpus file that packs a
      // paragraph into **Status:**) must NOT become the anchor — otherwise the inserted field
      // lands past the window and parseTypeHeader/parseSourceHeader miss it. Back off here.
      if (nextCum > HEADER_REGION_BYTES) break;
      anchorIdx = i; // extend the contiguous meta run
    } else if (anchorIdx !== -1) {
      break; // first non-meta line after the run — the run is over (contiguous only)
    }
    cum = nextCum;
  }
  // "None" dialect (no bold-label meta): anchor on the title line so the fields still
  // land in the header region, before the first section. (-1 → very top if no title.)
  if (anchorIdx === -1) anchorIdx = titleIdx;
  lines.splice(anchorIdx + 1, 0, ...toInsert);
  return lines.join('\n');
}

/**
 * Surgically strip STALE `**Verified:**` value(s) from a legacy KB file — the bounded
 * Spor 1 carve-out for the 14 files carrying `**Verified:** MCP <date>`. That non-date
 * value is read by parseVerifiedHeader as a bogus "verified" stamp, which drops the file
 * out of the full-pass worklist AND marks it falsely verified; stripping it is the OPPOSITE
 * of stamping verified. Acts on EVERY stale `**Verified:**` inside the 500-byte scan window
 * (the region parseVerifiedHeader actually reads) whose value is not a clean YYYY-MM(-DD)
 * date — including a stray duplicate sitting just below the first `---` rule, a corpus
 * artifact (9 mlops-genaiops files) that still poisons the read once the header copy is
 * gone. A clean date at the first scanned position stops the pass (a legitimately verified
 * file is left alone), and anything PAST the 500-byte window is genuine body content and is
 * never touched. On a pipe-delimited meta row it removes only the `**Verified:** <value>`
 * token plus one adjacent ` | ` separator, preserving the sibling tokens
 * (**Sist oppdatert:** etc.); on a standalone line it drops the whole line. Idempotent
 * (each pass removes one match; a re-run finds nothing to strip).
 *
 * @param {string} content — full existing file content
 * @returns {string} content with the stale in-window Verified value(s) removed, or unchanged
 */
export function normalizeStaleVerified(content) {
  let s = String(content ?? '');
  // Remove every stale (non-date) **Verified:** inside the scan window. Loop because a file
  // can carry two (header copy + stray body-dup below ---); each pass drops one, so it
  // terminates. A clean date at the first position ends the pass — parseVerifiedHeader reads
  // that first occurrence, so a dated first Verified means the file is legitimately verified.
  for (;;) {
    const head = s.slice(0, HEADER_REGION_BYTES);
    const m = /\*\*Verified:\*\*\s*(\S+)/i.exec(head);
    if (!m) return s; // nothing stale left in the window
    if (DATE_RE.test(m[1])) return s; // clean date at the read position → leave untouched
    const matchStart = m.index; // index of "**Verified:**" in s

    const lineStart = s.lastIndexOf('\n', matchStart - 1) + 1; // 0 if no preceding newline
    let lineEnd = s.indexOf('\n', matchStart);
    if (lineEnd === -1) lineEnd = s.length;
    const line = s.slice(lineStart, lineEnd);

    if (!line.includes(' | ')) {
      // Standalone `**Verified:** …` line — drop the whole line, including its newline.
      const cutEnd = lineEnd < s.length ? lineEnd + 1 : lineEnd;
      s = s.slice(0, lineStart) + s.slice(cutEnd);
      continue;
    }
    // Pipe row: remove only the `**Verified:** <value>` token + one adjacent ` | `.
    const nextSepRel = s.slice(matchStart, lineEnd).indexOf(' | ');
    const tokenEnd = nextSepRel === -1 ? lineEnd : matchStart + nextSepRel;
    if (s.slice(matchStart - 3, matchStart) === ' | ') {
      s = s.slice(0, matchStart - 3) + s.slice(tokenEnd); // drop the preceding separator
    } else if (nextSepRel !== -1) {
      s = s.slice(0, matchStart) + s.slice(tokenEnd + 3); // drop the following separator
    } else {
      s = s.slice(0, matchStart) + s.slice(tokenEnd); // degenerate — token only
    }
  }
}

const RE_TITLE = /^#\s+\S/m;
const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i;
const RE_STATUS = /\*\*Status:\*\*\s*\S/i;

/**
 * Validate that a candidate KB file carries the mandatory contract: a title, a
 * "Last updated:" header (so it is never dateless → never the 'unverified' bucket),
 * a Status field, and — for a large file (>100 lines, per N4) — a table of contents.
 * The TOC clause is the write-path regression gate: a regenerated large file that lost
 * its TOC fails here and the caller re-prompts, so KB-update can never silently strip a
 * TOC back out.
 *
 * Type-aware (Spor 3 Port 2): a `reference` file MUST additionally carry source +
 * verified + verified_by (born-verified). A file with no `**Type:**` header is treated
 * as a reference — the create path always stamps Type, and the legacy corpus (no Type)
 * is migrated under Spor 1, where this same contract is the target. Non-reference files
 * (template/methodology/regulatory) are out of correctness scope and need no source.
 *
 * @param {string} content — full file content
 * @returns {{valid: boolean, missing: string[]}}
 */
export function validateKbFile(content) {
  const s = String(content ?? '');
  const missing = [];
  if (!RE_TITLE.test(s)) missing.push('title');
  if (!RE_LAST_UPDATED.test(s)) missing.push('last_updated');
  if (!RE_STATUS.test(s)) missing.push('status');
  const type = parseTypeHeader(s) ?? DEFAULT_TYPE;
  if (type === 'reference') {
    if (parseSourceHeader(s) === null) missing.push('source');
    if (parseVerifiedHeader(s) === null) missing.push('verified');
    if (parseVerifiedByHeader(s) === null) missing.push('verified_by');
  }
  if (isLargeContent(s) && !hasToc(s)) missing.push('toc');
  return { valid: missing.length === 0, missing };
}

/**
 * Construct the lag-5 `change` object from a transformation result — the explicit,
 * documented seam between lag 4 and lag 5. Decouples lag 4's field names from
 * verify-out's, and supplies the safe defaults lag 5 expects (no refutations yet;
 * authority_source null until lag 3 backfills it).
 *
 * @param {{field: string, oldValue?: string, newValue: string, sourceUrl?: string,
 *          authoritySource?: string|null, refutations?: Array}} t
 * @returns {{field: string, old_value: string, new_value: string,
 *            source_url: string|null, authority_source: string|null, refutations: Array}}
 */
export function buildChange(t) {
  const o = t ?? {};
  return {
    field: o.field ?? '',
    old_value: o.oldValue ?? '',
    new_value: o.newValue ?? '',
    source_url: o.sourceUrl ?? null,
    authority_source: o.authoritySource ?? null,
    refutations: Array.isArray(o.refutations) ? o.refutations : [],
  };
}

/**
 * Resolve where a transformed file belongs, routing via the taxonomy's canonical
 * category → skill map (physical-disk truth — this is what replaced
 * category-skill-map.json). Returns null for an unknown category so the caller
 * must gate rather than silently misroute.
 *
 * @param {object} tax — loaded domain taxonomy
 * @param {string} category
 * @param {string} filename — kebab-case .md filename
 * @returns {string|null} repo-relative path, or null if the category is unknown
 */
export function resolveTargetPath(tax, category, filename) {
  const skill = getCategorySkill(tax, category);
  if (!skill) return null;
  return `skills/${skill}/references/${category}/${filename}`;
}