diff --git a/scripts/kb-update/lib/transform.mjs b/scripts/kb-update/lib/transform.mjs index 482081e..3ce70cb 100644 --- a/scripts/kb-update/lib/transform.mjs +++ b/scripts/kb-update/lib/transform.mjs @@ -50,6 +50,14 @@ const MS_LEARN_RE = /\b(?:learn|docs)\.microsoft\.com/i; const DATE_RE = /^\d{4}-\d{2}(?:-\d{2})?$/; +// The top-of-file region the kb-headers parsers scan. MIRRORS kb-headers.mjs:6 +// (HEADER_REGION_BYTES) — kept local rather than imported so this lib's only +// kb-headers dependency stays the read-only parser functions (same discipline as +// TOC_MIN_LINES mirroring eval.mjs). insertHeaderFields / normalizeStaleVerified use it +// to keep the fields they insert — and the stale Verified they strip — inside the window +// those parsers actually read. +const HEADER_REGION_BYTES = 500; + // A reference file longer than this many lines should carry a table of contents // (best-practices §Structure). MIRRORS eval.mjs N4_LARGE_FILE_LINES — kept in sync // behaviourally by test-transform's checkN4 parity assertion, NOT by importing across @@ -257,6 +265,122 @@ export function insertToc(content) { return [...before, ...block, ...lines.slice(idx)].join('\n'); } +// A bold-label meta line, e.g. **Category:** / **Kategori:** / **Status:** / +// **Sist oppdatert:** / **Confidence:** — and a pipe-delimited meta row, which also +// begins with a bold label. The non-greedy label body excludes `*` so it stops at the +// first `:**`. Language-agnostic by construction: it keys on the bold-label shape, not +// on a fixed label vocabulary. +const RE_META_LINE = /^\s*\*\*[^*\n]+?:\*\*/; + +/** + * Backfill the Port-1 `**Type:**` (and, for a reference, `**Source:**`) header field(s) + * into an EXISTING legacy KB file that lacks them — the Spor 1 migration primitive for + * the ~327 non-advisor files. Unlike buildKbHeader (which emits a COMPLETE new header), + * this inserts ONLY the missing field(s), placing them immediately after the last line of + * the first contiguous run of bold-label meta lines (RE_META_LINE — covers the + * **Category:** / **Kategori:** / pipe dialects). It never inserts before the first `## ` + * section or past the 500-byte scan window, so the fields land where kb-headers.* read + * them; the body is byte-identical. + * + * Per-field idempotent: skips `**Type:**` when parseTypeHeader is already set, skips + * `**Source:**` when parseSourceHeader is already set. A non-reference type NEVER receives + * a `**Source:**` (mirrors buildKbHeader's out-of-correctness-scope rule) — Type only. + * + * @param {string} content — full existing file content + * @param {{type?: string, source?: string}} meta — resolved type + (reference) authority URL + * @returns {string} content with the missing header field(s) inserted, or unchanged + * @throws if the type is not one of VALID_TYPES + */ +export function insertHeaderFields(content, meta) { + const s = String(content ?? ''); + const m = meta ?? {}; + const type = m.type ?? DEFAULT_TYPE; + if (!VALID_TYPES.includes(type)) { + throw new Error(`insertHeaderFields: invalid type '${type}' (expected ${VALID_TYPES.join('|')})`); + } + const isReference = type === 'reference'; + const src = m.source; + const hasSource = src !== undefined && src !== null && String(src).trim() !== ''; + const needType = parseTypeHeader(s) === null; + const needSource = isReference && hasSource && parseSourceHeader(s) === null; + if (!needType && !needSource) return s; + + const toInsert = []; + if (needType) toInsert.push(`**Type:** ${type}`); + if (needSource) toInsert.push(`**Source:** ${String(src).trim()}`); + + const lines = s.split('\n'); + let anchorIdx = -1; // last line of the first contiguous meta run + let titleIdx = -1; // fallback anchor for the "no meta line" dialect + let cum = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (/^##\s/.test(line) || /^---\s*$/.test(line)) break; // reached body / rule + if (titleIdx === -1 && /^#\s+\S/.test(line)) titleIdx = i; + if (RE_META_LINE.test(line)) { + anchorIdx = i; // extend the contiguous meta run + } else if (anchorIdx !== -1) { + break; // first non-meta line after the run — the run is over (contiguous only) + } + cum += line.length + 1; + if (cum > HEADER_REGION_BYTES) break; // never place fields past the scan window + } + // "None" dialect (no bold-label meta): anchor on the title line so the fields still + // land in the header region, before the first section. (-1 → very top if no title.) + if (anchorIdx === -1) anchorIdx = titleIdx; + lines.splice(anchorIdx + 1, 0, ...toInsert); + return lines.join('\n'); +} + +/** + * Surgically strip a STALE `**Verified:**` value from a legacy KB file's header — the + * bounded Spor 1 carve-out for the 14 files carrying `**Verified:** MCP `. That + * non-date value is read by parseVerifiedHeader as a bogus "verified" stamp, which drops + * the file out of the full-pass worklist AND marks it falsely verified; stripping it is + * the OPPOSITE of stamping verified. Acts ONLY on the first header-region `**Verified:**` + * (the exact occurrence parseVerifiedHeader reads) and ONLY when its value is not a clean + * YYYY-MM(-DD) date. It never touches anything at or below the first `---` rule, so a + * body-duplicate survives (removing body content is NOT part of the blessed byte-identity + * carve-out). On a pipe-delimited meta row it removes only the `**Verified:** ` + * token plus one adjacent ` | ` separator, preserving the sibling tokens + * (**Sist oppdatert:** etc.); on a standalone line it drops the whole line. Idempotent. + * + * @param {string} content — full existing file content + * @returns {string} content with the stale header Verified removed, or unchanged + */ +export function normalizeStaleVerified(content) { + const s = String(content ?? ''); + const head = s.slice(0, HEADER_REGION_BYTES); + const m = /\*\*Verified:\*\*\s*(\S+)/i.exec(head); + if (!m) return s; // no header Verified + if (DATE_RE.test(m[1])) return s; // clean date → leave untouched + const matchStart = m.index; // index of "**Verified:**" in s + // Guard: never act at or below the first `---` rule — body content is off-limits. + const rule = /^---\s*$/m.exec(s); + if (rule && matchStart >= rule.index) return s; // defensive (parser scans header, but be sure) + + const lineStart = s.lastIndexOf('\n', matchStart - 1) + 1; // 0 if no preceding newline + let lineEnd = s.indexOf('\n', matchStart); + if (lineEnd === -1) lineEnd = s.length; + const line = s.slice(lineStart, lineEnd); + + if (!line.includes(' | ')) { + // Standalone `**Verified:** …` line — drop the whole line, including its newline. + const cutEnd = lineEnd < s.length ? lineEnd + 1 : lineEnd; + return s.slice(0, lineStart) + s.slice(cutEnd); + } + // Pipe row: remove only the `**Verified:** ` token + one adjacent ` | `. + const nextSepRel = s.slice(matchStart, lineEnd).indexOf(' | '); + const tokenEnd = nextSepRel === -1 ? lineEnd : matchStart + nextSepRel; + if (s.slice(matchStart - 3, matchStart) === ' | ') { + return s.slice(0, matchStart - 3) + s.slice(tokenEnd); // drop the preceding separator + } + if (nextSepRel !== -1) { + return s.slice(0, matchStart) + s.slice(tokenEnd + 3); // drop the following separator + } + return s.slice(0, matchStart) + s.slice(tokenEnd); // degenerate — token only +} + const RE_TITLE = /^#\s+\S/m; const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i; const RE_STATUS = /\*\*Status:\*\*\s*\S/i; diff --git a/tests/kb-update/test-transform.test.mjs b/tests/kb-update/test-transform.test.mjs index 188d0d4..86687e5 100644 --- a/tests/kb-update/test-transform.test.mjs +++ b/tests/kb-update/test-transform.test.mjs @@ -24,6 +24,8 @@ import { buildToc, composeKbFile, insertToc, + insertHeaderFields, + normalizeStaleVerified, stampVerifiedMeta, } from '../../scripts/kb-update/lib/transform.mjs'; import { @@ -494,3 +496,215 @@ test('transform.mjs imports NO write-utils (pure transformer, never writes)', () assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/); assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/); }); + +// --- insertHeaderFields: Spor 1 migration primitive (backfill Type/Source) ------- +// Unlike buildKbHeader (which emits a COMPLETE new header), insertHeaderFields inserts +// only the missing Port-1 field(s) into an EXISTING legacy file's top-region meta block, +// immediately after the last line of the first contiguous run of bold-label meta lines +// (language-agnostic across the 3 dialects), never before the first `## ` section, never +// past the 500-byte scan window. Body byte-identical; per-field idempotent. + +// Dialect A — **Category:** (English labels), with a `---` header rule. +const HDR_CATEGORY = + '# MLOps Fundamentals\n\n' + + '**Last updated:** 2026-06-19\n' + + '**Status:** GA\n' + + '**Category:** MLOps & GenAIOps\n\n' + + '---\n\n' + + '## Introduksjon\n\nBrødtekst.\n'; + +// Dialect B — **Kategori:** (Norwegian labels), with a `---` header rule. +const HDR_KATEGORI = + '# Feedback Loops\n\n' + + '**Kategori:** MLOps & GenAIOps\n' + + '**Sist oppdatert:** 2026-06\n\n' + + '---\n\n' + + '## Introduksjon\n\nBrødtekst.\n'; + +// Dialect C — pipe-delimited meta row, NO `---` rule (the ai-center-of-excellence shape). +const HDR_PIPE = + '# AI Center of Excellence\n\n' + + '**Kategori:** Responsible AI & Governance\n' + + '**Opprettet:** 2026-04 | **Sist oppdatert:** 2026-06-19\n' + + '**Confidence:** HIGH (Microsoft CAF)\n\n' + + '## Introduksjon\n\nBrødtekst.\n'; + +// A large file (>100 lines) with a category header — for the combined insertToc oracle. +const HDR_LARGE = + '# Big Ref\n\n' + + '**Last updated:** 2026-06\n' + + '**Status:** GA\n' + + '**Category:** X\n\n' + + '---\n\n' + + LARGE_BODY; + +const MS_URL = 'https://learn.microsoft.com/azure/machine-learning/concept-mlops'; + +test('insertHeaderFields inserts Type+Source after the meta block (category dialect), body byte-identical', () => { + const out = insertHeaderFields(HDR_CATEGORY, { type: 'reference', source: MS_URL }); + assert.equal(parseTypeHeader(out), 'reference'); + assert.equal(parseSourceHeader(out), MS_URL); + // Type/Source sit inside the meta block: after **Category:**, before the `---` rule. + assert.ok(out.indexOf('**Category:**') < out.indexOf('**Type:**')); + assert.ok(out.indexOf('**Type:**') < out.indexOf('\n---')); + // body (from the first ## section) is byte-identical + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(HDR_CATEGORY)); +}); + +test('insertHeaderFields works on the Norwegian **Kategori:** dialect, body byte-identical', () => { + const out = insertHeaderFields(HDR_KATEGORI, { type: 'reference', source: MS_URL }); + assert.equal(parseTypeHeader(out), 'reference'); + assert.equal(parseSourceHeader(out), MS_URL); + assert.ok(out.indexOf('**Sist oppdatert:**') < out.indexOf('**Type:**')); + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(HDR_KATEGORI)); +}); + +test('insertHeaderFields works on the pipe-delimited dialect without a `---` rule, body byte-identical', () => { + const out = insertHeaderFields(HDR_PIPE, { type: 'reference', source: MS_URL }); + assert.equal(parseTypeHeader(out), 'reference'); + assert.equal(parseSourceHeader(out), MS_URL); + // inserted after the last meta line (**Confidence:**), before the first ## section + assert.ok(out.indexOf('**Confidence:**') < out.indexOf('**Type:**')); + assert.ok(out.indexOf('**Type:**') < out.indexOf('## Introduksjon')); + // the pipe meta row is untouched (siblings preserved) + assert.match(out, /\*\*Opprettet:\*\* 2026-04 \| \*\*Sist oppdatert:\*\* 2026-06-19/); + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(HDR_PIPE)); +}); + +test('insertHeaderFields for a non-reference type emits Type only, never Source', () => { + const out = insertHeaderFields(HDR_KATEGORI, { type: 'methodology', source: MS_URL }); + assert.equal(parseTypeHeader(out), 'methodology'); + assert.equal(parseSourceHeader(out), null); + assert.doesNotMatch(out, /\*\*Source:\*\*/); +}); + +test('insertHeaderFields is per-field idempotent (a re-run inserts nothing)', () => { + const once = insertHeaderFields(HDR_CATEGORY, { type: 'reference', source: MS_URL }); + const twice = insertHeaderFields(once, { type: 'reference', source: MS_URL }); + assert.equal(twice, once); + assert.equal((once.match(/\*\*Type:\*\*/g) || []).length, 1); + assert.equal((once.match(/\*\*Source:\*\*/g) || []).length, 1); +}); + +test('insertHeaderFields adds only the missing field when Type is already present', () => { + const hasType = + '# T\n\n**Status:** GA\n**Category:** X\n**Type:** reference\n\n---\n\n## A\n\ntekst\n'; + const out = insertHeaderFields(hasType, { type: 'reference', source: MS_URL }); + assert.equal((out.match(/\*\*Type:\*\*/g) || []).length, 1); // not duplicated + assert.equal(parseSourceHeader(out), MS_URL); // Source backfilled +}); + +test('insertHeaderFields keeps an existing **Source:** (the 7 pre-sourced files)', () => { + const hasSource = + '# T\n\n**Status:** GA\n**Type:** reference\n' + + '**Source:** https://learn.microsoft.com/existing\n\n---\n\n## A\n\ntekst\n'; + const out = insertHeaderFields(hasSource, { type: 'reference', source: MS_URL }); + assert.equal(parseSourceHeader(out), 'https://learn.microsoft.com/existing'); + assert.equal((out.match(/\*\*Source:\*\*/g) || []).length, 1); +}); + +test('insertHeaderFields rejects an unknown type', () => { + assert.throws(() => insertHeaderFields(HDR_CATEGORY, { type: 'blogpost' }), /type/i); +}); + +test('insertToc(insertHeaderFields(large)) → Type+Source+TOC, passes real checkN4, body byte-identical except header + one ## Innhold', () => { + const withHdr = insertHeaderFields(HDR_LARGE, { type: 'reference', source: MS_URL }); + const out = insertToc(withHdr); + assert.equal(parseTypeHeader(out), 'reference'); + assert.equal(parseSourceHeader(out), MS_URL); + const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: out }]); + assert.equal(n4.largeFiles, 1); + assert.equal(n4.withToc, 1); + assert.equal(n4.pass, true); + assert.equal((out.match(/^##\s+Innhold/gm) || []).length, 1); + // body from the first real section is byte-identical to the original body + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(HDR_LARGE)); +}); + +// --- normalizeStaleVerified: surgical carve-out for the 14 `**Verified:** MCP` files --- +// Acts ONLY on the first header-region **Verified:** (the one parseVerifiedHeader reads), +// ONLY when its value is not a clean YYYY-MM(-DD) date. Never touches at/below the first +// `---` (body-duplicate preserved). Pipe-safe: on a pipe row it removes only the Verified +// token + one adjacent ` | `, keeping siblings. Idempotent. + +const VERIFIED_STALE = + '# T\n\n**Last updated:** 2026-06-19\n**Verified:** MCP 2026-06-19\n' + + '**Status:** GA\n**Category:** X\n\n---\n\n## A\n\ntekst.\n'; + +const VERIFIED_CLEAN = + '# T\n\n**Last updated:** 2026-06\n**Verified:** 2026-06\n**Status:** GA\n\n---\n\n## A\n\ntekst\n'; + +// The mlops-fundamentals-overview.md shape: a stale header Verified AND a duplicate +// below the `---` rule (line-10 shape). The normalizer removes the header one only. +const VERIFIED_BODY_DUP = + '# MLOps Fundamentals\n\n**Last updated:** 2026-06-19\n**Verified:** MCP 2026-06-19\n' + + '**Status:** GA\n**Category:** MLOps\n\n---\n\n**Verified:** MCP 2026-06-19\n\n' + + '## Introduksjon\n\ntekst.\n'; + +// The ai-center-of-excellence-setup.md shape: Verified is the last token of a +// pipe-delimited meta row. +const VERIFIED_PIPE = + '# AI Center of Excellence\n\n**Kategori:** Responsible AI & Governance\n' + + '**Opprettet:** 2026-04 | **Sist oppdatert:** 2026-06-19 | **Verified:** MCP 2026-06-19\n' + + '**Confidence:** HIGH (Microsoft CAF)\n\n## Introduksjon\n\ntekst.\n'; + +test('normalizeStaleVerified removes a standalone stale **Verified:** MCP header line (parseVerifiedHeader → null)', () => { + const out = normalizeStaleVerified(VERIFIED_STALE); + assert.equal(parseVerifiedHeader(out), null); + assert.doesNotMatch(out, /\*\*Verified:\*\*/); + // siblings preserved + assert.match(out, /\*\*Last updated:\*\* 2026-06-19/); + assert.match(out, /\*\*Status:\*\* GA/); + // body untouched + const tail = (s) => s.slice(s.indexOf('## A')); + assert.equal(tail(out), tail(VERIFIED_STALE)); +}); + +test('normalizeStaleVerified leaves a clean YYYY-MM(-DD) Verified date untouched', () => { + assert.equal(normalizeStaleVerified(VERIFIED_CLEAN), VERIFIED_CLEAN); + assert.equal(parseVerifiedHeader(VERIFIED_CLEAN), '2026-06'); +}); + +test('normalizeStaleVerified removes the header **Verified:** but never the body-duplicate below ---', () => { + const out = normalizeStaleVerified(VERIFIED_BODY_DUP); + // exactly one **Verified:** remains … + assert.equal((out.match(/\*\*Verified:\*\*/g) || []).length, 1); + // … and it is the body one, below the `---` rule + const ruleIdx = out.indexOf('\n---'); + assert.ok(ruleIdx !== -1 && out.indexOf('**Verified:**') > ruleIdx, 'surviving Verified must be below ---'); + // header meta above the rule is intact + assert.match(out.slice(0, ruleIdx), /\*\*Last updated:\*\* 2026-06-19/); + // body from ## Introduksjon byte-identical + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(VERIFIED_BODY_DUP)); +}); + +test('normalizeStaleVerified on a pipe row removes only the Verified token, preserving siblings', () => { + const out = normalizeStaleVerified(VERIFIED_PIPE); + assert.equal(parseVerifiedHeader(out), null); + assert.match(out, /\*\*Opprettet:\*\* 2026-04/); + assert.match(out, /\*\*Sist oppdatert:\*\* 2026-06-19/); + assert.doesNotMatch(out, /\*\*Verified:\*\*/); + // no dangling separator left at the row end + assert.doesNotMatch(out, /2026-06-19 \|\s*$/m); + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(VERIFIED_PIPE)); +}); + +test('normalizeStaleVerified is idempotent', () => { + const once = normalizeStaleVerified(VERIFIED_STALE); + assert.equal(normalizeStaleVerified(once), once); + const oncePipe = normalizeStaleVerified(VERIFIED_PIPE); + assert.equal(normalizeStaleVerified(oncePipe), oncePipe); + const onceDup = normalizeStaleVerified(VERIFIED_BODY_DUP); + assert.equal(normalizeStaleVerified(onceDup), onceDup); +}); + +test('normalizeStaleVerified is a no-op when there is no header Verified', () => { + const clean = '# T\n\n**Status:** GA\n\n---\n\n## A\n\ntekst\n'; + assert.equal(normalizeStaleVerified(clean), clean); +});