feat(ms-ai-architect): Spor 1 — insertHeaderFields + surgical normalizeStaleVerified (3 dialekter, pipe-safe, ---aware, TDD) [skip-docs]
This commit is contained in:
parent
f8e420c773
commit
6b58afda3b
2 changed files with 338 additions and 0 deletions
|
|
@ -50,6 +50,14 @@ const MS_LEARN_RE = /\b(?:learn|docs)\.microsoft\.com/i;
|
|||
|
||||
const DATE_RE = /^\d{4}-\d{2}(?:-\d{2})?$/;
|
||||
|
||||
// The top-of-file region the kb-headers parsers scan. MIRRORS kb-headers.mjs:6
|
||||
// (HEADER_REGION_BYTES) — kept local rather than imported so this lib's only
|
||||
// kb-headers dependency stays the read-only parser functions (same discipline as
|
||||
// TOC_MIN_LINES mirroring eval.mjs). insertHeaderFields / normalizeStaleVerified use it
|
||||
// to keep the fields they insert — and the stale Verified they strip — inside the window
|
||||
// those parsers actually read.
|
||||
const HEADER_REGION_BYTES = 500;
|
||||
|
||||
// A reference file longer than this many lines should carry a table of contents
|
||||
// (best-practices §Structure). MIRRORS eval.mjs N4_LARGE_FILE_LINES — kept in sync
|
||||
// behaviourally by test-transform's checkN4 parity assertion, NOT by importing across
|
||||
|
|
@ -257,6 +265,122 @@ export function insertToc(content) {
|
|||
return [...before, ...block, ...lines.slice(idx)].join('\n');
|
||||
}
|
||||
|
||||
// A bold-label meta line, e.g. **Category:** / **Kategori:** / **Status:** /
|
||||
// **Sist oppdatert:** / **Confidence:** — and a pipe-delimited meta row, which also
|
||||
// begins with a bold label. The non-greedy label body excludes `*` so it stops at the
|
||||
// first `:**`. Language-agnostic by construction: it keys on the bold-label shape, not
|
||||
// on a fixed label vocabulary.
|
||||
const RE_META_LINE = /^\s*\*\*[^*\n]+?:\*\*/;
|
||||
|
||||
/**
|
||||
* Backfill the Port-1 `**Type:**` (and, for a reference, `**Source:**`) header field(s)
|
||||
* into an EXISTING legacy KB file that lacks them — the Spor 1 migration primitive for
|
||||
* the ~327 non-advisor files. Unlike buildKbHeader (which emits a COMPLETE new header),
|
||||
* this inserts ONLY the missing field(s), placing them immediately after the last line of
|
||||
* the first contiguous run of bold-label meta lines (RE_META_LINE — covers the
|
||||
* **Category:** / **Kategori:** / pipe dialects). It never inserts before the first `## `
|
||||
* section or past the 500-byte scan window, so the fields land where kb-headers.* read
|
||||
* them; the body is byte-identical.
|
||||
*
|
||||
* Per-field idempotent: skips `**Type:**` when parseTypeHeader is already set, skips
|
||||
* `**Source:**` when parseSourceHeader is already set. A non-reference type NEVER receives
|
||||
* a `**Source:**` (mirrors buildKbHeader's out-of-correctness-scope rule) — Type only.
|
||||
*
|
||||
* @param {string} content — full existing file content
|
||||
* @param {{type?: string, source?: string}} meta — resolved type + (reference) authority URL
|
||||
* @returns {string} content with the missing header field(s) inserted, or unchanged
|
||||
* @throws if the type is not one of VALID_TYPES
|
||||
*/
|
||||
export function insertHeaderFields(content, meta) {
|
||||
const s = String(content ?? '');
|
||||
const m = meta ?? {};
|
||||
const type = m.type ?? DEFAULT_TYPE;
|
||||
if (!VALID_TYPES.includes(type)) {
|
||||
throw new Error(`insertHeaderFields: invalid type '${type}' (expected ${VALID_TYPES.join('|')})`);
|
||||
}
|
||||
const isReference = type === 'reference';
|
||||
const src = m.source;
|
||||
const hasSource = src !== undefined && src !== null && String(src).trim() !== '';
|
||||
const needType = parseTypeHeader(s) === null;
|
||||
const needSource = isReference && hasSource && parseSourceHeader(s) === null;
|
||||
if (!needType && !needSource) return s;
|
||||
|
||||
const toInsert = [];
|
||||
if (needType) toInsert.push(`**Type:** ${type}`);
|
||||
if (needSource) toInsert.push(`**Source:** ${String(src).trim()}`);
|
||||
|
||||
const lines = s.split('\n');
|
||||
let anchorIdx = -1; // last line of the first contiguous meta run
|
||||
let titleIdx = -1; // fallback anchor for the "no meta line" dialect
|
||||
let cum = 0;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (/^##\s/.test(line) || /^---\s*$/.test(line)) break; // reached body / rule
|
||||
if (titleIdx === -1 && /^#\s+\S/.test(line)) titleIdx = i;
|
||||
if (RE_META_LINE.test(line)) {
|
||||
anchorIdx = i; // extend the contiguous meta run
|
||||
} else if (anchorIdx !== -1) {
|
||||
break; // first non-meta line after the run — the run is over (contiguous only)
|
||||
}
|
||||
cum += line.length + 1;
|
||||
if (cum > HEADER_REGION_BYTES) break; // never place fields past the scan window
|
||||
}
|
||||
// "None" dialect (no bold-label meta): anchor on the title line so the fields still
|
||||
// land in the header region, before the first section. (-1 → very top if no title.)
|
||||
if (anchorIdx === -1) anchorIdx = titleIdx;
|
||||
lines.splice(anchorIdx + 1, 0, ...toInsert);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Surgically strip a STALE `**Verified:**` value from a legacy KB file's header — the
|
||||
* bounded Spor 1 carve-out for the 14 files carrying `**Verified:** MCP <date>`. That
|
||||
* non-date value is read by parseVerifiedHeader as a bogus "verified" stamp, which drops
|
||||
* the file out of the full-pass worklist AND marks it falsely verified; stripping it is
|
||||
* the OPPOSITE of stamping verified. Acts ONLY on the first header-region `**Verified:**`
|
||||
* (the exact occurrence parseVerifiedHeader reads) and ONLY when its value is not a clean
|
||||
* YYYY-MM(-DD) date. It never touches anything at or below the first `---` rule, so a
|
||||
* body-duplicate survives (removing body content is NOT part of the blessed byte-identity
|
||||
* carve-out). On a pipe-delimited meta row it removes only the `**Verified:** <value>`
|
||||
* token plus one adjacent ` | ` separator, preserving the sibling tokens
|
||||
* (**Sist oppdatert:** etc.); on a standalone line it drops the whole line. Idempotent.
|
||||
*
|
||||
* @param {string} content — full existing file content
|
||||
* @returns {string} content with the stale header Verified removed, or unchanged
|
||||
*/
|
||||
export function normalizeStaleVerified(content) {
|
||||
const s = String(content ?? '');
|
||||
const head = s.slice(0, HEADER_REGION_BYTES);
|
||||
const m = /\*\*Verified:\*\*\s*(\S+)/i.exec(head);
|
||||
if (!m) return s; // no header Verified
|
||||
if (DATE_RE.test(m[1])) return s; // clean date → leave untouched
|
||||
const matchStart = m.index; // index of "**Verified:**" in s
|
||||
// Guard: never act at or below the first `---` rule — body content is off-limits.
|
||||
const rule = /^---\s*$/m.exec(s);
|
||||
if (rule && matchStart >= rule.index) return s; // defensive (parser scans header, but be sure)
|
||||
|
||||
const lineStart = s.lastIndexOf('\n', matchStart - 1) + 1; // 0 if no preceding newline
|
||||
let lineEnd = s.indexOf('\n', matchStart);
|
||||
if (lineEnd === -1) lineEnd = s.length;
|
||||
const line = s.slice(lineStart, lineEnd);
|
||||
|
||||
if (!line.includes(' | ')) {
|
||||
// Standalone `**Verified:** …` line — drop the whole line, including its newline.
|
||||
const cutEnd = lineEnd < s.length ? lineEnd + 1 : lineEnd;
|
||||
return s.slice(0, lineStart) + s.slice(cutEnd);
|
||||
}
|
||||
// Pipe row: remove only the `**Verified:** <value>` token + one adjacent ` | `.
|
||||
const nextSepRel = s.slice(matchStart, lineEnd).indexOf(' | ');
|
||||
const tokenEnd = nextSepRel === -1 ? lineEnd : matchStart + nextSepRel;
|
||||
if (s.slice(matchStart - 3, matchStart) === ' | ') {
|
||||
return s.slice(0, matchStart - 3) + s.slice(tokenEnd); // drop the preceding separator
|
||||
}
|
||||
if (nextSepRel !== -1) {
|
||||
return s.slice(0, matchStart) + s.slice(tokenEnd + 3); // drop the following separator
|
||||
}
|
||||
return s.slice(0, matchStart) + s.slice(tokenEnd); // degenerate — token only
|
||||
}
|
||||
|
||||
const RE_TITLE = /^#\s+\S/m;
|
||||
const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i;
|
||||
const RE_STATUS = /\*\*Status:\*\*\s*\S/i;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import {
|
|||
buildToc,
|
||||
composeKbFile,
|
||||
insertToc,
|
||||
insertHeaderFields,
|
||||
normalizeStaleVerified,
|
||||
stampVerifiedMeta,
|
||||
} from '../../scripts/kb-update/lib/transform.mjs';
|
||||
import {
|
||||
|
|
@ -494,3 +496,215 @@ test('transform.mjs imports NO write-utils (pure transformer, never writes)', ()
|
|||
assert.doesNotMatch(importLines, /from\s+'[^']*atomic-write[^']*'/);
|
||||
assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/);
|
||||
});
|
||||
|
||||
// --- insertHeaderFields: Spor 1 migration primitive (backfill Type/Source) -------
|
||||
// Unlike buildKbHeader (which emits a COMPLETE new header), insertHeaderFields inserts
|
||||
// only the missing Port-1 field(s) into an EXISTING legacy file's top-region meta block,
|
||||
// immediately after the last line of the first contiguous run of bold-label meta lines
|
||||
// (language-agnostic across the 3 dialects), never before the first `## ` section, never
|
||||
// past the 500-byte scan window. Body byte-identical; per-field idempotent.
|
||||
|
||||
// Dialect A — **Category:** (English labels), with a `---` header rule.
|
||||
const HDR_CATEGORY =
|
||||
'# MLOps Fundamentals\n\n' +
|
||||
'**Last updated:** 2026-06-19\n' +
|
||||
'**Status:** GA\n' +
|
||||
'**Category:** MLOps & GenAIOps\n\n' +
|
||||
'---\n\n' +
|
||||
'## Introduksjon\n\nBrødtekst.\n';
|
||||
|
||||
// Dialect B — **Kategori:** (Norwegian labels), with a `---` header rule.
|
||||
const HDR_KATEGORI =
|
||||
'# Feedback Loops\n\n' +
|
||||
'**Kategori:** MLOps & GenAIOps\n' +
|
||||
'**Sist oppdatert:** 2026-06\n\n' +
|
||||
'---\n\n' +
|
||||
'## Introduksjon\n\nBrødtekst.\n';
|
||||
|
||||
// Dialect C — pipe-delimited meta row, NO `---` rule (the ai-center-of-excellence shape).
|
||||
const HDR_PIPE =
|
||||
'# AI Center of Excellence\n\n' +
|
||||
'**Kategori:** Responsible AI & Governance\n' +
|
||||
'**Opprettet:** 2026-04 | **Sist oppdatert:** 2026-06-19\n' +
|
||||
'**Confidence:** HIGH (Microsoft CAF)\n\n' +
|
||||
'## Introduksjon\n\nBrødtekst.\n';
|
||||
|
||||
// A large file (>100 lines) with a category header — for the combined insertToc oracle.
|
||||
const HDR_LARGE =
|
||||
'# Big Ref\n\n' +
|
||||
'**Last updated:** 2026-06\n' +
|
||||
'**Status:** GA\n' +
|
||||
'**Category:** X\n\n' +
|
||||
'---\n\n' +
|
||||
LARGE_BODY;
|
||||
|
||||
const MS_URL = 'https://learn.microsoft.com/azure/machine-learning/concept-mlops';
|
||||
|
||||
test('insertHeaderFields inserts Type+Source after the meta block (category dialect), body byte-identical', () => {
|
||||
const out = insertHeaderFields(HDR_CATEGORY, { type: 'reference', source: MS_URL });
|
||||
assert.equal(parseTypeHeader(out), 'reference');
|
||||
assert.equal(parseSourceHeader(out), MS_URL);
|
||||
// Type/Source sit inside the meta block: after **Category:**, before the `---` rule.
|
||||
assert.ok(out.indexOf('**Category:**') < out.indexOf('**Type:**'));
|
||||
assert.ok(out.indexOf('**Type:**') < out.indexOf('\n---'));
|
||||
// body (from the first ## section) is byte-identical
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(HDR_CATEGORY));
|
||||
});
|
||||
|
||||
test('insertHeaderFields works on the Norwegian **Kategori:** dialect, body byte-identical', () => {
|
||||
const out = insertHeaderFields(HDR_KATEGORI, { type: 'reference', source: MS_URL });
|
||||
assert.equal(parseTypeHeader(out), 'reference');
|
||||
assert.equal(parseSourceHeader(out), MS_URL);
|
||||
assert.ok(out.indexOf('**Sist oppdatert:**') < out.indexOf('**Type:**'));
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(HDR_KATEGORI));
|
||||
});
|
||||
|
||||
test('insertHeaderFields works on the pipe-delimited dialect without a `---` rule, body byte-identical', () => {
|
||||
const out = insertHeaderFields(HDR_PIPE, { type: 'reference', source: MS_URL });
|
||||
assert.equal(parseTypeHeader(out), 'reference');
|
||||
assert.equal(parseSourceHeader(out), MS_URL);
|
||||
// inserted after the last meta line (**Confidence:**), before the first ## section
|
||||
assert.ok(out.indexOf('**Confidence:**') < out.indexOf('**Type:**'));
|
||||
assert.ok(out.indexOf('**Type:**') < out.indexOf('## Introduksjon'));
|
||||
// the pipe meta row is untouched (siblings preserved)
|
||||
assert.match(out, /\*\*Opprettet:\*\* 2026-04 \| \*\*Sist oppdatert:\*\* 2026-06-19/);
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(HDR_PIPE));
|
||||
});
|
||||
|
||||
test('insertHeaderFields for a non-reference type emits Type only, never Source', () => {
|
||||
const out = insertHeaderFields(HDR_KATEGORI, { type: 'methodology', source: MS_URL });
|
||||
assert.equal(parseTypeHeader(out), 'methodology');
|
||||
assert.equal(parseSourceHeader(out), null);
|
||||
assert.doesNotMatch(out, /\*\*Source:\*\*/);
|
||||
});
|
||||
|
||||
test('insertHeaderFields is per-field idempotent (a re-run inserts nothing)', () => {
|
||||
const once = insertHeaderFields(HDR_CATEGORY, { type: 'reference', source: MS_URL });
|
||||
const twice = insertHeaderFields(once, { type: 'reference', source: MS_URL });
|
||||
assert.equal(twice, once);
|
||||
assert.equal((once.match(/\*\*Type:\*\*/g) || []).length, 1);
|
||||
assert.equal((once.match(/\*\*Source:\*\*/g) || []).length, 1);
|
||||
});
|
||||
|
||||
test('insertHeaderFields adds only the missing field when Type is already present', () => {
|
||||
const hasType =
|
||||
'# T\n\n**Status:** GA\n**Category:** X\n**Type:** reference\n\n---\n\n## A\n\ntekst\n';
|
||||
const out = insertHeaderFields(hasType, { type: 'reference', source: MS_URL });
|
||||
assert.equal((out.match(/\*\*Type:\*\*/g) || []).length, 1); // not duplicated
|
||||
assert.equal(parseSourceHeader(out), MS_URL); // Source backfilled
|
||||
});
|
||||
|
||||
test('insertHeaderFields keeps an existing **Source:** (the 7 pre-sourced files)', () => {
|
||||
const hasSource =
|
||||
'# T\n\n**Status:** GA\n**Type:** reference\n' +
|
||||
'**Source:** https://learn.microsoft.com/existing\n\n---\n\n## A\n\ntekst\n';
|
||||
const out = insertHeaderFields(hasSource, { type: 'reference', source: MS_URL });
|
||||
assert.equal(parseSourceHeader(out), 'https://learn.microsoft.com/existing');
|
||||
assert.equal((out.match(/\*\*Source:\*\*/g) || []).length, 1);
|
||||
});
|
||||
|
||||
test('insertHeaderFields rejects an unknown type', () => {
|
||||
assert.throws(() => insertHeaderFields(HDR_CATEGORY, { type: 'blogpost' }), /type/i);
|
||||
});
|
||||
|
||||
test('insertToc(insertHeaderFields(large)) → Type+Source+TOC, passes real checkN4, body byte-identical except header + one ## Innhold', () => {
|
||||
const withHdr = insertHeaderFields(HDR_LARGE, { type: 'reference', source: MS_URL });
|
||||
const out = insertToc(withHdr);
|
||||
assert.equal(parseTypeHeader(out), 'reference');
|
||||
assert.equal(parseSourceHeader(out), MS_URL);
|
||||
const n4 = checkN4([{ path: 'skills/x/references/y/z.md', content: out }]);
|
||||
assert.equal(n4.largeFiles, 1);
|
||||
assert.equal(n4.withToc, 1);
|
||||
assert.equal(n4.pass, true);
|
||||
assert.equal((out.match(/^##\s+Innhold/gm) || []).length, 1);
|
||||
// body from the first real section is byte-identical to the original body
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(HDR_LARGE));
|
||||
});
|
||||
|
||||
// --- normalizeStaleVerified: surgical carve-out for the 14 `**Verified:** MCP` files ---
|
||||
// Acts ONLY on the first header-region **Verified:** (the one parseVerifiedHeader reads),
|
||||
// ONLY when its value is not a clean YYYY-MM(-DD) date. Never touches at/below the first
|
||||
// `---` (body-duplicate preserved). Pipe-safe: on a pipe row it removes only the Verified
|
||||
// token + one adjacent ` | `, keeping siblings. Idempotent.
|
||||
|
||||
const VERIFIED_STALE =
|
||||
'# T\n\n**Last updated:** 2026-06-19\n**Verified:** MCP 2026-06-19\n' +
|
||||
'**Status:** GA\n**Category:** X\n\n---\n\n## A\n\ntekst.\n';
|
||||
|
||||
const VERIFIED_CLEAN =
|
||||
'# T\n\n**Last updated:** 2026-06\n**Verified:** 2026-06\n**Status:** GA\n\n---\n\n## A\n\ntekst\n';
|
||||
|
||||
// The mlops-fundamentals-overview.md shape: a stale header Verified AND a duplicate
|
||||
// below the `---` rule (line-10 shape). The normalizer removes the header one only.
|
||||
const VERIFIED_BODY_DUP =
|
||||
'# MLOps Fundamentals\n\n**Last updated:** 2026-06-19\n**Verified:** MCP 2026-06-19\n' +
|
||||
'**Status:** GA\n**Category:** MLOps\n\n---\n\n**Verified:** MCP 2026-06-19\n\n' +
|
||||
'## Introduksjon\n\ntekst.\n';
|
||||
|
||||
// The ai-center-of-excellence-setup.md shape: Verified is the last token of a
|
||||
// pipe-delimited meta row.
|
||||
const VERIFIED_PIPE =
|
||||
'# AI Center of Excellence\n\n**Kategori:** Responsible AI & Governance\n' +
|
||||
'**Opprettet:** 2026-04 | **Sist oppdatert:** 2026-06-19 | **Verified:** MCP 2026-06-19\n' +
|
||||
'**Confidence:** HIGH (Microsoft CAF)\n\n## Introduksjon\n\ntekst.\n';
|
||||
|
||||
test('normalizeStaleVerified removes a standalone stale **Verified:** MCP header line (parseVerifiedHeader → null)', () => {
|
||||
const out = normalizeStaleVerified(VERIFIED_STALE);
|
||||
assert.equal(parseVerifiedHeader(out), null);
|
||||
assert.doesNotMatch(out, /\*\*Verified:\*\*/);
|
||||
// siblings preserved
|
||||
assert.match(out, /\*\*Last updated:\*\* 2026-06-19/);
|
||||
assert.match(out, /\*\*Status:\*\* GA/);
|
||||
// body untouched
|
||||
const tail = (s) => s.slice(s.indexOf('## A'));
|
||||
assert.equal(tail(out), tail(VERIFIED_STALE));
|
||||
});
|
||||
|
||||
test('normalizeStaleVerified leaves a clean YYYY-MM(-DD) Verified date untouched', () => {
|
||||
assert.equal(normalizeStaleVerified(VERIFIED_CLEAN), VERIFIED_CLEAN);
|
||||
assert.equal(parseVerifiedHeader(VERIFIED_CLEAN), '2026-06');
|
||||
});
|
||||
|
||||
test('normalizeStaleVerified removes the header **Verified:** but never the body-duplicate below ---', () => {
|
||||
const out = normalizeStaleVerified(VERIFIED_BODY_DUP);
|
||||
// exactly one **Verified:** remains …
|
||||
assert.equal((out.match(/\*\*Verified:\*\*/g) || []).length, 1);
|
||||
// … and it is the body one, below the `---` rule
|
||||
const ruleIdx = out.indexOf('\n---');
|
||||
assert.ok(ruleIdx !== -1 && out.indexOf('**Verified:**') > ruleIdx, 'surviving Verified must be below ---');
|
||||
// header meta above the rule is intact
|
||||
assert.match(out.slice(0, ruleIdx), /\*\*Last updated:\*\* 2026-06-19/);
|
||||
// body from ## Introduksjon byte-identical
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(VERIFIED_BODY_DUP));
|
||||
});
|
||||
|
||||
test('normalizeStaleVerified on a pipe row removes only the Verified token, preserving siblings', () => {
|
||||
const out = normalizeStaleVerified(VERIFIED_PIPE);
|
||||
assert.equal(parseVerifiedHeader(out), null);
|
||||
assert.match(out, /\*\*Opprettet:\*\* 2026-04/);
|
||||
assert.match(out, /\*\*Sist oppdatert:\*\* 2026-06-19/);
|
||||
assert.doesNotMatch(out, /\*\*Verified:\*\*/);
|
||||
// no dangling separator left at the row end
|
||||
assert.doesNotMatch(out, /2026-06-19 \|\s*$/m);
|
||||
const tail = (s) => s.slice(s.indexOf('## Introduksjon'));
|
||||
assert.equal(tail(out), tail(VERIFIED_PIPE));
|
||||
});
|
||||
|
||||
test('normalizeStaleVerified is idempotent', () => {
|
||||
const once = normalizeStaleVerified(VERIFIED_STALE);
|
||||
assert.equal(normalizeStaleVerified(once), once);
|
||||
const oncePipe = normalizeStaleVerified(VERIFIED_PIPE);
|
||||
assert.equal(normalizeStaleVerified(oncePipe), oncePipe);
|
||||
const onceDup = normalizeStaleVerified(VERIFIED_BODY_DUP);
|
||||
assert.equal(normalizeStaleVerified(onceDup), onceDup);
|
||||
});
|
||||
|
||||
test('normalizeStaleVerified is a no-op when there is no header Verified', () => {
|
||||
const clean = '# T\n\n**Status:** GA\n\n---\n\n## A\n\ntekst\n';
|
||||
assert.equal(normalizeStaleVerified(clean), clean);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue