fix(ms-ai-architect): registry-herding Fase 1a — +5 sitemap-prefiks + skjemaløs URL-ekstraksjon

Update-mekanismen lot siterte URL-er stå som not_in_sitemap fordi deres docset
aldri ble pollet, og skjemaløse citater (tabellceller: bare learn.microsoft.com/...
uten https://) ble aldri ekstrahert.

- Taxonomy: +5 docsets (graph, ai-builder, power-apps, power-automate,
  microsoftsearch) — hver ett child-sitemap, verifisert live mot indeksen.
  Bevist: 21/24 not_in_sitemap-URL-er i disse docsetene blir nå tracked.
- url-normalize: extractUrls fanger skjemaløse citater-med-sti (krever /path
  etter domenet → avviser bare-domene-prosa og JSON-eksempler); normalizeUrl
  kanonikaliserer scheme til https. Bevist: +19 nye URL-er ekstraherbare.
- Bugfix: backtick fra inline-kode-citat (`learn.microsoft.com/...`) lekket inn
  i URL-en — ekskludert i regex + trailing-strip.

TDD: tests/kb-update/test-url-normalize.test.mjs (ny, 14) + test-taxonomy
prefiks-count 18→23 + Fase 1a-assertion. Full suite 338/338, 0 regresjon.
Registry-refresh (build-registry --merge + poll) bevisst utsatt — unngår
552KB re-order-churn; effekten er empirisk verifisert read-only.

scripts/kb-update/data/domain-taxonomy.json

@@ -3,7 +3,7 @@
   "description": "Single source of truth (lag 0) for 'in-domain' classification. Consolidates four previously-divergent taxonomies into one: (a) poll-sitemaps TARGET_PREFIXES, (b) discover-new-urls INCLUDE/EXCLUDE, (c) category->skill ownership, (d) report-changes getFilePriority. Scripts READ this file; they no longer embed their own copies.",
   "provenance": {
     "category_skill": "PHYSICAL DISK is canonical (skills/<skill>/references/<category>/). category-skill-map.json drifted on 4 entries (copilot-extensibility, monitoring-observability, performance-scalability, prompt-engineering) and is no longer consumed by code.",
-    "sitemap_prefixes": "poll-sitemaps superset (18). discover-new-urls previously had only 12 — now reads this list for poll parity.",
+    "sitemap_prefixes": "poll-sitemaps superset (23). discover-new-urls previously had only 12 — now reads this list for poll parity. Fase 1a (2026-06-26) added 5 stack-relevant docsets — graph, ai-builder, power-apps, power-automate, microsoftsearch — each a single child sitemap verified live against the index, reclaiming cited URLs previously stranded as not_in_sitemap.",
     "notes": "dotnet_en-us_ deliberately excluded: 75 sitemaps, only ~12 matches — not worth weekly polling.",
     "course_products": "Product slugs enumerated LIVE from the Learn Platform API (C3.3, server-side /api/v1/modules?products=<slug> probe). Doc slug 'azure-ai-foundry' is DEAD (0 hits) → Foundry/AI-services courses surface under 'azure-openai'. Slug gotchas confirmed live: 'fabric' (not 'microsoft-fabric'), 'entra' (not 'microsoft-entra-id'), 'azure-cosmos-db' (not 'cosmos-db'), 'azure-cognitive-search' (not 'azure-ai-search'). 'azure-kubernetes-service' is live but excluded (no AI category → noise). Maps slug->category; the owning skill is DERIVED from category_skill via makeCourseClassifier (never stored, cannot diverge)."
   },
@@ -25,7 +25,12 @@
     "microsoft-365_en-us_",
     "training_en-us_",
     "cloud-computing_en-us_",
-    "privacy_en-us_"
+    "privacy_en-us_",
+    "graph_en-us_",
+    "ai-builder_en-us_",
+    "power-apps_en-us_",
+    "power-automate_en-us_",
+    "microsoftsearch_en-us_"
   ],
   "category_skill": {
     "agent-orchestration": "ms-ai-engineering",

scripts/kb-update/lib/url-normalize.mjs

@@ -4,6 +4,7 @@
 /**
  * Normalize a learn.microsoft.com URL to a canonical form.
  * Rules applied in order:
+ *   0. Canonicalise scheme — accept schema-less + http(s) citations, store https
  *   1. Strip trailing punctuation leaked from markdown
  *   2. Strip fragment (#anchor)
  *   3. Strip ?view= and other query params
@@ -18,8 +19,16 @@ export function normalizeUrl(raw) {
 
   let url = raw;
 
+  // 0. Canonicalise scheme — citations appear schema-less (table cells: bare
+  //    learn.microsoft.com/...) or as http; strip any leading scheme and force
+  //    https so a schema-less citation matches the same key as its https form
+  //    (and the https sitemap entries) instead of creating a duplicate entry.
+  url = url.replace(/^https?:\/\//i, '');
+  url = 'https://' + url;
+
   // 1. Strip trailing punctuation that leaked from markdown context
-  url = url.replace(/[.,;:!?'")}\]]+$/, '');
+  //    (incl. backtick from inline-code citations: `learn.microsoft.com/...`)
+  url = url.replace(/[.,;:!?'")}\]`]+$/, '');
 
   // 2. Strip fragment
   const hashIdx = url.indexOf('#');
@@ -43,18 +52,23 @@ export function normalizeUrl(raw) {
 
 /**
  * Extract all learn.microsoft.com URLs from a text string.
- * Handles all 5 citation formats found in reference files:
+ * Handles all citation formats found in reference files:
  *   - Markdown links: [text](https://learn.microsoft.com/...)
  *   - Bare URLs on their own line
  *   - URL: prefix format
  *   - Dash-bullet format
  *   - Table cell format
+ *   - Schema-less citations: bare learn.microsoft.com/<path> (no https://)
+ * The scheme is optional but a path (`/...`) after the domain is REQUIRED — this
+ * captures schema-less citations-with-path while rejecting bare-domain prose
+ * mentions ("(learn.microsoft.com, blogs...)") and JSON example strings
+ * ({"url": "learn.microsoft.com"}), which carry no trackable source.
  * @param {string} text
  * @returns {string[]} array of normalized unique URLs
  */
 export function extractUrls(text) {
   if (!text) return [];
-  const regex = /https:\/\/learn\.microsoft\.com[^\s)"'<>\]|]+/g;
+  const regex = /(?:https?:\/\/)?learn\.microsoft\.com\/[^\s)"'<>\]|`]+/g;
   const seen = new Set();
   const results = [];
   let match;

tests/kb-update/test-taxonomy.test.mjs

@@ -24,9 +24,9 @@ import {
 const tax = loadTaxonomy();
 
 // --- (a) sitemap prefixes — poll/discover parity ---
-test('sitemap_prefixes — 18 entries with poll/discover parity', () => {
+test('sitemap_prefixes — 23 entries with poll/discover parity', () => {
   const prefixes = getSitemapPrefixes(tax);
-  assert.equal(prefixes.length, 18);
+  assert.equal(prefixes.length, 23);
   // poll superset includes the 6 that discover previously lacked
   for (const p of ['microsoftteams_en-us_', 'sharepoint_en-us_', 'microsoft-365_en-us_',
                    'training_en-us_', 'cloud-computing_en-us_', 'privacy_en-us_']) {
@@ -37,6 +37,17 @@ test('sitemap_prefixes — 18 entries with poll/discover parity', () => {
   assert.ok(!prefixes.includes('dotnet_en-us_'));
 });
 
+// Fase 1a registry-herding: 5 stack-relevant docsets whose cited URLs were
+// stranded as not_in_sitemap because their child sitemaps were never polled.
+// Each is a single child sitemap (verified live against the sitemap index).
+test('sitemap_prefixes — Fase 1a additions (graph / power / ai-builder / search)', () => {
+  const prefixes = getSitemapPrefixes(tax);
+  for (const p of ['graph_en-us_', 'ai-builder_en-us_', 'power-apps_en-us_',
+                   'power-automate_en-us_', 'microsoftsearch_en-us_']) {
+    assert.ok(prefixes.includes(p), `missing Fase 1a prefix ${p}`);
+  }
+});
+
 // --- (c) category → skill — PHYSICAL DISK is canonical ---
 test('category_skill — disk-truth canonical (the 4 formerly stale entries)', () => {
   // category-skill-map.json had these as ms-ai-engineering; disk says otherwise.

tests/kb-update/test-url-normalize.test.mjs

@@ -0,0 +1,115 @@
+// tests/kb-update/test-url-normalize.test.mjs
+// Unit tests for scripts/kb-update/lib/url-normalize.mjs.
+// Pins the canonical-URL contract that build-registry ↔ poll-sitemaps matching
+// depends on, AND the Fase 1a registry-herding extension: schema-less citations
+// (learn.microsoft.com/... without the https:// prefix) must be extracted and
+// canonicalised to https://, while bare-domain prose mentions (no path) and
+// JSON example strings must NOT be captured.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { normalizeUrl, extractUrls } from '../../scripts/kb-update/lib/url-normalize.mjs';
+
+// --- normalizeUrl: existing contract (regression guard) ---
+test('normalizeUrl — strips locale, fragment, query, trailing slash; lowercases', () => {
+  assert.equal(
+    normalizeUrl('https://learn.microsoft.com/en-us/azure/AI-Foundry/Overview/?view=x#sec'),
+    'https://learn.microsoft.com/azure/ai-foundry/overview'
+  );
+});
+
+test('normalizeUrl — strips trailing markdown punctuation', () => {
+  assert.equal(
+    normalizeUrl('https://learn.microsoft.com/azure/foo).'),
+    'https://learn.microsoft.com/azure/foo'
+  );
+});
+
+test('normalizeUrl — non-learn URL returns null', () => {
+  assert.equal(normalizeUrl('https://example.com/azure/foo'), null);
+  assert.equal(normalizeUrl(''), null);
+  assert.equal(normalizeUrl(null), null);
+});
+
+// --- normalizeUrl: Fase 1a — schema-less canonicalisation ---
+test('normalizeUrl — schema-less citation gets https:// prepended', () => {
+  assert.equal(
+    normalizeUrl('learn.microsoft.com/azure/api-management/genai-gateway-capabilities'),
+    'https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities'
+  );
+});
+
+test('normalizeUrl — schema-less with en-us locale is also canonicalised', () => {
+  assert.equal(
+    normalizeUrl('learn.microsoft.com/en-us/compliance/assurance/assurance-artificial-intelligence'),
+    'https://learn.microsoft.com/compliance/assurance/assurance-artificial-intelligence'
+  );
+});
+
+test('normalizeUrl — idempotent on schema-less input', () => {
+  const once = normalizeUrl('learn.microsoft.com/azure/foo');
+  assert.equal(normalizeUrl(once), once);
+});
+
+// --- extractUrls: existing contract (regression guard) ---
+test('extractUrls — extracts https markdown-link URL', () => {
+  assert.deepEqual(
+    extractUrls('See [docs](https://learn.microsoft.com/azure/foo) here.'),
+    ['https://learn.microsoft.com/azure/foo']
+  );
+});
+
+test('extractUrls — dedups same URL across forms', () => {
+  const out = extractUrls(
+    'https://learn.microsoft.com/azure/foo and learn.microsoft.com/azure/foo'
+  );
+  assert.deepEqual(out, ['https://learn.microsoft.com/azure/foo']);
+});
+
+// --- extractUrls: Fase 1a — schema-less table-cell citations (the 2 named files) ---
+test('extractUrls — captures schema-less table-cell citation', () => {
+  const row = '| **GenAI Gateway** | desc | learn.microsoft.com/azure/api-management/genai-gateway-capabilities |';
+  assert.deepEqual(
+    extractUrls(row),
+    ['https://learn.microsoft.com/azure/api-management/genai-gateway-capabilities']
+  );
+});
+
+test('extractUrls — captures multiple schema-less rows', () => {
+  const text = [
+    '| A | learn.microsoft.com/en-us/compliance/assurance/assurance-artificial-intelligence |',
+    '| B | learn.microsoft.com/en-us/azure/cloud-adoption-framework/scenarios/ai/govern |',
+  ].join('\n');
+  assert.deepEqual(extractUrls(text), [
+    'https://learn.microsoft.com/compliance/assurance/assurance-artificial-intelligence',
+    'https://learn.microsoft.com/azure/cloud-adoption-framework/scenarios/ai/govern',
+  ]);
+});
+
+// --- extractUrls: must NOT capture bare-domain prose / JSON examples ---
+test('extractUrls — ignores bare-domain prose mention (no path)', () => {
+  assert.deepEqual(
+    extractUrls('All info fra offisiell dokumentasjon (learn.microsoft.com, blogs.microsoft.com).'),
+    []
+  );
+});
+
+test('extractUrls — ignores JSON example string with no path', () => {
+  assert.deepEqual(extractUrls('{"url": "learn.microsoft.com"}'), []);
+});
+
+test('extractUrls — strips wrapping backticks from inline-code citation', () => {
+  // `learn.microsoft.com/agent-framework/` inside prose backticks must not leak
+  // the backtick into the URL (would never match a sitemap entry).
+  assert.deepEqual(
+    extractUrls('verifisert mot `learn.microsoft.com/agent-framework/`).'),
+    ['https://learn.microsoft.com/agent-framework']
+  );
+});
+
+test('normalizeUrl — strips trailing backtick', () => {
+  assert.equal(
+    normalizeUrl('https://learn.microsoft.com/azure/foo`'),
+    'https://learn.microsoft.com/azure/foo'
+  );
+});