diff --git a/.gitignore b/.gitignore index 60c05d9..be8574b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,10 @@ node_modules/ # Runtime state .work/ org/ -scripts/kb-update/data/ +# Generated KB-update artifacts (registry, reports) are ignored, but the +# hand-authored taxonomy (single source of truth, lag 0) is tracked. +scripts/kb-update/data/* +!scripts/kb-update/data/domain-taxonomy.json .kb-backup/ .rollback-in-progress diff --git a/scripts/kb-update/data/domain-taxonomy.json b/scripts/kb-update/data/domain-taxonomy.json new file mode 100644 index 0000000..7202dbc --- /dev/null +++ b/scripts/kb-update/data/domain-taxonomy.json @@ -0,0 +1,95 @@ +{ + "version": 1, + "description": "Single source of truth (lag 0) for 'in-domain' classification. Consolidates four previously-divergent taxonomies into one: (a) poll-sitemaps TARGET_PREFIXES, (b) discover-new-urls INCLUDE/EXCLUDE, (c) category->skill ownership, (d) report-changes getFilePriority. Scripts READ this file; they no longer embed their own copies.", + "provenance": { + "category_skill": "PHYSICAL DISK is canonical (skills//references//). category-skill-map.json drifted on 4 entries (copilot-extensibility, monitoring-observability, performance-scalability, prompt-engineering) and is no longer consumed by code.", + "sitemap_prefixes": "poll-sitemaps superset (18). discover-new-urls previously had only 12 — now reads this list for poll parity.", + "notes": "dotnet_en-us_ deliberately excluded: 75 sitemaps, only ~12 matches — not worth weekly polling." + }, + "sitemap_prefixes": [ + "azure_en-us_", + "microsoft-copilot-studio_en-us_", + "security_en-us_", + "fabric_en-us_", + "power-platform_en-us_", + "ai_en-us_", + "copilot_en-us_", + "compliance_en-us_", + "agent-framework_en-us_", + "semantic-kernel_en-us_", + "entra_en-us_", + "purview_en-us_", + "microsoftteams_en-us_", + "sharepoint_en-us_", + "microsoft-365_en-us_", + "training_en-us_", + "cloud-computing_en-us_", + "privacy_en-us_" + ], + "category_skill": { + "agent-orchestration": "ms-ai-engineering", + "ai-security-engineering": "ms-ai-security", + "api-management": "ms-ai-engineering", + "architecture": "ms-ai-advisor", + "azure-ai-services": "ms-ai-engineering", + "bcdr": "ms-ai-infrastructure", + "copilot-extensibility": "ms-ai-advisor", + "cost-optimization": "ms-ai-security", + "data-engineering": "ms-ai-engineering", + "development": "ms-ai-advisor", + "hybrid-edge": "ms-ai-infrastructure", + "mlops-genaiops": "ms-ai-engineering", + "monitoring-observability": "ms-ai-governance", + "multi-modal": "ms-ai-engineering", + "norwegian-public-sector-governance": "ms-ai-governance", + "performance-scalability": "ms-ai-security", + "platforms": "ms-ai-advisor", + "prompt-engineering": "ms-ai-advisor", + "rag-architecture": "ms-ai-engineering", + "responsible-ai": "ms-ai-governance", + "security-scoring": "ms-ai-security" + }, + "relevance": { + "include": [ + { "pattern": "/azure/ai-foundry/", "category": "azure-ai-services" }, + { "pattern": "/azure/ai-services/", "category": "azure-ai-services" }, + { "pattern": "/azure/machine-learning/", "category": "mlops-genaiops" }, + { "pattern": "/azure/search/", "category": "rag-architecture" }, + { "pattern": "/azure/api-management/", "category": "api-management" }, + { "pattern": "/azure/azure-monitor/", "category": "monitoring-observability" }, + { "pattern": "/azure/well-architected/", "category": "architecture" }, + { "pattern": "/microsoft-copilot-studio/", "category": "copilot-extensibility" }, + { "pattern": "/copilot/microsoft-365/", "category": "copilot-extensibility" }, + { "pattern": "/microsoft-365-copilot/", "category": "copilot-extensibility" }, + { "pattern": "/security/(?:ai|benchmark|engineering)/", "category": "ai-security-engineering" }, + { "pattern": "/azure/defender-for-cloud/", "category": "ai-security-engineering" }, + { "pattern": "/purview/", "category": "responsible-ai" }, + { "pattern": "/semantic-kernel/", "category": "agent-orchestration" }, + { "pattern": "/agent-framework/", "category": "agent-orchestration" }, + { "pattern": "/fabric/(?:data-engineering|data-science|real-time-intelligence)/", "category": "data-engineering" }, + { "pattern": "/azure/cosmos-db/", "category": "data-engineering" }, + { "pattern": "/azure/databricks/", "category": "data-engineering" }, + { "pattern": "/entra/(?:identity|agent)/", "category": "ai-security-engineering" } + ], + "exclude": [ + "/training/", + "/credentials/", + "/legal/", + "/previous-versions/", + "/archive/", + "/samples/", + "/release-notes?/", + "/whats-new", + "/changelog", + "/migrate/" + ] + }, + "priority": { + "rules": [ + { "match": "cost|pricing|pris", "priority": "critical" }, + { "match": "responsible-ai|governance|ai-security-(?:engineering|scoring)", "priority": "high" }, + { "match": "platforms|copilot|azure-ai-services|agent-orchestration|rag|mlops|prompt-engineering|monitoring|performance", "priority": "medium" } + ], + "default": "low" + } +} diff --git a/scripts/kb-update/discover-new-urls.mjs b/scripts/kb-update/discover-new-urls.mjs index 33a4c23..d3f5913 100644 --- a/scripts/kb-update/discover-new-urls.mjs +++ b/scripts/kb-update/discover-new-urls.mjs @@ -8,74 +8,19 @@ import { fileURLToPath } from 'node:url'; import { normalizeUrl } from './lib/url-normalize.mjs'; import { loadRegistry, saveReport } from './lib/registry-io.mjs'; import { streamSitemap, fetchSitemapIndex } from './lib/sitemap-stream.mjs'; +import { loadTaxonomy, getSitemapPrefixes, makeClassifier } from './lib/taxonomy.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const DATA_DIR = join(__dirname, 'data'); const limitArg = process.argv.indexOf('--limit'); const limit = limitArg !== -1 ? parseInt(process.argv[limitArg + 1], 10) : Infinity; -// Relevance patterns — URL paths that indicate content this plugin should cover -const INCLUDE = [ - { pattern: /\/azure\/ai-foundry\//, skill: 'ms-ai-engineering', category: 'azure-ai-services' }, - { pattern: /\/azure\/ai-services\//, skill: 'ms-ai-engineering', category: 'azure-ai-services' }, - { pattern: /\/azure\/machine-learning\//, skill: 'ms-ai-engineering', category: 'mlops-genaiops' }, - { pattern: /\/azure\/search\//, skill: 'ms-ai-engineering', category: 'rag-architecture' }, - { pattern: /\/azure\/api-management\//, skill: 'ms-ai-engineering', category: 'api-management' }, - { pattern: /\/azure\/azure-monitor\//, skill: 'ms-ai-governance', category: 'monitoring-observability' }, - { pattern: /\/azure\/well-architected\//, skill: 'ms-ai-advisor', category: 'architecture' }, - { pattern: /\/microsoft-copilot-studio\//, skill: 'ms-ai-advisor', category: 'copilot-extensibility' }, - { pattern: /\/copilot\/microsoft-365\//, skill: 'ms-ai-advisor', category: 'copilot-extensibility' }, - { pattern: /\/microsoft-365-copilot\//, skill: 'ms-ai-advisor', category: 'copilot-extensibility' }, - { pattern: /\/security\/(?:ai|benchmark|engineering)\//, skill: 'ms-ai-security', category: 'ai-security-engineering' }, - { pattern: /\/azure\/defender-for-cloud\//, skill: 'ms-ai-security', category: 'ai-security-engineering' }, - { pattern: /\/purview\//, skill: 'ms-ai-governance', category: 'responsible-ai' }, - { pattern: /\/semantic-kernel\//, skill: 'ms-ai-engineering', category: 'agent-orchestration' }, - { pattern: /\/agent-framework\//, skill: 'ms-ai-engineering', category: 'agent-orchestration' }, - { pattern: /\/fabric\/(?:data-engineering|data-science|real-time-intelligence)\//, skill: 'ms-ai-engineering', category: 'data-engineering' }, - { pattern: /\/azure\/cosmos-db\//, skill: 'ms-ai-engineering', category: 'data-engineering' }, - { pattern: /\/azure\/databricks\//, skill: 'ms-ai-engineering', category: 'data-engineering' }, - { pattern: /\/entra\/(?:identity|agent)\//, skill: 'ms-ai-security', category: 'ai-security-engineering' }, -]; - -// Exclude patterns — skip even if they match INCLUDE -const EXCLUDE = [ - /\/training\//, - /\/credentials\//, - /\/legal\//, - /\/previous-versions\//, - /\/archive\//, - /\/samples\//, - /\/release-notes?\//, - /\/whats-new/, - /\/changelog/, - /\/migrate\//, -]; - -function classifyUrl(url) { - if (EXCLUDE.some(p => p.test(url))) return null; - for (const rule of INCLUDE) { - if (rule.pattern.test(url)) { - return { skill: rule.skill, category: rule.category }; - } - } - return null; -} - -// Target sitemaps for discovery — same as poller, minus dotnet -const TARGET_PREFIXES = [ - 'azure_en-us_', - 'microsoft-copilot-studio_en-us_', - 'security_en-us_', - 'fabric_en-us_', - 'power-platform_en-us_', - 'ai_en-us_', - 'copilot_en-us_', - 'compliance_en-us_', - 'agent-framework_en-us_', - 'semantic-kernel_en-us_', - 'entra_en-us_', - 'purview_en-us_', -]; +// Relevance + sitemap targets come from the consolidated taxonomy (lag 0). +// classifyUrl: exclude wins, first include match, skill derived from the +// canonical category->skill map — no embedded skill assignments here. +const taxonomy = loadTaxonomy(DATA_DIR); +const classifyUrl = makeClassifier(taxonomy); +const TARGET_PREFIXES = getSitemapPrefixes(taxonomy); function extractChildName(loc) { const match = loc.match(/\/_sitemaps\/([^/]+)\.xml$/); diff --git a/scripts/kb-update/lib/taxonomy.mjs b/scripts/kb-update/lib/taxonomy.mjs new file mode 100644 index 0000000..c0e11c4 --- /dev/null +++ b/scripts/kb-update/lib/taxonomy.mjs @@ -0,0 +1,85 @@ +// taxonomy.mjs — Loader + accessors for data/domain-taxonomy.json (lag 0). +// Zero dependencies. The taxonomy is the single source of truth for in-domain +// classification: which sitemaps to poll, which URLs are relevant, which +// skill/category owns content, and update priority. poll-sitemaps, +// discover-new-urls and report-changes READ this instead of embedding copies. + +import { readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_DATA_DIR = join(__dirname, '..', 'data'); + +/** + * Load the domain taxonomy from disk. + * @param {string} [dataDir] — defaults to ../data/ relative to lib/ + * @returns {object} parsed taxonomy + */ +export function loadTaxonomy(dataDir = DEFAULT_DATA_DIR) { + const path = join(dataDir, 'domain-taxonomy.json'); + return JSON.parse(readFileSync(path, 'utf8')); +} + +/** + * Target sitemap child-name prefixes to scan/poll. + * @param {object} tax + * @returns {string[]} + */ +export function getSitemapPrefixes(tax) { + return tax.sitemap_prefixes; +} + +/** + * Canonical owning skill for a KB category (physical-disk truth). + * @param {object} tax + * @param {string} category + * @returns {string|null} skill directory name, or null if unknown + */ +export function getCategorySkill(tax, category) { + return tax.category_skill[category] ?? null; +} + +/** + * Build a URL classifier mirroring discover-new-urls semantics: + * exclude wins; first include match → { skill, category } with skill DERIVED + * from the canonical category_skill map (no embedded skill assignments). + * @param {object} tax + * @returns {(url: string) => ({skill: string|null, category: string}|null)} + */ +export function makeClassifier(tax) { + const include = tax.relevance.include.map((r) => ({ + re: new RegExp(r.pattern), + category: r.category, + })); + const exclude = tax.relevance.exclude.map((p) => new RegExp(p)); + + return function classifyUrl(url) { + if (exclude.some((re) => re.test(url))) return null; + for (const rule of include) { + if (rule.re.test(url)) { + return { skill: tax.category_skill[rule.category] ?? null, category: rule.category }; + } + } + return null; + }; +} + +/** + * Build a file-priority function mirroring report-changes getFilePriority: + * ordered regex rules over the lowercased path; first match wins; else default. + * @param {object} tax + * @returns {(filePath: string) => string} + */ +export function makePriorityFn(tax) { + const rules = tax.priority.rules.map((r) => ({ re: new RegExp(r.match), priority: r.priority })); + const fallback = tax.priority.default; + + return function getFilePriority(filePath) { + const lower = filePath.toLowerCase(); + for (const r of rules) { + if (r.re.test(lower)) return r.priority; + } + return fallback; + }; +} diff --git a/scripts/kb-update/poll-sitemaps.mjs b/scripts/kb-update/poll-sitemaps.mjs index b8726c4..1d7eeed 100644 --- a/scripts/kb-update/poll-sitemaps.mjs +++ b/scripts/kb-update/poll-sitemaps.mjs @@ -8,36 +8,16 @@ import { fileURLToPath } from 'node:url'; import { normalizeUrl } from './lib/url-normalize.mjs'; import { loadRegistry, saveRegistry } from './lib/registry-io.mjs'; import { fetchSitemapIndex, streamSitemap } from './lib/sitemap-stream.mjs'; +import { loadTaxonomy, getSitemapPrefixes } from './lib/taxonomy.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const DATA_DIR = join(__dirname, 'data'); const force = process.argv.includes('--force'); const verbose = process.argv.includes('--verbose'); -// Target child sitemaps — covers all URL path prefixes in the registry. -// Derived from analyzing which sitemaps contain our 1342 tracked URLs. -const TARGET_PREFIXES = [ - 'azure_en-us_', - 'microsoft-copilot-studio_en-us_', - 'security_en-us_', - 'fabric_en-us_', - 'power-platform_en-us_', - 'ai_en-us_', - 'copilot_en-us_', - 'compliance_en-us_', - 'agent-framework_en-us_', - 'semantic-kernel_en-us_', - 'entra_en-us_', - 'purview_en-us_', - 'microsoftteams_en-us_', - 'sharepoint_en-us_', - 'microsoft-365_en-us_', - 'training_en-us_', - 'cloud-computing_en-us_', - 'privacy_en-us_', - // dotnet_en-us_ excluded: 75 sitemaps, only 12 matches. Not worth weekly polling. - // Re-enable with --force if needed. -]; +// Target sitemap prefixes come from the consolidated taxonomy (lag 0) — the +// single source of truth shared with discover-new-urls. +const TARGET_PREFIXES = getSitemapPrefixes(loadTaxonomy(DATA_DIR)); function extractChildName(loc) { // https://learn.microsoft.com/_sitemaps/azure_en-us_7.xml → azure_en-us_7 diff --git a/scripts/kb-update/report-changes.mjs b/scripts/kb-update/report-changes.mjs index e0c9437..ca6d99c 100644 --- a/scripts/kb-update/report-changes.mjs +++ b/scripts/kb-update/report-changes.mjs @@ -7,20 +7,15 @@ import { readFileSync, existsSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { loadRegistry, saveReport } from './lib/registry-io.mjs'; +import { loadTaxonomy, makePriorityFn } from './lib/taxonomy.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = join(__dirname, '..', '..'); const DATA_DIR = join(__dirname, 'data'); const jsonOnly = process.argv.includes('--json'); -// Priority classification by file path patterns -function getFilePriority(filePath) { - const lower = filePath.toLowerCase(); - if (/cost|pricing|pris/.test(lower)) return 'critical'; - if (/responsible-ai|governance|ai-security-(?:engineering|scoring)/.test(lower)) return 'high'; - if (/platforms|copilot|azure-ai-services|agent-orchestration|rag|mlops|prompt-engineering|monitoring|performance/.test(lower)) return 'medium'; - return 'low'; -} +// Priority classification by file path patterns — rules from the taxonomy (lag 0). +const getFilePriority = makePriorityFn(loadTaxonomy(DATA_DIR)); // Parse "Last updated:" header from a reference file const LAST_UPDATED_PATTERNS = [ diff --git a/tests/kb-update/test-taxonomy.test.mjs b/tests/kb-update/test-taxonomy.test.mjs new file mode 100644 index 0000000..45b6ad7 --- /dev/null +++ b/tests/kb-update/test-taxonomy.test.mjs @@ -0,0 +1,125 @@ +// tests/kb-update/test-taxonomy.test.mjs +// Unit tests for scripts/kb-update/lib/taxonomy.mjs + data/domain-taxonomy.json. +// The taxonomy is the single source of truth (lag 0) consolidating the four +// previously-divergent taxonomies: poll TARGET_PREFIXES, discover INCLUDE/EXCLUDE, +// category-skill-map.json, and report-changes getFilePriority. These tests pin the +// behaviour the refactored scripts must reproduce, plus the no-divergence invariant. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + loadTaxonomy, + getSitemapPrefixes, + getCategorySkill, + makeClassifier, + makePriorityFn, +} from '../../scripts/kb-update/lib/taxonomy.mjs'; + +const tax = loadTaxonomy(); + +// --- (a) sitemap prefixes — poll/discover parity --- +test('sitemap_prefixes — 18 entries with poll/discover parity', () => { + const prefixes = getSitemapPrefixes(tax); + assert.equal(prefixes.length, 18); + // poll superset includes the 6 that discover previously lacked + for (const p of ['microsoftteams_en-us_', 'sharepoint_en-us_', 'microsoft-365_en-us_', + 'training_en-us_', 'cloud-computing_en-us_', 'privacy_en-us_']) { + assert.ok(prefixes.includes(p), `missing prefix ${p}`); + } + assert.ok(prefixes.includes('azure_en-us_')); + // dotnet is deliberately excluded (75 sitemaps, 12 matches) + assert.ok(!prefixes.includes('dotnet_en-us_')); +}); + +// --- (c) category → skill — PHYSICAL DISK is canonical --- +test('category_skill — disk-truth canonical (the 4 formerly stale entries)', () => { + // category-skill-map.json had these as ms-ai-engineering; disk says otherwise. + assert.equal(getCategorySkill(tax, 'monitoring-observability'), 'ms-ai-governance'); + assert.equal(getCategorySkill(tax, 'copilot-extensibility'), 'ms-ai-advisor'); + assert.equal(getCategorySkill(tax, 'prompt-engineering'), 'ms-ai-advisor'); + assert.equal(getCategorySkill(tax, 'performance-scalability'), 'ms-ai-security'); + // unchanged ones + assert.equal(getCategorySkill(tax, 'rag-architecture'), 'ms-ai-engineering'); + assert.equal(getCategorySkill(tax, 'responsible-ai'), 'ms-ai-governance'); + assert.equal(getCategorySkill(tax, 'cost-optimization'), 'ms-ai-security'); + // unknown category → null + assert.equal(getCategorySkill(tax, 'does-not-exist'), null); +}); + +// --- (b) classifyUrl — reproduces discover's classification exactly --- +test('classifyUrl — include rules map to correct {skill, category}', () => { + const classify = makeClassifier(tax); + assert.deepEqual( + classify('https://learn.microsoft.com/azure/ai-foundry/openai/how-to/manage-costs'), + { skill: 'ms-ai-engineering', category: 'azure-ai-services' } + ); + // monitoring → governance (was a divergence; derivation reproduces discover's original) + assert.deepEqual( + classify('https://learn.microsoft.com/azure/azure-monitor/logs/foo'), + { skill: 'ms-ai-governance', category: 'monitoring-observability' } + ); + // copilot-studio → advisor (was a divergence; derivation reproduces discover's original) + assert.deepEqual( + classify('https://learn.microsoft.com/microsoft-copilot-studio/fundamentals-what-is'), + { skill: 'ms-ai-advisor', category: 'copilot-extensibility' } + ); + // alternation pattern + assert.deepEqual( + classify('https://learn.microsoft.com/security/benchmark/azure/foo'), + { skill: 'ms-ai-security', category: 'ai-security-engineering' } + ); +}); + +test('classifyUrl — exclude wins over include and non-matches return null', () => { + const classify = makeClassifier(tax); + // /samples/ excluded even though /azure/ai-foundry/ would include + assert.equal(classify('https://learn.microsoft.com/azure/ai-foundry/samples/quickstart'), null); + // /training/ excluded + assert.equal(classify('https://learn.microsoft.com/training/modules/intro'), null); + // unrelated path + assert.equal(classify('https://learn.microsoft.com/azure/virtual-machines/overview'), null); +}); + +test('INVARIANT — every include category resolves and skill === category_skill[category]', () => { + const classify = makeClassifier(tax); + for (const rule of tax.relevance.include) { + const skill = getCategorySkill(tax, rule.category); + assert.ok(skill, `include category ${rule.category} absent from category_skill`); + } + // probe each rule via a synthetic URL and assert no skill/category divergence + const probes = [ + ['https://learn.microsoft.com/azure/ai-foundry/x', 'azure-ai-services'], + ['https://learn.microsoft.com/azure/machine-learning/x', 'mlops-genaiops'], + ['https://learn.microsoft.com/azure/search/x', 'rag-architecture'], + ['https://learn.microsoft.com/azure/azure-monitor/x', 'monitoring-observability'], + ['https://learn.microsoft.com/azure/well-architected/x', 'architecture'], + ['https://learn.microsoft.com/microsoft-copilot-studio/x', 'copilot-extensibility'], + ['https://learn.microsoft.com/purview/x', 'responsible-ai'], + ['https://learn.microsoft.com/agent-framework/x', 'agent-orchestration'], + ['https://learn.microsoft.com/azure/cosmos-db/x', 'data-engineering'], + ]; + for (const [url, expectedCat] of probes) { + const r = classify(url); + assert.ok(r, `no classification for ${url}`); + assert.equal(r.category, expectedCat); + assert.equal(r.skill, getCategorySkill(tax, r.category), `divergence at ${url}`); + } +}); + +// --- (d) getFilePriority — reproduces report-changes ordering exactly --- +test('getFilePriority — pattern → priority, first match wins', () => { + const prio = makePriorityFn(tax); + assert.equal(prio('skills/ms-ai-security/references/cost-optimization/budget.md'), 'critical'); + assert.equal(prio('skills/ms-ai-governance/references/responsible-ai/fairness.md'), 'high'); + // 'governance' substring → high + assert.equal(prio('skills/ms-ai-governance/references/norwegian-public-sector-governance/x.md'), 'high'); + assert.equal(prio('skills/ms-ai-advisor/references/platforms/copilot-studio.md'), 'medium'); + // no pattern → low + assert.equal(prio('skills/ms-ai-engineering/references/data-engineering/cosmos.md'), 'low'); +}); + +test('getFilePriority — order: cost (critical) outranks governance (high)', () => { + const prio = makePriorityFn(tax); + // contains both 'cost' and 'governance' — critical rule is evaluated first + assert.equal(prio('skills/x/references/cost-governance-tradeoffs.md'), 'critical'); +});