ms-ai-architect/scripts/kb-update/discover-new-urls.mjs

#!/usr/bin/env node
// discover-new-urls.mjs — Find relevant Microsoft Learn pages not yet in the registry.
// Scans sitemaps for URLs matching relevance patterns, suggests skill/category mapping.
// Usage: node discover-new-urls.mjs [--limit N]

import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { normalizeUrl } from './lib/url-normalize.mjs';
import { loadRegistry, saveReport } from './lib/registry-io.mjs';
import { streamSitemap, fetchSitemapIndex } from './lib/sitemap-stream.mjs';
import { loadTaxonomy, getSitemapPrefixes, makeClassifier } from './lib/taxonomy.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const DATA_DIR = join(__dirname, 'data');
const limitArg = process.argv.indexOf('--limit');
const limit = limitArg !== -1 ? parseInt(process.argv[limitArg + 1], 10) : Infinity;

// Relevance + sitemap targets come from the consolidated taxonomy (lag 0).
// classifyUrl: exclude wins, first include match, skill derived from the
// canonical category->skill map — no embedded skill assignments here.
const taxonomy = loadTaxonomy(DATA_DIR);
const classifyUrl = makeClassifier(taxonomy);
const TARGET_PREFIXES = getSitemapPrefixes(taxonomy);

function extractChildName(loc) {
  const match = loc.match(/\/_sitemaps\/([^/]+)\.xml$/);
  return match ? match[1] : null;
}

// --- Main ---
async function main() {
  const registry = loadRegistry(DATA_DIR);
  const knownUrls = new Set(Object.keys(registry.urls));
  console.log(`Registry: ${knownUrls.size} known URLs`);

  console.log('Fetching sitemap index...');
  const indexEntries = await fetchSitemapIndex();

  const targetChildren = indexEntries
    .filter(e => {
      const name = extractChildName(e.loc);
      return name && TARGET_PREFIXES.some(p => name.startsWith(p));
    });

  console.log(`Scanning ${targetChildren.length} sitemaps for new relevant URLs...`);

  const candidates = [];
  const bySkill = {};

  for (const child of targetChildren) {
    const childName = extractChildName(child.loc);
    let foundInChild = 0;

    try {
      for await (const entry of streamSitemap(child.loc)) {
        const normalized = normalizeUrl(entry.loc);
        if (!normalized || knownUrls.has(normalized)) continue;

        const classification = classifyUrl(normalized);
        if (!classification) continue;

        candidates.push({
          url: normalized,
          lastmod: entry.lastmod,
          sitemap: childName,
          suggested_skill: classification.skill,
          suggested_category: classification.category,
        });
        knownUrls.add(normalized); // Prevent dupes across sitemaps

        bySkill[classification.skill] = (bySkill[classification.skill] || 0) + 1;
        foundInChild++;

        if (candidates.length >= limit) break;
      }
    } catch (err) {
      console.error(`  ERROR scanning ${childName}: ${err.message}`);
    }

    if (foundInChild > 0) {
      console.log(`  ${childName}: ${foundInChild} new candidates`);
    }
    if (candidates.length >= limit) break;
  }

  // Sort by lastmod descending (newest first)
  candidates.sort((a, b) => (b.lastmod || '').localeCompare(a.lastmod || ''));

  const report = {
    generated_at: new Date().toISOString().split('T')[0],
    new_candidates: candidates.length,
    by_suggested_skill: bySkill,
    candidates,
  };

  saveReport('discovery-report.json', report, DATA_DIR);

  console.log(`\n=== Discovery Report ===`);
  console.log(`New relevant URLs found: ${candidates.length}`);
  console.log('By skill:', JSON.stringify(bySkill, null, 2));
  if (candidates.length > 0) {
    console.log('\nNewest 10:');
    for (const c of candidates.slice(0, 10)) {
      console.log(`  [${c.suggested_skill}/${c.suggested_category}] ${c.url}`);
      console.log(`    lastmod: ${c.lastmod}`);
    }
  }
}

main().catch(err => {
  console.error('Fatal error:', err.message);
  process.exit(1);
});