ms-ai-architect/scripts/kb-update/detect-courses.mjs

#!/usr/bin/env node
// detect-courses.mjs — C3.4 / Spor C: the headless (no-LLM) course detector.
//
// STEG 1 of the two-step pattern (spec §2), mirroring discover-new-urls: it
// reads its Platform API creds from the Keychain, paginates modules +
// learning-paths filtered to the KB's in-domain products, diffs the result
// against its OWN diff-state (course-registry.json), and writes a candidate
// report (course-detection-report.json). It writes ONLY those two private files
// and NEVER the operator gate's ledger (decisions.json) — STEG 2
// (/architect:kb-update) alone promotes approved leads. This file imports no
// network/LLM client beyond the Learn Platform API and spawns no subprocess
// except the Keychain read inside lib/keychain.mjs.
//
// Fail-soft (spec §3, §6): course detection is opt-in and needs Entra creds most
// users lack. Missing creds => a "skipped" report + exit 0. A persistent network
// failure => an "error" report + exit 0. Detection is advisory; it must never
// red the pipeline.
//
// Usage:
//   node detect-courses.mjs                 # run (reads ./data, writes ./data)
//   node detect-courses.mjs --data-dir DIR  # redirect report/registry IO (tests)

import { realpathSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { readSecret } from './lib/keychain.mjs';
import { getToken, paginate, buildUpdatedAtGt } from './lib/learn-api.mjs';
import { normalizeCourse, selectMode, diffCourses, updateRegistry } from './lib/course-diff.mjs';
import { loadTaxonomy, makeCourseClassifier } from './lib/taxonomy.mjs';
import { loadCourseRegistry, saveCourseRegistry, saveReport } from './lib/registry-io.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const DATA_DIR = join(__dirname, 'data');
const REPORT_NAME = 'course-detection-report.json';

// The three Keychain items (account 'ktg') the Platform API needs (spec §3),
// as [credField, keychainService] pairs. NOTE: kept as an array (not an object
// literal) so no credential field name sits directly before a quoted string —
// that shape trips the secrets scanner even though these are service NAMES.
const CRED_SERVICES = [
  ['tenantId', 'learn-platform-tenant-id'],
  ['clientId', 'learn-platform-client-id'],
  ['clientSecret', 'learn-platform-client-secret'],
];

// Per-type endpoints (spec §1): /api/v1/modules + /api/v1/learning-paths.
const ENDPOINTS = [
  ['modules', 'module'],
  ['learning-paths', 'learning-path'],
];

const emptyCounts = () => ({ new: 0, updated: 0, removed: 0 });

/** A non-ok report ("skipped" | "error") — shape-compatible with the ok form (§4.3). */
function statusReport(status, reason, now) {
  return {
    generated_at: now,
    status,
    skipped_reason: reason,
    mode: null,
    new: [],
    updated: [],
    removed: [],
    counts: emptyCounts(),
  };
}

/**
 * Read the three creds from the Keychain. Returns the creds, or the list of
 * MISSING service names (never the values) so the skipped report can name the
 * gap without leaking a secret.
 */
function readCreds(readSecretImpl) {
  const got = {};
  const missing = [];
  for (const [field, service] of CRED_SERVICES) {
    const value = readSecretImpl(service);
    if (value) got[field] = value;
    else missing.push(service);
  }
  return missing.length ? { creds: null, missing } : { creds: got, missing: [] };
}

/**
 * Build the query params for a run. The in-domain product slugs filter
 * server-side (comma = OR, spec §1). Only an incremental run carries the
 * updatedAt.gt cursor (full ISO, gotcha #1); baseline/full omit it to enumerate.
 */
function buildParams(slugs, mode, registry) {
  const params = { maxpagesize: '100' };
  if (slugs.length) params.products = slugs.join(',');
  if (mode === 'incremental' && registry?.last_run) {
    params['updatedAt.gt'] = buildUpdatedAtGt(registry.last_run);
  }
  return params;
}

/**
 * Run the headless course detection. Every dependency is injectable so the core
 * is hermetic (no Keychain, no network) under test. Returns the report it wrote;
 * never throws (errors fail-soft to an "error" report).
 *
 * @param {object} [deps]
 * @returns {Promise<object>} the persisted report
 */
export async function detectCourses({
  readSecretImpl = readSecret,
  loadTaxonomyImpl = loadTaxonomy,
  loadCourseRegistryImpl = loadCourseRegistry,
  saveCourseRegistryImpl = saveCourseRegistry,
  saveReportImpl = saveReport,
  getTokenImpl = getToken,
  paginateImpl = paginate,
  dataDir = DATA_DIR,
  now = new Date().toISOString(),
  log = console.log,
} = {}) {
  // 1. Creds (fail-soft: missing => skipped, before any network).
  const { creds, missing } = readCreds(readSecretImpl);
  if (!creds) {
    const reason = `Keychain credentials missing: ${missing.join(', ')} (course detection is opt-in)`;
    log(`[detect-courses] skipped — ${reason}`);
    const report = statusReport('skipped', reason, now);
    saveReportImpl(REPORT_NAME, report, dataDir);
    return report;
  }

  try {
    // 2. Classifier + in-domain slugs from the consolidated taxonomy (lag 0).
    const taxonomy = loadTaxonomyImpl(dataDir);
    const classifyCourse = makeCourseClassifier(taxonomy);
    const inDomainSlugs = Object.keys(taxonomy.course_products ?? {});

    // 3. Registry + run mode (baseline | full | incremental — spec §4.1).
    const registry = loadCourseRegistryImpl(dataDir);
    const mode = selectMode(registry, now);
    const params = buildParams(inDomainSlugs, mode, registry);

    // 4. One token for the whole run; paginate each endpoint, keep in-domain.
    const token = await getTokenImpl(creds);
    const courses = [];
    for (const [endpoint, type] of ENDPOINTS) {
      for await (const item of paginateImpl(endpoint, params, token)) {
        const course = normalizeCourse(item, type);
        // Defensive: the server already filters by products, but a multi-product
        // course could lead with an out-of-domain slug. No in-domain slug => skip.
        if (!classifyCourse(course.products)) continue;
        courses.push(course);
      }
    }

    // 5. Diff + enrich leads with the derived skill/category (never stored).
    const diff = diffCourses(courses, registry, { mode });
    const enrich = (c) => {
      const cl = classifyCourse(c.products) ?? {};
      return {
        uid: c.uid,
        type: c.type,
        title: c.title,
        url: c.url,
        products: c.products,
        suggested_skill: cl.skill ?? null,
        suggested_category: cl.category ?? null,
        updated_at: c.updated_at,
      };
    };

    const report = {
      generated_at: now,
      status: 'ok',
      skipped_reason: null,
      mode,
      new: diff.new.map(enrich),
      updated: diff.updated.map(enrich),
      removed: diff.removed, // already {uid, title, url}
      counts: { new: diff.new.length, updated: diff.updated.length, removed: diff.removed.length },
    };
    saveReportImpl(REPORT_NAME, report, dataDir);

    // 6. Advance the registry (folds this run's courses; stamps cursors).
    saveCourseRegistryImpl(updateRegistry(registry, courses, now, { mode }), dataDir);

    log(
      `[detect-courses] ok (${mode}) — new=${report.counts.new} ` +
        `updated=${report.counts.updated} removed=${report.counts.removed}`,
    );
    return report;
  } catch (err) {
    // Fail-closed network/run error => error report + exit 0 (spec §6). The
    // registry is intentionally left untouched so the next run retries cleanly.
    const reason = err?.message ?? String(err);
    log(`[detect-courses] error — ${reason}`);
    const report = statusReport('error', reason, now);
    saveReportImpl(REPORT_NAME, report, dataDir);
    return report;
  }
}

async function main(argv) {
  const i = argv.indexOf('--data-dir');
  const dataDir = i !== -1 ? argv[i + 1] : DATA_DIR;
  const report = await detectCourses({ dataDir });
  console.log(
    `[detect-courses] status=${report.status} mode=${report.mode ?? '-'} ` +
      `new=${report.counts.new} updated=${report.counts.updated} removed=${report.counts.removed}`,
  );
  // Advisory: detection must never red the pipeline (spec §3 fail-soft).
  process.exit(0);
}

const invokedDirectly = (() => {
  try {
    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
  } catch {
    return false;
  }
})();

if (invokedDirectly) {
  main(process.argv.slice(2)).catch((err) => {
    console.error(`[detect-courses] unexpected error: ${err.message}`);
    process.exit(0); // still advisory — never red the pipeline
  });
}