ms-ai-architect/tests/kb-update/test-detect-courses.test.mjs

// tests/kb-update/test-detect-courses.test.mjs
// C3.4 — the detector that binds C3.1 (keychain + learn-api), C3.2 (course-diff
// + course-registry) and C3.3 (makeCourseClassifier) into the Claude-free STEG 1.
// Every dependency is injectable so the core is hermetic (no Keychain, no
// network); the CLI is then proven end-to-end via a subprocess with a shadowed
// `security` binary. Contract under test (spec §3 fail-soft, §4.3 report shape):
//   - missing creds        => status:"skipped", a skipped report, exit 0
//   - a network/run error  => status:"error" (never throws), no registry write
//   - a normal run         => status:"ok", §4.3-shaped report + registry advanced

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { detectCourses } from '../../scripts/kb-update/detect-courses.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = join(__dirname, '..', '..');
const NOW = '2026-06-23T12:00:00Z';

// A small in-domain taxonomy: slug -> category, skill DERIVED from category_skill.
const FAKE_TAX = {
  category_skill: { 'azure-ai-services': 'ms-ai-engineering', platforms: 'ms-ai-advisor' },
  course_products: { 'azure-openai': 'azure-ai-services', 'power-automate': 'platforms' },
};

// Registry with a known course + recent cursors => selectMode picks 'incremental'.
const REGISTRY = {
  version: 1,
  created_at: '2026-06-01T00:00:00Z',
  last_run: '2026-06-20T00:00:00Z',
  last_full_enum: '2026-06-19T00:00:00Z',
  courses: {
    'learn.module-known': {
      type: 'module',
      title: 'Known module',
      url: 'https://learn.microsoft.com/training/modules/known',
      products: ['azure-openai'],
      updated_at: '2026-06-10T00:00:00Z',
      first_seen: '2026-06-01T00:00:00Z',
      last_seen: '2026-06-20T00:00:00Z',
    },
  },
};

function makePaginate(pagesByEndpoint, captured) {
  return async function* paginate(endpoint, params /* , token */) {
    captured[endpoint] = params;
    for (const item of pagesByEndpoint[endpoint] ?? []) yield item;
  };
}

// ---------------------------------------------------------------------------
// (1) Fail-soft: missing creds => skipped, no network touched
// ---------------------------------------------------------------------------

test('detectCourses — all creds missing => status "skipped", writes a skipped report, no network', async () => {
  const writes = [];
  const report = await detectCourses({
    readSecretImpl: () => null,
    saveReportImpl: (name, data) => writes.push([name, data]),
    getTokenImpl: () => assert.fail('getToken must not run without creds'),
    paginateImpl: () => assert.fail('paginate must not run without creds'),
    now: NOW,
    log: () => {},
  });
  assert.equal(report.status, 'skipped');
  assert.ok(report.skipped_reason, 'a human reason is recorded');
  assert.equal(report.generated_at, NOW);
  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
  assert.equal(writes.length, 1);
  assert.equal(writes[0][0], 'course-detection-report.json');
  assert.equal(writes[0][1].status, 'skipped');
});

test('detectCourses — a single missing cred is enough to skip (partial creds)', async () => {
  const report = await detectCourses({
    readSecretImpl: (service) => (service === 'learn-platform-client-secret' ? null : 'present'),
    saveReportImpl: () => {},
    getTokenImpl: () => assert.fail('must not run'),
    paginateImpl: () => assert.fail('must not run'),
    now: NOW,
    log: () => {},
  });
  assert.equal(report.status, 'skipped');
});

// ---------------------------------------------------------------------------
// (2) Error path: a run failure fail-softs to status "error", never throws,
//     and does NOT advance the registry.
// ---------------------------------------------------------------------------

test('detectCourses — a network error yields status "error" (no throw) and leaves the registry untouched', async () => {
  let registryWritten = false;
  let saved = null;
  const report = await detectCourses({
    readSecretImpl: () => 'present',
    loadTaxonomyImpl: () => FAKE_TAX,
    loadCourseRegistryImpl: () => REGISTRY,
    getTokenImpl: async () => {
      throw new Error('HTTP 500 boom');
    },
    saveReportImpl: (_name, data) => {
      saved = data;
    },
    saveCourseRegistryImpl: () => {
      registryWritten = true;
    },
    now: NOW,
    log: () => {},
  });
  assert.equal(report.status, 'error');
  assert.match(report.skipped_reason, /boom/);
  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
  assert.equal(saved.status, 'error', 'the error report is persisted');
  assert.equal(registryWritten, false, 'no registry write when the run errored');
});

// ---------------------------------------------------------------------------
// (3) Happy path: incremental run produces a §4.3-shaped report and advances
//     the registry.
// ---------------------------------------------------------------------------

test('detectCourses — incremental run: §4.3 report shape, new/updated leads, registry advanced', async () => {
  const captured = {};
  const pages = {
    modules: [
      // brand-new UID => new lead
      {
        id: 'learn.module-new',
        title: 'New AOAI course',
        url: 'https://learn.microsoft.com/training/modules/new',
        updatedAt: '2026-06-22T00:00:00Z',
        products: [{ id: 'azure-openai' }],
      },
      // known UID with a newer timestamp => updated lead
      {
        id: 'learn.module-known',
        title: 'Known module',
        url: 'https://learn.microsoft.com/training/modules/known',
        updatedAt: '2026-06-21T00:00:00Z',
        products: [{ id: 'azure-openai' }],
      },
    ],
    'learning-paths': [],
  };
  let savedReport = null;
  let savedRegistry = null;
  const report = await detectCourses({
    readSecretImpl: () => 'present',
    loadTaxonomyImpl: () => FAKE_TAX,
    loadCourseRegistryImpl: () => REGISTRY,
    getTokenImpl: async () => 'token-abc',
    paginateImpl: makePaginate(pages, captured),
    saveReportImpl: (_name, data) => {
      savedReport = data;
    },
    saveCourseRegistryImpl: (reg) => {
      savedRegistry = reg;
    },
    now: NOW,
    log: () => {},
  });

  // status + counts
  assert.equal(report.status, 'ok');
  assert.equal(report.skipped_reason, null);
  assert.equal(report.mode, 'incremental');
  assert.deepEqual(report.counts, { new: 1, updated: 1, removed: 0 });

  // §4.3 item shape on the new lead, incl. derived skill/category
  const lead = report.new[0];
  for (const k of ['uid', 'title', 'url', 'products', 'suggested_skill', 'suggested_category', 'updated_at']) {
    assert.ok(k in lead, `new lead exposes "${k}"`);
  }
  assert.equal(lead.uid, 'learn.module-new');
  assert.equal(lead.suggested_skill, 'ms-ai-engineering');
  assert.equal(lead.suggested_category, 'azure-ai-services');

  // the updated lead is the known UID
  assert.equal(report.updated[0].uid, 'learn.module-known');
  assert.equal(report.removed.length, 0);

  // incremental run pins the updatedAt.gt cursor (full ISO) from last_run, and
  // filters server-side by the in-domain product slugs.
  assert.equal(captured.modules['updatedAt.gt'], '2026-06-20T00:00:00.000Z');
  assert.match(captured.modules.products, /azure-openai/);
  assert.match(captured.modules.products, /power-automate/);

  // registry advanced: both UIDs present, last_run = now
  assert.equal(savedReport.status, 'ok');
  assert.equal(savedRegistry.last_run, NOW);
  assert.ok(savedRegistry.courses['learn.module-new']);
  assert.ok(savedRegistry.courses['learn.module-known']);
});

test('detectCourses — baseline run (no last_run) emits no leads but establishes the registry', async () => {
  const pages = {
    modules: [
      {
        id: 'learn.module-a',
        title: 'A',
        url: 'u-a',
        updatedAt: '2026-06-22T00:00:00Z',
        products: [{ id: 'azure-openai' }],
      },
    ],
    'learning-paths': [],
  };
  let savedRegistry = null;
  const captured = {};
  const report = await detectCourses({
    readSecretImpl: () => 'present',
    loadTaxonomyImpl: () => FAKE_TAX,
    loadCourseRegistryImpl: () => ({ version: 1, created_at: null, last_run: null, last_full_enum: null, courses: {} }),
    getTokenImpl: async () => 'token-abc',
    paginateImpl: makePaginate(pages, captured),
    saveReportImpl: () => {},
    saveCourseRegistryImpl: (reg) => {
      savedRegistry = reg;
    },
    now: NOW,
    log: () => {},
  });
  assert.equal(report.mode, 'baseline');
  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }, 'baseline emits no leads (§8 b)');
  // baseline omits the updatedAt.gt cursor (full enumeration)
  assert.equal(captured.modules['updatedAt.gt'], undefined);
  // but it DID fold the API set into the registry + stamp last_full_enum
  assert.ok(savedRegistry.courses['learn.module-a']);
  assert.equal(savedRegistry.last_full_enum, NOW);
});

// ---------------------------------------------------------------------------
// (4) The real CLI fails soft to exit 0 — proven hermetically by shadowing the
//     `security` binary on PATH so readSecret returns null (no Keychain, no net).
// ---------------------------------------------------------------------------

test('detect-courses.mjs CLI — missing creds => skipped report + exit 0 (no network)', { skip: process.platform === 'win32' }, () => {
  const tmp = mkdtempSync(join(tmpdir(), 'detect-courses-'));
  try {
    const dataDir = join(tmp, 'data');
    const bin = join(tmp, 'bin');
    mkdirSync(dataDir, { recursive: true });
    mkdirSync(bin, { recursive: true });
    // A `security` that always fails => keychain.readSecret returns null => skip.
    const stub = join(bin, 'security');
    writeFileSync(stub, '#!/bin/sh\nexit 1\n');
    chmodSync(stub, 0o755);

    const script = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'detect-courses.mjs');
    const res = spawnSync('node', [script, '--data-dir', dataDir], {
      env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
      encoding: 'utf8',
    });
    assert.equal(res.status, 0, `fail-soft must exit 0 (stderr: ${res.stderr})`);
    const report = JSON.parse(readFileSync(join(dataDir, 'course-detection-report.json'), 'utf8'));
    assert.equal(report.status, 'skipped');
  } finally {
    rmSync(tmp, { recursive: true, force: true });
  }
});