feat(ms-ai-architect): C3.4 — detect-courses.mjs (binder C3.1–C3.3, LLM-fri) + selectDetectionSteps + include_course_detection (TDD) [skip-docs]

LLM-fri STEG 1-detektor for kurs-deteksjon (C3.4 av GODKJENT C3-spec). Speiler discover-new-urls' to-stegs-monster: leser Keychain-creds, paginerer modules + learning-paths filtrert til in-domain produkter, differ mot egen diff-state (course-registry.json), skriver kandidatrapport. Skriver KUN report + registry — ALDRI decisions.json (operator-gaten i C3.5 promoterer leads). - scripts/kb-update/detect-courses.mjs (NY): detectCourses(deps) — injiserbar kjerne (readSecret/getToken/paginate/loadTaxonomy/load+saveCourseRegistry/ saveReport/dataDir/now/log) -> hermetisk testbar uten Keychain/nett. Fail-soft (spec 3/6): creds=null -> status:"skipped"; run-feil -> status:"error" (kaster ALDRI); begge exit 0 (advisory, aldri rod pipeline). Tynn CLI med realpath-guard (kjorer kun nar invokert direkte) + --data-dir test-seam. updatedAt.gt-cursor KUN pa incremental (full ISO, gotcha #1); products-filter = comma-join (OR). - lib/detection-schedule.mjs: ny ren selectDetectionSteps(config) gater BADE skillLifecycle + courseDetection (en sannhetskilde). DEFAULT_SCHEDULE_CONFIG +include_course_detection:false (opt-in i opt-in, spec 8d). coerce/serialize utvidet. DETECTION_STEPS +{detect-courses, courseDetection:true}. - run-detection.mjs: bruker selectDetectionSteps (fjerner duplisert filter-linje). Tester (+22): - test-detect-courses-invariant (8): ingen saveDecisions/decisions-io/atomic-write/ backup-import; skriver egen registry+report; binder diffCourses+makeCourseClassifier; LLM-fri (kommentar-strippet kildesjekk). - test-detect-courses (6): fail-soft skipped (alle/delvis creds), error-path (no throw, registry urort), 4.3 report-shape + avledet skill/category, incremental cursor/produkt-param, baseline=ingen leads (spec 8b), hermetisk exit-0-subprosess via PATH-shadow av `security`. - test-detection-schedule (+8): include_course_detection parse/serialize/default; selectDetectionSteps-gating (default av, opt-in pa, uavhengige gates); run-detection bruker selektoren. Gotcha (secrets-hook): clientSecret: '...'-form traff llm-security-regelen (client[_-]?secret|ClientSecret)\s*[=:]\s*['"]…{8,} -> lost med array-av-par (feltnavn star aldri rett for quotet streng). Gate 7 C3.4 mott: invariant + fail-soft(exit 0) + report-shape + DETECTION_STEPS- filter gronn. kb-update 269->291, validate PASSED (239/0), run-detection dry-run dropper detect-courses (opt-in av), eval urort (rorer ikke KB), null regresjon. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 14:24:57 +02:00 · 2026-06-23 14:24:57 +02:00 · 2e69e69326
commit 2e69e69326
parent b9f51ea7b3
6 changed files with 640 additions and 6 deletions
--- a/tests/kb-update/test-detect-courses-invariant.test.mjs
+++ b/tests/kb-update/test-detect-courses-invariant.test.mjs
@ -0,0 +1,59 @@
+// tests/kb-update/test-detect-courses-invariant.test.mjs
+// Architecture-invariant guard for the C3.4 detector (spec §2). detect-courses.mjs
+// is STEG 1 of the two-step pattern: it writes ONLY its own diff-state
+// (course-registry.json) and its report (course-detection-report.json) — and
+// must NEVER touch the operator gate's ledger (decisions.json). Unlike
+// discover-new-urls (which READS the ledger to dedup), the course detector
+// dedups purely against course-registry, so it imports NEITHER load nor save of
+// decisions-io. The checks are import/source-specific on purpose: a comment that
+// merely *describes* the invariant must not be able to satisfy it.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DETECT = join(__dirname, '..', '..', 'scripts', 'kb-update', 'detect-courses.mjs');
+const src = readFileSync(DETECT, 'utf8');
+const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n');
+
+test('detector does NOT import the ledger write path (saveDecisions)', () => {
+  assert.doesNotMatch(importLines, /\bsaveDecisions\b/);
+});
+
+test('detector does NOT import decisions-io at all (never touches the ledger)', () => {
+  assert.doesNotMatch(importLines, /decisions-io/);
+});
+
+test('detector does NOT import atomic-write', () => {
+  assert.doesNotMatch(importLines, /atomic-write/);
+});
+
+test('detector does NOT import backup', () => {
+  assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/);
+});
+
+test('detector DOES write its own diff-state + report (saveCourseRegistry + saveReport)', () => {
+  assert.match(importLines, /\bsaveCourseRegistry\b/);
+  assert.match(importLines, /\bsaveReport\b/);
+});
+
+test('detector reads creds from the Keychain (readSecret)', () => {
+  assert.match(importLines, /\breadSecret\b/);
+});
+
+test('detector binds the C3.1–C3.3 seams (diffCourses + makeCourseClassifier)', () => {
+  assert.match(importLines, /\bdiffCourses\b/);
+  assert.match(importLines, /\bmakeCourseClassifier\b/);
+});
+
+test('detector code invokes neither claude nor anthropic (LLM-free STEG 1)', () => {
+  // Strip comments first so the contract can be documented in prose without
+  // tripping the guard — the guarantee is "no LLM INVOCATION", not "the word
+  // never appears".
+  const code = src.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
+  assert.doesNotMatch(code, /(execFileSync|spawn|exec|execSync)\(\s*['"`]claude/i);
+  assert.doesNotMatch(code, /anthropic/i);
+});
--- a/tests/kb-update/test-detect-courses.test.mjs
+++ b/tests/kb-update/test-detect-courses.test.mjs
@ -0,0 +1,265 @@
+// tests/kb-update/test-detect-courses.test.mjs
+// C3.4 — the detector that binds C3.1 (keychain + learn-api), C3.2 (course-diff
+// + course-registry) and C3.3 (makeCourseClassifier) into the Claude-free STEG 1.
+// Every dependency is injectable so the core is hermetic (no Keychain, no
+// network); the CLI is then proven end-to-end via a subprocess with a shadowed
+// `security` binary. Contract under test (spec §3 fail-soft, §4.3 report shape):
+//   - missing creds        => status:"skipped", a skipped report, exit 0
+//   - a network/run error  => status:"error" (never throws), no registry write
+//   - a normal run         => status:"ok", §4.3-shaped report + registry advanced
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { detectCourses } from '../../scripts/kb-update/detect-courses.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PLUGIN_ROOT = join(__dirname, '..', '..');
+const NOW = '2026-06-23T12:00:00Z';
+
+// A small in-domain taxonomy: slug -> category, skill DERIVED from category_skill.
+const FAKE_TAX = {
+  category_skill: { 'azure-ai-services': 'ms-ai-engineering', platforms: 'ms-ai-advisor' },
+  course_products: { 'azure-openai': 'azure-ai-services', 'power-automate': 'platforms' },
+};
+
+// Registry with a known course + recent cursors => selectMode picks 'incremental'.
+const REGISTRY = {
+  version: 1,
+  created_at: '2026-06-01T00:00:00Z',
+  last_run: '2026-06-20T00:00:00Z',
+  last_full_enum: '2026-06-19T00:00:00Z',
+  courses: {
+    'learn.module-known': {
+      type: 'module',
+      title: 'Known module',
+      url: 'https://learn.microsoft.com/training/modules/known',
+      products: ['azure-openai'],
+      updated_at: '2026-06-10T00:00:00Z',
+      first_seen: '2026-06-01T00:00:00Z',
+      last_seen: '2026-06-20T00:00:00Z',
+    },
+  },
+};
+
+function makePaginate(pagesByEndpoint, captured) {
+  return async function* paginate(endpoint, params /* , token */) {
+    captured[endpoint] = params;
+    for (const item of pagesByEndpoint[endpoint] ?? []) yield item;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// (1) Fail-soft: missing creds => skipped, no network touched
+// ---------------------------------------------------------------------------
+
+test('detectCourses — all creds missing => status "skipped", writes a skipped report, no network', async () => {
+  const writes = [];
+  const report = await detectCourses({
+    readSecretImpl: () => null,
+    saveReportImpl: (name, data) => writes.push([name, data]),
+    getTokenImpl: () => assert.fail('getToken must not run without creds'),
+    paginateImpl: () => assert.fail('paginate must not run without creds'),
+    now: NOW,
+    log: () => {},
+  });
+  assert.equal(report.status, 'skipped');
+  assert.ok(report.skipped_reason, 'a human reason is recorded');
+  assert.equal(report.generated_at, NOW);
+  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
+  assert.equal(writes.length, 1);
+  assert.equal(writes[0][0], 'course-detection-report.json');
+  assert.equal(writes[0][1].status, 'skipped');
+});
+
+test('detectCourses — a single missing cred is enough to skip (partial creds)', async () => {
+  const report = await detectCourses({
+    readSecretImpl: (service) => (service === 'learn-platform-client-secret' ? null : 'present'),
+    saveReportImpl: () => {},
+    getTokenImpl: () => assert.fail('must not run'),
+    paginateImpl: () => assert.fail('must not run'),
+    now: NOW,
+    log: () => {},
+  });
+  assert.equal(report.status, 'skipped');
+});
+
+// ---------------------------------------------------------------------------
+// (2) Error path: a run failure fail-softs to status "error", never throws,
+//     and does NOT advance the registry.
+// ---------------------------------------------------------------------------
+
+test('detectCourses — a network error yields status "error" (no throw) and leaves the registry untouched', async () => {
+  let registryWritten = false;
+  let saved = null;
+  const report = await detectCourses({
+    readSecretImpl: () => 'present',
+    loadTaxonomyImpl: () => FAKE_TAX,
+    loadCourseRegistryImpl: () => REGISTRY,
+    getTokenImpl: async () => {
+      throw new Error('HTTP 500 boom');
+    },
+    saveReportImpl: (_name, data) => {
+      saved = data;
+    },
+    saveCourseRegistryImpl: () => {
+      registryWritten = true;
+    },
+    now: NOW,
+    log: () => {},
+  });
+  assert.equal(report.status, 'error');
+  assert.match(report.skipped_reason, /boom/);
+  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
+  assert.equal(saved.status, 'error', 'the error report is persisted');
+  assert.equal(registryWritten, false, 'no registry write when the run errored');
+});
+
+// ---------------------------------------------------------------------------
+// (3) Happy path: incremental run produces a §4.3-shaped report and advances
+//     the registry.
+// ---------------------------------------------------------------------------
+
+test('detectCourses — incremental run: §4.3 report shape, new/updated leads, registry advanced', async () => {
+  const captured = {};
+  const pages = {
+    modules: [
+      // brand-new UID => new lead
+      {
+        id: 'learn.module-new',
+        title: 'New AOAI course',
+        url: 'https://learn.microsoft.com/training/modules/new',
+        updatedAt: '2026-06-22T00:00:00Z',
+        products: [{ id: 'azure-openai' }],
+      },
+      // known UID with a newer timestamp => updated lead
+      {
+        id: 'learn.module-known',
+        title: 'Known module',
+        url: 'https://learn.microsoft.com/training/modules/known',
+        updatedAt: '2026-06-21T00:00:00Z',
+        products: [{ id: 'azure-openai' }],
+      },
+    ],
+    'learning-paths': [],
+  };
+  let savedReport = null;
+  let savedRegistry = null;
+  const report = await detectCourses({
+    readSecretImpl: () => 'present',
+    loadTaxonomyImpl: () => FAKE_TAX,
+    loadCourseRegistryImpl: () => REGISTRY,
+    getTokenImpl: async () => 'token-abc',
+    paginateImpl: makePaginate(pages, captured),
+    saveReportImpl: (_name, data) => {
+      savedReport = data;
+    },
+    saveCourseRegistryImpl: (reg) => {
+      savedRegistry = reg;
+    },
+    now: NOW,
+    log: () => {},
+  });
+
+  // status + counts
+  assert.equal(report.status, 'ok');
+  assert.equal(report.skipped_reason, null);
+  assert.equal(report.mode, 'incremental');
+  assert.deepEqual(report.counts, { new: 1, updated: 1, removed: 0 });
+
+  // §4.3 item shape on the new lead, incl. derived skill/category
+  const lead = report.new[0];
+  for (const k of ['uid', 'title', 'url', 'products', 'suggested_skill', 'suggested_category', 'updated_at']) {
+    assert.ok(k in lead, `new lead exposes "${k}"`);
+  }
+  assert.equal(lead.uid, 'learn.module-new');
+  assert.equal(lead.suggested_skill, 'ms-ai-engineering');
+  assert.equal(lead.suggested_category, 'azure-ai-services');
+
+  // the updated lead is the known UID
+  assert.equal(report.updated[0].uid, 'learn.module-known');
+  assert.equal(report.removed.length, 0);
+
+  // incremental run pins the updatedAt.gt cursor (full ISO) from last_run, and
+  // filters server-side by the in-domain product slugs.
+  assert.equal(captured.modules['updatedAt.gt'], '2026-06-20T00:00:00.000Z');
+  assert.match(captured.modules.products, /azure-openai/);
+  assert.match(captured.modules.products, /power-automate/);
+
+  // registry advanced: both UIDs present, last_run = now
+  assert.equal(savedReport.status, 'ok');
+  assert.equal(savedRegistry.last_run, NOW);
+  assert.ok(savedRegistry.courses['learn.module-new']);
+  assert.ok(savedRegistry.courses['learn.module-known']);
+});
+
+test('detectCourses — baseline run (no last_run) emits no leads but establishes the registry', async () => {
+  const pages = {
+    modules: [
+      {
+        id: 'learn.module-a',
+        title: 'A',
+        url: 'u-a',
+        updatedAt: '2026-06-22T00:00:00Z',
+        products: [{ id: 'azure-openai' }],
+      },
+    ],
+    'learning-paths': [],
+  };
+  let savedRegistry = null;
+  const captured = {};
+  const report = await detectCourses({
+    readSecretImpl: () => 'present',
+    loadTaxonomyImpl: () => FAKE_TAX,
+    loadCourseRegistryImpl: () => ({ version: 1, created_at: null, last_run: null, last_full_enum: null, courses: {} }),
+    getTokenImpl: async () => 'token-abc',
+    paginateImpl: makePaginate(pages, captured),
+    saveReportImpl: () => {},
+    saveCourseRegistryImpl: (reg) => {
+      savedRegistry = reg;
+    },
+    now: NOW,
+    log: () => {},
+  });
+  assert.equal(report.mode, 'baseline');
+  assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }, 'baseline emits no leads (§8 b)');
+  // baseline omits the updatedAt.gt cursor (full enumeration)
+  assert.equal(captured.modules['updatedAt.gt'], undefined);
+  // but it DID fold the API set into the registry + stamp last_full_enum
+  assert.ok(savedRegistry.courses['learn.module-a']);
+  assert.equal(savedRegistry.last_full_enum, NOW);
+});
+
+// ---------------------------------------------------------------------------
+// (4) The real CLI fails soft to exit 0 — proven hermetically by shadowing the
+//     `security` binary on PATH so readSecret returns null (no Keychain, no net).
+// ---------------------------------------------------------------------------
+
+test('detect-courses.mjs CLI — missing creds => skipped report + exit 0 (no network)', { skip: process.platform === 'win32' }, () => {
+  const tmp = mkdtempSync(join(tmpdir(), 'detect-courses-'));
+  try {
+    const dataDir = join(tmp, 'data');
+    const bin = join(tmp, 'bin');
+    mkdirSync(dataDir, { recursive: true });
+    mkdirSync(bin, { recursive: true });
+    // A `security` that always fails => keychain.readSecret returns null => skip.
+    const stub = join(bin, 'security');
+    writeFileSync(stub, '#!/bin/sh\nexit 1\n');
+    chmodSync(stub, 0o755);
+
+    const script = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'detect-courses.mjs');
+    const res = spawnSync('node', [script, '--data-dir', dataDir], {
+      env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
+      encoding: 'utf8',
+    });
+    assert.equal(res.status, 0, `fail-soft must exit 0 (stderr: ${res.stderr})`);
+    const report = JSON.parse(readFileSync(join(dataDir, 'course-detection-report.json'), 'utf8'));
+    assert.equal(report.status, 'skipped');
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
--- a/tests/kb-update/test-detection-schedule.test.mjs
+++ b/tests/kb-update/test-detection-schedule.test.mjs
@ -23,6 +23,7 @@ import {
  serializeScheduleConfig,
  loadScheduleConfig,
  shouldRunDetection,
+  selectDetectionSteps,
  summarizeSkillLifecycle,
  daysSinceLastPoll,
 } from '../../scripts/kb-update/lib/detection-schedule.mjs';
@ -42,6 +43,30 @@ test('DEFAULT_SCHEDULE_CONFIG — opt-in: disabled by default', () => {
  assert.equal(DEFAULT_SCHEDULE_CONFIG.include_skill_lifecycle, true);
 });

+// --- C3.4: include_course_detection (opt-in inside opt-in; default OFF) ---
+
+test('DEFAULT_SCHEDULE_CONFIG — include_course_detection defaults to false (§8 d)', () => {
+  assert.equal(DEFAULT_SCHEDULE_CONFIG.include_course_detection, false,
+    'course detection needs Keychain creds almost nobody has => opt-in, default OFF');
+});
+
+test('parseScheduleConfig — reads include_course_detection: true', () => {
+  const cfg = parseScheduleConfig('scheduled_detection:\n  include_course_detection: true\n');
+  assert.equal(cfg.include_course_detection, true);
+});
+
+test('parseScheduleConfig — a non-boolean include_course_detection stays OFF', () => {
+  const cfg = parseScheduleConfig('scheduled_detection:\n  include_course_detection: maybe\n');
+  assert.equal(cfg.include_course_detection, false);
+});
+
+test('serializeScheduleConfig — emits + round-trips include_course_detection', () => {
+  assert.match(serializeScheduleConfig({}), /\n {2}include_course_detection: false/,
+    'serialized at its OFF default');
+  const cfg = parseScheduleConfig(serializeScheduleConfig({ include_course_detection: true }));
+  assert.equal(cfg.include_course_detection, true);
+});
+
 test('parseScheduleConfig — empty/absent text returns the OFF default', () => {
  assert.deepEqual(parseScheduleConfig(''), DEFAULT_SCHEDULE_CONFIG);
  assert.deepEqual(parseScheduleConfig('# just some markdown, no frontmatter'), DEFAULT_SCHEDULE_CONFIG);
@ -353,6 +378,7 @@ const KNOWN_DETECTION_SCRIPTS = new Set([
  'report-changes.mjs',
  'discover-new-urls.mjs',
  'detect-skill-lifecycle.mjs',
+  'detect-courses.mjs',
  'build-registry.mjs',
 ]);

@ -365,6 +391,37 @@ test('DETECTION_STEPS — only the known pure-node detection scripts, never clau
  }
  // skill-lifecycle detection must be present and flagged so config can exclude it
  assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-skill-lifecycle.mjs' && s.skillLifecycle === true));
+  // C3.4: course detection present and flagged so config can exclude it (opt-in)
+  assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true));
+});
+
+// --- C3.4: selectDetectionSteps — the pure gate run-detection.mjs applies ---
+
+test('selectDetectionSteps — drops detect-courses when include_course_detection is OFF (default)', () => {
+  const steps = selectDetectionSteps(DEFAULT_SCHEDULE_CONFIG);
+  assert.ok(!steps.some((s) => s.courseDetection), 'course detection excluded by default');
+  // the always-on steps survive
+  assert.ok(steps.some((s) => s.script === 'poll-sitemaps.mjs'));
+});
+
+test('selectDetectionSteps — includes detect-courses when opted in', () => {
+  const steps = selectDetectionSteps({ ...DEFAULT_SCHEDULE_CONFIG, include_course_detection: true });
+  assert.ok(steps.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true));
+});
+
+test('selectDetectionSteps — the two opt-in gates are independent', () => {
+  const onlyCourses = selectDetectionSteps({
+    ...DEFAULT_SCHEDULE_CONFIG,
+    include_skill_lifecycle: false,
+    include_course_detection: true,
+  });
+  assert.ok(!onlyCourses.some((s) => s.skillLifecycle), 'skill-lifecycle gated OFF');
+  assert.ok(onlyCourses.some((s) => s.courseDetection), 'course detection gated ON');
+});
+
+test('run-detection.mjs source — gates detection via the shared selector (no duplicated filter)', () => {
+  const src = readFileSync(join(PLUGIN_ROOT, 'scripts', 'kb-update', 'run-detection.mjs'), 'utf8');
+  assert.match(src, /selectDetectionSteps\(/, 'run-detection uses the pure step selector');
 });

 test('run-detection.mjs source — spawns node, NEVER claude (the ToS boundary made structural)', () => {