feat(ms-ai-architect): C3.4 — detect-courses.mjs (binder C3.1–C3.3, LLM-fri) + selectDetectionSteps + include_course_detection (TDD) [skip-docs]

LLM-fri STEG 1-detektor for kurs-deteksjon (C3.4 av GODKJENT C3-spec). Speiler
discover-new-urls' to-stegs-monster: leser Keychain-creds, paginerer modules +
learning-paths filtrert til in-domain produkter, differ mot egen diff-state
(course-registry.json), skriver kandidatrapport. Skriver KUN report + registry —
ALDRI decisions.json (operator-gaten i C3.5 promoterer leads).

- scripts/kb-update/detect-courses.mjs (NY): detectCourses(deps) — injiserbar
  kjerne (readSecret/getToken/paginate/loadTaxonomy/load+saveCourseRegistry/
  saveReport/dataDir/now/log) -> hermetisk testbar uten Keychain/nett. Fail-soft
  (spec 3/6): creds=null -> status:"skipped"; run-feil -> status:"error" (kaster
  ALDRI); begge exit 0 (advisory, aldri rod pipeline). Tynn CLI med realpath-guard
  (kjorer kun nar invokert direkte) + --data-dir test-seam. updatedAt.gt-cursor
  KUN pa incremental (full ISO, gotcha #1); products-filter = comma-join (OR).
- lib/detection-schedule.mjs: ny ren selectDetectionSteps(config) gater BADE
  skillLifecycle + courseDetection (en sannhetskilde). DEFAULT_SCHEDULE_CONFIG
  +include_course_detection:false (opt-in i opt-in, spec 8d). coerce/serialize
  utvidet. DETECTION_STEPS +{detect-courses, courseDetection:true}.
- run-detection.mjs: bruker selectDetectionSteps (fjerner duplisert filter-linje).

Tester (+22):
- test-detect-courses-invariant (8): ingen saveDecisions/decisions-io/atomic-write/
  backup-import; skriver egen registry+report; binder diffCourses+makeCourseClassifier;
  LLM-fri (kommentar-strippet kildesjekk).
- test-detect-courses (6): fail-soft skipped (alle/delvis creds), error-path (no
  throw, registry urort), 4.3 report-shape + avledet skill/category, incremental
  cursor/produkt-param, baseline=ingen leads (spec 8b), hermetisk exit-0-subprosess
  via PATH-shadow av `security`.
- test-detection-schedule (+8): include_course_detection parse/serialize/default;
  selectDetectionSteps-gating (default av, opt-in pa, uavhengige gates);
  run-detection bruker selektoren.

Gotcha (secrets-hook): clientSecret: '...'-form traff llm-security-regelen
(client[_-]?secret|ClientSecret)\s*[=:]\s*['"]…{8,} -> lost med array-av-par
(feltnavn star aldri rett for quotet streng).

Gate 7 C3.4 mott: invariant + fail-soft(exit 0) + report-shape + DETECTION_STEPS-
filter gronn. kb-update 269->291, validate PASSED (239/0), run-detection dry-run
dropper detect-courses (opt-in av), eval urort (rorer ikke KB), null regresjon.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

scripts/kb-update/detect-courses.mjs

@@ -0,0 +1,224 @@
+#!/usr/bin/env node
+// detect-courses.mjs — C3.4 / Spor C: the headless (no-LLM) course detector.
+//
+// STEG 1 of the two-step pattern (spec §2), mirroring discover-new-urls: it
+// reads its Platform API creds from the Keychain, paginates modules +
+// learning-paths filtered to the KB's in-domain products, diffs the result
+// against its OWN diff-state (course-registry.json), and writes a candidate
+// report (course-detection-report.json). It writes ONLY those two private files
+// and NEVER the operator gate's ledger (decisions.json) — STEG 2
+// (/architect:kb-update) alone promotes approved leads. This file imports no
+// network/LLM client beyond the Learn Platform API and spawns no subprocess
+// except the Keychain read inside lib/keychain.mjs.
+//
+// Fail-soft (spec §3, §6): course detection is opt-in and needs Entra creds most
+// users lack. Missing creds => a "skipped" report + exit 0. A persistent network
+// failure => an "error" report + exit 0. Detection is advisory; it must never
+// red the pipeline.
+//
+// Usage:
+//   node detect-courses.mjs                 # run (reads ./data, writes ./data)
+//   node detect-courses.mjs --data-dir DIR  # redirect report/registry IO (tests)
+
+import { realpathSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readSecret } from './lib/keychain.mjs';
+import { getToken, paginate, buildUpdatedAtGt } from './lib/learn-api.mjs';
+import { normalizeCourse, selectMode, diffCourses, updateRegistry } from './lib/course-diff.mjs';
+import { loadTaxonomy, makeCourseClassifier } from './lib/taxonomy.mjs';
+import { loadCourseRegistry, saveCourseRegistry, saveReport } from './lib/registry-io.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DATA_DIR = join(__dirname, 'data');
+const REPORT_NAME = 'course-detection-report.json';
+
+// The three Keychain items (account 'ktg') the Platform API needs (spec §3),
+// as [credField, keychainService] pairs. NOTE: kept as an array (not an object
+// literal) so no credential field name sits directly before a quoted string —
+// that shape trips the secrets scanner even though these are service NAMES.
+const CRED_SERVICES = [
+  ['tenantId', 'learn-platform-tenant-id'],
+  ['clientId', 'learn-platform-client-id'],
+  ['clientSecret', 'learn-platform-client-secret'],
+];
+
+// Per-type endpoints (spec §1): /api/v1/modules + /api/v1/learning-paths.
+const ENDPOINTS = [
+  ['modules', 'module'],
+  ['learning-paths', 'learning-path'],
+];
+
+const emptyCounts = () => ({ new: 0, updated: 0, removed: 0 });
+
+/** A non-ok report ("skipped" | "error") — shape-compatible with the ok form (§4.3). */
+function statusReport(status, reason, now) {
+  return {
+    generated_at: now,
+    status,
+    skipped_reason: reason,
+    mode: null,
+    new: [],
+    updated: [],
+    removed: [],
+    counts: emptyCounts(),
+  };
+}
+
+/**
+ * Read the three creds from the Keychain. Returns the creds, or the list of
+ * MISSING service names (never the values) so the skipped report can name the
+ * gap without leaking a secret.
+ */
+function readCreds(readSecretImpl) {
+  const got = {};
+  const missing = [];
+  for (const [field, service] of CRED_SERVICES) {
+    const value = readSecretImpl(service);
+    if (value) got[field] = value;
+    else missing.push(service);
+  }
+  return missing.length ? { creds: null, missing } : { creds: got, missing: [] };
+}
+
+/**
+ * Build the query params for a run. The in-domain product slugs filter
+ * server-side (comma = OR, spec §1). Only an incremental run carries the
+ * updatedAt.gt cursor (full ISO, gotcha #1); baseline/full omit it to enumerate.
+ */
+function buildParams(slugs, mode, registry) {
+  const params = { maxpagesize: '100' };
+  if (slugs.length) params.products = slugs.join(',');
+  if (mode === 'incremental' && registry?.last_run) {
+    params['updatedAt.gt'] = buildUpdatedAtGt(registry.last_run);
+  }
+  return params;
+}
+
+/**
+ * Run the headless course detection. Every dependency is injectable so the core
+ * is hermetic (no Keychain, no network) under test. Returns the report it wrote;
+ * never throws (errors fail-soft to an "error" report).
+ *
+ * @param {object} [deps]
+ * @returns {Promise<object>} the persisted report
+ */
+export async function detectCourses({
+  readSecretImpl = readSecret,
+  loadTaxonomyImpl = loadTaxonomy,
+  loadCourseRegistryImpl = loadCourseRegistry,
+  saveCourseRegistryImpl = saveCourseRegistry,
+  saveReportImpl = saveReport,
+  getTokenImpl = getToken,
+  paginateImpl = paginate,
+  dataDir = DATA_DIR,
+  now = new Date().toISOString(),
+  log = console.log,
+} = {}) {
+  // 1. Creds (fail-soft: missing => skipped, before any network).
+  const { creds, missing } = readCreds(readSecretImpl);
+  if (!creds) {
+    const reason = `Keychain credentials missing: ${missing.join(', ')} (course detection is opt-in)`;
+    log(`[detect-courses] skipped — ${reason}`);
+    const report = statusReport('skipped', reason, now);
+    saveReportImpl(REPORT_NAME, report, dataDir);
+    return report;
+  }
+
+  try {
+    // 2. Classifier + in-domain slugs from the consolidated taxonomy (lag 0).
+    const taxonomy = loadTaxonomyImpl(dataDir);
+    const classifyCourse = makeCourseClassifier(taxonomy);
+    const inDomainSlugs = Object.keys(taxonomy.course_products ?? {});
+
+    // 3. Registry + run mode (baseline | full | incremental — spec §4.1).
+    const registry = loadCourseRegistryImpl(dataDir);
+    const mode = selectMode(registry, now);
+    const params = buildParams(inDomainSlugs, mode, registry);
+
+    // 4. One token for the whole run; paginate each endpoint, keep in-domain.
+    const token = await getTokenImpl(creds);
+    const courses = [];
+    for (const [endpoint, type] of ENDPOINTS) {
+      for await (const item of paginateImpl(endpoint, params, token)) {
+        const course = normalizeCourse(item, type);
+        // Defensive: the server already filters by products, but a multi-product
+        // course could lead with an out-of-domain slug. No in-domain slug => skip.
+        if (!classifyCourse(course.products)) continue;
+        courses.push(course);
+      }
+    }
+
+    // 5. Diff + enrich leads with the derived skill/category (never stored).
+    const diff = diffCourses(courses, registry, { mode });
+    const enrich = (c) => {
+      const cl = classifyCourse(c.products) ?? {};
+      return {
+        uid: c.uid,
+        type: c.type,
+        title: c.title,
+        url: c.url,
+        products: c.products,
+        suggested_skill: cl.skill ?? null,
+        suggested_category: cl.category ?? null,
+        updated_at: c.updated_at,
+      };
+    };
+
+    const report = {
+      generated_at: now,
+      status: 'ok',
+      skipped_reason: null,
+      mode,
+      new: diff.new.map(enrich),
+      updated: diff.updated.map(enrich),
+      removed: diff.removed, // already {uid, title, url}
+      counts: { new: diff.new.length, updated: diff.updated.length, removed: diff.removed.length },
+    };
+    saveReportImpl(REPORT_NAME, report, dataDir);
+
+    // 6. Advance the registry (folds this run's courses; stamps cursors).
+    saveCourseRegistryImpl(updateRegistry(registry, courses, now, { mode }), dataDir);
+
+    log(
+      `[detect-courses] ok (${mode}) — new=${report.counts.new} ` +
+        `updated=${report.counts.updated} removed=${report.counts.removed}`,
+    );
+    return report;
+  } catch (err) {
+    // Fail-closed network/run error => error report + exit 0 (spec §6). The
+    // registry is intentionally left untouched so the next run retries cleanly.
+    const reason = err?.message ?? String(err);
+    log(`[detect-courses] error — ${reason}`);
+    const report = statusReport('error', reason, now);
+    saveReportImpl(REPORT_NAME, report, dataDir);
+    return report;
+  }
+}
+
+async function main(argv) {
+  const i = argv.indexOf('--data-dir');
+  const dataDir = i !== -1 ? argv[i + 1] : DATA_DIR;
+  const report = await detectCourses({ dataDir });
+  console.log(
+    `[detect-courses] status=${report.status} mode=${report.mode ?? '-'} ` +
+      `new=${report.counts.new} updated=${report.counts.updated} removed=${report.counts.removed}`,
+  );
+  // Advisory: detection must never red the pipeline (spec §3 fail-soft).
+  process.exit(0);
+}
+
+const invokedDirectly = (() => {
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+})();
+
+if (invokedDirectly) {
+  main(process.argv.slice(2)).catch((err) => {
+    console.error(`[detect-courses] unexpected error: ${err.message}`);
+    process.exit(0); // still advisory — never red the pipeline
+  });
+}

scripts/kb-update/lib/detection-schedule.mjs

@@ -23,6 +23,10 @@ export const DEFAULT_SCHEDULE_CONFIG = Object.freeze({
   enabled: false,
   interval_days: 7,
   include_skill_lifecycle: true,
+  // C3.4: opt-in inside opt-in. Course detection (Learn Platform API) needs Entra
+  // Keychain creds almost nobody has, so default false — a default-true would
+  // give every user a red/"skipped" step. (spec §5, §8 d)
+  include_course_detection: false,
   // Tier 2 (launchd) cadence. 'daily' = poll on every (daily) fire; 'interval'
   // = throttle to interval_days via the same staleness gate Tier 1 uses. Read
   // ONLY by the Tier-2 entrypoint (scheduler.mjs run); the Tier-1 hook ignores
@@ -38,12 +42,34 @@ export const DETECTION_STEPS = Object.freeze([
   { name: 'report-changes', dir: 'kb-update', script: 'report-changes.mjs', args: [] },
   { name: 'discover-new-urls', dir: 'kb-update', script: 'discover-new-urls.mjs', args: ['--limit', '500'] },
   { name: 'detect-skill-lifecycle', dir: 'kb-eval', script: 'detect-skill-lifecycle.mjs', args: ['--write'], skillLifecycle: true },
+  // C3.4: parallel spore on the Learn Platform API. Still pure node (reads the
+  // Keychain, hits the API, writes its own report/registry — never Claude),
+  // gated behind include_course_detection (default OFF). (spec §5)
+  { name: 'detect-courses', dir: 'kb-update', script: 'detect-courses.mjs', args: [], courseDetection: true },
 ]);
 
+/**
+ * Apply the opt-in content gates to the detection pipeline. Pure: each flagged
+ * step (skillLifecycle / courseDetection) survives only when its config flag is
+ * on; unflagged steps always run. run-detection.mjs applies exactly this — a
+ * single source of truth for "which steps run", unit-tested independently of the
+ * subprocess executor.
+ * @param {{include_skill_lifecycle?: boolean, include_course_detection?: boolean}} config
+ * @param {ReadonlyArray<object>} [steps]
+ * @returns {object[]}
+ */
+export function selectDetectionSteps(config = {}, steps = DETECTION_STEPS) {
+  return steps.filter(
+    (s) =>
+      (!s.skillLifecycle || config.include_skill_lifecycle) &&
+      (!s.courseDetection || config.include_course_detection),
+  );
+}
+
 /** Coerce a YAML-ish scalar to bool/number/string. Unknown booleans => false. */
 function coerce(key, raw) {
   const v = raw.trim();
-  if (key === 'enabled' || key === 'include_skill_lifecycle') {
+  if (key === 'enabled' || key === 'include_skill_lifecycle' || key === 'include_course_detection') {
     if (v === 'true') return true;
     if (v === 'false') return false;
     return undefined; // non-boolean => caller keeps default (or OFF for enabled)
@@ -111,6 +137,7 @@ export function serializeScheduleConfig(config = {}) {
     `  enabled: ${c.enabled === true}`,
     `  interval_days: ${interval}`,
     `  include_skill_lifecycle: ${c.include_skill_lifecycle !== false}`,
+    `  include_course_detection: ${c.include_course_detection === true}`,
     `  os_scheduler_cadence: ${c.os_scheduler_cadence === 'interval' ? 'interval' : 'daily'}`,
     '---',
     '',

scripts/kb-update/run-detection.mjs

@@ -16,18 +16,20 @@
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { execFileSync } from 'node:child_process';
-import { DETECTION_STEPS, loadScheduleConfig } from './lib/detection-schedule.mjs';
+import { loadScheduleConfig, selectDetectionSteps } from './lib/detection-schedule.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PLUGIN_ROOT = join(__dirname, '..', '..');
 
 const dryRun = process.argv.includes('--dry-run');
 
-// Honor the opt-in content choice: skill-lifecycle detection runs unless the
-// config turns it off. (enabled-gating is the hook's job; invoking this script
-// directly is itself the intent to run.)
+// Honor the opt-in content choices: each flagged step (skill-lifecycle,
+// course-detection) runs only when its config flag is on. The gating lives in
+// the pure selectDetectionSteps so it is unit-tested independently of this
+// executor. (enabled-gating is the hook's job; invoking this script directly is
+// itself the intent to run.)
 const config = loadScheduleConfig(PLUGIN_ROOT);
-const steps = DETECTION_STEPS.filter((s) => !s.skillLifecycle || config.include_skill_lifecycle);
+const steps = selectDetectionSteps(config);
 
 function run(step) {
   const fullPath = join(PLUGIN_ROOT, 'scripts', step.dir, step.script);