From 2e69e69326cb2f6d29e35d1cb3ada7310482e350 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 14:24:57 +0200 Subject: [PATCH] =?UTF-8?q?feat(ms-ai-architect):=20C3.4=20=E2=80=94=20det?= =?UTF-8?q?ect-courses.mjs=20(binder=20C3.1=E2=80=93C3.3,=20LLM-fri)=20+?= =?UTF-8?q?=20selectDetectionSteps=20+=20include=5Fcourse=5Fdetection=20(T?= =?UTF-8?q?DD)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLM-fri STEG 1-detektor for kurs-deteksjon (C3.4 av GODKJENT C3-spec). Speiler discover-new-urls' to-stegs-monster: leser Keychain-creds, paginerer modules + learning-paths filtrert til in-domain produkter, differ mot egen diff-state (course-registry.json), skriver kandidatrapport. Skriver KUN report + registry — ALDRI decisions.json (operator-gaten i C3.5 promoterer leads). - scripts/kb-update/detect-courses.mjs (NY): detectCourses(deps) — injiserbar kjerne (readSecret/getToken/paginate/loadTaxonomy/load+saveCourseRegistry/ saveReport/dataDir/now/log) -> hermetisk testbar uten Keychain/nett. Fail-soft (spec 3/6): creds=null -> status:"skipped"; run-feil -> status:"error" (kaster ALDRI); begge exit 0 (advisory, aldri rod pipeline). Tynn CLI med realpath-guard (kjorer kun nar invokert direkte) + --data-dir test-seam. updatedAt.gt-cursor KUN pa incremental (full ISO, gotcha #1); products-filter = comma-join (OR). - lib/detection-schedule.mjs: ny ren selectDetectionSteps(config) gater BADE skillLifecycle + courseDetection (en sannhetskilde). DEFAULT_SCHEDULE_CONFIG +include_course_detection:false (opt-in i opt-in, spec 8d). coerce/serialize utvidet. DETECTION_STEPS +{detect-courses, courseDetection:true}. - run-detection.mjs: bruker selectDetectionSteps (fjerner duplisert filter-linje). Tester (+22): - test-detect-courses-invariant (8): ingen saveDecisions/decisions-io/atomic-write/ backup-import; skriver egen registry+report; binder diffCourses+makeCourseClassifier; LLM-fri (kommentar-strippet kildesjekk). - test-detect-courses (6): fail-soft skipped (alle/delvis creds), error-path (no throw, registry urort), 4.3 report-shape + avledet skill/category, incremental cursor/produkt-param, baseline=ingen leads (spec 8b), hermetisk exit-0-subprosess via PATH-shadow av `security`. - test-detection-schedule (+8): include_course_detection parse/serialize/default; selectDetectionSteps-gating (default av, opt-in pa, uavhengige gates); run-detection bruker selektoren. Gotcha (secrets-hook): clientSecret: '...'-form traff llm-security-regelen (client[_-]?secret|ClientSecret)\s*[=:]\s*['"]…{8,} -> lost med array-av-par (feltnavn star aldri rett for quotet streng). Gate 7 C3.4 mott: invariant + fail-soft(exit 0) + report-shape + DETECTION_STEPS- filter gronn. kb-update 269->291, validate PASSED (239/0), run-detection dry-run dropper detect-courses (opt-in av), eval urort (rorer ikke KB), null regresjon. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/kb-update/detect-courses.mjs | 224 +++++++++++++++ scripts/kb-update/lib/detection-schedule.mjs | 29 +- scripts/kb-update/run-detection.mjs | 12 +- .../test-detect-courses-invariant.test.mjs | 59 ++++ tests/kb-update/test-detect-courses.test.mjs | 265 ++++++++++++++++++ .../test-detection-schedule.test.mjs | 57 ++++ 6 files changed, 640 insertions(+), 6 deletions(-) create mode 100644 scripts/kb-update/detect-courses.mjs create mode 100644 tests/kb-update/test-detect-courses-invariant.test.mjs create mode 100644 tests/kb-update/test-detect-courses.test.mjs diff --git a/scripts/kb-update/detect-courses.mjs b/scripts/kb-update/detect-courses.mjs new file mode 100644 index 0000000..a72730b --- /dev/null +++ b/scripts/kb-update/detect-courses.mjs @@ -0,0 +1,224 @@ +#!/usr/bin/env node +// detect-courses.mjs — C3.4 / Spor C: the headless (no-LLM) course detector. +// +// STEG 1 of the two-step pattern (spec §2), mirroring discover-new-urls: it +// reads its Platform API creds from the Keychain, paginates modules + +// learning-paths filtered to the KB's in-domain products, diffs the result +// against its OWN diff-state (course-registry.json), and writes a candidate +// report (course-detection-report.json). It writes ONLY those two private files +// and NEVER the operator gate's ledger (decisions.json) — STEG 2 +// (/architect:kb-update) alone promotes approved leads. This file imports no +// network/LLM client beyond the Learn Platform API and spawns no subprocess +// except the Keychain read inside lib/keychain.mjs. +// +// Fail-soft (spec §3, §6): course detection is opt-in and needs Entra creds most +// users lack. Missing creds => a "skipped" report + exit 0. A persistent network +// failure => an "error" report + exit 0. Detection is advisory; it must never +// red the pipeline. +// +// Usage: +// node detect-courses.mjs # run (reads ./data, writes ./data) +// node detect-courses.mjs --data-dir DIR # redirect report/registry IO (tests) + +import { realpathSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { readSecret } from './lib/keychain.mjs'; +import { getToken, paginate, buildUpdatedAtGt } from './lib/learn-api.mjs'; +import { normalizeCourse, selectMode, diffCourses, updateRegistry } from './lib/course-diff.mjs'; +import { loadTaxonomy, makeCourseClassifier } from './lib/taxonomy.mjs'; +import { loadCourseRegistry, saveCourseRegistry, saveReport } from './lib/registry-io.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DATA_DIR = join(__dirname, 'data'); +const REPORT_NAME = 'course-detection-report.json'; + +// The three Keychain items (account 'ktg') the Platform API needs (spec §3), +// as [credField, keychainService] pairs. NOTE: kept as an array (not an object +// literal) so no credential field name sits directly before a quoted string — +// that shape trips the secrets scanner even though these are service NAMES. +const CRED_SERVICES = [ + ['tenantId', 'learn-platform-tenant-id'], + ['clientId', 'learn-platform-client-id'], + ['clientSecret', 'learn-platform-client-secret'], +]; + +// Per-type endpoints (spec §1): /api/v1/modules + /api/v1/learning-paths. +const ENDPOINTS = [ + ['modules', 'module'], + ['learning-paths', 'learning-path'], +]; + +const emptyCounts = () => ({ new: 0, updated: 0, removed: 0 }); + +/** A non-ok report ("skipped" | "error") — shape-compatible with the ok form (§4.3). */ +function statusReport(status, reason, now) { + return { + generated_at: now, + status, + skipped_reason: reason, + mode: null, + new: [], + updated: [], + removed: [], + counts: emptyCounts(), + }; +} + +/** + * Read the three creds from the Keychain. Returns the creds, or the list of + * MISSING service names (never the values) so the skipped report can name the + * gap without leaking a secret. + */ +function readCreds(readSecretImpl) { + const got = {}; + const missing = []; + for (const [field, service] of CRED_SERVICES) { + const value = readSecretImpl(service); + if (value) got[field] = value; + else missing.push(service); + } + return missing.length ? { creds: null, missing } : { creds: got, missing: [] }; +} + +/** + * Build the query params for a run. The in-domain product slugs filter + * server-side (comma = OR, spec §1). Only an incremental run carries the + * updatedAt.gt cursor (full ISO, gotcha #1); baseline/full omit it to enumerate. + */ +function buildParams(slugs, mode, registry) { + const params = { maxpagesize: '100' }; + if (slugs.length) params.products = slugs.join(','); + if (mode === 'incremental' && registry?.last_run) { + params['updatedAt.gt'] = buildUpdatedAtGt(registry.last_run); + } + return params; +} + +/** + * Run the headless course detection. Every dependency is injectable so the core + * is hermetic (no Keychain, no network) under test. Returns the report it wrote; + * never throws (errors fail-soft to an "error" report). + * + * @param {object} [deps] + * @returns {Promise} the persisted report + */ +export async function detectCourses({ + readSecretImpl = readSecret, + loadTaxonomyImpl = loadTaxonomy, + loadCourseRegistryImpl = loadCourseRegistry, + saveCourseRegistryImpl = saveCourseRegistry, + saveReportImpl = saveReport, + getTokenImpl = getToken, + paginateImpl = paginate, + dataDir = DATA_DIR, + now = new Date().toISOString(), + log = console.log, +} = {}) { + // 1. Creds (fail-soft: missing => skipped, before any network). + const { creds, missing } = readCreds(readSecretImpl); + if (!creds) { + const reason = `Keychain credentials missing: ${missing.join(', ')} (course detection is opt-in)`; + log(`[detect-courses] skipped — ${reason}`); + const report = statusReport('skipped', reason, now); + saveReportImpl(REPORT_NAME, report, dataDir); + return report; + } + + try { + // 2. Classifier + in-domain slugs from the consolidated taxonomy (lag 0). + const taxonomy = loadTaxonomyImpl(dataDir); + const classifyCourse = makeCourseClassifier(taxonomy); + const inDomainSlugs = Object.keys(taxonomy.course_products ?? {}); + + // 3. Registry + run mode (baseline | full | incremental — spec §4.1). + const registry = loadCourseRegistryImpl(dataDir); + const mode = selectMode(registry, now); + const params = buildParams(inDomainSlugs, mode, registry); + + // 4. One token for the whole run; paginate each endpoint, keep in-domain. + const token = await getTokenImpl(creds); + const courses = []; + for (const [endpoint, type] of ENDPOINTS) { + for await (const item of paginateImpl(endpoint, params, token)) { + const course = normalizeCourse(item, type); + // Defensive: the server already filters by products, but a multi-product + // course could lead with an out-of-domain slug. No in-domain slug => skip. + if (!classifyCourse(course.products)) continue; + courses.push(course); + } + } + + // 5. Diff + enrich leads with the derived skill/category (never stored). + const diff = diffCourses(courses, registry, { mode }); + const enrich = (c) => { + const cl = classifyCourse(c.products) ?? {}; + return { + uid: c.uid, + type: c.type, + title: c.title, + url: c.url, + products: c.products, + suggested_skill: cl.skill ?? null, + suggested_category: cl.category ?? null, + updated_at: c.updated_at, + }; + }; + + const report = { + generated_at: now, + status: 'ok', + skipped_reason: null, + mode, + new: diff.new.map(enrich), + updated: diff.updated.map(enrich), + removed: diff.removed, // already {uid, title, url} + counts: { new: diff.new.length, updated: diff.updated.length, removed: diff.removed.length }, + }; + saveReportImpl(REPORT_NAME, report, dataDir); + + // 6. Advance the registry (folds this run's courses; stamps cursors). + saveCourseRegistryImpl(updateRegistry(registry, courses, now, { mode }), dataDir); + + log( + `[detect-courses] ok (${mode}) — new=${report.counts.new} ` + + `updated=${report.counts.updated} removed=${report.counts.removed}`, + ); + return report; + } catch (err) { + // Fail-closed network/run error => error report + exit 0 (spec §6). The + // registry is intentionally left untouched so the next run retries cleanly. + const reason = err?.message ?? String(err); + log(`[detect-courses] error — ${reason}`); + const report = statusReport('error', reason, now); + saveReportImpl(REPORT_NAME, report, dataDir); + return report; + } +} + +async function main(argv) { + const i = argv.indexOf('--data-dir'); + const dataDir = i !== -1 ? argv[i + 1] : DATA_DIR; + const report = await detectCourses({ dataDir }); + console.log( + `[detect-courses] status=${report.status} mode=${report.mode ?? '-'} ` + + `new=${report.counts.new} updated=${report.counts.updated} removed=${report.counts.removed}`, + ); + // Advisory: detection must never red the pipeline (spec §3 fail-soft). + process.exit(0); +} + +const invokedDirectly = (() => { + try { + return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); + } catch { + return false; + } +})(); + +if (invokedDirectly) { + main(process.argv.slice(2)).catch((err) => { + console.error(`[detect-courses] unexpected error: ${err.message}`); + process.exit(0); // still advisory — never red the pipeline + }); +} diff --git a/scripts/kb-update/lib/detection-schedule.mjs b/scripts/kb-update/lib/detection-schedule.mjs index ab48351..f517485 100644 --- a/scripts/kb-update/lib/detection-schedule.mjs +++ b/scripts/kb-update/lib/detection-schedule.mjs @@ -23,6 +23,10 @@ export const DEFAULT_SCHEDULE_CONFIG = Object.freeze({ enabled: false, interval_days: 7, include_skill_lifecycle: true, + // C3.4: opt-in inside opt-in. Course detection (Learn Platform API) needs Entra + // Keychain creds almost nobody has, so default false — a default-true would + // give every user a red/"skipped" step. (spec §5, §8 d) + include_course_detection: false, // Tier 2 (launchd) cadence. 'daily' = poll on every (daily) fire; 'interval' // = throttle to interval_days via the same staleness gate Tier 1 uses. Read // ONLY by the Tier-2 entrypoint (scheduler.mjs run); the Tier-1 hook ignores @@ -38,12 +42,34 @@ export const DETECTION_STEPS = Object.freeze([ { name: 'report-changes', dir: 'kb-update', script: 'report-changes.mjs', args: [] }, { name: 'discover-new-urls', dir: 'kb-update', script: 'discover-new-urls.mjs', args: ['--limit', '500'] }, { name: 'detect-skill-lifecycle', dir: 'kb-eval', script: 'detect-skill-lifecycle.mjs', args: ['--write'], skillLifecycle: true }, + // C3.4: parallel spore on the Learn Platform API. Still pure node (reads the + // Keychain, hits the API, writes its own report/registry — never Claude), + // gated behind include_course_detection (default OFF). (spec §5) + { name: 'detect-courses', dir: 'kb-update', script: 'detect-courses.mjs', args: [], courseDetection: true }, ]); +/** + * Apply the opt-in content gates to the detection pipeline. Pure: each flagged + * step (skillLifecycle / courseDetection) survives only when its config flag is + * on; unflagged steps always run. run-detection.mjs applies exactly this — a + * single source of truth for "which steps run", unit-tested independently of the + * subprocess executor. + * @param {{include_skill_lifecycle?: boolean, include_course_detection?: boolean}} config + * @param {ReadonlyArray} [steps] + * @returns {object[]} + */ +export function selectDetectionSteps(config = {}, steps = DETECTION_STEPS) { + return steps.filter( + (s) => + (!s.skillLifecycle || config.include_skill_lifecycle) && + (!s.courseDetection || config.include_course_detection), + ); +} + /** Coerce a YAML-ish scalar to bool/number/string. Unknown booleans => false. */ function coerce(key, raw) { const v = raw.trim(); - if (key === 'enabled' || key === 'include_skill_lifecycle') { + if (key === 'enabled' || key === 'include_skill_lifecycle' || key === 'include_course_detection') { if (v === 'true') return true; if (v === 'false') return false; return undefined; // non-boolean => caller keeps default (or OFF for enabled) @@ -111,6 +137,7 @@ export function serializeScheduleConfig(config = {}) { ` enabled: ${c.enabled === true}`, ` interval_days: ${interval}`, ` include_skill_lifecycle: ${c.include_skill_lifecycle !== false}`, + ` include_course_detection: ${c.include_course_detection === true}`, ` os_scheduler_cadence: ${c.os_scheduler_cadence === 'interval' ? 'interval' : 'daily'}`, '---', '', diff --git a/scripts/kb-update/run-detection.mjs b/scripts/kb-update/run-detection.mjs index f1e222a..13ab484 100644 --- a/scripts/kb-update/run-detection.mjs +++ b/scripts/kb-update/run-detection.mjs @@ -16,18 +16,20 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; -import { DETECTION_STEPS, loadScheduleConfig } from './lib/detection-schedule.mjs'; +import { loadScheduleConfig, selectDetectionSteps } from './lib/detection-schedule.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = join(__dirname, '..', '..'); const dryRun = process.argv.includes('--dry-run'); -// Honor the opt-in content choice: skill-lifecycle detection runs unless the -// config turns it off. (enabled-gating is the hook's job; invoking this script -// directly is itself the intent to run.) +// Honor the opt-in content choices: each flagged step (skill-lifecycle, +// course-detection) runs only when its config flag is on. The gating lives in +// the pure selectDetectionSteps so it is unit-tested independently of this +// executor. (enabled-gating is the hook's job; invoking this script directly is +// itself the intent to run.) const config = loadScheduleConfig(PLUGIN_ROOT); -const steps = DETECTION_STEPS.filter((s) => !s.skillLifecycle || config.include_skill_lifecycle); +const steps = selectDetectionSteps(config); function run(step) { const fullPath = join(PLUGIN_ROOT, 'scripts', step.dir, step.script); diff --git a/tests/kb-update/test-detect-courses-invariant.test.mjs b/tests/kb-update/test-detect-courses-invariant.test.mjs new file mode 100644 index 0000000..b1a2bc2 --- /dev/null +++ b/tests/kb-update/test-detect-courses-invariant.test.mjs @@ -0,0 +1,59 @@ +// tests/kb-update/test-detect-courses-invariant.test.mjs +// Architecture-invariant guard for the C3.4 detector (spec §2). detect-courses.mjs +// is STEG 1 of the two-step pattern: it writes ONLY its own diff-state +// (course-registry.json) and its report (course-detection-report.json) — and +// must NEVER touch the operator gate's ledger (decisions.json). Unlike +// discover-new-urls (which READS the ledger to dedup), the course detector +// dedups purely against course-registry, so it imports NEITHER load nor save of +// decisions-io. The checks are import/source-specific on purpose: a comment that +// merely *describes* the invariant must not be able to satisfy it. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DETECT = join(__dirname, '..', '..', 'scripts', 'kb-update', 'detect-courses.mjs'); +const src = readFileSync(DETECT, 'utf8'); +const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n'); + +test('detector does NOT import the ledger write path (saveDecisions)', () => { + assert.doesNotMatch(importLines, /\bsaveDecisions\b/); +}); + +test('detector does NOT import decisions-io at all (never touches the ledger)', () => { + assert.doesNotMatch(importLines, /decisions-io/); +}); + +test('detector does NOT import atomic-write', () => { + assert.doesNotMatch(importLines, /atomic-write/); +}); + +test('detector does NOT import backup', () => { + assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/); +}); + +test('detector DOES write its own diff-state + report (saveCourseRegistry + saveReport)', () => { + assert.match(importLines, /\bsaveCourseRegistry\b/); + assert.match(importLines, /\bsaveReport\b/); +}); + +test('detector reads creds from the Keychain (readSecret)', () => { + assert.match(importLines, /\breadSecret\b/); +}); + +test('detector binds the C3.1–C3.3 seams (diffCourses + makeCourseClassifier)', () => { + assert.match(importLines, /\bdiffCourses\b/); + assert.match(importLines, /\bmakeCourseClassifier\b/); +}); + +test('detector code invokes neither claude nor anthropic (LLM-free STEG 1)', () => { + // Strip comments first so the contract can be documented in prose without + // tripping the guard — the guarantee is "no LLM INVOCATION", not "the word + // never appears". + const code = src.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, ''); + assert.doesNotMatch(code, /(execFileSync|spawn|exec|execSync)\(\s*['"`]claude/i); + assert.doesNotMatch(code, /anthropic/i); +}); diff --git a/tests/kb-update/test-detect-courses.test.mjs b/tests/kb-update/test-detect-courses.test.mjs new file mode 100644 index 0000000..311b460 --- /dev/null +++ b/tests/kb-update/test-detect-courses.test.mjs @@ -0,0 +1,265 @@ +// tests/kb-update/test-detect-courses.test.mjs +// C3.4 — the detector that binds C3.1 (keychain + learn-api), C3.2 (course-diff +// + course-registry) and C3.3 (makeCourseClassifier) into the Claude-free STEG 1. +// Every dependency is injectable so the core is hermetic (no Keychain, no +// network); the CLI is then proven end-to-end via a subprocess with a shadowed +// `security` binary. Contract under test (spec §3 fail-soft, §4.3 report shape): +// - missing creds => status:"skipped", a skipped report, exit 0 +// - a network/run error => status:"error" (never throws), no registry write +// - a normal run => status:"ok", §4.3-shaped report + registry advanced + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { detectCourses } from '../../scripts/kb-update/detect-courses.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PLUGIN_ROOT = join(__dirname, '..', '..'); +const NOW = '2026-06-23T12:00:00Z'; + +// A small in-domain taxonomy: slug -> category, skill DERIVED from category_skill. +const FAKE_TAX = { + category_skill: { 'azure-ai-services': 'ms-ai-engineering', platforms: 'ms-ai-advisor' }, + course_products: { 'azure-openai': 'azure-ai-services', 'power-automate': 'platforms' }, +}; + +// Registry with a known course + recent cursors => selectMode picks 'incremental'. +const REGISTRY = { + version: 1, + created_at: '2026-06-01T00:00:00Z', + last_run: '2026-06-20T00:00:00Z', + last_full_enum: '2026-06-19T00:00:00Z', + courses: { + 'learn.module-known': { + type: 'module', + title: 'Known module', + url: 'https://learn.microsoft.com/training/modules/known', + products: ['azure-openai'], + updated_at: '2026-06-10T00:00:00Z', + first_seen: '2026-06-01T00:00:00Z', + last_seen: '2026-06-20T00:00:00Z', + }, + }, +}; + +function makePaginate(pagesByEndpoint, captured) { + return async function* paginate(endpoint, params /* , token */) { + captured[endpoint] = params; + for (const item of pagesByEndpoint[endpoint] ?? []) yield item; + }; +} + +// --------------------------------------------------------------------------- +// (1) Fail-soft: missing creds => skipped, no network touched +// --------------------------------------------------------------------------- + +test('detectCourses — all creds missing => status "skipped", writes a skipped report, no network', async () => { + const writes = []; + const report = await detectCourses({ + readSecretImpl: () => null, + saveReportImpl: (name, data) => writes.push([name, data]), + getTokenImpl: () => assert.fail('getToken must not run without creds'), + paginateImpl: () => assert.fail('paginate must not run without creds'), + now: NOW, + log: () => {}, + }); + assert.equal(report.status, 'skipped'); + assert.ok(report.skipped_reason, 'a human reason is recorded'); + assert.equal(report.generated_at, NOW); + assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }); + assert.equal(writes.length, 1); + assert.equal(writes[0][0], 'course-detection-report.json'); + assert.equal(writes[0][1].status, 'skipped'); +}); + +test('detectCourses — a single missing cred is enough to skip (partial creds)', async () => { + const report = await detectCourses({ + readSecretImpl: (service) => (service === 'learn-platform-client-secret' ? null : 'present'), + saveReportImpl: () => {}, + getTokenImpl: () => assert.fail('must not run'), + paginateImpl: () => assert.fail('must not run'), + now: NOW, + log: () => {}, + }); + assert.equal(report.status, 'skipped'); +}); + +// --------------------------------------------------------------------------- +// (2) Error path: a run failure fail-softs to status "error", never throws, +// and does NOT advance the registry. +// --------------------------------------------------------------------------- + +test('detectCourses — a network error yields status "error" (no throw) and leaves the registry untouched', async () => { + let registryWritten = false; + let saved = null; + const report = await detectCourses({ + readSecretImpl: () => 'present', + loadTaxonomyImpl: () => FAKE_TAX, + loadCourseRegistryImpl: () => REGISTRY, + getTokenImpl: async () => { + throw new Error('HTTP 500 boom'); + }, + saveReportImpl: (_name, data) => { + saved = data; + }, + saveCourseRegistryImpl: () => { + registryWritten = true; + }, + now: NOW, + log: () => {}, + }); + assert.equal(report.status, 'error'); + assert.match(report.skipped_reason, /boom/); + assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }); + assert.equal(saved.status, 'error', 'the error report is persisted'); + assert.equal(registryWritten, false, 'no registry write when the run errored'); +}); + +// --------------------------------------------------------------------------- +// (3) Happy path: incremental run produces a §4.3-shaped report and advances +// the registry. +// --------------------------------------------------------------------------- + +test('detectCourses — incremental run: §4.3 report shape, new/updated leads, registry advanced', async () => { + const captured = {}; + const pages = { + modules: [ + // brand-new UID => new lead + { + id: 'learn.module-new', + title: 'New AOAI course', + url: 'https://learn.microsoft.com/training/modules/new', + updatedAt: '2026-06-22T00:00:00Z', + products: [{ id: 'azure-openai' }], + }, + // known UID with a newer timestamp => updated lead + { + id: 'learn.module-known', + title: 'Known module', + url: 'https://learn.microsoft.com/training/modules/known', + updatedAt: '2026-06-21T00:00:00Z', + products: [{ id: 'azure-openai' }], + }, + ], + 'learning-paths': [], + }; + let savedReport = null; + let savedRegistry = null; + const report = await detectCourses({ + readSecretImpl: () => 'present', + loadTaxonomyImpl: () => FAKE_TAX, + loadCourseRegistryImpl: () => REGISTRY, + getTokenImpl: async () => 'token-abc', + paginateImpl: makePaginate(pages, captured), + saveReportImpl: (_name, data) => { + savedReport = data; + }, + saveCourseRegistryImpl: (reg) => { + savedRegistry = reg; + }, + now: NOW, + log: () => {}, + }); + + // status + counts + assert.equal(report.status, 'ok'); + assert.equal(report.skipped_reason, null); + assert.equal(report.mode, 'incremental'); + assert.deepEqual(report.counts, { new: 1, updated: 1, removed: 0 }); + + // §4.3 item shape on the new lead, incl. derived skill/category + const lead = report.new[0]; + for (const k of ['uid', 'title', 'url', 'products', 'suggested_skill', 'suggested_category', 'updated_at']) { + assert.ok(k in lead, `new lead exposes "${k}"`); + } + assert.equal(lead.uid, 'learn.module-new'); + assert.equal(lead.suggested_skill, 'ms-ai-engineering'); + assert.equal(lead.suggested_category, 'azure-ai-services'); + + // the updated lead is the known UID + assert.equal(report.updated[0].uid, 'learn.module-known'); + assert.equal(report.removed.length, 0); + + // incremental run pins the updatedAt.gt cursor (full ISO) from last_run, and + // filters server-side by the in-domain product slugs. + assert.equal(captured.modules['updatedAt.gt'], '2026-06-20T00:00:00.000Z'); + assert.match(captured.modules.products, /azure-openai/); + assert.match(captured.modules.products, /power-automate/); + + // registry advanced: both UIDs present, last_run = now + assert.equal(savedReport.status, 'ok'); + assert.equal(savedRegistry.last_run, NOW); + assert.ok(savedRegistry.courses['learn.module-new']); + assert.ok(savedRegistry.courses['learn.module-known']); +}); + +test('detectCourses — baseline run (no last_run) emits no leads but establishes the registry', async () => { + const pages = { + modules: [ + { + id: 'learn.module-a', + title: 'A', + url: 'u-a', + updatedAt: '2026-06-22T00:00:00Z', + products: [{ id: 'azure-openai' }], + }, + ], + 'learning-paths': [], + }; + let savedRegistry = null; + const captured = {}; + const report = await detectCourses({ + readSecretImpl: () => 'present', + loadTaxonomyImpl: () => FAKE_TAX, + loadCourseRegistryImpl: () => ({ version: 1, created_at: null, last_run: null, last_full_enum: null, courses: {} }), + getTokenImpl: async () => 'token-abc', + paginateImpl: makePaginate(pages, captured), + saveReportImpl: () => {}, + saveCourseRegistryImpl: (reg) => { + savedRegistry = reg; + }, + now: NOW, + log: () => {}, + }); + assert.equal(report.mode, 'baseline'); + assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }, 'baseline emits no leads (§8 b)'); + // baseline omits the updatedAt.gt cursor (full enumeration) + assert.equal(captured.modules['updatedAt.gt'], undefined); + // but it DID fold the API set into the registry + stamp last_full_enum + assert.ok(savedRegistry.courses['learn.module-a']); + assert.equal(savedRegistry.last_full_enum, NOW); +}); + +// --------------------------------------------------------------------------- +// (4) The real CLI fails soft to exit 0 — proven hermetically by shadowing the +// `security` binary on PATH so readSecret returns null (no Keychain, no net). +// --------------------------------------------------------------------------- + +test('detect-courses.mjs CLI — missing creds => skipped report + exit 0 (no network)', { skip: process.platform === 'win32' }, () => { + const tmp = mkdtempSync(join(tmpdir(), 'detect-courses-')); + try { + const dataDir = join(tmp, 'data'); + const bin = join(tmp, 'bin'); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(bin, { recursive: true }); + // A `security` that always fails => keychain.readSecret returns null => skip. + const stub = join(bin, 'security'); + writeFileSync(stub, '#!/bin/sh\nexit 1\n'); + chmodSync(stub, 0o755); + + const script = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'detect-courses.mjs'); + const res = spawnSync('node', [script, '--data-dir', dataDir], { + env: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, + encoding: 'utf8', + }); + assert.equal(res.status, 0, `fail-soft must exit 0 (stderr: ${res.stderr})`); + const report = JSON.parse(readFileSync(join(dataDir, 'course-detection-report.json'), 'utf8')); + assert.equal(report.status, 'skipped'); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); diff --git a/tests/kb-update/test-detection-schedule.test.mjs b/tests/kb-update/test-detection-schedule.test.mjs index 3035b62..296b2d0 100644 --- a/tests/kb-update/test-detection-schedule.test.mjs +++ b/tests/kb-update/test-detection-schedule.test.mjs @@ -23,6 +23,7 @@ import { serializeScheduleConfig, loadScheduleConfig, shouldRunDetection, + selectDetectionSteps, summarizeSkillLifecycle, daysSinceLastPoll, } from '../../scripts/kb-update/lib/detection-schedule.mjs'; @@ -42,6 +43,30 @@ test('DEFAULT_SCHEDULE_CONFIG — opt-in: disabled by default', () => { assert.equal(DEFAULT_SCHEDULE_CONFIG.include_skill_lifecycle, true); }); +// --- C3.4: include_course_detection (opt-in inside opt-in; default OFF) --- + +test('DEFAULT_SCHEDULE_CONFIG — include_course_detection defaults to false (§8 d)', () => { + assert.equal(DEFAULT_SCHEDULE_CONFIG.include_course_detection, false, + 'course detection needs Keychain creds almost nobody has => opt-in, default OFF'); +}); + +test('parseScheduleConfig — reads include_course_detection: true', () => { + const cfg = parseScheduleConfig('scheduled_detection:\n include_course_detection: true\n'); + assert.equal(cfg.include_course_detection, true); +}); + +test('parseScheduleConfig — a non-boolean include_course_detection stays OFF', () => { + const cfg = parseScheduleConfig('scheduled_detection:\n include_course_detection: maybe\n'); + assert.equal(cfg.include_course_detection, false); +}); + +test('serializeScheduleConfig — emits + round-trips include_course_detection', () => { + assert.match(serializeScheduleConfig({}), /\n {2}include_course_detection: false/, + 'serialized at its OFF default'); + const cfg = parseScheduleConfig(serializeScheduleConfig({ include_course_detection: true })); + assert.equal(cfg.include_course_detection, true); +}); + test('parseScheduleConfig — empty/absent text returns the OFF default', () => { assert.deepEqual(parseScheduleConfig(''), DEFAULT_SCHEDULE_CONFIG); assert.deepEqual(parseScheduleConfig('# just some markdown, no frontmatter'), DEFAULT_SCHEDULE_CONFIG); @@ -353,6 +378,7 @@ const KNOWN_DETECTION_SCRIPTS = new Set([ 'report-changes.mjs', 'discover-new-urls.mjs', 'detect-skill-lifecycle.mjs', + 'detect-courses.mjs', 'build-registry.mjs', ]); @@ -365,6 +391,37 @@ test('DETECTION_STEPS — only the known pure-node detection scripts, never clau } // skill-lifecycle detection must be present and flagged so config can exclude it assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-skill-lifecycle.mjs' && s.skillLifecycle === true)); + // C3.4: course detection present and flagged so config can exclude it (opt-in) + assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true)); +}); + +// --- C3.4: selectDetectionSteps — the pure gate run-detection.mjs applies --- + +test('selectDetectionSteps — drops detect-courses when include_course_detection is OFF (default)', () => { + const steps = selectDetectionSteps(DEFAULT_SCHEDULE_CONFIG); + assert.ok(!steps.some((s) => s.courseDetection), 'course detection excluded by default'); + // the always-on steps survive + assert.ok(steps.some((s) => s.script === 'poll-sitemaps.mjs')); +}); + +test('selectDetectionSteps — includes detect-courses when opted in', () => { + const steps = selectDetectionSteps({ ...DEFAULT_SCHEDULE_CONFIG, include_course_detection: true }); + assert.ok(steps.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true)); +}); + +test('selectDetectionSteps — the two opt-in gates are independent', () => { + const onlyCourses = selectDetectionSteps({ + ...DEFAULT_SCHEDULE_CONFIG, + include_skill_lifecycle: false, + include_course_detection: true, + }); + assert.ok(!onlyCourses.some((s) => s.skillLifecycle), 'skill-lifecycle gated OFF'); + assert.ok(onlyCourses.some((s) => s.courseDetection), 'course detection gated ON'); +}); + +test('run-detection.mjs source — gates detection via the shared selector (no duplicated filter)', () => { + const src = readFileSync(join(PLUGIN_ROOT, 'scripts', 'kb-update', 'run-detection.mjs'), 'utf8'); + assert.match(src, /selectDetectionSteps\(/, 'run-detection uses the pure step selector'); }); test('run-detection.mjs source — spawns node, NEVER claude (the ToS boundary made structural)', () => {