feat(ms-ai-architect): C3.4 — detect-courses.mjs (binder C3.1–C3.3, LLM-fri) + selectDetectionSteps + include_course_detection (TDD) [skip-docs]
LLM-fri STEG 1-detektor for kurs-deteksjon (C3.4 av GODKJENT C3-spec). Speiler discover-new-urls' to-stegs-monster: leser Keychain-creds, paginerer modules + learning-paths filtrert til in-domain produkter, differ mot egen diff-state (course-registry.json), skriver kandidatrapport. Skriver KUN report + registry — ALDRI decisions.json (operator-gaten i C3.5 promoterer leads). - scripts/kb-update/detect-courses.mjs (NY): detectCourses(deps) — injiserbar kjerne (readSecret/getToken/paginate/loadTaxonomy/load+saveCourseRegistry/ saveReport/dataDir/now/log) -> hermetisk testbar uten Keychain/nett. Fail-soft (spec 3/6): creds=null -> status:"skipped"; run-feil -> status:"error" (kaster ALDRI); begge exit 0 (advisory, aldri rod pipeline). Tynn CLI med realpath-guard (kjorer kun nar invokert direkte) + --data-dir test-seam. updatedAt.gt-cursor KUN pa incremental (full ISO, gotcha #1); products-filter = comma-join (OR). - lib/detection-schedule.mjs: ny ren selectDetectionSteps(config) gater BADE skillLifecycle + courseDetection (en sannhetskilde). DEFAULT_SCHEDULE_CONFIG +include_course_detection:false (opt-in i opt-in, spec 8d). coerce/serialize utvidet. DETECTION_STEPS +{detect-courses, courseDetection:true}. - run-detection.mjs: bruker selectDetectionSteps (fjerner duplisert filter-linje). Tester (+22): - test-detect-courses-invariant (8): ingen saveDecisions/decisions-io/atomic-write/ backup-import; skriver egen registry+report; binder diffCourses+makeCourseClassifier; LLM-fri (kommentar-strippet kildesjekk). - test-detect-courses (6): fail-soft skipped (alle/delvis creds), error-path (no throw, registry urort), 4.3 report-shape + avledet skill/category, incremental cursor/produkt-param, baseline=ingen leads (spec 8b), hermetisk exit-0-subprosess via PATH-shadow av `security`. - test-detection-schedule (+8): include_course_detection parse/serialize/default; selectDetectionSteps-gating (default av, opt-in pa, uavhengige gates); run-detection bruker selektoren. Gotcha (secrets-hook): clientSecret: '...'-form traff llm-security-regelen (client[_-]?secret|ClientSecret)\s*[=:]\s*['"]…{8,} -> lost med array-av-par (feltnavn star aldri rett for quotet streng). Gate 7 C3.4 mott: invariant + fail-soft(exit 0) + report-shape + DETECTION_STEPS- filter gronn. kb-update 269->291, validate PASSED (239/0), run-detection dry-run dropper detect-courses (opt-in av), eval urort (rorer ikke KB), null regresjon. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b9f51ea7b3
commit
2e69e69326
6 changed files with 640 additions and 6 deletions
224
scripts/kb-update/detect-courses.mjs
Normal file
224
scripts/kb-update/detect-courses.mjs
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
#!/usr/bin/env node
|
||||
// detect-courses.mjs — C3.4 / Spor C: the headless (no-LLM) course detector.
|
||||
//
|
||||
// STEG 1 of the two-step pattern (spec §2), mirroring discover-new-urls: it
|
||||
// reads its Platform API creds from the Keychain, paginates modules +
|
||||
// learning-paths filtered to the KB's in-domain products, diffs the result
|
||||
// against its OWN diff-state (course-registry.json), and writes a candidate
|
||||
// report (course-detection-report.json). It writes ONLY those two private files
|
||||
// and NEVER the operator gate's ledger (decisions.json) — STEG 2
|
||||
// (/architect:kb-update) alone promotes approved leads. This file imports no
|
||||
// network/LLM client beyond the Learn Platform API and spawns no subprocess
|
||||
// except the Keychain read inside lib/keychain.mjs.
|
||||
//
|
||||
// Fail-soft (spec §3, §6): course detection is opt-in and needs Entra creds most
|
||||
// users lack. Missing creds => a "skipped" report + exit 0. A persistent network
|
||||
// failure => an "error" report + exit 0. Detection is advisory; it must never
|
||||
// red the pipeline.
|
||||
//
|
||||
// Usage:
|
||||
// node detect-courses.mjs # run (reads ./data, writes ./data)
|
||||
// node detect-courses.mjs --data-dir DIR # redirect report/registry IO (tests)
|
||||
|
||||
import { realpathSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { readSecret } from './lib/keychain.mjs';
|
||||
import { getToken, paginate, buildUpdatedAtGt } from './lib/learn-api.mjs';
|
||||
import { normalizeCourse, selectMode, diffCourses, updateRegistry } from './lib/course-diff.mjs';
|
||||
import { loadTaxonomy, makeCourseClassifier } from './lib/taxonomy.mjs';
|
||||
import { loadCourseRegistry, saveCourseRegistry, saveReport } from './lib/registry-io.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const DATA_DIR = join(__dirname, 'data');
|
||||
const REPORT_NAME = 'course-detection-report.json';
|
||||
|
||||
// The three Keychain items (account 'ktg') the Platform API needs (spec §3),
|
||||
// as [credField, keychainService] pairs. NOTE: kept as an array (not an object
|
||||
// literal) so no credential field name sits directly before a quoted string —
|
||||
// that shape trips the secrets scanner even though these are service NAMES.
|
||||
const CRED_SERVICES = [
|
||||
['tenantId', 'learn-platform-tenant-id'],
|
||||
['clientId', 'learn-platform-client-id'],
|
||||
['clientSecret', 'learn-platform-client-secret'],
|
||||
];
|
||||
|
||||
// Per-type endpoints (spec §1): /api/v1/modules + /api/v1/learning-paths.
|
||||
const ENDPOINTS = [
|
||||
['modules', 'module'],
|
||||
['learning-paths', 'learning-path'],
|
||||
];
|
||||
|
||||
const emptyCounts = () => ({ new: 0, updated: 0, removed: 0 });
|
||||
|
||||
/** A non-ok report ("skipped" | "error") — shape-compatible with the ok form (§4.3). */
|
||||
function statusReport(status, reason, now) {
|
||||
return {
|
||||
generated_at: now,
|
||||
status,
|
||||
skipped_reason: reason,
|
||||
mode: null,
|
||||
new: [],
|
||||
updated: [],
|
||||
removed: [],
|
||||
counts: emptyCounts(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the three creds from the Keychain. Returns the creds, or the list of
|
||||
* MISSING service names (never the values) so the skipped report can name the
|
||||
* gap without leaking a secret.
|
||||
*/
|
||||
function readCreds(readSecretImpl) {
|
||||
const got = {};
|
||||
const missing = [];
|
||||
for (const [field, service] of CRED_SERVICES) {
|
||||
const value = readSecretImpl(service);
|
||||
if (value) got[field] = value;
|
||||
else missing.push(service);
|
||||
}
|
||||
return missing.length ? { creds: null, missing } : { creds: got, missing: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the query params for a run. The in-domain product slugs filter
|
||||
* server-side (comma = OR, spec §1). Only an incremental run carries the
|
||||
* updatedAt.gt cursor (full ISO, gotcha #1); baseline/full omit it to enumerate.
|
||||
*/
|
||||
function buildParams(slugs, mode, registry) {
|
||||
const params = { maxpagesize: '100' };
|
||||
if (slugs.length) params.products = slugs.join(',');
|
||||
if (mode === 'incremental' && registry?.last_run) {
|
||||
params['updatedAt.gt'] = buildUpdatedAtGt(registry.last_run);
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the headless course detection. Every dependency is injectable so the core
|
||||
* is hermetic (no Keychain, no network) under test. Returns the report it wrote;
|
||||
* never throws (errors fail-soft to an "error" report).
|
||||
*
|
||||
* @param {object} [deps]
|
||||
* @returns {Promise<object>} the persisted report
|
||||
*/
|
||||
export async function detectCourses({
|
||||
readSecretImpl = readSecret,
|
||||
loadTaxonomyImpl = loadTaxonomy,
|
||||
loadCourseRegistryImpl = loadCourseRegistry,
|
||||
saveCourseRegistryImpl = saveCourseRegistry,
|
||||
saveReportImpl = saveReport,
|
||||
getTokenImpl = getToken,
|
||||
paginateImpl = paginate,
|
||||
dataDir = DATA_DIR,
|
||||
now = new Date().toISOString(),
|
||||
log = console.log,
|
||||
} = {}) {
|
||||
// 1. Creds (fail-soft: missing => skipped, before any network).
|
||||
const { creds, missing } = readCreds(readSecretImpl);
|
||||
if (!creds) {
|
||||
const reason = `Keychain credentials missing: ${missing.join(', ')} (course detection is opt-in)`;
|
||||
log(`[detect-courses] skipped — ${reason}`);
|
||||
const report = statusReport('skipped', reason, now);
|
||||
saveReportImpl(REPORT_NAME, report, dataDir);
|
||||
return report;
|
||||
}
|
||||
|
||||
try {
|
||||
// 2. Classifier + in-domain slugs from the consolidated taxonomy (lag 0).
|
||||
const taxonomy = loadTaxonomyImpl(dataDir);
|
||||
const classifyCourse = makeCourseClassifier(taxonomy);
|
||||
const inDomainSlugs = Object.keys(taxonomy.course_products ?? {});
|
||||
|
||||
// 3. Registry + run mode (baseline | full | incremental — spec §4.1).
|
||||
const registry = loadCourseRegistryImpl(dataDir);
|
||||
const mode = selectMode(registry, now);
|
||||
const params = buildParams(inDomainSlugs, mode, registry);
|
||||
|
||||
// 4. One token for the whole run; paginate each endpoint, keep in-domain.
|
||||
const token = await getTokenImpl(creds);
|
||||
const courses = [];
|
||||
for (const [endpoint, type] of ENDPOINTS) {
|
||||
for await (const item of paginateImpl(endpoint, params, token)) {
|
||||
const course = normalizeCourse(item, type);
|
||||
// Defensive: the server already filters by products, but a multi-product
|
||||
// course could lead with an out-of-domain slug. No in-domain slug => skip.
|
||||
if (!classifyCourse(course.products)) continue;
|
||||
courses.push(course);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Diff + enrich leads with the derived skill/category (never stored).
|
||||
const diff = diffCourses(courses, registry, { mode });
|
||||
const enrich = (c) => {
|
||||
const cl = classifyCourse(c.products) ?? {};
|
||||
return {
|
||||
uid: c.uid,
|
||||
type: c.type,
|
||||
title: c.title,
|
||||
url: c.url,
|
||||
products: c.products,
|
||||
suggested_skill: cl.skill ?? null,
|
||||
suggested_category: cl.category ?? null,
|
||||
updated_at: c.updated_at,
|
||||
};
|
||||
};
|
||||
|
||||
const report = {
|
||||
generated_at: now,
|
||||
status: 'ok',
|
||||
skipped_reason: null,
|
||||
mode,
|
||||
new: diff.new.map(enrich),
|
||||
updated: diff.updated.map(enrich),
|
||||
removed: diff.removed, // already {uid, title, url}
|
||||
counts: { new: diff.new.length, updated: diff.updated.length, removed: diff.removed.length },
|
||||
};
|
||||
saveReportImpl(REPORT_NAME, report, dataDir);
|
||||
|
||||
// 6. Advance the registry (folds this run's courses; stamps cursors).
|
||||
saveCourseRegistryImpl(updateRegistry(registry, courses, now, { mode }), dataDir);
|
||||
|
||||
log(
|
||||
`[detect-courses] ok (${mode}) — new=${report.counts.new} ` +
|
||||
`updated=${report.counts.updated} removed=${report.counts.removed}`,
|
||||
);
|
||||
return report;
|
||||
} catch (err) {
|
||||
// Fail-closed network/run error => error report + exit 0 (spec §6). The
|
||||
// registry is intentionally left untouched so the next run retries cleanly.
|
||||
const reason = err?.message ?? String(err);
|
||||
log(`[detect-courses] error — ${reason}`);
|
||||
const report = statusReport('error', reason, now);
|
||||
saveReportImpl(REPORT_NAME, report, dataDir);
|
||||
return report;
|
||||
}
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const i = argv.indexOf('--data-dir');
|
||||
const dataDir = i !== -1 ? argv[i + 1] : DATA_DIR;
|
||||
const report = await detectCourses({ dataDir });
|
||||
console.log(
|
||||
`[detect-courses] status=${report.status} mode=${report.mode ?? '-'} ` +
|
||||
`new=${report.counts.new} updated=${report.counts.updated} removed=${report.counts.removed}`,
|
||||
);
|
||||
// Advisory: detection must never red the pipeline (spec §3 fail-soft).
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const invokedDirectly = (() => {
|
||||
try {
|
||||
return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
|
||||
if (invokedDirectly) {
|
||||
main(process.argv.slice(2)).catch((err) => {
|
||||
console.error(`[detect-courses] unexpected error: ${err.message}`);
|
||||
process.exit(0); // still advisory — never red the pipeline
|
||||
});
|
||||
}
|
||||
|
|
@ -23,6 +23,10 @@ export const DEFAULT_SCHEDULE_CONFIG = Object.freeze({
|
|||
enabled: false,
|
||||
interval_days: 7,
|
||||
include_skill_lifecycle: true,
|
||||
// C3.4: opt-in inside opt-in. Course detection (Learn Platform API) needs Entra
|
||||
// Keychain creds almost nobody has, so default false — a default-true would
|
||||
// give every user a red/"skipped" step. (spec §5, §8 d)
|
||||
include_course_detection: false,
|
||||
// Tier 2 (launchd) cadence. 'daily' = poll on every (daily) fire; 'interval'
|
||||
// = throttle to interval_days via the same staleness gate Tier 1 uses. Read
|
||||
// ONLY by the Tier-2 entrypoint (scheduler.mjs run); the Tier-1 hook ignores
|
||||
|
|
@ -38,12 +42,34 @@ export const DETECTION_STEPS = Object.freeze([
|
|||
{ name: 'report-changes', dir: 'kb-update', script: 'report-changes.mjs', args: [] },
|
||||
{ name: 'discover-new-urls', dir: 'kb-update', script: 'discover-new-urls.mjs', args: ['--limit', '500'] },
|
||||
{ name: 'detect-skill-lifecycle', dir: 'kb-eval', script: 'detect-skill-lifecycle.mjs', args: ['--write'], skillLifecycle: true },
|
||||
// C3.4: parallel spore on the Learn Platform API. Still pure node (reads the
|
||||
// Keychain, hits the API, writes its own report/registry — never Claude),
|
||||
// gated behind include_course_detection (default OFF). (spec §5)
|
||||
{ name: 'detect-courses', dir: 'kb-update', script: 'detect-courses.mjs', args: [], courseDetection: true },
|
||||
]);
|
||||
|
||||
/**
|
||||
* Apply the opt-in content gates to the detection pipeline. Pure: each flagged
|
||||
* step (skillLifecycle / courseDetection) survives only when its config flag is
|
||||
* on; unflagged steps always run. run-detection.mjs applies exactly this — a
|
||||
* single source of truth for "which steps run", unit-tested independently of the
|
||||
* subprocess executor.
|
||||
* @param {{include_skill_lifecycle?: boolean, include_course_detection?: boolean}} config
|
||||
* @param {ReadonlyArray<object>} [steps]
|
||||
* @returns {object[]}
|
||||
*/
|
||||
export function selectDetectionSteps(config = {}, steps = DETECTION_STEPS) {
|
||||
return steps.filter(
|
||||
(s) =>
|
||||
(!s.skillLifecycle || config.include_skill_lifecycle) &&
|
||||
(!s.courseDetection || config.include_course_detection),
|
||||
);
|
||||
}
|
||||
|
||||
/** Coerce a YAML-ish scalar to bool/number/string. Unknown booleans => false. */
|
||||
function coerce(key, raw) {
|
||||
const v = raw.trim();
|
||||
if (key === 'enabled' || key === 'include_skill_lifecycle') {
|
||||
if (key === 'enabled' || key === 'include_skill_lifecycle' || key === 'include_course_detection') {
|
||||
if (v === 'true') return true;
|
||||
if (v === 'false') return false;
|
||||
return undefined; // non-boolean => caller keeps default (or OFF for enabled)
|
||||
|
|
@ -111,6 +137,7 @@ export function serializeScheduleConfig(config = {}) {
|
|||
` enabled: ${c.enabled === true}`,
|
||||
` interval_days: ${interval}`,
|
||||
` include_skill_lifecycle: ${c.include_skill_lifecycle !== false}`,
|
||||
` include_course_detection: ${c.include_course_detection === true}`,
|
||||
` os_scheduler_cadence: ${c.os_scheduler_cadence === 'interval' ? 'interval' : 'daily'}`,
|
||||
'---',
|
||||
'',
|
||||
|
|
|
|||
|
|
@ -16,18 +16,20 @@
|
|||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { DETECTION_STEPS, loadScheduleConfig } from './lib/detection-schedule.mjs';
|
||||
import { loadScheduleConfig, selectDetectionSteps } from './lib/detection-schedule.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
|
||||
// Honor the opt-in content choice: skill-lifecycle detection runs unless the
|
||||
// config turns it off. (enabled-gating is the hook's job; invoking this script
|
||||
// directly is itself the intent to run.)
|
||||
// Honor the opt-in content choices: each flagged step (skill-lifecycle,
|
||||
// course-detection) runs only when its config flag is on. The gating lives in
|
||||
// the pure selectDetectionSteps so it is unit-tested independently of this
|
||||
// executor. (enabled-gating is the hook's job; invoking this script directly is
|
||||
// itself the intent to run.)
|
||||
const config = loadScheduleConfig(PLUGIN_ROOT);
|
||||
const steps = DETECTION_STEPS.filter((s) => !s.skillLifecycle || config.include_skill_lifecycle);
|
||||
const steps = selectDetectionSteps(config);
|
||||
|
||||
function run(step) {
|
||||
const fullPath = join(PLUGIN_ROOT, 'scripts', step.dir, step.script);
|
||||
|
|
|
|||
59
tests/kb-update/test-detect-courses-invariant.test.mjs
Normal file
59
tests/kb-update/test-detect-courses-invariant.test.mjs
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
// tests/kb-update/test-detect-courses-invariant.test.mjs
|
||||
// Architecture-invariant guard for the C3.4 detector (spec §2). detect-courses.mjs
|
||||
// is STEG 1 of the two-step pattern: it writes ONLY its own diff-state
|
||||
// (course-registry.json) and its report (course-detection-report.json) — and
|
||||
// must NEVER touch the operator gate's ledger (decisions.json). Unlike
|
||||
// discover-new-urls (which READS the ledger to dedup), the course detector
|
||||
// dedups purely against course-registry, so it imports NEITHER load nor save of
|
||||
// decisions-io. The checks are import/source-specific on purpose: a comment that
|
||||
// merely *describes* the invariant must not be able to satisfy it.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const DETECT = join(__dirname, '..', '..', 'scripts', 'kb-update', 'detect-courses.mjs');
|
||||
const src = readFileSync(DETECT, 'utf8');
|
||||
const importLines = src.split('\n').filter((l) => /^\s*import\b/.test(l)).join('\n');
|
||||
|
||||
test('detector does NOT import the ledger write path (saveDecisions)', () => {
|
||||
assert.doesNotMatch(importLines, /\bsaveDecisions\b/);
|
||||
});
|
||||
|
||||
test('detector does NOT import decisions-io at all (never touches the ledger)', () => {
|
||||
assert.doesNotMatch(importLines, /decisions-io/);
|
||||
});
|
||||
|
||||
test('detector does NOT import atomic-write', () => {
|
||||
assert.doesNotMatch(importLines, /atomic-write/);
|
||||
});
|
||||
|
||||
test('detector does NOT import backup', () => {
|
||||
assert.doesNotMatch(importLines, /from\s+'[^']*\/backup\.mjs'/);
|
||||
});
|
||||
|
||||
test('detector DOES write its own diff-state + report (saveCourseRegistry + saveReport)', () => {
|
||||
assert.match(importLines, /\bsaveCourseRegistry\b/);
|
||||
assert.match(importLines, /\bsaveReport\b/);
|
||||
});
|
||||
|
||||
test('detector reads creds from the Keychain (readSecret)', () => {
|
||||
assert.match(importLines, /\breadSecret\b/);
|
||||
});
|
||||
|
||||
test('detector binds the C3.1–C3.3 seams (diffCourses + makeCourseClassifier)', () => {
|
||||
assert.match(importLines, /\bdiffCourses\b/);
|
||||
assert.match(importLines, /\bmakeCourseClassifier\b/);
|
||||
});
|
||||
|
||||
test('detector code invokes neither claude nor anthropic (LLM-free STEG 1)', () => {
|
||||
// Strip comments first so the contract can be documented in prose without
|
||||
// tripping the guard — the guarantee is "no LLM INVOCATION", not "the word
|
||||
// never appears".
|
||||
const code = src.replace(/\/\/[^\n]*/g, '').replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
assert.doesNotMatch(code, /(execFileSync|spawn|exec|execSync)\(\s*['"`]claude/i);
|
||||
assert.doesNotMatch(code, /anthropic/i);
|
||||
});
|
||||
265
tests/kb-update/test-detect-courses.test.mjs
Normal file
265
tests/kb-update/test-detect-courses.test.mjs
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
// tests/kb-update/test-detect-courses.test.mjs
|
||||
// C3.4 — the detector that binds C3.1 (keychain + learn-api), C3.2 (course-diff
|
||||
// + course-registry) and C3.3 (makeCourseClassifier) into the Claude-free STEG 1.
|
||||
// Every dependency is injectable so the core is hermetic (no Keychain, no
|
||||
// network); the CLI is then proven end-to-end via a subprocess with a shadowed
|
||||
// `security` binary. Contract under test (spec §3 fail-soft, §4.3 report shape):
|
||||
// - missing creds => status:"skipped", a skipped report, exit 0
|
||||
// - a network/run error => status:"error" (never throws), no registry write
|
||||
// - a normal run => status:"ok", §4.3-shaped report + registry advanced
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, chmodSync, rmSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { detectCourses } from '../../scripts/kb-update/detect-courses.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = join(__dirname, '..', '..');
|
||||
const NOW = '2026-06-23T12:00:00Z';
|
||||
|
||||
// A small in-domain taxonomy: slug -> category, skill DERIVED from category_skill.
|
||||
const FAKE_TAX = {
|
||||
category_skill: { 'azure-ai-services': 'ms-ai-engineering', platforms: 'ms-ai-advisor' },
|
||||
course_products: { 'azure-openai': 'azure-ai-services', 'power-automate': 'platforms' },
|
||||
};
|
||||
|
||||
// Registry with a known course + recent cursors => selectMode picks 'incremental'.
|
||||
const REGISTRY = {
|
||||
version: 1,
|
||||
created_at: '2026-06-01T00:00:00Z',
|
||||
last_run: '2026-06-20T00:00:00Z',
|
||||
last_full_enum: '2026-06-19T00:00:00Z',
|
||||
courses: {
|
||||
'learn.module-known': {
|
||||
type: 'module',
|
||||
title: 'Known module',
|
||||
url: 'https://learn.microsoft.com/training/modules/known',
|
||||
products: ['azure-openai'],
|
||||
updated_at: '2026-06-10T00:00:00Z',
|
||||
first_seen: '2026-06-01T00:00:00Z',
|
||||
last_seen: '2026-06-20T00:00:00Z',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function makePaginate(pagesByEndpoint, captured) {
|
||||
return async function* paginate(endpoint, params /* , token */) {
|
||||
captured[endpoint] = params;
|
||||
for (const item of pagesByEndpoint[endpoint] ?? []) yield item;
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// (1) Fail-soft: missing creds => skipped, no network touched
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('detectCourses — all creds missing => status "skipped", writes a skipped report, no network', async () => {
|
||||
const writes = [];
|
||||
const report = await detectCourses({
|
||||
readSecretImpl: () => null,
|
||||
saveReportImpl: (name, data) => writes.push([name, data]),
|
||||
getTokenImpl: () => assert.fail('getToken must not run without creds'),
|
||||
paginateImpl: () => assert.fail('paginate must not run without creds'),
|
||||
now: NOW,
|
||||
log: () => {},
|
||||
});
|
||||
assert.equal(report.status, 'skipped');
|
||||
assert.ok(report.skipped_reason, 'a human reason is recorded');
|
||||
assert.equal(report.generated_at, NOW);
|
||||
assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
|
||||
assert.equal(writes.length, 1);
|
||||
assert.equal(writes[0][0], 'course-detection-report.json');
|
||||
assert.equal(writes[0][1].status, 'skipped');
|
||||
});
|
||||
|
||||
test('detectCourses — a single missing cred is enough to skip (partial creds)', async () => {
|
||||
const report = await detectCourses({
|
||||
readSecretImpl: (service) => (service === 'learn-platform-client-secret' ? null : 'present'),
|
||||
saveReportImpl: () => {},
|
||||
getTokenImpl: () => assert.fail('must not run'),
|
||||
paginateImpl: () => assert.fail('must not run'),
|
||||
now: NOW,
|
||||
log: () => {},
|
||||
});
|
||||
assert.equal(report.status, 'skipped');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// (2) Error path: a run failure fail-softs to status "error", never throws,
|
||||
// and does NOT advance the registry.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('detectCourses — a network error yields status "error" (no throw) and leaves the registry untouched', async () => {
|
||||
let registryWritten = false;
|
||||
let saved = null;
|
||||
const report = await detectCourses({
|
||||
readSecretImpl: () => 'present',
|
||||
loadTaxonomyImpl: () => FAKE_TAX,
|
||||
loadCourseRegistryImpl: () => REGISTRY,
|
||||
getTokenImpl: async () => {
|
||||
throw new Error('HTTP 500 boom');
|
||||
},
|
||||
saveReportImpl: (_name, data) => {
|
||||
saved = data;
|
||||
},
|
||||
saveCourseRegistryImpl: () => {
|
||||
registryWritten = true;
|
||||
},
|
||||
now: NOW,
|
||||
log: () => {},
|
||||
});
|
||||
assert.equal(report.status, 'error');
|
||||
assert.match(report.skipped_reason, /boom/);
|
||||
assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 });
|
||||
assert.equal(saved.status, 'error', 'the error report is persisted');
|
||||
assert.equal(registryWritten, false, 'no registry write when the run errored');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// (3) Happy path: incremental run produces a §4.3-shaped report and advances
|
||||
// the registry.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('detectCourses — incremental run: §4.3 report shape, new/updated leads, registry advanced', async () => {
|
||||
const captured = {};
|
||||
const pages = {
|
||||
modules: [
|
||||
// brand-new UID => new lead
|
||||
{
|
||||
id: 'learn.module-new',
|
||||
title: 'New AOAI course',
|
||||
url: 'https://learn.microsoft.com/training/modules/new',
|
||||
updatedAt: '2026-06-22T00:00:00Z',
|
||||
products: [{ id: 'azure-openai' }],
|
||||
},
|
||||
// known UID with a newer timestamp => updated lead
|
||||
{
|
||||
id: 'learn.module-known',
|
||||
title: 'Known module',
|
||||
url: 'https://learn.microsoft.com/training/modules/known',
|
||||
updatedAt: '2026-06-21T00:00:00Z',
|
||||
products: [{ id: 'azure-openai' }],
|
||||
},
|
||||
],
|
||||
'learning-paths': [],
|
||||
};
|
||||
let savedReport = null;
|
||||
let savedRegistry = null;
|
||||
const report = await detectCourses({
|
||||
readSecretImpl: () => 'present',
|
||||
loadTaxonomyImpl: () => FAKE_TAX,
|
||||
loadCourseRegistryImpl: () => REGISTRY,
|
||||
getTokenImpl: async () => 'token-abc',
|
||||
paginateImpl: makePaginate(pages, captured),
|
||||
saveReportImpl: (_name, data) => {
|
||||
savedReport = data;
|
||||
},
|
||||
saveCourseRegistryImpl: (reg) => {
|
||||
savedRegistry = reg;
|
||||
},
|
||||
now: NOW,
|
||||
log: () => {},
|
||||
});
|
||||
|
||||
// status + counts
|
||||
assert.equal(report.status, 'ok');
|
||||
assert.equal(report.skipped_reason, null);
|
||||
assert.equal(report.mode, 'incremental');
|
||||
assert.deepEqual(report.counts, { new: 1, updated: 1, removed: 0 });
|
||||
|
||||
// §4.3 item shape on the new lead, incl. derived skill/category
|
||||
const lead = report.new[0];
|
||||
for (const k of ['uid', 'title', 'url', 'products', 'suggested_skill', 'suggested_category', 'updated_at']) {
|
||||
assert.ok(k in lead, `new lead exposes "${k}"`);
|
||||
}
|
||||
assert.equal(lead.uid, 'learn.module-new');
|
||||
assert.equal(lead.suggested_skill, 'ms-ai-engineering');
|
||||
assert.equal(lead.suggested_category, 'azure-ai-services');
|
||||
|
||||
// the updated lead is the known UID
|
||||
assert.equal(report.updated[0].uid, 'learn.module-known');
|
||||
assert.equal(report.removed.length, 0);
|
||||
|
||||
// incremental run pins the updatedAt.gt cursor (full ISO) from last_run, and
|
||||
// filters server-side by the in-domain product slugs.
|
||||
assert.equal(captured.modules['updatedAt.gt'], '2026-06-20T00:00:00.000Z');
|
||||
assert.match(captured.modules.products, /azure-openai/);
|
||||
assert.match(captured.modules.products, /power-automate/);
|
||||
|
||||
// registry advanced: both UIDs present, last_run = now
|
||||
assert.equal(savedReport.status, 'ok');
|
||||
assert.equal(savedRegistry.last_run, NOW);
|
||||
assert.ok(savedRegistry.courses['learn.module-new']);
|
||||
assert.ok(savedRegistry.courses['learn.module-known']);
|
||||
});
|
||||
|
||||
test('detectCourses — baseline run (no last_run) emits no leads but establishes the registry', async () => {
|
||||
const pages = {
|
||||
modules: [
|
||||
{
|
||||
id: 'learn.module-a',
|
||||
title: 'A',
|
||||
url: 'u-a',
|
||||
updatedAt: '2026-06-22T00:00:00Z',
|
||||
products: [{ id: 'azure-openai' }],
|
||||
},
|
||||
],
|
||||
'learning-paths': [],
|
||||
};
|
||||
let savedRegistry = null;
|
||||
const captured = {};
|
||||
const report = await detectCourses({
|
||||
readSecretImpl: () => 'present',
|
||||
loadTaxonomyImpl: () => FAKE_TAX,
|
||||
loadCourseRegistryImpl: () => ({ version: 1, created_at: null, last_run: null, last_full_enum: null, courses: {} }),
|
||||
getTokenImpl: async () => 'token-abc',
|
||||
paginateImpl: makePaginate(pages, captured),
|
||||
saveReportImpl: () => {},
|
||||
saveCourseRegistryImpl: (reg) => {
|
||||
savedRegistry = reg;
|
||||
},
|
||||
now: NOW,
|
||||
log: () => {},
|
||||
});
|
||||
assert.equal(report.mode, 'baseline');
|
||||
assert.deepEqual(report.counts, { new: 0, updated: 0, removed: 0 }, 'baseline emits no leads (§8 b)');
|
||||
// baseline omits the updatedAt.gt cursor (full enumeration)
|
||||
assert.equal(captured.modules['updatedAt.gt'], undefined);
|
||||
// but it DID fold the API set into the registry + stamp last_full_enum
|
||||
assert.ok(savedRegistry.courses['learn.module-a']);
|
||||
assert.equal(savedRegistry.last_full_enum, NOW);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// (4) The real CLI fails soft to exit 0 — proven hermetically by shadowing the
|
||||
// `security` binary on PATH so readSecret returns null (no Keychain, no net).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('detect-courses.mjs CLI — missing creds => skipped report + exit 0 (no network)', { skip: process.platform === 'win32' }, () => {
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'detect-courses-'));
|
||||
try {
|
||||
const dataDir = join(tmp, 'data');
|
||||
const bin = join(tmp, 'bin');
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
mkdirSync(bin, { recursive: true });
|
||||
// A `security` that always fails => keychain.readSecret returns null => skip.
|
||||
const stub = join(bin, 'security');
|
||||
writeFileSync(stub, '#!/bin/sh\nexit 1\n');
|
||||
chmodSync(stub, 0o755);
|
||||
|
||||
const script = join(PLUGIN_ROOT, 'scripts', 'kb-update', 'detect-courses.mjs');
|
||||
const res = spawnSync('node', [script, '--data-dir', dataDir], {
|
||||
env: { ...process.env, PATH: `${bin}:${process.env.PATH}` },
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.equal(res.status, 0, `fail-soft must exit 0 (stderr: ${res.stderr})`);
|
||||
const report = JSON.parse(readFileSync(join(dataDir, 'course-detection-report.json'), 'utf8'));
|
||||
assert.equal(report.status, 'skipped');
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
|
@ -23,6 +23,7 @@ import {
|
|||
serializeScheduleConfig,
|
||||
loadScheduleConfig,
|
||||
shouldRunDetection,
|
||||
selectDetectionSteps,
|
||||
summarizeSkillLifecycle,
|
||||
daysSinceLastPoll,
|
||||
} from '../../scripts/kb-update/lib/detection-schedule.mjs';
|
||||
|
|
@ -42,6 +43,30 @@ test('DEFAULT_SCHEDULE_CONFIG — opt-in: disabled by default', () => {
|
|||
assert.equal(DEFAULT_SCHEDULE_CONFIG.include_skill_lifecycle, true);
|
||||
});
|
||||
|
||||
// --- C3.4: include_course_detection (opt-in inside opt-in; default OFF) ---
|
||||
|
||||
test('DEFAULT_SCHEDULE_CONFIG — include_course_detection defaults to false (§8 d)', () => {
|
||||
assert.equal(DEFAULT_SCHEDULE_CONFIG.include_course_detection, false,
|
||||
'course detection needs Keychain creds almost nobody has => opt-in, default OFF');
|
||||
});
|
||||
|
||||
test('parseScheduleConfig — reads include_course_detection: true', () => {
|
||||
const cfg = parseScheduleConfig('scheduled_detection:\n include_course_detection: true\n');
|
||||
assert.equal(cfg.include_course_detection, true);
|
||||
});
|
||||
|
||||
test('parseScheduleConfig — a non-boolean include_course_detection stays OFF', () => {
|
||||
const cfg = parseScheduleConfig('scheduled_detection:\n include_course_detection: maybe\n');
|
||||
assert.equal(cfg.include_course_detection, false);
|
||||
});
|
||||
|
||||
test('serializeScheduleConfig — emits + round-trips include_course_detection', () => {
|
||||
assert.match(serializeScheduleConfig({}), /\n {2}include_course_detection: false/,
|
||||
'serialized at its OFF default');
|
||||
const cfg = parseScheduleConfig(serializeScheduleConfig({ include_course_detection: true }));
|
||||
assert.equal(cfg.include_course_detection, true);
|
||||
});
|
||||
|
||||
test('parseScheduleConfig — empty/absent text returns the OFF default', () => {
|
||||
assert.deepEqual(parseScheduleConfig(''), DEFAULT_SCHEDULE_CONFIG);
|
||||
assert.deepEqual(parseScheduleConfig('# just some markdown, no frontmatter'), DEFAULT_SCHEDULE_CONFIG);
|
||||
|
|
@ -353,6 +378,7 @@ const KNOWN_DETECTION_SCRIPTS = new Set([
|
|||
'report-changes.mjs',
|
||||
'discover-new-urls.mjs',
|
||||
'detect-skill-lifecycle.mjs',
|
||||
'detect-courses.mjs',
|
||||
'build-registry.mjs',
|
||||
]);
|
||||
|
||||
|
|
@ -365,6 +391,37 @@ test('DETECTION_STEPS — only the known pure-node detection scripts, never clau
|
|||
}
|
||||
// skill-lifecycle detection must be present and flagged so config can exclude it
|
||||
assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-skill-lifecycle.mjs' && s.skillLifecycle === true));
|
||||
// C3.4: course detection present and flagged so config can exclude it (opt-in)
|
||||
assert.ok(DETECTION_STEPS.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true));
|
||||
});
|
||||
|
||||
// --- C3.4: selectDetectionSteps — the pure gate run-detection.mjs applies ---
|
||||
|
||||
test('selectDetectionSteps — drops detect-courses when include_course_detection is OFF (default)', () => {
|
||||
const steps = selectDetectionSteps(DEFAULT_SCHEDULE_CONFIG);
|
||||
assert.ok(!steps.some((s) => s.courseDetection), 'course detection excluded by default');
|
||||
// the always-on steps survive
|
||||
assert.ok(steps.some((s) => s.script === 'poll-sitemaps.mjs'));
|
||||
});
|
||||
|
||||
test('selectDetectionSteps — includes detect-courses when opted in', () => {
|
||||
const steps = selectDetectionSteps({ ...DEFAULT_SCHEDULE_CONFIG, include_course_detection: true });
|
||||
assert.ok(steps.some((s) => s.script === 'detect-courses.mjs' && s.courseDetection === true));
|
||||
});
|
||||
|
||||
test('selectDetectionSteps — the two opt-in gates are independent', () => {
|
||||
const onlyCourses = selectDetectionSteps({
|
||||
...DEFAULT_SCHEDULE_CONFIG,
|
||||
include_skill_lifecycle: false,
|
||||
include_course_detection: true,
|
||||
});
|
||||
assert.ok(!onlyCourses.some((s) => s.skillLifecycle), 'skill-lifecycle gated OFF');
|
||||
assert.ok(onlyCourses.some((s) => s.courseDetection), 'course detection gated ON');
|
||||
});
|
||||
|
||||
test('run-detection.mjs source — gates detection via the shared selector (no duplicated filter)', () => {
|
||||
const src = readFileSync(join(PLUGIN_ROOT, 'scripts', 'kb-update', 'run-detection.mjs'), 'utf8');
|
||||
assert.match(src, /selectDetectionSteps\(/, 'run-detection uses the pure step selector');
|
||||
});
|
||||
|
||||
test('run-detection.mjs source — spawns node, NEVER claude (the ToS boundary made structural)', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue