ms-ai-architect/scripts/kb-update/lib/course-diff.mjs

// course-diff.mjs — PURE course diff + registry-fold + run-mode selector (C3.2).
// Zero imports: writes NO files, reads NO secrets, never touches the ledger or
// KB. The detector (C3.4) loads/saves course-registry.json via registry-io and
// drives this module — keeping the diff logic pure and unit-testable.
// See c3-course-detection-plan.md §4.1 (registry schema), §4.2 (diff semantics),
// and the §8 decisions: (b) baseline emits no leads, (c) `removed` is computed
// ONLY on a full enumeration.

// A full enumeration is forced when the last one is missing or older than this.
// Only a full enum (no updatedAt.gt filter) can correctly detect retired courses.
export const FULL_ENUM_MAX_AGE_DAYS = 30;
const DAY_MS = 86_400_000;

/**
 * Normalize a raw Platform API item into the registry/lead shape. Keeps the diff
 * agnostic of the API field names (id→uid, updatedAt→updated_at, products[].id).
 * @param {{id: string, title?: string, url?: string, updatedAt?: string, products?: Array<{id: string}>}} apiItem
 * @param {'module'|'learning-path'} type — which endpoint the item came from
 * @returns {{uid: string, type: string, title: string, url: string, products: string[], updated_at: string|null}}
 */
export function normalizeCourse(apiItem, type) {
  return {
    uid: apiItem.id,
    type,
    title: apiItem.title ?? '',
    url: apiItem.url ?? '',
    products: (apiItem.products ?? []).map((p) => p.id).filter(Boolean),
    updated_at: apiItem.updatedAt ?? null,
  };
}

/**
 * Is a full enumeration due? True when the registry has never run a full enum
 * (last_full_enum missing) or the last one is strictly older than maxAgeDays.
 * Fails toward `true` on an unparseable timestamp — a full enum is a safe
 * superset of an incremental one.
 * @param {{last_full_enum?: string|null}} registry
 * @param {string} now — full ISO datetime
 * @param {{maxAgeDays?: number}} [opts]
 * @returns {boolean}
 */
export function isFullEnumDue(registry, now, { maxAgeDays = FULL_ENUM_MAX_AGE_DAYS } = {}) {
  if (!registry || !registry.last_full_enum) return true;
  const last = new Date(registry.last_full_enum).getTime();
  const t = new Date(now).getTime();
  if (!Number.isFinite(last) || !Number.isFinite(t)) return true;
  return t - last > maxAgeDays * DAY_MS;
}

/**
 * Pick the run mode from registry state (§4.1's three run types):
 *   - 'baseline'    — first ever run (no last_run): establish the registry, emit no leads.
 *   - 'full'        — a full enumeration is due: new/updated + removed.
 *   - 'incremental' — normal run: updatedAt.gt fetch, new/updated, removed:[].
 * @param {{last_run?: string|null, last_full_enum?: string|null}} registry
 * @param {string} now — full ISO datetime
 * @param {{maxAgeDays?: number}} [opts]
 * @returns {'baseline'|'full'|'incremental'}
 */
export function selectMode(registry, now, opts = {}) {
  if (!registry || !registry.last_run) return 'baseline';
  return isFullEnumDue(registry, now, opts) ? 'full' : 'incremental';
}

// Strictly newer than what we last stored (an equal or older timestamp is a no-op).
function isNewer(apiUpdatedAt, storedUpdatedAt) {
  if (!apiUpdatedAt) return false;
  if (!storedUpdatedAt) return true;
  return new Date(apiUpdatedAt).getTime() > new Date(storedUpdatedAt).getTime();
}

/**
 * Diff a set of (already-normalized, already-in-domain) courses against the
 * registry and classify leads. Leads are the normalized course objects; the
 * detector enriches them with suggested_skill/category downstream.
 *
 * @param {Array<object>} courses — normalized courses from the API for this run
 * @param {{courses?: object}} registry
 * @param {{mode: 'baseline'|'full'|'incremental'}} options
 * @returns {{new: object[], updated: object[], removed: Array<{uid: string, title: string, url: string}>}}
 */
export function diffCourses(courses, registry, { mode } = {}) {
  // §8 (b): a baseline run establishes the registry without emitting any leads —
  // flagging hundreds of "new" courses on day one trains the operator that the
  // tool is noise, not signal.
  if (mode === 'baseline') return { new: [], updated: [], removed: [] };

  const known = registry?.courses ?? {};
  const newLeads = [];
  const updatedLeads = [];
  for (const c of courses) {
    const prev = known[c.uid];
    if (!prev) newLeads.push(c);
    else if (isNewer(c.updated_at, prev.updated_at)) updatedLeads.push(c);
  }

  // §4.2 correctness pin: `removed` is computed ONLY on a full enumeration. An
  // incremental updatedAt.gt response omits everything unchanged, so "in
  // registry, not in response" is true for nearly the whole registry — a naive
  // removed-diff would be a false-positive flood. Incremental forces removed:[].
  let removed = [];
  if (mode === 'full') {
    const apiUids = new Set(courses.map((c) => c.uid));
    removed = Object.entries(known)
      .filter(([uid]) => !apiUids.has(uid))
      .map(([uid, e]) => ({ uid, title: e.title ?? '', url: e.url ?? '' }));
  }
  return { new: newLeads, updated: updatedLeads, removed };
}

// Build a stored entry, preserving first_seen across runs and advancing last_seen.
function foldEntry(prev, c, now) {
  return {
    type: c.type,
    title: c.title,
    url: c.url,
    products: c.products,
    updated_at: c.updated_at,
    first_seen: prev?.first_seen ?? now,
    last_seen: now,
  };
}

/**
 * Fold this run's courses into the registry. Pure — returns a new registry,
 * never mutates the input.
 *   - incremental: merge (keep untouched UIDs, add/update the ones returned).
 *   - baseline | full: the API set IS the complete registry — rebuild from it,
 *     dropping UIDs absent from the API (retired courses) so they aren't
 *     re-flagged as removed on the next full enum. Stamps last_full_enum = now.
 * Every run advances last_run = now and seeds created_at when missing.
 *
 * @param {object} registry
 * @param {Array<object>} courses — normalized courses from the API for this run
 * @param {string} now — full ISO datetime
 * @param {{mode: 'baseline'|'full'|'incremental'}} options
 * @returns {object} new registry
 */
export function updateRegistry(registry, courses, now, { mode } = {}) {
  const base = registry ?? { version: 1, created_at: null, last_run: null, last_full_enum: null, courses: {} };
  const prevCourses = base.courses ?? {};
  const isFull = mode === 'baseline' || mode === 'full';

  // incremental keeps the prior set; baseline/full rebuild from the API set only.
  const nextCourses = isFull ? {} : { ...prevCourses };
  for (const c of courses) nextCourses[c.uid] = foldEntry(prevCourses[c.uid], c, now);

  return {
    ...base,
    version: base.version ?? 1,
    created_at: base.created_at ?? now,
    last_run: now,
    last_full_enum: isFull ? now : base.last_full_enum,
    courses: nextCourses,
  };
}