ms-ai-architect/scripts/kb-update/lib/verified-staleness.mjs

// verified-staleness.mjs — Spor 3 Port 3, INCREMENTAL pass (the cheap, lastmod-
// visible half of the cadence-judge). A PURE classifier: it computes which
// `reference` files have a source that changed AFTER they were last verified,
// and NEVER touches disk (mirrors verify-out.mjs — the I/O lives in the CLI).
//
// The expensive periodic full-pass (P3d) catches the drift class this misses
// (base-rate showed staleness-recall 0/40); this incremental pass is the cheap
// floor that catches what the sitemap already tells us. The reporting floor
// (P3b SessionStart signal) reads this classifier's output.
//
// Scope = the Port 1 frontmatter contract:
//   - type 'reference'                          → in correctness scope
//   - type template|methodology|regulatory      → out-of-scope (exempt)
//   - no Type header (legacy corpus)            → unmigrated (Spor 1 territory)
//
// Conservative by design (mirrors verify-out): when in doubt, flag. A YYYY-MM
// verify date normalizes to the 1st (so a mid-month source change still counts
// as stale), and the comparison is strict `>` so an equal date is fresh — the
// same convention report-changes uses for **Last updated:**.

const REFERENCE_TYPE = 'reference';
const OUT_OF_SCOPE_TYPES = new Set(['template', 'methodology', 'regulatory']);

/**
 * Invert the URL registry into a per-reference-file newest-source-lastmod map.
 * Mirrors report-changes' grouping loop, but pure so both the incremental pass
 * and the full-pass (P3d) share one inversion. Only `tracked` URLs that carry a
 * sitemap_lastmod contribute (a missing lastmod or non-tracked status is not a
 * currency signal).
 *
 * @param {object} registry — url-registry.json shape ({ urls: { url: entry } })
 * @returns {Map<string, string>} refFilePath → newest sitemap_lastmod (YYYY-MM-DD)
 */
export function buildFileSourceMap(registry) {
  const map = new Map();
  const urls = (registry && registry.urls) || {};
  for (const entry of Object.values(urls)) {
    if (!entry || entry.status !== 'tracked' || !entry.sitemap_lastmod) continue;
    for (const refFile of entry.reference_files || []) {
      const cur = map.get(refFile);
      if (!cur || entry.sitemap_lastmod > cur) map.set(refFile, entry.sitemap_lastmod);
    }
  }
  return map;
}

/** YYYY-MM → YYYY-MM-01; full date as-is. Mirrors report-changes' parseLastUpdated. */
function normalizeDate(d) {
  if (!d) return null;
  return d.length === 7 ? d + '-01' : d;
}

/**
 * Classify reference files for lastmod-visible drift since last verification.
 *
 * @param {Array<{path: string, type: string|null, verified: string|null,
 *                newestSourceLastmod: string|null}>} files
 *   - type:                 Port 1 Type header (lowercased), or null if absent
 *   - verified:             Port 1 Verified date (YYYY-MM|YYYY-MM-DD), or null
 *   - newestSourceLastmod:  newest sitemap_lastmod across the file's tracked
 *                           source URLs (YYYY-MM-DD), or null if no tracked source
 * @returns {{flagged: Array<{path,reason,verified,newestSourceLastmod}>,
 *            counts: {total,fresh,stale,unverified,unmigrated,outOfScope}}}
 */
export function classifyVerifiedStaleness(files) {
  const list = Array.isArray(files) ? files : [];
  const flagged = [];
  const counts = { total: list.length, fresh: 0, stale: 0, unverified: 0, unmigrated: 0, outOfScope: 0 };

  for (const file of list) {
    const type = file.type == null ? null : String(file.type).toLowerCase();

    // Out of correctness scope — template/methodology/regulatory are exempt.
    if (type && OUT_OF_SCOPE_TYPES.has(type)) {
      counts.outOfScope++;
      continue;
    }

    // No Type header — legacy file predating the Port 1 contract. Not judged
    // here (Spor 1 migrates it); counted so the gap is an honest number.
    if (type !== REFERENCE_TYPE) {
      counts.unmigrated++;
      continue;
    }

    // Declared reference. The contract requires born-verified; a missing verify
    // date is a breach create-guard should have caught → flag for attention.
    if (file.verified == null) {
      counts.unverified++;
      flagged.push({ path: file.path, reason: 'unverified', verified: null, newestSourceLastmod: file.newestSourceLastmod ?? null });
      continue;
    }

    // Lastmod-visible drift: source changed strictly after we verified.
    const verifiedNorm = normalizeDate(file.verified);
    if (file.newestSourceLastmod && file.newestSourceLastmod > verifiedNorm) {
      counts.stale++;
      flagged.push({ path: file.path, reason: 'stale-since-verified', verified: file.verified, newestSourceLastmod: file.newestSourceLastmod });
      continue;
    }

    counts.fresh++;
  }

  return { flagged, counts };
}

const REASON_ORDER = { 'stale-since-verified': 0, unverified: 1 };

/**
 * Assemble the full incremental staleness report. PURE: the caller injects a
 * header reader so the registry inversion + classification + shaping stay
 * testable without disk. The CLI wraps this with loadRegistry + a readFileSync-
 * backed reader + saveReport.
 *
 * @param {object} registry — url-registry.json shape
 * @param {(path: string) => ({type: string|null, verified: string|null}|null)} headerReader
 *   returns the file's Type/Verified headers, or null if the file is absent
 *   (deleted on disk) — absent files are skipped entirely, not counted.
 * @param {{today: string}} opts — today as YYYY-MM-DD (injected; Date.now is impure)
 * @returns {{generated_at, last_poll, scope, counts, flagged}}
 */
export function buildStalenessReport(registry, headerReader, { today }) {
  const sourceMap = buildFileSourceMap(registry);
  const files = [];
  for (const [path, newestSourceLastmod] of sourceMap) {
    const headers = headerReader(path);
    if (!headers) continue; // deleted on disk — registry hygiene, not staleness
    files.push({ path, type: headers.type ?? null, verified: headers.verified ?? null, newestSourceLastmod });
  }

  const { flagged, counts } = classifyVerifiedStaleness(files);
  flagged.sort((a, b) => {
    const r = (REASON_ORDER[a.reason] ?? 9) - (REASON_ORDER[b.reason] ?? 9);
    if (r !== 0) return r;
    return String(b.newestSourceLastmod ?? '').localeCompare(String(a.newestSourceLastmod ?? ''));
  });

  return {
    generated_at: today,
    last_poll: (registry && registry.last_poll) || null,
    scope: 'incremental (reference files with tracked sources)',
    counts,
    flagged,
  };
}