ms-ai-architect/scripts/okf-consume/lib/bundle-cut.mjs

// bundle-cut.mjs — the DETERMINISTIC half of the OKF bundle consumption skill.
//
// Consumption contract (llm-ingestion-okf, docs/plan/okf-bundle-consumption-contract.md
// @ 01e4170) C4: "the script cuts, the agent judges". This module is the script.
// It reads, ranks and CUTS a bundle to a bounded context, and it always hands
// back a receipt for what it held back — a silent cut is C3's failure with
// extra steps.
//
// BOUNDARY against llm-ingestion-okf: nothing here writes, indexes, checks or
// retrieves in the sense that package owns. The OKF *envelope* is parsed by the
// package itself (materialize.parse_frontmatter, called as a subprocess by the
// driver); what this module parses out of the document BODY is the user's own
// corpus frontmatter, which OKF preserves verbatim but does not define.
//
// Pure functions only: no fs, no spawn, no clock. Everything is handed in.

// Weights locked in the pre-registration BEFORE any arm was run. Changing one
// invalidates the measurement, not just the code.
export const FIELD_WEIGHTS = Object.freeze({ tittel: 3, h2: 2, sjanger: 2, label: 1 });

// The bound the cut targets, in characters: 3x the plugin's existing 4000-char
// org-context budget (DEFAULT_MAX_TOTAL_LEN in kb-update/lib/user-data.mjs),
// because this cut delivers WHOLE documents rather than one-line field summaries.
export const AGENT_BUDGET = 12000;

// Same window A3-max used in the bake-off, so the arms stay comparable.
export const HEAD_SCAN_LINES = 16;

// Minimum shared prefix for two tokens to count as the same word. 4 is what
// Norwegian inflection needs (vedtak/vedtaket) without collapsing vei/veileder.
const MIN_PREFIX = 4;

// Standard Norwegian stop words (bokmaal + nynorsk). Only the 4-char-and-longer
// entries can ever fire, since shorter tokens are dropped on length anyway.
export const NORSKE_STOPPORD = new Set([
  'alle', 'andre', 'annet', 'bare', 'begge', 'blir', 'blitt', 'bare', 'begge',
  'denne', 'dere', 'deres', 'deim', 'deira', 'deires', 'dette', 'disse', 'ditt',
  'dykk', 'dykkar', 'eller', 'elles', 'etter', 'fordi', 'hadde', 'hans', 'henne',
  'hennar', 'hennes', 'here', 'hoss', 'hossen', 'hvem', 'hvilke', 'hvilken',
  'hvis', 'hvor', 'hvordan', 'hvorfor', 'ikke', 'ikkje', 'ingen', 'ingi', 'inkje',
  'inni', 'kine', 'korleis', 'korso', 'kunne', 'kvar', 'kvarhelst', 'kvifor',
  'mange', 'medan', 'meget', 'mellom', 'mine', 'mitt', 'mykje', 'noen', 'noka',
  'nokon', 'nokor', 'nokre', 'over', 'samme', 'siden', 'sidan', 'sine', 'sitt',
  'skal', 'skulle', 'slik', 'somme', 'somt', 'sånn', 'uten', 'vart', 'varte',
  'vere', 'verte', 'vore', 'vors', 'vort', 'ville', 'være', 'vært',
]);

/**
 * NFC -> lowercase -> split on anything that is not a letter or digit ->
 * drop tokens under MIN_PREFIX chars -> drop stop words.
 */
export function tokenize(text) {
  if (!text) return [];
  return String(text)
    .normalize('NFC')
    .toLowerCase()
    .split(/[^\p{L}\p{N}]+/u)
    .filter((t) => t.length >= MIN_PREFIX && !NORSKE_STOPPORD.has(t));
}

/** Two tokens are the same word if one prefixes the other by at least MIN_PREFIX. */
export function tokensMatch(a, b) {
  if (!a || !b) return false;
  const [short, long] = a.length <= b.length ? [a, b] : [b, a];
  if (short.length < MIN_PREFIX) return false;
  return long.startsWith(short);
}

// POST-HOC (pre-registration amendment 1): Norwegian is a compounding language,
// so "vektorindeks" and "vektorrepresentasjoner" are the same subject while
// neither prefixes the other. This matcher adds a shared-substring rule on top
// of the pre-registered prefix rule. It is a SUPERSET of tokensMatch and is
// labelled post-hoc everywhere it is reported.
const MIN_COMMON_SUBSTRING = 6;

export function tokensMatchCompound(a, b) {
  if (tokensMatch(a, b)) return true;
  if (!a || !b) return false;
  for (let len = Math.min(a.length, b.length); len >= MIN_COMMON_SUBSTRING; len -= 1) {
    for (let i = 0; i + len <= a.length; i += 1) {
      if (b.includes(a.slice(i, i + len))) return true;
    }
  }
  return false;
}

/**
 * Read the bundle index. DEFAULT's link template is '- [{label}]({target})';
 * the policy allows prose, so non-matching lines are skipped rather than an error.
 *
 * The index is read because DEFAULT sets entries_match_directory=False — the
 * index is AUTHORED, so the directory is not its denominator and a listing is
 * not a check (contract paragraph 2, checklist item 9).
 */
export function parseIndex(indexMd) {
  const re = /^- \[([^\]]*)\]\(([^)]+)\)$/;
  return String(indexMd ?? '')
    .split('\n')
    .map((line) => re.exec(line.trim()))
    .filter(Boolean)
    .map((m) => ({ label: m[1], target: m[2] }));
}

// The OKF envelope is recognised by a key only it writes. Recognising it by
// "starts with ---" would eat the user's own frontmatter on a non-bundle file.
const OKF_ENVELOPE_KEY = /^source_sha256:/m;

const PRESERVED_KEYS = ['id', 'tittel', 'sjanger', 'dato', 'status', 'superseded_by'];

/**
 * Lift the user's preserved frontmatter and H2 headings out of the first
 * HEAD_SCAN_LINES lines of the document BODY (the OKF envelope, if present,
 * is stripped first and does not count against the window).
 *
 * A key that is absent is ABSENT — it is never defaulted. C5: absence of a
 * conditionally-written field is a measurement, not the negation of what the
 * field asserts.
 */
export function headScan(text, lines = HEAD_SCAN_LINES) {
  let rest = String(text ?? '');
  if (rest.startsWith('---\n')) {
    const end = rest.indexOf('\n---', 3);
    if (end !== -1) {
      const block = rest.slice(0, end);
      if (OKF_ENVELOPE_KEY.test(block)) rest = rest.slice(end + 4);
    }
  }
  const window = rest.split('\n').slice(0, lines);
  const out = { h2: [] };
  for (const line of window) {
    const h2 = /^##\s+(.+?)\s*$/.exec(line);
    if (h2) { out.h2.push(h2[1]); continue; }
    const kv = /^([a-zA-Z_]+):\s*(.+?)\s*$/.exec(line);
    if (kv && PRESERVED_KEYS.includes(kv[1])) out[kv[1]] = kv[2];
  }
  return out;
}

/**
 * Score = sum over DISTINCT query tokens of the highest field weight that token
 * hit. Distinct, so a repeated word cannot inflate a document's rank.
 */
export function scoreCandidate(queryTokens, candidate, matcher = tokensMatch) {
  const fields = [
    [FIELD_WEIGHTS.tittel, tokenize(candidate.tittel)],
    [FIELD_WEIGHTS.h2, (candidate.h2 ?? []).flatMap((h) => tokenize(h))],
    [FIELD_WEIGHTS.sjanger, tokenize(candidate.sjanger)],
    [FIELD_WEIGHTS.label, tokenize(candidate.label)],
  ];
  let total = 0;
  for (const q of new Set(queryTokens)) {
    let best = 0;
    for (const [weight, tokens] of fields) {
      if (weight > best && tokens.some((t) => matcher(q, t))) best = weight;
    }
    total += best;
  }
  return total;
}

/**
 * Take scored candidates and cut them to the budget, newest-first on ties.
 *
 * Returns the delivered documents AND the receipt: how many bundle documents
 * were not delivered, out of how many there were. The denominator is the whole
 * candidate set, never the shortlist — reporting against the shortlist is how a
 * cut launders itself into looking like a complete read.
 */
export function cutToBudget({ candidates, budget = AGENT_BUDGET, k = Infinity, applySupersession = false }) {
  const denominator = candidates.length;
  let superseded = 0;
  let statusUnknown = 0;

  const eligible = candidates.filter((c) => {
    if (applySupersession) {
      if (c.status === 'erstattet') { superseded += 1; return false; }
      if (c.status === undefined) statusUnknown += 1;
    }
    return true;
  });

  const ranked = eligible
    .map((c, i) => ({ c, i }))
    .filter(({ c }) => (c.score ?? 0) > 0)
    .sort((a, b) => (b.c.score - a.c.score)
      || String(b.c.dato ?? '').localeCompare(String(a.c.dato ?? ''))
      || (a.i - b.i))
    .map(({ c }) => c);

  const delivered = [];
  let agentChars = 0;
  for (const c of ranked) {
    if (delivered.length >= k) break;
    const size = (c.body ?? '').length;
    if (agentChars + size > budget) break;
    delivered.push(c);
    agentChars += size;
  }

  return {
    delivered,
    agentChars,
    denominator,
    unread: denominator - delivered.length,
    superseded,
    statusUnknown,
  };
}