ms-ai-architect/scripts/kb-eval/lib/o2-return-check.mjs

// o2-return-check.mjs — the machine half of R11 §10 measurement #2.
//
// O2 candidacy is a PROSE judgement (docs/r11-tiered-fix-design.md §5): conditions
// 2 and 3 require a human to read the remainder. This module does not attempt
// that. It bounds the two failure modes that a human reviewing 46 proposals
// cannot cheaply catch, and that would otherwise waste the human's attention:
//
//   V1  the quoted file text must occur VERBATIM in the named file. An agent that
//       invented the text, or transliterated æ/ø/å, did not read the file — and
//       its remainder is then a proposal about a file that does not exist.
//   V2  the proposed remainder must be obtainable from that text by DELETING
//       characters only. This is the machine-checkable half of condition 1
//       ("asserts strictly less"); the semantic half stays human.
//   V2b V2 alone is too weak: deleting a leading word and recapitalising the next
//       ("Automatically add" -> "Add") still passes, because the capital already
//       existed inside the deleted word. A word-level, case-sensitive check
//       catches it. Recapitalising after a subtraction is defensible — but it is
//       a text change, and the human must SEE it rather than have it pass as
//       "pure deletion".
//   V3  schema completeness and verdict/condition coherence.
//
// A proposal that fails V1 or V2 is not an O2 candidate whatever its prose says.

const REQUIRED_FIELDS = [
  'idx', 'file', 'line', 'real_line', 'locator_failed',
  'file_text_verbatim', 'failing_part', 'proposed_remainder',
  'cond1_strictly_less', 'cond2_remainder_not_misleading',
  'cond3_nothing_confirmed_removed', 'verdict', 'o3_reason', 'confidence',
];

const VERDICTS = new Set(['O2_CANDIDATE', 'O3']);

/** Collapse runs of whitespace — a subtraction legitimately closes the gap it leaves. */
const normalise = (s) => String(s).replace(/\s+/g, ' ').trim();

/**
 * Is `sub` obtainable from `full` by deleting characters only?
 * Case- and diacritic-sensitive by design: a transliterated or recased remainder
 * is a rewrite, not a subtraction.
 */
export function isDeletionOnly(full, sub) {
  const f = normalise(full);
  const s = normalise(sub);
  if (s.length >= f.length) return false;
  let cursor = 0;
  for (const ch of s) {
    cursor = f.indexOf(ch, cursor);
    if (cursor === -1) return false;
    cursor += 1;
  }
  return true;
}

const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}'’-]*/gu;

/** Word forms present in `sub` but absent from `full`, case-sensitively. */
export function novelWordForms(full, sub) {
  const before = new Set(String(full).match(WORD_RE) || []);
  const after = new Set(String(sub).match(WORD_RE) || []);
  return [...after].filter((w) => !before.has(w));
}

/**
 * Check one classification row.
 * @param {object} row      a return record (see docs/r11-pilot-results.md §9)
 * @param {(rel: string) => string} readFile reads a repo-relative file, throws if absent
 * @returns {Array<{check: string, detail: string}>} empty when the row is clean
 */
export function checkRow(row, readFile) {
  const findings = [];
  const add = (check, detail) => findings.push({ check, detail });

  for (const field of REQUIRED_FIELDS) {
    if (!(field in row)) add('V3', `missing field: ${field}`);
  }
  if (!VERDICTS.has(row.verdict)) add('V3', `unknown verdict: ${row.verdict}`);

  // V1 — applies to EVERY row, not just the O2 candidates. An O3 verdict resting
  // on invented file text is just as wrong, it is merely wrong in the safe
  // direction, and the corpus-wide numbers count both.
  if (row.file_text_verbatim) {
    let text;
    try {
      text = readFile(row.file);
    } catch (err) {
      add('V1', `unreadable file: ${err.message}`);
      return findings;
    }
    if (!text.includes(row.file_text_verbatim)) {
      const loose = normalise(text).includes(normalise(row.file_text_verbatim));
      add('V1', loose
        ? 'file_text_verbatim matches only after whitespace normalisation'
        : 'file_text_verbatim NOT FOUND in the file');
    }
  } else if (!row.locator_failed) {
    add('V1', 'no file_text_verbatim and locator_failed is false');
  }

  if (row.verdict !== 'O2_CANDIDATE') return findings;

  if (row.locator_failed) add('V3', 'O2_CANDIDATE with locator_failed');
  if (row.cond1_strictly_less?.holds !== true) add('V3', 'O2_CANDIDATE but condition 1 does not hold');

  if (!row.proposed_remainder) {
    add('V2', 'O2_CANDIDATE with no proposed_remainder');
  } else if (!isDeletionOnly(row.file_text_verbatim || '', row.proposed_remainder)) {
    add('V2', 'proposed_remainder is not deletion-only (adds or reorders characters)');
  } else {
    const novel = novelWordForms(row.file_text_verbatim || '', row.proposed_remainder);
    if (novel.length) {
      add('V2b', `proposed_remainder introduces word forms absent from the original: ${novel.join(', ')}`);
    }
  }

  return findings;
}