ms-ai-architect/scripts/kb-eval/classify-fix-ops.mjs

#!/usr/bin/env node
// classify-fix-ops.mjs — R11 pilot runner (docs/r11-tiered-fix-design.md §10).
//
// Runs the fix-operation classifier over the pilot population: the files
// carrying >= 7 `not_grounded` flags, the densest available sample. Produces the
// four §10 measurements — the O1/O3 split, the R8 breakdown, the typed abort
// distribution, and the per-flag record needed to re-analyse without re-running.
//
// READ-ONLY over the corpus and the ledger. It never edits a KB file and never
// touches judge-pass-manifest.json — §8's single-writer state is untouched. The
// only write is its own report, and only with --write.
//
// Usage:  node scripts/kb-eval/classify-fix-ops.mjs [--write] [--threshold N] [--examples N]

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

import { ABORT_CODES, classifyFlag } from './lib/fix-op.mjs';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO = path.resolve(__dirname, '..', '..');
const DATA = path.join(__dirname, 'data');

const argv = process.argv.slice(2);
const flagArg = (name, fallback) => {
  const i = argv.indexOf(name);
  return i === -1 ? fallback : Number(argv[i + 1]);
};
const THRESHOLD = flagArg('--threshold', 7);
const EXAMPLES = flagArg('--examples', 3);

const ledger = JSON.parse(fs.readFileSync(path.join(DATA, 'judge-pass-manifest.json'), 'utf8'));

const ng = (rec) => (rec.flags || []).filter((f) => f.judge_verdict === 'not_grounded');
const population = ledger.files.filter((rec) => ng(rec).length >= THRESHOLD);

// Two passes over the same population. The canonical one applies the context
// condition; the §4-only pass exists purely to MEASURE what that condition
// removes — it is never a source of proposals, because four of the six swaps it
// admits on this population are wrong edits (see lib/fix-op.mjs).
const items = [];
const s4Only = [];
for (const rec of population) {
  const text = fs.readFileSync(path.join(REPO, rec.file), 'utf8');
  for (const flag of ng(rec)) {
    const verdict = classifyFlag(flag, text);
    s4Only.push(classifyFlag(flag, text, { contextCheck: false }));
    items.push({
      id: flag.id,
      file: flag.file,
      line: flag.line,
      rule: flag.rule || '(none)',
      claim: flag.claim,
      evidence_url: flag.evidence_url,
      evidence_quote: flag.evidence_quote,
      reason: flag.reason,
      op: verdict.op,
      code: verdict.code,
      detail: verdict.detail,
      proposal: verdict.proposal,
    });
  }
}

// ------------------------------------------------------------------ measurements

const tally = (rows, key) =>
  rows.reduce((acc, r) => {
    const k = typeof key === 'function' ? key(r) : r[key];
    acc[k] = (acc[k] || 0) + 1;
    return acc;
  }, {});

const o1 = items.filter((i) => i.op === 'O1');
const o3 = items.filter((i) => i.op === 'O3');
const byCode = tally(o3, 'code');
const byRule = tally(items, 'rule');
const r8 = items.filter((i) => i.rule === 'R8');

// LOCATOR_MISS / LOCATOR_AMBIGUOUS are a FIXABLE engineering gap (the locator did
// not find the value the claim asserts). Every other abort is intrinsic to the
// flag: no swappable value, no same-type replacement in the cited quote, or a
// claim that is not a value swap at all. The distinction is what tells the
// operator whether more engineering would move the O1 number.
const LOCATOR_CODES = new Set([ABORT_CODES.LOCATOR_MISS, ABORT_CODES.LOCATOR_AMBIGUOUS]);
const locatorAborts = o3.filter((i) => LOCATOR_CODES.has(i.code)).length;

// §4-as-written is a NUMERIC-path measurement: it is the baseline the context
// condition (§4a) was added against. §4b status swaps do not run through the
// context condition at all, so counting them here would silently inflate the
// baseline and break the comparison with the pilot's hand-verified 6.
const isStatus = (v) => v.op === 'O1' && v.proposal.type === 'status';
const s4O1 = s4Only.filter((v) => v.op === 'O1' && !isStatus(v)).length;
const o1Numeric = o1.filter((i) => i.proposal.type !== 'status').length;

// §4b (ratified 2026-08-03): the STATUS_SYNONYM class, split into what the closed
// synonym table proves and why the remainder still aborts. The abort REASON
// sub-distribution is the actionable part — NO_COMPLETE_FILE_LABEL is a corpus
// shape, SOURCE_STATUS_AMBIGUOUS is a quote shape, FILE_ALREADY_MATCHES means the
// flag was never a status mismatch in the first place.
const statusProposals = items.filter((i) => i.op === 'O1' && i.proposal.type === 'status');
const statusAborts = o3.filter((i) => i.code === ABORT_CODES.STATUS_SYNONYM);

// O1 precision is NOT uniform across token types, and this split is the pilot's
// operational conclusion. Hand-verified over the whole not_grounded population:
// every iso_date swap is an `api-version=` bump in a URL or code sample and all
// were correct; the number/version swaps mutilated identifiers instead
// ("AI-900" -> "AI-901", "gpt-4o" -> "gpt-5.1o" twice, a Java agent DOWNgrade),
// because a matching identifier prefix ("AI-", "gpt-") satisfies the context
// condition while the digit is part of a name rather than a quantity.
const O1_HAND_VERIFIED_TYPES = new Set(['iso_date']);
const byType = tally(o1, (i) => i.proposal.type);
const recommended = o1.filter((i) => O1_HAND_VERIFIED_TYPES.has(i.proposal.type));
const pct = (n) => `${((n / items.length) * 100).toFixed(1)} %`;

const report = {
  _meta: {
    purpose:
      'R11 pilot measurement (§10): fix-operation classification over the densest not_grounded sample. Read-only — no KB file and no ledger record was written.',
    contract: 'docs/r11-tiered-fix-design.md §3/§4/§10',
    classifier: 'scripts/kb-eval/lib/fix-op.mjs (the O1 driver with writes disabled)',
    ledger: 'scripts/kb-eval/data/judge-pass-manifest.json',
    ledger_records: ledger.files.length,
    threshold: `not_grounded >= ${THRESHOLD} (source_silent excluded, per §10)`,
    generated_from: 'ledger snapshot at run time — counts are re-derived, never read from a plan',
    disclaimer_two_202s:
      "This population is 202 flags. §3's '202 flags whose claim and quote contain a numeric token' is a DIFFERENT 202, measured over the full 712-flag population. Do not conflate them.",
  },
  population: { files: population.length, flags: items.length },
  s4_as_written: {
    O1: s4O1,
    note:
      'What §4 exactly as written would admit on the NUMERIC path (§4b status swaps excluded — they never run through the context condition). NOT a source of proposals: on the >=7 pilot all 6 were hand-verified and 4 were wrong edits (unit crossing, metric crossing, two mutilated identifiers) — measured precision 2/6. Runs at other thresholds carry no hand-verification.',
  },
  status_synonym: {
    contract: '§4b — the closed synonym table, ratified 2026-08-03',
    class_total: statusProposals.length + statusAborts.length,
    proven: statusProposals.length,
    aborts: tally(statusAborts, (i) => (i.detail && i.detail.reason) || '(unspecified)'),
    hand_verified:
      THRESHOLD === 7
        ? 'All 5 hand-judged 2026-08-03 (docs/r11-pilot-results.md appendix B). Four carry the source phrasing on the row\'s OWN subject and are correct. One (security-copilot-integration.md:94) harvests a "(Preview)" marker that belongs to a DIFFERENT agent in an enumerated quote — the same provenance-without-referent defect that falsified §4. Its outcome is plausibly right; its proof is not.'
        : 'hand-verification was done on the >=7 pilot only',
    applicability:
      'REVIEW-GRADE, NOT APPLY-GRADE. status is deliberately absent from o1_recommended: §4b binds the table, the completeness of the file label and the written value, and nothing about whether the source phrasing refers to the row\'s subject. A referent condition is an open operator decision.',
  },
  o1_by_type: byType,
  o1_recommended: {
    count: recommended.length,
    types: [...O1_HAND_VERIFIED_TYPES],
    note:
      'The only O1 class that survived hand-verification: iso_date, which in this corpus is always an api-version bump inside a URL or code sample. number/version proposals are NOT safe to apply — they mutilate product, model and certification identifiers.',
  },
  split: { O1: o1.length, O2: 0, O3: o3.length, O2_note: 'O2 requires operator ratification (§5); until then every non-O1 item is O3 by design.' },
  abort_codes: byCode,
  locator_aborts: { count: locatorAborts, note: 'fixable engineering gap — every other abort is intrinsic to the flag' },
  by_rule: byRule,
  r8: { total: r8.length, O1: r8.filter((i) => i.op === 'O1').length, codes: tally(r8.filter((i) => i.op === 'O3'), 'code') },
  items,
};

// ---------------------------------------------------------------------- output

console.log(`R11 pilot — ${population.length} files / ${items.length} not_grounded flags (threshold >= ${THRESHOLD})`);
console.log(`ledger: ${ledger.files.length} records\n`);
console.log(`O1 (provable value swap): ${o1.length}  (${pct(o1.length)})`);
console.log(`O3 (human):              ${o3.length}  (${pct(o3.length)})`);
console.log(`O2:                      0  (unratified — §5)`);
const handNote =
  THRESHOLD === 7
    ? ' — all 6 hand-verified: 4 are wrong edits (unit crossing, metric crossing, two mutilated identifiers)'
    : ' (hand-verification was done on the >=7 pilot only)';
console.log(`\n§4 as written would admit ${s4O1} on the numeric path${handNote}. Context condition removes ${s4O1 - o1Numeric}.`);
console.log(
  `§4b status class: ${report.status_synonym.class_total} flags -> ${report.status_synonym.proven} proven, ` +
    `${JSON.stringify(report.status_synonym.aborts)} — REVIEW-grade, not applied by any driver.\n`,
);
console.log('abort codes:');
for (const [code, n] of Object.entries(byCode).sort((a, b) => b[1] - a[1])) {
  console.log(`  ${code.padEnd(20)} ${String(n).padStart(4)}  ${pct(n)}`);
}
console.log(`\nO1 by token type: ${JSON.stringify(byType)}`);
console.log(`O1 hand-verified-safe class (iso_date / api-version): ${recommended.length}  — the rest mutilate identifiers, do NOT apply`);
console.log(`\nlocator aborts (fixable): ${locatorAborts}   intrinsic aborts: ${o3.length - locatorAborts}`);
console.log(`\nrule distribution: ${JSON.stringify(byRule)}`);
console.log(`R8: ${r8.length} flags — O1 ${report.r8.O1}, aborts ${JSON.stringify(report.r8.codes)}`);

if (EXAMPLES > 0 && o1.length > 0) {
  console.log(`\n--- ${Math.min(EXAMPLES, o1.length)} proven O1 proposals ---`);
  for (const i of o1.slice(0, EXAMPLES)) {
    console.log(`\n${i.file}:${i.proposal.line}  [${i.rule}]  ${i.token || i.proposal.token} -> ${i.proposal.replacement}`);
    console.log(`  - ${i.proposal.before}`);
    console.log(`  + ${i.proposal.after}`);
    console.log(`  quote: ${i.proposal.evidence_quote.slice(0, 160)}`);
  }
}

if (argv.includes('--write')) {
  const out = path.join(DATA, 'r11-pilot-classification.json');
  fs.writeFileSync(out, JSON.stringify(report, null, 2) + '\n');
  console.log(`\nwrote ${out}`);
} else {
  console.log('\n(dry run — pass --write to persist r11-pilot-classification.json)');
}