feat(ms-ai-architect): per-fil judge-pass-manifest (appendJudgedFile + pendingFiles resume-skip, data/ git-tracked, TDD) [skip-docs]

scripts/kb-eval/judge-pass-manifest.mjs

@@ -0,0 +1,201 @@
+#!/usr/bin/env node
+// judge-pass-manifest.mjs — R7–R10 full-pass judge LEDGER (Spor 3 pre-pass infra, R6 Step 4).
+//
+// The durable per-file record of the corpus judge pass: one entry per never-verified reference
+// file, carrying its per-file verdict (born-verified `pass` → surgical stamp, or `flagged` →
+// R11 fix work-list), the judge provenance, and any not-grounded claim flags. Mirrors
+// scripts/kb-eval/data/spor0-fix-manifest.json in shape ({_meta, <records[]>}) and discipline
+// (git-tracked under data/ so drift is diff-reviewable; hand/subagent-maintained).
+//
+// Durability + resume are the whole point (brief NFR): R7 calls appendJudgedFile after EVERY
+// judged file and writes atomically, so a crash on file 200/243 loses at most ONE file; a
+// resume calls pendingFiles(manifest, worklist) and re-processes only the remainder. The
+// aggregate-in-memory-write-once alternative loses a whole økt on a crash — rejected.
+//
+// Pure core: validateManifest / mergeBatch / appendJudgedFile / pendingFiles are disk-free and
+// deterministic (path-sorted output). IO-shell: --json / --write [--merge] [--out <path>].
+//
+// Exit codes: 0 ok · 2 usage error · 3 re-entrance guard (judged ledger, no --merge)
+
+import { readFileSync, existsSync, realpathSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Pinned: the single git-tracked ledger downstream R7–R10 batches append to (verified NOT
+// under .gitignore, so the judged ledger is review-visible — brief preference).
+export const DEFAULT_MANIFEST_PATH = join(__dirname, 'data', 'judge-pass-manifest.json');
+
+// A file's per-pass verdict: `pass` = every judgeable claim grounded (AND) → born-verified
+// stamp; `flagged` = ≥1 not-grounded claim OR unstampable → R11 work-list. Nothing else.
+export const VALID_VERDICTS = ['pass', 'flagged'];
+
+const KNOWN_FLAGS = new Set(['--json', '--write', '--merge', '--out']);
+
+const byFilePath = (a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0);
+
+/** A fresh, empty ledger scaffold. Deterministic (no live timestamp) so --write is stable. */
+export function scaffoldManifest() {
+  return {
+    _meta: {
+      purpose:
+        'R7–R10 full-pass judge ledger: one record per never-verified reference file — ' +
+        'born-verified pass (surgical stamp) or flagged (R11 work-list). Durable per file.',
+      derived_from: 'scripts/kb-update/data/full-pass-worklist.json (243 due, all never-verified)',
+      cadence: 'R7–R10 (5 økter × ~49, 8–10 samtidige)',
+      batch: null,
+      count: 0,
+      generated: null,
+    },
+    files: [],
+  };
+}
+
+/**
+ * Validate the ledger shape. Pure. Returns {valid, errors[]}. Enforces the born-verified
+ * contract at rest: a `pass` record MUST carry verified + verified_by; a `flagged` record MUST
+ * carry ≥1 flag. Duplicate file records are an error (one record per file).
+ */
+export function validateManifest(manifest) {
+  const errors = [];
+  if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) {
+    return { valid: false, errors: ['manifest must be an object'] };
+  }
+  if (!manifest._meta || typeof manifest._meta !== 'object') errors.push('_meta must be an object');
+  if (!Array.isArray(manifest.files)) {
+    errors.push('files must be an array');
+    return { valid: false, errors };
+  }
+  const seen = new Set();
+  for (const rec of manifest.files) {
+    if (!rec || typeof rec !== 'object') { errors.push('file record must be an object'); continue; }
+    if (typeof rec.file !== 'string' || rec.file.trim() === '') { errors.push("record missing 'file'"); continue; }
+    if (seen.has(rec.file)) errors.push(`duplicate file record: ${rec.file}`);
+    seen.add(rec.file);
+    if (!VALID_VERDICTS.includes(rec.per_file_verdict)) {
+      errors.push(`${rec.file}: per_file_verdict must be one of ${VALID_VERDICTS.join('|')}`);
+    }
+    if (!Array.isArray(rec.flags)) errors.push(`${rec.file}: flags must be an array`);
+    if (rec.per_file_verdict === 'pass' && (!rec.verified || !rec.verified_by)) {
+      errors.push(`${rec.file}: a pass record must carry verified + verified_by`);
+    }
+    if (rec.per_file_verdict === 'flagged' && !(Array.isArray(rec.flags) && rec.flags.length > 0)) {
+      errors.push(`${rec.file}: a flagged record must carry ≥1 flag`);
+    }
+  }
+  return { valid: errors.length === 0, errors };
+}
+
+/**
+ * Add or update ONE file record, returning a NEW manifest (pure). The durability primitive:
+ * R7 calls this after each judged file, then writes atomically → max one file lost on crash.
+ * An existing record for the same `file` is replaced (idempotent re-judge). Files stay
+ * path-sorted and _meta.count is refreshed, so the on-disk ledger is deterministic + diff-clean.
+ */
+export function appendJudgedFile(manifest, record) {
+  if (!record || typeof record.file !== 'string' || record.file.trim() === '') {
+    throw new Error('appendJudgedFile: record.file is required');
+  }
+  const base = manifest && typeof manifest === 'object' && !Array.isArray(manifest) ? manifest : scaffoldManifest();
+  const kept = (Array.isArray(base.files) ? base.files : []).filter((r) => r.file !== record.file);
+  const files = [...kept, record].sort(byFilePath);
+  return { ...base, _meta: { ...(base._meta ?? {}), count: files.length }, files };
+}
+
+/**
+ * Merge a batch of file records into an existing ledger, preserving already-judged records
+ * verbatim (a judged entry is authoritative — a re-run never silently re-judges it). Records
+ * for not-yet-judged files are added/updated. Pure; output path-sorted, _meta.count refreshed.
+ * @param {object} existing — the current ledger
+ * @param {Array|object} incoming — a batch (array of records, or a manifest with files[])
+ */
+export function mergeBatch(existing, incoming) {
+  const existingFiles = Array.isArray(existing?.files) ? existing.files : [];
+  const judged = new Set(
+    existingFiles.filter((r) => VALID_VERDICTS.includes(r?.per_file_verdict)).map((r) => r.file),
+  );
+  const byFile = new Map(existingFiles.map((r) => [r.file, r]));
+  const incomingFiles = Array.isArray(incoming) ? incoming : Array.isArray(incoming?.files) ? incoming.files : [];
+  for (const rec of incomingFiles) {
+    if (!rec || typeof rec.file !== 'string') continue;
+    if (judged.has(rec.file)) continue; // judged wins — preserve verbatim
+    byFile.set(rec.file, rec);
+  }
+  const files = [...byFile.values()].sort(byFilePath);
+  return { ...(existing ?? {}), _meta: { ...(existing?._meta ?? {}), count: files.length }, files };
+}
+
+/**
+ * Resume primitive: given the full worklist of file paths, return those NOT yet judged (no
+ * ledger record with a per_file_verdict set). R7 resume processes only these — a re-run after
+ * a crash skips every already-judged file. Pure; preserves worklist order.
+ */
+export function pendingFiles(manifest, worklistPaths) {
+  const judged = new Set(
+    (manifest?.files ?? [])
+      .filter((r) => r && VALID_VERDICTS.includes(r.per_file_verdict))
+      .map((r) => r.file),
+  );
+  return (worklistPaths ?? []).filter((p) => !judged.has(p));
+}
+
+function hasJudged(manifest) {
+  return Array.isArray(manifest?.files) && manifest.files.some((r) => VALID_VERDICTS.includes(r?.per_file_verdict));
+}
+
+function usageExit(msg, code) {
+  process.stderr.write(`${msg}\nusage: judge-pass-manifest.mjs [--json] [--write] [--merge] [--out <path>]\n`);
+  process.exit(code);
+}
+
+function main(argv) {
+  const args = argv.slice(2);
+  let outPath = DEFAULT_MANIFEST_PATH;
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--out') {
+      if (!args[i + 1]) usageExit('--out requires a path', 2);
+      outPath = args[++i];
+      continue;
+    }
+    if (!KNOWN_FLAGS.has(args[i])) usageExit(`unknown flag: ${args[i]}`, 2);
+  }
+  const json = args.includes('--json');
+  const write = args.includes('--write');
+  const merge = args.includes('--merge');
+
+  const existing = existsSync(outPath) ? JSON.parse(readFileSync(outPath, 'utf8')) : null;
+  if (existing) {
+    const check = validateManifest(existing);
+    if (!check.valid) usageExit(`invalid manifest at ${outPath}:\n  ${check.errors.join('\n  ')}`, 2);
+  }
+  let manifest = existing ?? scaffoldManifest();
+
+  if (write) {
+    if (existing && hasJudged(existing) && !merge) {
+      process.stderr.write(
+        `refusing --write: ledger at ${outPath} already carries judged record(s); a fresh write ` +
+        'would clobber the judged pass. Re-run with --merge to preserve it.\n',
+      );
+      process.exit(3);
+    }
+    manifest = mergeBatch(manifest, []); // normalize: path-sort + refresh count, judged preserved
+    atomicWriteJson(outPath, manifest);
+  }
+
+  if (json) {
+    process.stdout.write(JSON.stringify(manifest, null, 2) + '\n');
+  } else {
+    console.log(`judge-pass-manifest: ${manifest.files.length} record(s)` + (write ? ` → ${outPath}` : ''));
+  }
+}
+
+const isMain = (() => {
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+})();
+if (isMain) main(process.argv);

tests/kb-eval/test-judge-pass-manifest.test.mjs

@@ -0,0 +1,172 @@
+// tests/kb-eval/test-judge-pass-manifest.test.mjs
+// R6 Step 4 — the R7–R10 judge-pass LEDGER: a durable per-file record of the corpus judge
+// pass. The two NFR-load-bearing primitives are appendJudgedFile (write ONE record after each
+// judged file → max one file lost on crash) and pendingFiles (resume = skip already-judged).
+// Pure core (validateManifest / mergeBatch / appendJudgedFile / pendingFiles) + an IO-shell
+// CLI (--json / --write [--merge], exit 3 re-entrance guard, exit 2 unknown flag), mirroring
+// the classify-ref-type.mjs manifest-CLI shape.
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { atomicWriteJson } from '../../scripts/kb-update/lib/atomic-write.mjs';
+import {
+  scaffoldManifest,
+  validateManifest,
+  appendJudgedFile,
+  mergeBatch,
+  pendingFiles,
+  DEFAULT_MANIFEST_PATH,
+} from '../../scripts/kb-eval/judge-pass-manifest.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'judge-pass-manifest.mjs');
+
+// A born-verified `pass` record: every judgeable claim grounded (AND) → surgical stamp.
+const PASS_REC = {
+  file: 'skills/ms-ai-engineering/references/rag/azure-ai-search.md',
+  batch: 1, judged_at: '2026-07-04', per_file_verdict: 'pass',
+  claim_count: 9, verified: '2026-07-04', verified_by: 'judge-v3.1', flags: [],
+};
+// A `flagged` record: ≥1 not-grounded claim → R11 work-list, never a stamp.
+const FLAG_REC = {
+  file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md',
+  batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged',
+  claim_count: 8, verified: null, verified_by: null,
+  flags: [{ id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3', line: 42, claim: 'X', disposition: 'outdated' }],
+};
+
+test('appendJudgedFile → validateManifest valid, files path-sorted, count refreshed', () => {
+  let m = scaffoldManifest();
+  m = appendJudgedFile(m, FLAG_REC);   // skills/ms-ai-advisor/...
+  m = appendJudgedFile(m, PASS_REC);   // skills/ms-ai-engineering/...
+  const v = validateManifest(m);
+  assert.equal(v.valid, true, v.errors.join('; '));
+  const paths = m.files.map((r) => r.file);
+  assert.deepEqual(paths, [...paths].sort(), 'files must be path-sorted (deterministic ledger)');
+  assert.equal(m._meta.count, 2);
+});
+
+test('validateManifest rejects a pass without a stamp, a bad verdict, a flagged without flags, and duplicates', () => {
+  assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'pass', flags: [] }] }).valid, false);
+  assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'maybe', flags: [] }] }).valid, false);
+  assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'flagged', flags: [] }] }).valid, false);
+  assert.equal(validateManifest({ _meta: {}, files: [PASS_REC, PASS_REC] }).valid, false);
+  assert.equal(validateManifest(scaffoldManifest()).valid, true); // empty scaffold is valid
+});
+
+test('appendJudgedFile is idempotent per file (re-judge replaces, never duplicates)', () => {
+  let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
+  const rejudged = { ...PASS_REC, judged_at: '2026-07-05', verified: '2026-07-05' };
+  m = appendJudgedFile(m, rejudged);
+  assert.equal(m.files.length, 1);
+  assert.equal(m.files[0].verified, '2026-07-05');
+});
+
+test('appendJudgedFile persists ONE record durably, readable after (crash-resume simulation)', () => {
+  const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
+  const out = join(tmp, 'ledger.json');
+  try {
+    // file 1 judged → write atomically → "crash" → reload → file 2 judged → write again
+    atomicWriteJson(out, appendJudgedFile(scaffoldManifest(), PASS_REC));
+    const reloaded = JSON.parse(readFileSync(out, 'utf8'));
+    assert.equal(reloaded.files.length, 1);
+    assert.equal(reloaded.files[0].file, PASS_REC.file);
+    atomicWriteJson(out, appendJudgedFile(reloaded, FLAG_REC));
+    const after = JSON.parse(readFileSync(out, 'utf8'));
+    assert.equal(after.files.length, 2);
+    assert.equal(validateManifest(after).valid, true);
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('pendingFiles returns only not-yet-judged worklist files (resume skips judged)', () => {
+  const NEW = 'skills/ms-ai-security/references/x/new.md';
+  const worklist = [PASS_REC.file, FLAG_REC.file, NEW];
+  let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
+  m = appendJudgedFile(m, FLAG_REC);
+  assert.deepEqual(pendingFiles(m, worklist), [NEW]);
+  assert.deepEqual(pendingFiles(scaffoldManifest(), worklist), worklist, 'nothing judged → all pending');
+});
+
+test('mergeBatch adds not-yet-judged records and preserves judged verbatim (judged wins)', () => {
+  const judgedLedger = appendJudgedFile(scaffoldManifest(), PASS_REC);
+  const incoming = [
+    // same file as the judged record, different verdict → must NOT overwrite the judged entry
+    { ...PASS_REC, per_file_verdict: 'flagged', verified: null, verified_by: null,
+      flags: [{ id: 'z', judge_verdict: 'not_grounded', rule: 'R1' }] },
+    FLAG_REC, // a new file → added
+  ];
+  const merged = mergeBatch(judgedLedger, incoming);
+  assert.deepEqual(merged.files.find((r) => r.file === PASS_REC.file), PASS_REC, 'judged preserved verbatim');
+  assert.ok(merged.files.some((r) => r.file === FLAG_REC.file), 'new record added');
+  assert.equal(merged._meta.count, 2);
+});
+
+test('DEFAULT_MANIFEST_PATH is a git-tracked-able path under scripts/kb-eval/data/', () => {
+  assert.ok(DEFAULT_MANIFEST_PATH.includes(join('scripts', 'kb-eval', 'data')), DEFAULT_MANIFEST_PATH);
+  assert.ok(DEFAULT_MANIFEST_PATH.endsWith('judge-pass-manifest.json'));
+});
+
+// --- IO-shell CLI ------------------------------------------------------------------
+
+test('CLI --write persists a parseable, schema-valid ledger to --out', () => {
+  const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
+  const out = join(tmp, 'ledger.json');
+  try {
+    execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
+    const m = JSON.parse(readFileSync(out, 'utf8'));
+    assert.equal(validateManifest(m).valid, true);
+    assert.equal(Array.isArray(m.files), true);
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('CLI --write refuses (exit 3, file byte-identical) when the ledger already carries judged records', () => {
+  const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
+  const out = join(tmp, 'ledger.json');
+  try {
+    writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
+    const before = readFileSync(out, 'utf8');
+    let status = 0;
+    try {
+      execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
+    } catch (err) {
+      status = err.status;
+    }
+    assert.equal(status, 3);
+    assert.equal(readFileSync(out, 'utf8'), before, 'ledger must be untouched on re-entrance refusal');
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('CLI --write --merge preserves judged records verbatim', () => {
+  const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
+  const out = join(tmp, 'ledger.json');
+  try {
+    writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
+    execFileSync('node', [CLI, '--write', '--merge', '--out', out], { encoding: 'utf8' });
+    const m = JSON.parse(readFileSync(out, 'utf8'));
+    assert.equal(m.files.length, 1);
+    assert.deepEqual(m.files[0], PASS_REC);
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('CLI unknown flag → usage error exit 2', () => {
+  let status = 0;
+  try {
+    execFileSync('node', [CLI, '--bogus'], { encoding: 'utf8' });
+  } catch (err) {
+    status = err.status;
+  }
+  assert.equal(status, 2);
+});