diff --git a/scripts/kb-eval/judge-pass-manifest.mjs b/scripts/kb-eval/judge-pass-manifest.mjs new file mode 100755 index 0000000..97ff98a --- /dev/null +++ b/scripts/kb-eval/judge-pass-manifest.mjs @@ -0,0 +1,201 @@ +#!/usr/bin/env node +// judge-pass-manifest.mjs — R7–R10 full-pass judge LEDGER (Spor 3 pre-pass infra, R6 Step 4). +// +// The durable per-file record of the corpus judge pass: one entry per never-verified reference +// file, carrying its per-file verdict (born-verified `pass` → surgical stamp, or `flagged` → +// R11 fix work-list), the judge provenance, and any not-grounded claim flags. Mirrors +// scripts/kb-eval/data/spor0-fix-manifest.json in shape ({_meta, }) and discipline +// (git-tracked under data/ so drift is diff-reviewable; hand/subagent-maintained). +// +// Durability + resume are the whole point (brief NFR): R7 calls appendJudgedFile after EVERY +// judged file and writes atomically, so a crash on file 200/243 loses at most ONE file; a +// resume calls pendingFiles(manifest, worklist) and re-processes only the remainder. The +// aggregate-in-memory-write-once alternative loses a whole økt on a crash — rejected. +// +// Pure core: validateManifest / mergeBatch / appendJudgedFile / pendingFiles are disk-free and +// deterministic (path-sorted output). IO-shell: --json / --write [--merge] [--out ]. +// +// Exit codes: 0 ok · 2 usage error · 3 re-entrance guard (judged ledger, no --merge) + +import { readFileSync, existsSync, realpathSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { atomicWriteJson } from '../kb-update/lib/atomic-write.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// Pinned: the single git-tracked ledger downstream R7–R10 batches append to (verified NOT +// under .gitignore, so the judged ledger is review-visible — brief preference). +export const DEFAULT_MANIFEST_PATH = join(__dirname, 'data', 'judge-pass-manifest.json'); + +// A file's per-pass verdict: `pass` = every judgeable claim grounded (AND) → born-verified +// stamp; `flagged` = ≥1 not-grounded claim OR unstampable → R11 work-list. Nothing else. +export const VALID_VERDICTS = ['pass', 'flagged']; + +const KNOWN_FLAGS = new Set(['--json', '--write', '--merge', '--out']); + +const byFilePath = (a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0); + +/** A fresh, empty ledger scaffold. Deterministic (no live timestamp) so --write is stable. */ +export function scaffoldManifest() { + return { + _meta: { + purpose: + 'R7–R10 full-pass judge ledger: one record per never-verified reference file — ' + + 'born-verified pass (surgical stamp) or flagged (R11 work-list). Durable per file.', + derived_from: 'scripts/kb-update/data/full-pass-worklist.json (243 due, all never-verified)', + cadence: 'R7–R10 (5 økter × ~49, 8–10 samtidige)', + batch: null, + count: 0, + generated: null, + }, + files: [], + }; +} + +/** + * Validate the ledger shape. Pure. Returns {valid, errors[]}. Enforces the born-verified + * contract at rest: a `pass` record MUST carry verified + verified_by; a `flagged` record MUST + * carry ≥1 flag. Duplicate file records are an error (one record per file). + */ +export function validateManifest(manifest) { + const errors = []; + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + return { valid: false, errors: ['manifest must be an object'] }; + } + if (!manifest._meta || typeof manifest._meta !== 'object') errors.push('_meta must be an object'); + if (!Array.isArray(manifest.files)) { + errors.push('files must be an array'); + return { valid: false, errors }; + } + const seen = new Set(); + for (const rec of manifest.files) { + if (!rec || typeof rec !== 'object') { errors.push('file record must be an object'); continue; } + if (typeof rec.file !== 'string' || rec.file.trim() === '') { errors.push("record missing 'file'"); continue; } + if (seen.has(rec.file)) errors.push(`duplicate file record: ${rec.file}`); + seen.add(rec.file); + if (!VALID_VERDICTS.includes(rec.per_file_verdict)) { + errors.push(`${rec.file}: per_file_verdict must be one of ${VALID_VERDICTS.join('|')}`); + } + if (!Array.isArray(rec.flags)) errors.push(`${rec.file}: flags must be an array`); + if (rec.per_file_verdict === 'pass' && (!rec.verified || !rec.verified_by)) { + errors.push(`${rec.file}: a pass record must carry verified + verified_by`); + } + if (rec.per_file_verdict === 'flagged' && !(Array.isArray(rec.flags) && rec.flags.length > 0)) { + errors.push(`${rec.file}: a flagged record must carry ≥1 flag`); + } + } + return { valid: errors.length === 0, errors }; +} + +/** + * Add or update ONE file record, returning a NEW manifest (pure). The durability primitive: + * R7 calls this after each judged file, then writes atomically → max one file lost on crash. + * An existing record for the same `file` is replaced (idempotent re-judge). Files stay + * path-sorted and _meta.count is refreshed, so the on-disk ledger is deterministic + diff-clean. + */ +export function appendJudgedFile(manifest, record) { + if (!record || typeof record.file !== 'string' || record.file.trim() === '') { + throw new Error('appendJudgedFile: record.file is required'); + } + const base = manifest && typeof manifest === 'object' && !Array.isArray(manifest) ? manifest : scaffoldManifest(); + const kept = (Array.isArray(base.files) ? base.files : []).filter((r) => r.file !== record.file); + const files = [...kept, record].sort(byFilePath); + return { ...base, _meta: { ...(base._meta ?? {}), count: files.length }, files }; +} + +/** + * Merge a batch of file records into an existing ledger, preserving already-judged records + * verbatim (a judged entry is authoritative — a re-run never silently re-judges it). Records + * for not-yet-judged files are added/updated. Pure; output path-sorted, _meta.count refreshed. + * @param {object} existing — the current ledger + * @param {Array|object} incoming — a batch (array of records, or a manifest with files[]) + */ +export function mergeBatch(existing, incoming) { + const existingFiles = Array.isArray(existing?.files) ? existing.files : []; + const judged = new Set( + existingFiles.filter((r) => VALID_VERDICTS.includes(r?.per_file_verdict)).map((r) => r.file), + ); + const byFile = new Map(existingFiles.map((r) => [r.file, r])); + const incomingFiles = Array.isArray(incoming) ? incoming : Array.isArray(incoming?.files) ? incoming.files : []; + for (const rec of incomingFiles) { + if (!rec || typeof rec.file !== 'string') continue; + if (judged.has(rec.file)) continue; // judged wins — preserve verbatim + byFile.set(rec.file, rec); + } + const files = [...byFile.values()].sort(byFilePath); + return { ...(existing ?? {}), _meta: { ...(existing?._meta ?? {}), count: files.length }, files }; +} + +/** + * Resume primitive: given the full worklist of file paths, return those NOT yet judged (no + * ledger record with a per_file_verdict set). R7 resume processes only these — a re-run after + * a crash skips every already-judged file. Pure; preserves worklist order. + */ +export function pendingFiles(manifest, worklistPaths) { + const judged = new Set( + (manifest?.files ?? []) + .filter((r) => r && VALID_VERDICTS.includes(r.per_file_verdict)) + .map((r) => r.file), + ); + return (worklistPaths ?? []).filter((p) => !judged.has(p)); +} + +function hasJudged(manifest) { + return Array.isArray(manifest?.files) && manifest.files.some((r) => VALID_VERDICTS.includes(r?.per_file_verdict)); +} + +function usageExit(msg, code) { + process.stderr.write(`${msg}\nusage: judge-pass-manifest.mjs [--json] [--write] [--merge] [--out ]\n`); + process.exit(code); +} + +function main(argv) { + const args = argv.slice(2); + let outPath = DEFAULT_MANIFEST_PATH; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--out') { + if (!args[i + 1]) usageExit('--out requires a path', 2); + outPath = args[++i]; + continue; + } + if (!KNOWN_FLAGS.has(args[i])) usageExit(`unknown flag: ${args[i]}`, 2); + } + const json = args.includes('--json'); + const write = args.includes('--write'); + const merge = args.includes('--merge'); + + const existing = existsSync(outPath) ? JSON.parse(readFileSync(outPath, 'utf8')) : null; + if (existing) { + const check = validateManifest(existing); + if (!check.valid) usageExit(`invalid manifest at ${outPath}:\n ${check.errors.join('\n ')}`, 2); + } + let manifest = existing ?? scaffoldManifest(); + + if (write) { + if (existing && hasJudged(existing) && !merge) { + process.stderr.write( + `refusing --write: ledger at ${outPath} already carries judged record(s); a fresh write ` + + 'would clobber the judged pass. Re-run with --merge to preserve it.\n', + ); + process.exit(3); + } + manifest = mergeBatch(manifest, []); // normalize: path-sort + refresh count, judged preserved + atomicWriteJson(outPath, manifest); + } + + if (json) { + process.stdout.write(JSON.stringify(manifest, null, 2) + '\n'); + } else { + console.log(`judge-pass-manifest: ${manifest.files.length} record(s)` + (write ? ` → ${outPath}` : '')); + } +} + +const isMain = (() => { + try { + return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); + } catch { + return false; + } +})(); +if (isMain) main(process.argv); diff --git a/tests/kb-eval/test-judge-pass-manifest.test.mjs b/tests/kb-eval/test-judge-pass-manifest.test.mjs new file mode 100644 index 0000000..b268de9 --- /dev/null +++ b/tests/kb-eval/test-judge-pass-manifest.test.mjs @@ -0,0 +1,172 @@ +// tests/kb-eval/test-judge-pass-manifest.test.mjs +// R6 Step 4 — the R7–R10 judge-pass LEDGER: a durable per-file record of the corpus judge +// pass. The two NFR-load-bearing primitives are appendJudgedFile (write ONE record after each +// judged file → max one file lost on crash) and pendingFiles (resume = skip already-judged). +// Pure core (validateManifest / mergeBatch / appendJudgedFile / pendingFiles) + an IO-shell +// CLI (--json / --write [--merge], exit 3 re-entrance guard, exit 2 unknown flag), mirroring +// the classify-ref-type.mjs manifest-CLI shape. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { atomicWriteJson } from '../../scripts/kb-update/lib/atomic-write.mjs'; +import { + scaffoldManifest, + validateManifest, + appendJudgedFile, + mergeBatch, + pendingFiles, + DEFAULT_MANIFEST_PATH, +} from '../../scripts/kb-eval/judge-pass-manifest.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'judge-pass-manifest.mjs'); + +// A born-verified `pass` record: every judgeable claim grounded (AND) → surgical stamp. +const PASS_REC = { + file: 'skills/ms-ai-engineering/references/rag/azure-ai-search.md', + batch: 1, judged_at: '2026-07-04', per_file_verdict: 'pass', + claim_count: 9, verified: '2026-07-04', verified_by: 'judge-v3.1', flags: [], +}; +// A `flagged` record: ≥1 not-grounded claim → R11 work-list, never a stamp. +const FLAG_REC = { + file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md', + batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged', + claim_count: 8, verified: null, verified_by: null, + flags: [{ id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3', line: 42, claim: 'X', disposition: 'outdated' }], +}; + +test('appendJudgedFile → validateManifest valid, files path-sorted, count refreshed', () => { + let m = scaffoldManifest(); + m = appendJudgedFile(m, FLAG_REC); // skills/ms-ai-advisor/... + m = appendJudgedFile(m, PASS_REC); // skills/ms-ai-engineering/... + const v = validateManifest(m); + assert.equal(v.valid, true, v.errors.join('; ')); + const paths = m.files.map((r) => r.file); + assert.deepEqual(paths, [...paths].sort(), 'files must be path-sorted (deterministic ledger)'); + assert.equal(m._meta.count, 2); +}); + +test('validateManifest rejects a pass without a stamp, a bad verdict, a flagged without flags, and duplicates', () => { + assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'pass', flags: [] }] }).valid, false); + assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'maybe', flags: [] }] }).valid, false); + assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'flagged', flags: [] }] }).valid, false); + assert.equal(validateManifest({ _meta: {}, files: [PASS_REC, PASS_REC] }).valid, false); + assert.equal(validateManifest(scaffoldManifest()).valid, true); // empty scaffold is valid +}); + +test('appendJudgedFile is idempotent per file (re-judge replaces, never duplicates)', () => { + let m = appendJudgedFile(scaffoldManifest(), PASS_REC); + const rejudged = { ...PASS_REC, judged_at: '2026-07-05', verified: '2026-07-05' }; + m = appendJudgedFile(m, rejudged); + assert.equal(m.files.length, 1); + assert.equal(m.files[0].verified, '2026-07-05'); +}); + +test('appendJudgedFile persists ONE record durably, readable after (crash-resume simulation)', () => { + const tmp = mkdtempSync(join(tmpdir(), 'jpm-')); + const out = join(tmp, 'ledger.json'); + try { + // file 1 judged → write atomically → "crash" → reload → file 2 judged → write again + atomicWriteJson(out, appendJudgedFile(scaffoldManifest(), PASS_REC)); + const reloaded = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(reloaded.files.length, 1); + assert.equal(reloaded.files[0].file, PASS_REC.file); + atomicWriteJson(out, appendJudgedFile(reloaded, FLAG_REC)); + const after = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(after.files.length, 2); + assert.equal(validateManifest(after).valid, true); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test('pendingFiles returns only not-yet-judged worklist files (resume skips judged)', () => { + const NEW = 'skills/ms-ai-security/references/x/new.md'; + const worklist = [PASS_REC.file, FLAG_REC.file, NEW]; + let m = appendJudgedFile(scaffoldManifest(), PASS_REC); + m = appendJudgedFile(m, FLAG_REC); + assert.deepEqual(pendingFiles(m, worklist), [NEW]); + assert.deepEqual(pendingFiles(scaffoldManifest(), worklist), worklist, 'nothing judged → all pending'); +}); + +test('mergeBatch adds not-yet-judged records and preserves judged verbatim (judged wins)', () => { + const judgedLedger = appendJudgedFile(scaffoldManifest(), PASS_REC); + const incoming = [ + // same file as the judged record, different verdict → must NOT overwrite the judged entry + { ...PASS_REC, per_file_verdict: 'flagged', verified: null, verified_by: null, + flags: [{ id: 'z', judge_verdict: 'not_grounded', rule: 'R1' }] }, + FLAG_REC, // a new file → added + ]; + const merged = mergeBatch(judgedLedger, incoming); + assert.deepEqual(merged.files.find((r) => r.file === PASS_REC.file), PASS_REC, 'judged preserved verbatim'); + assert.ok(merged.files.some((r) => r.file === FLAG_REC.file), 'new record added'); + assert.equal(merged._meta.count, 2); +}); + +test('DEFAULT_MANIFEST_PATH is a git-tracked-able path under scripts/kb-eval/data/', () => { + assert.ok(DEFAULT_MANIFEST_PATH.includes(join('scripts', 'kb-eval', 'data')), DEFAULT_MANIFEST_PATH); + assert.ok(DEFAULT_MANIFEST_PATH.endsWith('judge-pass-manifest.json')); +}); + +// --- IO-shell CLI ------------------------------------------------------------------ + +test('CLI --write persists a parseable, schema-valid ledger to --out', () => { + const tmp = mkdtempSync(join(tmpdir(), 'jpm-')); + const out = join(tmp, 'ledger.json'); + try { + execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' }); + const m = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(validateManifest(m).valid, true); + assert.equal(Array.isArray(m.files), true); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test('CLI --write refuses (exit 3, file byte-identical) when the ledger already carries judged records', () => { + const tmp = mkdtempSync(join(tmpdir(), 'jpm-')); + const out = join(tmp, 'ledger.json'); + try { + writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n'); + const before = readFileSync(out, 'utf8'); + let status = 0; + try { + execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' }); + } catch (err) { + status = err.status; + } + assert.equal(status, 3); + assert.equal(readFileSync(out, 'utf8'), before, 'ledger must be untouched on re-entrance refusal'); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test('CLI --write --merge preserves judged records verbatim', () => { + const tmp = mkdtempSync(join(tmpdir(), 'jpm-')); + const out = join(tmp, 'ledger.json'); + try { + writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n'); + execFileSync('node', [CLI, '--write', '--merge', '--out', out], { encoding: 'utf8' }); + const m = JSON.parse(readFileSync(out, 'utf8')); + assert.equal(m.files.length, 1); + assert.deepEqual(m.files[0], PASS_REC); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + +test('CLI unknown flag → usage error exit 2', () => { + let status = 0; + try { + execFileSync('node', [CLI, '--bogus'], { encoding: 'utf8' }); + } catch (err) { + status = err.status; + } + assert.equal(status, 2); +});