231 lines
11 KiB
JavaScript
231 lines
11 KiB
JavaScript
// tests/kb-eval/test-judge-pass-manifest.test.mjs
|
||
// R6 Step 4 — the R7–R10 judge-pass LEDGER: a durable per-file record of the corpus judge
|
||
// pass. The two NFR-load-bearing primitives are appendJudgedFile (write ONE record after each
|
||
// judged file → max one file lost on crash) and pendingFiles (resume = skip already-judged).
|
||
// Pure core (validateManifest / mergeBatch / appendJudgedFile / pendingFiles) + an IO-shell
|
||
// CLI (--json / --write [--merge], exit 3 re-entrance guard, exit 2 unknown flag), mirroring
|
||
// the classify-ref-type.mjs manifest-CLI shape.
|
||
|
||
import { test } from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { execFileSync } from 'node:child_process';
|
||
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { dirname, join } from 'node:path';
|
||
import { atomicWriteJson } from '../../scripts/kb-update/lib/atomic-write.mjs';
|
||
import {
|
||
scaffoldManifest,
|
||
validateManifest,
|
||
appendJudgedFile,
|
||
mergeBatch,
|
||
pendingFiles,
|
||
DEFAULT_MANIFEST_PATH,
|
||
} from '../../scripts/kb-eval/judge-pass-manifest.mjs';
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'judge-pass-manifest.mjs');
|
||
|
||
// A born-verified `pass` record: every judgeable claim grounded (AND) → surgical stamp.
|
||
const PASS_REC = {
|
||
file: 'skills/ms-ai-engineering/references/rag/azure-ai-search.md',
|
||
batch: 1, judged_at: '2026-07-04', per_file_verdict: 'pass',
|
||
claim_count: 9, verified: '2026-07-04', verified_by: 'judge-v3.1', flags: [],
|
||
};
|
||
// A `flagged` record: ≥1 not-grounded claim → R11 work-list, never a stamp.
|
||
const FLAG_REC = {
|
||
file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md',
|
||
batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged',
|
||
claim_count: 8, verified: null, verified_by: null,
|
||
flags: [{ id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3', line: 42, claim: 'X', disposition: 'outdated' }],
|
||
};
|
||
|
||
test('appendJudgedFile → validateManifest valid, files path-sorted, count refreshed', () => {
|
||
let m = scaffoldManifest();
|
||
m = appendJudgedFile(m, FLAG_REC); // skills/ms-ai-advisor/...
|
||
m = appendJudgedFile(m, PASS_REC); // skills/ms-ai-engineering/...
|
||
const v = validateManifest(m);
|
||
assert.equal(v.valid, true, v.errors.join('; '));
|
||
const paths = m.files.map((r) => r.file);
|
||
assert.deepEqual(paths, [...paths].sort(), 'files must be path-sorted (deterministic ledger)');
|
||
assert.equal(m._meta.count, 2);
|
||
});
|
||
|
||
test('validateManifest rejects a pass without a stamp, a bad verdict, a flagged without flags, and duplicates', () => {
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'pass', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'maybe', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'flagged', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [PASS_REC, PASS_REC] }).valid, false);
|
||
assert.equal(validateManifest(scaffoldManifest()).valid, true); // empty scaffold is valid
|
||
});
|
||
|
||
test('appendJudgedFile is idempotent per file (re-judge replaces, never duplicates)', () => {
|
||
let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
const rejudged = { ...PASS_REC, judged_at: '2026-07-05', verified: '2026-07-05' };
|
||
m = appendJudgedFile(m, rejudged);
|
||
assert.equal(m.files.length, 1);
|
||
assert.equal(m.files[0].verified, '2026-07-05');
|
||
});
|
||
|
||
test('appendJudgedFile persists ONE record durably, readable after (crash-resume simulation)', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
// file 1 judged → write atomically → "crash" → reload → file 2 judged → write again
|
||
atomicWriteJson(out, appendJudgedFile(scaffoldManifest(), PASS_REC));
|
||
const reloaded = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(reloaded.files.length, 1);
|
||
assert.equal(reloaded.files[0].file, PASS_REC.file);
|
||
atomicWriteJson(out, appendJudgedFile(reloaded, FLAG_REC));
|
||
const after = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(after.files.length, 2);
|
||
assert.equal(validateManifest(after).valid, true);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('pendingFiles returns only not-yet-judged worklist files (resume skips judged)', () => {
|
||
const NEW = 'skills/ms-ai-security/references/x/new.md';
|
||
const worklist = [PASS_REC.file, FLAG_REC.file, NEW];
|
||
let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
m = appendJudgedFile(m, FLAG_REC);
|
||
assert.deepEqual(pendingFiles(m, worklist), [NEW]);
|
||
assert.deepEqual(pendingFiles(scaffoldManifest(), worklist), worklist, 'nothing judged → all pending');
|
||
});
|
||
|
||
test('mergeBatch adds not-yet-judged records and preserves judged verbatim (judged wins)', () => {
|
||
const judgedLedger = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
const incoming = [
|
||
// same file as the judged record, different verdict → must NOT overwrite the judged entry
|
||
{ ...PASS_REC, per_file_verdict: 'flagged', verified: null, verified_by: null,
|
||
flags: [{ id: 'z', judge_verdict: 'not_grounded', rule: 'R1' }] },
|
||
FLAG_REC, // a new file → added
|
||
];
|
||
const merged = mergeBatch(judgedLedger, incoming);
|
||
assert.deepEqual(merged.files.find((r) => r.file === PASS_REC.file), PASS_REC, 'judged preserved verbatim');
|
||
assert.ok(merged.files.some((r) => r.file === FLAG_REC.file), 'new record added');
|
||
assert.equal(merged._meta.count, 2);
|
||
});
|
||
|
||
test('DEFAULT_MANIFEST_PATH is a git-tracked-able path under scripts/kb-eval/data/', () => {
|
||
assert.ok(DEFAULT_MANIFEST_PATH.includes(join('scripts', 'kb-eval', 'data')), DEFAULT_MANIFEST_PATH);
|
||
assert.ok(DEFAULT_MANIFEST_PATH.endsWith('judge-pass-manifest.json'));
|
||
});
|
||
|
||
// --- IO-shell CLI ------------------------------------------------------------------
|
||
|
||
test('CLI --write persists a parseable, schema-valid ledger to --out', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
|
||
const m = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(validateManifest(m).valid, true);
|
||
assert.equal(Array.isArray(m.files), true);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI --write refuses (exit 3, file byte-identical) when the ledger already carries judged records', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
|
||
const before = readFileSync(out, 'utf8');
|
||
let status = 0;
|
||
try {
|
||
execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
|
||
} catch (err) {
|
||
status = err.status;
|
||
}
|
||
assert.equal(status, 3);
|
||
assert.equal(readFileSync(out, 'utf8'), before, 'ledger must be untouched on re-entrance refusal');
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI --write --merge preserves judged records verbatim', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
|
||
execFileSync('node', [CLI, '--write', '--merge', '--out', out], { encoding: 'utf8' });
|
||
const m = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(m.files.length, 1);
|
||
assert.deepEqual(m.files[0], PASS_REC);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI unknown flag → usage error exit 2', () => {
|
||
let status = 0;
|
||
try {
|
||
execFileSync('node', [CLI, '--bogus'], { encoding: 'utf8' });
|
||
} catch (err) {
|
||
status = err.status;
|
||
}
|
||
assert.equal(status, 2);
|
||
});
|
||
|
||
// --- R11 flag format (R6 Step 5): the judge-pass → fix-work-list contract -----------
|
||
// A flag = a judged claim the pass could not ground. The pass augments the judge output
|
||
// {id, judge_verdict, rule, evidence_url, evidence_quote, reason} with {file, line, claim,
|
||
// disposition}. Two vocabularies must never be conflated: judge_verdict (grounded/
|
||
// not_grounded/source_silent) vs disposition (base-rate VERDICTS; ERROR_VERDICTS = R11 fix
|
||
// targets). Spec: docs/r11-flag-format-2026-07.md.
|
||
|
||
const JUDGE_VERDICTS = ['grounded', 'not_grounded', 'source_silent'];
|
||
const DISPOSITIONS = ['correct', 'outdated', 'wrong', 'unsourced']; // mirrors base-rate VERDICTS
|
||
const ERROR_DISPOSITIONS = ['outdated', 'wrong']; // mirrors base-rate ERROR_VERDICTS (R11 fix targets)
|
||
|
||
const FLAG = {
|
||
id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3',
|
||
evidence_url: 'https://learn.microsoft.com/azure/ai-foundry/models', evidence_quote: 'GPT-4o mini context 128k',
|
||
reason: 'File claims X tokens; source states Y.',
|
||
file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md', line: 42,
|
||
claim: 'DeepSeek-R1 131072 tokens', disposition: 'outdated',
|
||
};
|
||
|
||
test('flag record carries the judge output fields + the four augmented fields', () => {
|
||
for (const k of ['id', 'judge_verdict', 'rule', 'evidence_url', 'evidence_quote', 'reason', 'file', 'line', 'claim', 'disposition']) {
|
||
assert.ok(k in FLAG, `flag record missing '${k}'`);
|
||
}
|
||
assert.ok(JUDGE_VERDICTS.includes(FLAG.judge_verdict) && FLAG.judge_verdict !== 'grounded', 'a flag is never grounded');
|
||
assert.match(FLAG.rule, /^(R[1-8])?$/); // R1-R8 or empty
|
||
assert.ok(DISPOSITIONS.includes(FLAG.disposition));
|
||
});
|
||
|
||
test('a flagged ledger record built from a flag validates (Step 4 ↔ Step 5 contract)', () => {
|
||
const rec = {
|
||
file: FLAG.file, batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged',
|
||
claim_count: 8, verified: null, verified_by: null, flags: [FLAG],
|
||
};
|
||
assert.equal(validateManifest(appendJudgedFile(scaffoldManifest(), rec)).valid, true);
|
||
});
|
||
|
||
test('ERROR dispositions (outdated/wrong) are the R11 fix targets; unsourced/correct are not', () => {
|
||
assert.deepEqual(ERROR_DISPOSITIONS, ['outdated', 'wrong']);
|
||
assert.ok(!ERROR_DISPOSITIONS.includes('unsourced'));
|
||
assert.ok(!ERROR_DISPOSITIONS.includes('correct'));
|
||
});
|
||
|
||
test('disposition vocabulary is parity-checked against base-rate.mjs source (no silent drift)', () => {
|
||
// base-rate.mjs is the source of truth for the correctness vocabulary; the flag disposition
|
||
// reuses it (its enum is module-private, so we assert against the source rather than duplicate
|
||
// a live copy). If base-rate's vocabulary changes, this fails → drift is caught.
|
||
const src = readFileSync(join(__dirname, '..', '..', 'scripts', 'kb-eval', 'lib', 'base-rate.mjs'), 'utf8');
|
||
for (const v of DISPOSITIONS) assert.match(src, new RegExp(`'${v}'`), `base-rate must define '${v}'`);
|
||
assert.match(src, /ERROR_VERDICTS\s*=\s*new Set\(\[\s*'outdated'\s*,\s*'wrong'\s*\]\)/);
|
||
});
|
||
|
||
test('the R11 flag-format spec documents judge_verdict + the disposition mapping', () => {
|
||
const doc = readFileSync(join(__dirname, '..', '..', 'docs', 'r11-flag-format-2026-07.md'), 'utf8');
|
||
assert.match(doc, /judge_verdict/);
|
||
assert.match(doc, /not_grounded/);
|
||
assert.match(doc, /disposition/);
|
||
for (const v of DISPOSITIONS) assert.ok(doc.includes(v), `doc must document disposition '${v}'`);
|
||
});
|