172 lines
7.9 KiB
JavaScript
172 lines
7.9 KiB
JavaScript
// tests/kb-eval/test-judge-pass-manifest.test.mjs
|
||
// R6 Step 4 — the R7–R10 judge-pass LEDGER: a durable per-file record of the corpus judge
|
||
// pass. The two NFR-load-bearing primitives are appendJudgedFile (write ONE record after each
|
||
// judged file → max one file lost on crash) and pendingFiles (resume = skip already-judged).
|
||
// Pure core (validateManifest / mergeBatch / appendJudgedFile / pendingFiles) + an IO-shell
|
||
// CLI (--json / --write [--merge], exit 3 re-entrance guard, exit 2 unknown flag), mirroring
|
||
// the classify-ref-type.mjs manifest-CLI shape.
|
||
|
||
import { test } from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { execFileSync } from 'node:child_process';
|
||
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { dirname, join } from 'node:path';
|
||
import { atomicWriteJson } from '../../scripts/kb-update/lib/atomic-write.mjs';
|
||
import {
|
||
scaffoldManifest,
|
||
validateManifest,
|
||
appendJudgedFile,
|
||
mergeBatch,
|
||
pendingFiles,
|
||
DEFAULT_MANIFEST_PATH,
|
||
} from '../../scripts/kb-eval/judge-pass-manifest.mjs';
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'judge-pass-manifest.mjs');
|
||
|
||
// A born-verified `pass` record: every judgeable claim grounded (AND) → surgical stamp.
|
||
const PASS_REC = {
|
||
file: 'skills/ms-ai-engineering/references/rag/azure-ai-search.md',
|
||
batch: 1, judged_at: '2026-07-04', per_file_verdict: 'pass',
|
||
claim_count: 9, verified: '2026-07-04', verified_by: 'judge-v3.1', flags: [],
|
||
};
|
||
// A `flagged` record: ≥1 not-grounded claim → R11 work-list, never a stamp.
|
||
const FLAG_REC = {
|
||
file: 'skills/ms-ai-advisor/references/platforms/azure-ai-foundry.md',
|
||
batch: 1, judged_at: '2026-07-04', per_file_verdict: 'flagged',
|
||
claim_count: 8, verified: null, verified_by: null,
|
||
flags: [{ id: 'azure-ai-foundry.md#3', judge_verdict: 'not_grounded', rule: 'R3', line: 42, claim: 'X', disposition: 'outdated' }],
|
||
};
|
||
|
||
test('appendJudgedFile → validateManifest valid, files path-sorted, count refreshed', () => {
|
||
let m = scaffoldManifest();
|
||
m = appendJudgedFile(m, FLAG_REC); // skills/ms-ai-advisor/...
|
||
m = appendJudgedFile(m, PASS_REC); // skills/ms-ai-engineering/...
|
||
const v = validateManifest(m);
|
||
assert.equal(v.valid, true, v.errors.join('; '));
|
||
const paths = m.files.map((r) => r.file);
|
||
assert.deepEqual(paths, [...paths].sort(), 'files must be path-sorted (deterministic ledger)');
|
||
assert.equal(m._meta.count, 2);
|
||
});
|
||
|
||
test('validateManifest rejects a pass without a stamp, a bad verdict, a flagged without flags, and duplicates', () => {
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'pass', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'maybe', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [{ file: 'a.md', per_file_verdict: 'flagged', flags: [] }] }).valid, false);
|
||
assert.equal(validateManifest({ _meta: {}, files: [PASS_REC, PASS_REC] }).valid, false);
|
||
assert.equal(validateManifest(scaffoldManifest()).valid, true); // empty scaffold is valid
|
||
});
|
||
|
||
test('appendJudgedFile is idempotent per file (re-judge replaces, never duplicates)', () => {
|
||
let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
const rejudged = { ...PASS_REC, judged_at: '2026-07-05', verified: '2026-07-05' };
|
||
m = appendJudgedFile(m, rejudged);
|
||
assert.equal(m.files.length, 1);
|
||
assert.equal(m.files[0].verified, '2026-07-05');
|
||
});
|
||
|
||
test('appendJudgedFile persists ONE record durably, readable after (crash-resume simulation)', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
// file 1 judged → write atomically → "crash" → reload → file 2 judged → write again
|
||
atomicWriteJson(out, appendJudgedFile(scaffoldManifest(), PASS_REC));
|
||
const reloaded = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(reloaded.files.length, 1);
|
||
assert.equal(reloaded.files[0].file, PASS_REC.file);
|
||
atomicWriteJson(out, appendJudgedFile(reloaded, FLAG_REC));
|
||
const after = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(after.files.length, 2);
|
||
assert.equal(validateManifest(after).valid, true);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('pendingFiles returns only not-yet-judged worklist files (resume skips judged)', () => {
|
||
const NEW = 'skills/ms-ai-security/references/x/new.md';
|
||
const worklist = [PASS_REC.file, FLAG_REC.file, NEW];
|
||
let m = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
m = appendJudgedFile(m, FLAG_REC);
|
||
assert.deepEqual(pendingFiles(m, worklist), [NEW]);
|
||
assert.deepEqual(pendingFiles(scaffoldManifest(), worklist), worklist, 'nothing judged → all pending');
|
||
});
|
||
|
||
test('mergeBatch adds not-yet-judged records and preserves judged verbatim (judged wins)', () => {
|
||
const judgedLedger = appendJudgedFile(scaffoldManifest(), PASS_REC);
|
||
const incoming = [
|
||
// same file as the judged record, different verdict → must NOT overwrite the judged entry
|
||
{ ...PASS_REC, per_file_verdict: 'flagged', verified: null, verified_by: null,
|
||
flags: [{ id: 'z', judge_verdict: 'not_grounded', rule: 'R1' }] },
|
||
FLAG_REC, // a new file → added
|
||
];
|
||
const merged = mergeBatch(judgedLedger, incoming);
|
||
assert.deepEqual(merged.files.find((r) => r.file === PASS_REC.file), PASS_REC, 'judged preserved verbatim');
|
||
assert.ok(merged.files.some((r) => r.file === FLAG_REC.file), 'new record added');
|
||
assert.equal(merged._meta.count, 2);
|
||
});
|
||
|
||
test('DEFAULT_MANIFEST_PATH is a git-tracked-able path under scripts/kb-eval/data/', () => {
|
||
assert.ok(DEFAULT_MANIFEST_PATH.includes(join('scripts', 'kb-eval', 'data')), DEFAULT_MANIFEST_PATH);
|
||
assert.ok(DEFAULT_MANIFEST_PATH.endsWith('judge-pass-manifest.json'));
|
||
});
|
||
|
||
// --- IO-shell CLI ------------------------------------------------------------------
|
||
|
||
test('CLI --write persists a parseable, schema-valid ledger to --out', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
|
||
const m = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(validateManifest(m).valid, true);
|
||
assert.equal(Array.isArray(m.files), true);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI --write refuses (exit 3, file byte-identical) when the ledger already carries judged records', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
|
||
const before = readFileSync(out, 'utf8');
|
||
let status = 0;
|
||
try {
|
||
execFileSync('node', [CLI, '--write', '--out', out], { encoding: 'utf8' });
|
||
} catch (err) {
|
||
status = err.status;
|
||
}
|
||
assert.equal(status, 3);
|
||
assert.equal(readFileSync(out, 'utf8'), before, 'ledger must be untouched on re-entrance refusal');
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI --write --merge preserves judged records verbatim', () => {
|
||
const tmp = mkdtempSync(join(tmpdir(), 'jpm-'));
|
||
const out = join(tmp, 'ledger.json');
|
||
try {
|
||
writeFileSync(out, JSON.stringify(appendJudgedFile(scaffoldManifest(), PASS_REC), null, 2) + '\n');
|
||
execFileSync('node', [CLI, '--write', '--merge', '--out', out], { encoding: 'utf8' });
|
||
const m = JSON.parse(readFileSync(out, 'utf8'));
|
||
assert.equal(m.files.length, 1);
|
||
assert.deepEqual(m.files[0], PASS_REC);
|
||
} finally {
|
||
rmSync(tmp, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test('CLI unknown flag → usage error exit 2', () => {
|
||
let status = 0;
|
||
try {
|
||
execFileSync('node', [CLI, '--bogus'], { encoding: 'utf8' });
|
||
} catch (err) {
|
||
status = err.status;
|
||
}
|
||
assert.equal(status, 2);
|
||
});
|