71 lines
3.4 KiB
JavaScript
71 lines
3.4 KiB
JavaScript
// test-build-judge-payloads.test.mjs — R7.1: build-judge-payloads.mjs must accept an
|
||
// alternate claims manifest via --claims <path>, so the R7–R10 corpus pass can feed its
|
||
// per-batch extracted claims through the SAME deterministic payload construction the
|
||
// bake-off used (same grouping, same <FILE>/<CLAIMS> substitution — apples-to-apples).
|
||
// Default (no --claims) stays the bake-off manifest data/judge-bakeoff-claims.json.
|
||
|
||
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { execFileSync } from 'node:child_process';
|
||
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
|
||
import { tmpdir } from 'node:os';
|
||
import { join, dirname } from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||
const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'build-judge-payloads.mjs');
|
||
|
||
function withTmp(fn) {
|
||
const dir = mkdtempSync(join(tmpdir(), 'bjp-'));
|
||
try { return fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); }
|
||
}
|
||
|
||
const TEMPLATE = 'Judge file <FILE> now.\n\n<CLAIMS>\n';
|
||
const CLAIMS = {
|
||
_meta: { source: 'r7-test' },
|
||
claims: [
|
||
{ id: 'a.md#1', file: 'skills/x/references/a.md', line: 10, claim: 'v1', claim_type: 'version', evidence_url: 'https://learn.microsoft.com/a' },
|
||
{ id: 'b.md#1', file: 'skills/x/references/b.md', line: 5, claim: 'GA', claim_type: 'status', evidence_url: null },
|
||
{ id: 'a.md#2', file: 'skills/x/references/a.md', line: 20, claim: 'r2', claim_type: 'region', evidence_url: 'https://learn.microsoft.com/a2' },
|
||
],
|
||
};
|
||
|
||
test('--claims <path> builds payloads from an alternate manifest (R7 corpus batches)', () => {
|
||
withTmp((dir) => {
|
||
const claimsPath = join(dir, 'r7-claims.json');
|
||
const promptPath = join(dir, 'prompt.md');
|
||
writeFileSync(claimsPath, JSON.stringify(CLAIMS));
|
||
writeFileSync(promptPath, TEMPLATE);
|
||
const outName = 'r7-payloads-test.json';
|
||
const outPath = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', outName);
|
||
try {
|
||
const stdout = execFileSync(
|
||
'node', [CLI, '--prompt', promptPath, '--claims', claimsPath, '--out', outName, '--write'],
|
||
{ encoding: 'utf8' },
|
||
);
|
||
assert.match(stdout, /2 per-file payloads, 3 claims total/);
|
||
const payloads = JSON.parse(readFileSync(outPath, 'utf8'));
|
||
assert.equal(payloads.length, 2);
|
||
// Grouping preserves manifest order; both a.md claims land in one payload.
|
||
assert.equal(payloads[0].file, 'skills/x/references/a.md');
|
||
assert.equal(payloads[0].claim_count, 2);
|
||
assert.equal(payloads[1].file, 'skills/x/references/b.md');
|
||
// Substitution: <FILE> and <CLAIMS> replaced, claims JSON embedded verbatim fields.
|
||
assert.ok(payloads[0].prompt.includes('Judge file skills/x/references/a.md now.'));
|
||
assert.ok(payloads[0].prompt.includes('"id": "a.md#1"'));
|
||
assert.ok(!payloads[0].prompt.includes('<CLAIMS>'));
|
||
} finally {
|
||
if (existsSync(outPath)) rmSync(outPath);
|
||
}
|
||
});
|
||
});
|
||
|
||
test('--claims with a missing file exits non-zero (no silent fallback to the bake-off manifest)', () => {
|
||
withTmp((dir) => {
|
||
const promptPath = join(dir, 'prompt.md');
|
||
writeFileSync(promptPath, TEMPLATE);
|
||
assert.throws(() =>
|
||
execFileSync('node', [CLI, '--prompt', promptPath, '--claims', join(dir, 'nope.json')], { encoding: 'utf8', stdio: 'pipe' }),
|
||
);
|
||
});
|
||
});
|