// test-build-judge-payloads.test.mjs — R7.1: build-judge-payloads.mjs must accept an // alternate claims manifest via --claims , so the R7–R10 corpus pass can feed its // per-batch extracted claims through the SAME deterministic payload construction the // bake-off used (same grouping, same / substitution — apples-to-apples). // Default (no --claims) stays the bake-off manifest data/judge-bakeoff-claims.json. import test from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'build-judge-payloads.mjs'); function withTmp(fn) { const dir = mkdtempSync(join(tmpdir(), 'bjp-')); try { return fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); } } const TEMPLATE = 'Judge file now.\n\n\n'; const CLAIMS = { _meta: { source: 'r7-test' }, claims: [ { id: 'a.md#1', file: 'skills/x/references/a.md', line: 10, claim: 'v1', claim_type: 'version', evidence_url: 'https://learn.microsoft.com/a' }, { id: 'b.md#1', file: 'skills/x/references/b.md', line: 5, claim: 'GA', claim_type: 'status', evidence_url: null }, { id: 'a.md#2', file: 'skills/x/references/a.md', line: 20, claim: 'r2', claim_type: 'region', evidence_url: 'https://learn.microsoft.com/a2' }, ], }; test('--claims builds payloads from an alternate manifest (R7 corpus batches)', () => { withTmp((dir) => { const claimsPath = join(dir, 'r7-claims.json'); const promptPath = join(dir, 'prompt.md'); writeFileSync(claimsPath, JSON.stringify(CLAIMS)); writeFileSync(promptPath, TEMPLATE); const outName = 'r7-payloads-test.json'; const outPath = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', outName); try { const stdout = execFileSync( 'node', [CLI, '--prompt', promptPath, '--claims', claimsPath, '--out', outName, '--write'], { encoding: 'utf8' }, ); assert.match(stdout, /2 per-file payloads, 3 claims total/); const payloads = JSON.parse(readFileSync(outPath, 'utf8')); assert.equal(payloads.length, 2); // Grouping preserves manifest order; both a.md claims land in one payload. assert.equal(payloads[0].file, 'skills/x/references/a.md'); assert.equal(payloads[0].claim_count, 2); assert.equal(payloads[1].file, 'skills/x/references/b.md'); // Substitution: and replaced, claims JSON embedded verbatim fields. assert.ok(payloads[0].prompt.includes('Judge file skills/x/references/a.md now.')); assert.ok(payloads[0].prompt.includes('"id": "a.md#1"')); assert.ok(!payloads[0].prompt.includes('')); } finally { if (existsSync(outPath)) rmSync(outPath); } }); }); test('--claims with a missing file exits non-zero (no silent fallback to the bake-off manifest)', () => { withTmp((dir) => { const promptPath = join(dir, 'prompt.md'); writeFileSync(promptPath, TEMPLATE); assert.throws(() => execFileSync('node', [CLI, '--prompt', promptPath, '--claims', join(dir, 'nope.json')], { encoding: 'utf8', stdio: 'pipe' }), ); }); });