98 lines
4.2 KiB
JavaScript
98 lines
4.2 KiB
JavaScript
#!/usr/bin/env node
|
||
// build-judge-payloads.mjs — deterministic fan-out prep for the per-claim
|
||
// groundedness judge bake-off. Turns the blind claim manifest + a judge prompt
|
||
// template into 45 ready-to-dispatch subagent payloads (one per file), so the
|
||
// fan-out is reproducible instead of hand-assembled in main context.
|
||
//
|
||
// The v3 fan-out grouped claims by file MANUALLY at dispatch time; that made the
|
||
// exact payloads non-reproducible. This script fixes the construction: same prompt,
|
||
// same per-file claim grouping, every run — so v3.1 (and any future arm) is
|
||
// apples-to-apples and a fresh session can resume with one command, no improvising.
|
||
//
|
||
// Pure string assembly — no LLM, no network, no math. Reads:
|
||
// <--claims> (default data/judge-bakeoff-claims.json — the bake-off blind manifest;
|
||
// R7–R10 corpus batches pass their per-batch extracted claims manifest here)
|
||
// <--prompt> (judge-claim-prompt-vN.md, with <FILE>/<CLAIMS>)
|
||
// Writes (with --write):
|
||
// data/<--out> (array of {file, claim_count, prompt})
|
||
//
|
||
// Usage:
|
||
// node scripts/kb-eval/build-judge-payloads.mjs --prompt judge-claim-prompt-v3.1.md \
|
||
// [--claims <path>] --out judge-bakeoff-payloads-v3.1.json [--write]
|
||
// (default: print per-file claim counts + a sanity sample; --write persists)
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const DATA = path.join(__dirname, 'data');
|
||
|
||
function flag(name, def) {
|
||
const i = process.argv.indexOf(name);
|
||
return i >= 0 ? process.argv[i + 1] : def;
|
||
}
|
||
|
||
const promptName = flag('--prompt');
|
||
if (!promptName) {
|
||
console.error('error: --prompt <judge-claim-prompt-vN.md> is required');
|
||
process.exit(2);
|
||
}
|
||
const promptPath = path.isAbsolute(promptName) ? promptName : path.join(__dirname, promptName);
|
||
if (!fs.existsSync(promptPath)) {
|
||
console.error(`error: prompt template not found: ${promptPath}`);
|
||
process.exit(2);
|
||
}
|
||
const template = fs.readFileSync(promptPath, 'utf8');
|
||
if (!template.includes('<FILE>') || !template.includes('<CLAIMS>')) {
|
||
console.error('error: prompt template must contain both <FILE> and <CLAIMS> placeholders');
|
||
process.exit(2);
|
||
}
|
||
|
||
const claimsFlag = flag('--claims');
|
||
const claimsPath = claimsFlag
|
||
? (path.isAbsolute(claimsFlag) ? claimsFlag : path.resolve(process.cwd(), claimsFlag))
|
||
: path.join(DATA, 'judge-bakeoff-claims.json');
|
||
if (!fs.existsSync(claimsPath)) {
|
||
console.error(`error: claims manifest not found: ${claimsPath}`);
|
||
process.exit(2);
|
||
}
|
||
const manifest = JSON.parse(fs.readFileSync(claimsPath, 'utf8'));
|
||
const claims = manifest.claims || [];
|
||
|
||
// Group by file, preserving manifest order (deterministic).
|
||
const byFile = new Map();
|
||
for (const c of claims) {
|
||
if (!byFile.has(c.file)) byFile.set(c.file, []);
|
||
byFile.get(c.file).push({
|
||
id: c.id,
|
||
claim: c.claim,
|
||
claim_type: c.claim_type,
|
||
evidence_url: c.evidence_url || null,
|
||
});
|
||
}
|
||
|
||
const payloads = [];
|
||
for (const [file, fileClaims] of byFile) {
|
||
const claimsBlock = JSON.stringify(fileClaims, null, 2);
|
||
const prompt = template.split('<FILE>').join(file).split('<CLAIMS>').join(claimsBlock);
|
||
payloads.push({ file, claim_count: fileClaims.length, prompt });
|
||
}
|
||
|
||
const totalClaims = payloads.reduce((s, p) => s + p.claim_count, 0);
|
||
|
||
if (process.argv.includes('--write')) {
|
||
const outName = flag('--out', `judge-bakeoff-payloads-${path.basename(promptName).replace(/^judge-claim-prompt-/, '').replace(/\.md$/, '')}.json`);
|
||
const outPath = path.join(DATA, outName);
|
||
fs.writeFileSync(outPath, JSON.stringify(payloads, null, 2) + '\n');
|
||
console.log(`wrote ${outPath}`);
|
||
console.log(`${payloads.length} per-file payloads, ${totalClaims} claims total, prompt=${promptName}`);
|
||
} else {
|
||
console.log(`prompt template: ${promptName}`);
|
||
console.log(`${payloads.length} files, ${totalClaims} claims total`);
|
||
console.log('per-file claim counts:');
|
||
for (const p of payloads) console.log(` ${p.claim_count.toString().padStart(2)} ${p.file}`);
|
||
console.log(`\nsample payload[0] head (file=${payloads[0].file}):`);
|
||
console.log(payloads[0].prompt.slice(0, 200) + ' …');
|
||
console.log('\n(dry run — pass --write to persist)');
|
||
}
|