open/ms-ai-architect
1
0
Fork 0
Code Issues Pull requests Releases Activity

feat(ms-ai-architect): R7.1 — build-judge-payloads får --claims <path> (per-batch korpus-manifest, TDD; default = bakeoff-manifestet uendret) [skip-docs]

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-07-18 11:16:22 +02:00
commit 312c3c0369
2 changed files with 83 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
scripts/kb-eval/build-judge-payloads.mjs
View file

@ -10,14 +10,15 @@
// apples-to-apples and a fresh session can resume with one command, no improvising.
//
// Pure string assembly — no LLM, no network, no math. Reads:
// data/judge-bakeoff-claims.json (blind manifest from extract-judge-claims.mjs)
// <--claims> (default data/judge-bakeoff-claims.json — the bake-off blind manifest;
// R7–R10 corpus batches pass their per-batch extracted claims manifest here)
// <--prompt> (judge-claim-prompt-vN.md, with <FILE>/<CLAIMS>)
// Writes (with --write):
// data/<--out> (array of {file, claim_count, prompt})
//
// Usage:
// node scripts/kb-eval/build-judge-payloads.mjs --prompt judge-claim-prompt-v3.1.md \
// --out judge-bakeoff-payloads-v3.1.json [--write]
// [--claims <path>] --out judge-bakeoff-payloads-v3.1.json [--write]
// (default: print per-file claim counts + a sanity sample; --write persists)
import fs from 'node:fs';
@ -48,7 +49,15 @@ if (!template.includes('<FILE>') || !template.includes('<CLAIMS>')) {
process.exit(2);
}
const manifest = JSON.parse(fs.readFileSync(path.join(DATA, 'judge-bakeoff-claims.json'), 'utf8'));
const claimsFlag = flag('--claims');
const claimsPath = claimsFlag
? (path.isAbsolute(claimsFlag) ? claimsFlag : path.resolve(process.cwd(), claimsFlag))
: path.join(DATA, 'judge-bakeoff-claims.json');
if (!fs.existsSync(claimsPath)) {
console.error(`error: claims manifest not found: ${claimsPath}`);
process.exit(2);
}
const manifest = JSON.parse(fs.readFileSync(claimsPath, 'utf8'));
const claims = manifest.claims || [];
// Group by file, preserving manifest order (deterministic).