feat(ms-ai-architect): R7.1 — build-judge-payloads får --claims <path> (per-batch korpus-manifest, TDD; default = bakeoff-manifestet uendret) [skip-docs]

scripts/kb-eval/build-judge-payloads.mjs

@@ -10,14 +10,15 @@
 // apples-to-apples and a fresh session can resume with one command, no improvising.
 //
 // Pure string assembly — no LLM, no network, no math. Reads:
-//   data/judge-bakeoff-claims.json   (blind manifest from extract-judge-claims.mjs)
+//   <--claims> (default data/judge-bakeoff-claims.json — the bake-off blind manifest;
+//               R7–R10 corpus batches pass their per-batch extracted claims manifest here)
 //   <--prompt>                        (judge-claim-prompt-vN.md, with <FILE>/<CLAIMS>)
 // Writes (with --write):
 //   data/<--out>                      (array of {file, claim_count, prompt})
 //
 // Usage:
 //   node scripts/kb-eval/build-judge-payloads.mjs --prompt judge-claim-prompt-v3.1.md \
-//        --out judge-bakeoff-payloads-v3.1.json [--write]
+//        [--claims <path>] --out judge-bakeoff-payloads-v3.1.json [--write]
 //   (default: print per-file claim counts + a sanity sample; --write persists)
 
 import fs from 'node:fs';
@@ -48,7 +49,15 @@ if (!template.includes('<FILE>') || !template.includes('<CLAIMS>')) {
   process.exit(2);
 }
 
-const manifest = JSON.parse(fs.readFileSync(path.join(DATA, 'judge-bakeoff-claims.json'), 'utf8'));
+const claimsFlag = flag('--claims');
+const claimsPath = claimsFlag
+  ? (path.isAbsolute(claimsFlag) ? claimsFlag : path.resolve(process.cwd(), claimsFlag))
+  : path.join(DATA, 'judge-bakeoff-claims.json');
+if (!fs.existsSync(claimsPath)) {
+  console.error(`error: claims manifest not found: ${claimsPath}`);
+  process.exit(2);
+}
+const manifest = JSON.parse(fs.readFileSync(claimsPath, 'utf8'));
 const claims = manifest.claims || [];
 
 // Group by file, preserving manifest order (deterministic).

tests/kb-eval/test-build-judge-payloads.test.mjs

@@ -0,0 +1,71 @@
+// test-build-judge-payloads.test.mjs — R7.1: build-judge-payloads.mjs must accept an
+// alternate claims manifest via --claims <path>, so the R7–R10 corpus pass can feed its
+// per-batch extracted claims through the SAME deterministic payload construction the
+// bake-off used (same grouping, same <FILE>/<CLAIMS> substitution — apples-to-apples).
+// Default (no --claims) stays the bake-off manifest data/judge-bakeoff-claims.json.
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CLI = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'build-judge-payloads.mjs');
+
+function withTmp(fn) {
+  const dir = mkdtempSync(join(tmpdir(), 'bjp-'));
+  try { return fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); }
+}
+
+const TEMPLATE = 'Judge file <FILE> now.\n\n<CLAIMS>\n';
+const CLAIMS = {
+  _meta: { source: 'r7-test' },
+  claims: [
+    { id: 'a.md#1', file: 'skills/x/references/a.md', line: 10, claim: 'v1', claim_type: 'version', evidence_url: 'https://learn.microsoft.com/a' },
+    { id: 'b.md#1', file: 'skills/x/references/b.md', line: 5, claim: 'GA', claim_type: 'status', evidence_url: null },
+    { id: 'a.md#2', file: 'skills/x/references/a.md', line: 20, claim: 'r2', claim_type: 'region', evidence_url: 'https://learn.microsoft.com/a2' },
+  ],
+};
+
+test('--claims <path> builds payloads from an alternate manifest (R7 corpus batches)', () => {
+  withTmp((dir) => {
+    const claimsPath = join(dir, 'r7-claims.json');
+    const promptPath = join(dir, 'prompt.md');
+    writeFileSync(claimsPath, JSON.stringify(CLAIMS));
+    writeFileSync(promptPath, TEMPLATE);
+    const outName = 'r7-payloads-test.json';
+    const outPath = join(__dirname, '..', '..', 'scripts', 'kb-eval', 'data', outName);
+    try {
+      const stdout = execFileSync(
+        'node', [CLI, '--prompt', promptPath, '--claims', claimsPath, '--out', outName, '--write'],
+        { encoding: 'utf8' },
+      );
+      assert.match(stdout, /2 per-file payloads, 3 claims total/);
+      const payloads = JSON.parse(readFileSync(outPath, 'utf8'));
+      assert.equal(payloads.length, 2);
+      // Grouping preserves manifest order; both a.md claims land in one payload.
+      assert.equal(payloads[0].file, 'skills/x/references/a.md');
+      assert.equal(payloads[0].claim_count, 2);
+      assert.equal(payloads[1].file, 'skills/x/references/b.md');
+      // Substitution: <FILE> and <CLAIMS> replaced, claims JSON embedded verbatim fields.
+      assert.ok(payloads[0].prompt.includes('Judge file skills/x/references/a.md now.'));
+      assert.ok(payloads[0].prompt.includes('"id": "a.md#1"'));
+      assert.ok(!payloads[0].prompt.includes('<CLAIMS>'));
+    } finally {
+      if (existsSync(outPath)) rmSync(outPath);
+    }
+  });
+});
+
+test('--claims with a missing file exits non-zero (no silent fallback to the bake-off manifest)', () => {
+  withTmp((dir) => {
+    const promptPath = join(dir, 'prompt.md');
+    writeFileSync(promptPath, TEMPLATE);
+    assert.throws(() =>
+      execFileSync('node', [CLI, '--prompt', promptPath, '--claims', join(dir, 'nope.json')], { encoding: 'utf8', stdio: 'pipe' }),
+    );
+  });
+});