feat(voyage): S10 part B — NW2 full bake-off (rich fixture) → verdict POSITIVE
Run the full T2 §5 prose-vs-Workflow /trekreview bake-off (operator GO, choice "a"): 3 runs/arm on a rich-finding JWT-auth fixture, resolving the smoke's 0-finding limitation. Deliverables: - tests/fixtures/bakeoff-rich/ — JWT-auth brief + diff with 5 seeded blatant, brief-traceable issues (varied severity/rule_key, one dual-flaggable). - scripts/bakeoff-armA-merge.mjs — Arm A (prose) validate (NW1) + triplet-dedup, matching Arm B's dedup exactly. - scripts/bakeoff-fidelity.mjs — cross-arm + within-arm + granularity-ladder fidelity analysis over the structured arm outputs. - docs/T2-bakeoff-results.md §Full run — the T2 §5 verdict. Result (3 runs/arm, both arms ran the coordinator): - Verdict fidelity EQUIVALENT — all 6 runs BLOCK, cross-arm verdict-match 1.0. - Finding-set: substrate is fidelity-neutral. Cross-arm jaccard 0.41 (triplet) → 0.71 (file,rule_key) → 1.0 (file); cross-arm ≈ within-arm at every granularity. Issue coverage 5/5 in 6/6 runs. Low triplet jaccard is line-citation noise shared by both arms, not a substrate effect. - Token +4.4% (Arm B vs A; <=+15%). Classifier interference 0 at 9-agent concurrency. JSON-robustness: Arm B schema-forced; Arm A 6/6 valid via NW1. - VERDICT POSITIVE → S11 proceeds with opt-in --workflow flag. Caveat (per plan posture): strict triplet-jaccard>=0.7 flag is 0/9, a metric-calibration artifact (both arms sub-0.7 against themselves), not a regression. Residual: F4 auto/bypass explicit-mode check (mode not settable in-session). Suite green (662/660 pass, 2 skip); plugin validate clean (modulo the pre-existing root-CLAUDE.md warning). No production code changed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
This commit is contained in:
parent
869bf318d2
commit
f7c8aa45ab
6 changed files with 537 additions and 4 deletions
101
scripts/bakeoff-armA-merge.mjs
Normal file
101
scripts/bakeoff-armA-merge.mjs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
// scripts/bakeoff-armA-merge.mjs
|
||||
// NW2 bake-off — Arm A (prose path) merge step.
|
||||
//
|
||||
// Arm A is the CURRENT /trekreview Phase 5–6 substrate: main spawns the two
|
||||
// reviewers FOREGROUND (Agent tool), each emits a trailing fenced ```json block
|
||||
// of findings (the prose contract), and main hand-validates + dedups before
|
||||
// spawning the coordinator. This script is that hand-validate + dedup, made
|
||||
// reproducible:
|
||||
//
|
||||
// for each reviewer raw-output file:
|
||||
// validateReviewerOutput() (NW1 — extract last json fence, parse, schema-check)
|
||||
// collect findings from VALID reviewers
|
||||
// triplet-dedup by (file,line,rule_key) [SAME logic as Arm B's dedupByTriplet,
|
||||
// so the two arms dedup identically —
|
||||
// triplet-only, no jaccard pass-2]
|
||||
// → emit { merged, validation, raw_finding_count, deduped_count }
|
||||
//
|
||||
// The merged findings are then handed to the review-coordinator (spawned by main),
|
||||
// mirroring Arm B's coordinatorPrompt input. JSON-robustness metric = the
|
||||
// `validation` report (parse/schema failures + which codes).
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/bakeoff-armA-merge.mjs [--json] <reviewerA.txt> <reviewerB.txt> ...
|
||||
// → stdout: merged findings JSON (or full report with --json). Exit 0 always
|
||||
// (validation failures are reported in-band, not as a nonzero exit).
|
||||
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { validateReviewerOutput } from '../lib/review/findings-schema.mjs';
|
||||
|
||||
// Triplet dedup — byte-for-byte the same key + first-wins policy as
|
||||
// scripts/trekreview-armB.workflow.mjs `dedupByTriplet`, so neither arm gets a
|
||||
// dedup advantage. Triplet-only (file,line,rule_key); NO jaccard pass-2.
|
||||
export function dedupByTriplet(findings) {
|
||||
const seen = new Map();
|
||||
for (const f of findings) {
|
||||
const key = `${f.file} ${f.line} ${f.rule_key}`;
|
||||
if (!seen.has(key)) seen.set(key, f);
|
||||
}
|
||||
return [...seen.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge an array of reviewer raw-output texts into the coordinator's input.
|
||||
* @param {Array<{label?: string, text: string}>} reviewerOutputs
|
||||
* @returns {{ merged, validation, raw_finding_count, deduped_count }}
|
||||
*/
|
||||
export function mergeArmA(reviewerOutputs) {
|
||||
const validation = [];
|
||||
const allFindings = [];
|
||||
for (const { label, text } of reviewerOutputs) {
|
||||
const r = validateReviewerOutput(text);
|
||||
validation.push({
|
||||
label: label || null,
|
||||
valid: r.valid,
|
||||
finding_count: Array.isArray(r.parsed?.findings) ? r.parsed.findings.length : 0,
|
||||
error_codes: (r.errors || []).map((e) => e.code),
|
||||
warning_codes: (r.warnings || []).map((w) => w.code),
|
||||
});
|
||||
// Mirror Phase 5: only VALID reviewer output flows to the coordinator.
|
||||
if (r.valid && Array.isArray(r.parsed?.findings)) {
|
||||
allFindings.push(...r.parsed.findings);
|
||||
}
|
||||
}
|
||||
const merged = dedupByTriplet(allFindings);
|
||||
return {
|
||||
merged,
|
||||
validation,
|
||||
raw_finding_count: allFindings.length,
|
||||
deduped_count: merged.length,
|
||||
};
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const argv = process.argv.slice(2);
|
||||
const asJson = argv.includes('--json');
|
||||
const files = argv.filter((a) => !a.startsWith('--'));
|
||||
if (files.length === 0) {
|
||||
process.stderr.write('Usage: bakeoff-armA-merge.mjs [--json] <reviewer-output> ...\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const outputs = files.map((f) => {
|
||||
if (!existsSync(f)) {
|
||||
process.stderr.write(`bakeoff-armA-merge: file not found: ${f}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
return { label: f, text: readFileSync(f, 'utf-8') };
|
||||
});
|
||||
const result = mergeArmA(outputs);
|
||||
if (asJson) {
|
||||
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify(result.merged, null, 2) + '\n');
|
||||
process.stderr.write(
|
||||
`arm-A merge: ${result.raw_finding_count} raw → ${result.deduped_count} deduped; ` +
|
||||
`validation ${result.validation.map((v) => `${v.label}:${v.valid ? 'OK' : v.error_codes.join('/')}`).join(' ')}\n`,
|
||||
);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
123
scripts/bakeoff-fidelity.mjs
Normal file
123
scripts/bakeoff-fidelity.mjs
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// scripts/bakeoff-fidelity.mjs
|
||||
// NW2 bake-off — fidelity analysis across ≥3 runs/arm (S10 part B).
|
||||
//
|
||||
// Consumes the per-run structured arm outputs ({verdict, findings:[...]}) and
|
||||
// computes the T2 §5 fidelity picture:
|
||||
//
|
||||
// - CROSS-ARM (PRIMARY): every (Ai, Bj) pair via fidelityDiffStructured
|
||||
// (lib/review/fidelity-diff.mjs) → median jaccard, verdict-match rate,
|
||||
// equivalence rate, severity/rule_key mismatch tallies.
|
||||
// - WITHIN-ARM (context): each arm's own run-to-run variance (Ai vs Aj),
|
||||
// so cross-arm divergence can be read against each arm's intrinsic noise.
|
||||
// - Distributions: per-arm verdicts + finding counts.
|
||||
//
|
||||
// Pure analysis over JSON the live runs produced — no LLM, deterministic.
|
||||
//
|
||||
// Usage:
|
||||
// node scripts/bakeoff-fidelity.mjs --armA a1.json a2.json a3.json \
|
||||
// --armB b1.json b2.json b3.json [--json]
|
||||
// Each *.json is a structured arm result: {"verdict": "...", "findings": [...]}.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fidelityDiffStructured } from '../lib/review/fidelity-diff.mjs';
|
||||
|
||||
function median(nums) {
|
||||
if (nums.length === 0) return null;
|
||||
const s = [...nums].sort((a, b) => a - b);
|
||||
const mid = Math.floor(s.length / 2);
|
||||
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
|
||||
}
|
||||
|
||||
function round(n, d = 4) {
|
||||
return n === null ? null : Number(n.toFixed(d));
|
||||
}
|
||||
|
||||
function pairwise(arms, label) {
|
||||
const out = [];
|
||||
for (let i = 0; i < arms.length; i++) {
|
||||
for (let j = i + 1; j < arms.length; j++) {
|
||||
const d = fidelityDiffStructured(arms[i].result, arms[j].result);
|
||||
out.push({ pair: `${label}${i + 1}×${label}${j + 1}`, jaccard: round(d.jaccard), verdictMatch: d.verdictMatch, equivalent: d.equivalent });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Array<{name, result:{verdict,findings}}>} armA
|
||||
* @param {Array<{name, result:{verdict,findings}}>} armB
|
||||
*/
|
||||
export function analyze(armA, armB) {
|
||||
// Cross-arm — the substrate comparison.
|
||||
const cross = [];
|
||||
for (let i = 0; i < armA.length; i++) {
|
||||
for (let j = 0; j < armB.length; j++) {
|
||||
const d = fidelityDiffStructured(armA[i].result, armB[j].result);
|
||||
cross.push({
|
||||
pair: `A${i + 1}×B${j + 1}`,
|
||||
verdictA: d.verdictA, verdictB: d.verdictB, verdictMatch: d.verdictMatch,
|
||||
jaccard: round(d.jaccard), countA: d.countA, countB: d.countB,
|
||||
severityMismatches: d.severityMismatches.length,
|
||||
ruleKeyMismatches: d.ruleKeyMismatches.length,
|
||||
equivalent: d.equivalent,
|
||||
});
|
||||
}
|
||||
}
|
||||
const crossJ = cross.map((c) => c.jaccard);
|
||||
const summary = {
|
||||
runs: { armA: armA.length, armB: armB.length },
|
||||
cross_arm: {
|
||||
median_jaccard: round(median(crossJ)),
|
||||
min_jaccard: round(Math.min(...crossJ)),
|
||||
max_jaccard: round(Math.max(...crossJ)),
|
||||
verdict_match_rate: round(cross.filter((c) => c.verdictMatch).length / cross.length, 3),
|
||||
equivalent_rate: round(cross.filter((c) => c.equivalent).length / cross.length, 3),
|
||||
severity_mismatch_pairs: cross.filter((c) => c.severityMismatches > 0).length,
|
||||
rulekey_mismatch_pairs: cross.filter((c) => c.ruleKeyMismatches > 0).length,
|
||||
},
|
||||
within_arm_A: pairwise(armA, 'A'),
|
||||
within_arm_B: pairwise(armB, 'B'),
|
||||
distributions: {
|
||||
armA_verdicts: armA.map((a) => a.result.verdict),
|
||||
armB_verdicts: armB.map((b) => b.result.verdict),
|
||||
armA_counts: armA.map((a) => (a.result.findings || []).length),
|
||||
armB_counts: armB.map((b) => (b.result.findings || []).length),
|
||||
},
|
||||
};
|
||||
return { summary, cross };
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const argv = process.argv.slice(2);
|
||||
const asJson = argv.includes('--json');
|
||||
function collect(flag) {
|
||||
const i = argv.indexOf(flag);
|
||||
if (i === -1) return [];
|
||||
const files = [];
|
||||
for (let k = i + 1; k < argv.length && !argv[k].startsWith('--'); k++) files.push(argv[k]);
|
||||
return files.map((f) => ({ name: f, result: JSON.parse(readFileSync(f, 'utf-8')) }));
|
||||
}
|
||||
const armA = collect('--armA');
|
||||
const armB = collect('--armB');
|
||||
if (armA.length === 0 || armB.length === 0) {
|
||||
process.stderr.write('Usage: bakeoff-fidelity.mjs --armA a*.json --armB b*.json [--json]\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const { summary, cross } = analyze(armA, armB);
|
||||
if (asJson) {
|
||||
process.stdout.write(JSON.stringify({ summary, cross }, null, 2) + '\n');
|
||||
} else {
|
||||
const x = summary.cross_arm;
|
||||
process.stdout.write('NW2 bake-off fidelity (cross-arm = PRIMARY)\n');
|
||||
process.stdout.write(` runs: A=${summary.runs.armA} B=${summary.runs.armB}\n`);
|
||||
process.stdout.write(` median jaccard: ${x.median_jaccard} (min ${x.min_jaccard}, max ${x.max_jaccard})\n`);
|
||||
process.stdout.write(` verdict-match rate: ${x.verdict_match_rate}\n`);
|
||||
process.stdout.write(` equivalent rate: ${x.equivalent_rate}\n`);
|
||||
process.stdout.write(` severity-mismatch pairs: ${x.severity_mismatch_pairs}; rule_key-mismatch pairs: ${x.rulekey_mismatch_pairs}\n`);
|
||||
process.stdout.write(` armA verdicts: ${summary.distributions.armA_verdicts.join(',')} | counts ${summary.distributions.armA_counts.join(',')}\n`);
|
||||
process.stdout.write(` armB verdicts: ${summary.distributions.armB_verdicts.join(',')} | counts ${summary.distributions.armB_counts.join(',')}\n`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue