Run the full T2 §5 prose-vs-Workflow /trekreview bake-off (operator GO, choice "a"): 3 runs/arm on a rich-finding JWT-auth fixture, resolving the smoke's 0-finding limitation. Deliverables: - tests/fixtures/bakeoff-rich/ — JWT-auth brief + diff with 5 seeded blatant, brief-traceable issues (varied severity/rule_key, one dual-flaggable). - scripts/bakeoff-armA-merge.mjs — Arm A (prose) validate (NW1) + triplet-dedup, matching Arm B's dedup exactly. - scripts/bakeoff-fidelity.mjs — cross-arm + within-arm + granularity-ladder fidelity analysis over the structured arm outputs. - docs/T2-bakeoff-results.md §Full run — the T2 §5 verdict. Result (3 runs/arm, both arms ran the coordinator): - Verdict fidelity EQUIVALENT — all 6 runs BLOCK, cross-arm verdict-match 1.0. - Finding-set: substrate is fidelity-neutral. Cross-arm jaccard 0.41 (triplet) → 0.71 (file,rule_key) → 1.0 (file); cross-arm ≈ within-arm at every granularity. Issue coverage 5/5 in 6/6 runs. Low triplet jaccard is line-citation noise shared by both arms, not a substrate effect. - Token +4.4% (Arm B vs A; <=+15%). Classifier interference 0 at 9-agent concurrency. JSON-robustness: Arm B schema-forced; Arm A 6/6 valid via NW1. - VERDICT POSITIVE → S11 proceeds with opt-in --workflow flag. Caveat (per plan posture): strict triplet-jaccard>=0.7 flag is 0/9, a metric-calibration artifact (both arms sub-0.7 against themselves), not a regression. Residual: F4 auto/bypass explicit-mode check (mode not settable in-session). Suite green (662/660 pass, 2 skip); plugin validate clean (modulo the pre-existing root-CLAUDE.md warning). No production code changed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
101 lines
4.1 KiB
JavaScript
101 lines
4.1 KiB
JavaScript
// scripts/bakeoff-armA-merge.mjs
|
||
// NW2 bake-off — Arm A (prose path) merge step.
|
||
//
|
||
// Arm A is the CURRENT /trekreview Phase 5–6 substrate: main spawns the two
|
||
// reviewers FOREGROUND (Agent tool), each emits a trailing fenced ```json block
|
||
// of findings (the prose contract), and main hand-validates + dedups before
|
||
// spawning the coordinator. This script is that hand-validate + dedup, made
|
||
// reproducible:
|
||
//
|
||
// for each reviewer raw-output file:
|
||
// validateReviewerOutput() (NW1 — extract last json fence, parse, schema-check)
|
||
// collect findings from VALID reviewers
|
||
// triplet-dedup by (file,line,rule_key) [SAME logic as Arm B's dedupByTriplet,
|
||
// so the two arms dedup identically —
|
||
// triplet-only, no jaccard pass-2]
|
||
// → emit { merged, validation, raw_finding_count, deduped_count }
|
||
//
|
||
// The merged findings are then handed to the review-coordinator (spawned by main),
|
||
// mirroring Arm B's coordinatorPrompt input. JSON-robustness metric = the
|
||
// `validation` report (parse/schema failures + which codes).
|
||
//
|
||
// Usage:
|
||
// node scripts/bakeoff-armA-merge.mjs [--json] <reviewerA.txt> <reviewerB.txt> ...
|
||
// → stdout: merged findings JSON (or full report with --json). Exit 0 always
|
||
// (validation failures are reported in-band, not as a nonzero exit).
|
||
|
||
import { readFileSync, existsSync } from 'node:fs';
|
||
import { validateReviewerOutput } from '../lib/review/findings-schema.mjs';
|
||
|
||
// Triplet dedup — byte-for-byte the same key + first-wins policy as
|
||
// scripts/trekreview-armB.workflow.mjs `dedupByTriplet`, so neither arm gets a
|
||
// dedup advantage. Triplet-only (file,line,rule_key); NO jaccard pass-2.
|
||
export function dedupByTriplet(findings) {
|
||
const seen = new Map();
|
||
for (const f of findings) {
|
||
const key = `${f.file} ${f.line} ${f.rule_key}`;
|
||
if (!seen.has(key)) seen.set(key, f);
|
||
}
|
||
return [...seen.values()];
|
||
}
|
||
|
||
/**
|
||
* Merge an array of reviewer raw-output texts into the coordinator's input.
|
||
* @param {Array<{label?: string, text: string}>} reviewerOutputs
|
||
* @returns {{ merged, validation, raw_finding_count, deduped_count }}
|
||
*/
|
||
export function mergeArmA(reviewerOutputs) {
|
||
const validation = [];
|
||
const allFindings = [];
|
||
for (const { label, text } of reviewerOutputs) {
|
||
const r = validateReviewerOutput(text);
|
||
validation.push({
|
||
label: label || null,
|
||
valid: r.valid,
|
||
finding_count: Array.isArray(r.parsed?.findings) ? r.parsed.findings.length : 0,
|
||
error_codes: (r.errors || []).map((e) => e.code),
|
||
warning_codes: (r.warnings || []).map((w) => w.code),
|
||
});
|
||
// Mirror Phase 5: only VALID reviewer output flows to the coordinator.
|
||
if (r.valid && Array.isArray(r.parsed?.findings)) {
|
||
allFindings.push(...r.parsed.findings);
|
||
}
|
||
}
|
||
const merged = dedupByTriplet(allFindings);
|
||
return {
|
||
merged,
|
||
validation,
|
||
raw_finding_count: allFindings.length,
|
||
deduped_count: merged.length,
|
||
};
|
||
}
|
||
|
||
// ---- CLI shim ----------------------------------------------------------------
|
||
|
||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||
const argv = process.argv.slice(2);
|
||
const asJson = argv.includes('--json');
|
||
const files = argv.filter((a) => !a.startsWith('--'));
|
||
if (files.length === 0) {
|
||
process.stderr.write('Usage: bakeoff-armA-merge.mjs [--json] <reviewer-output> ...\n');
|
||
process.exit(2);
|
||
}
|
||
const outputs = files.map((f) => {
|
||
if (!existsSync(f)) {
|
||
process.stderr.write(`bakeoff-armA-merge: file not found: ${f}\n`);
|
||
process.exit(2);
|
||
}
|
||
return { label: f, text: readFileSync(f, 'utf-8') };
|
||
});
|
||
const result = mergeArmA(outputs);
|
||
if (asJson) {
|
||
process.stdout.write(JSON.stringify(result, null, 2) + '\n');
|
||
} else {
|
||
process.stdout.write(JSON.stringify(result.merged, null, 2) + '\n');
|
||
process.stderr.write(
|
||
`arm-A merge: ${result.raw_finding_count} raw → ${result.deduped_count} deduped; ` +
|
||
`validation ${result.validation.map((v) => `${v.label}:${v.valid ? 'OK' : v.error_codes.join('/')}`).join(' ')}\n`,
|
||
);
|
||
}
|
||
process.exit(0);
|
||
}
|