voyage/scripts/trekreview-armB.workflow.mjs
Kjell Tore Guttormsen 869bf318d2 feat(voyage): S10 — NW2 part A (Workflow port + fidelity harness + smoke)
Build the prose-vs-Workflow bake-off machinery for /trekreview Phase 5-6 and
run a 1-run/arm smoke to de-risk before the full measurement (operator posture:
build + smoke, then pause for go/no-go on the full >=3-runs/arm run).

New:
- lib/review/fidelity-diff.mjs (+ tests) — the PRIMARY metric: parse two
  review.md (or two structured arm outputs) and compare verdict + jaccard over
  (file,line,rule_key)-IDs + per-finding severity/rule_key. Reuses jaccard +
  frontmatter + NW1 findings-schema + finding-id. fidelityDiffStructured avoids
  rendering review.md per run.
- scripts/trekreview-armB.workflow.mjs — Arm B: Phase 5-6 as a Workflow
  (parallel([conformance, correctness]) schema-forced -> JS dedup-by-triplet ->
  agent(review-coordinator) verdict schema). Path-based input via args (reviewers
  carry Read). Inlines dedup + the 12-key rule_key enum (scripts have no imports).
- tests/fixtures/bakeoff/ — committable fixture: real diff of b149538 (NW1) +
  brief reconstructed from plan S9. Both arms review the same pinned input.
- docs/T2-bakeoff-results.md — smoke results + verdict + go/no-go recommendation.

Smoke result: SMOKE PASS. Arm B runs the full pipeline (3 agents) with ZERO
classifier interference; fidelity EQUIVALENT to Arm A at the verdict level
(both ALLOW; jaccard 1.0). Caveat: the clean TDD'd fixture yielded ~0 findings,
so finding-SET fidelity was not stressed (only verdict fidelity proven). A
reviewer-level divergence appeared (Arm B raised 1 raw finding, coordinator
filtered it; Arm A raised 0) — to be quantified in the full run on a
richer-finding-surface fixture. NOT the T2 §5 POSITIVE/NEGATIVE verdict.

Suite 647 -> 662 (660 pass / 2 skip / 0 fail; +15 fidelity-diff). claude plugin
validate clean (known root-CLAUDE.md warning only). Plan: docs/W1-narrow-wins-plan.md S10.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
2026-06-18 14:40:19 +02:00

179 lines
7.4 KiB
JavaScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// scripts/trekreview-armB.workflow.mjs
// NW2 bake-off — Arm B: /trekreview Phase 56 reimplemented as a Workflow.
//
// This is the T2 §5 Arm B substrate: where the prose path (Arm A) has main
// spawn two reviewers, hand-parse their trailing JSON, JS-dedup, then spawn the
// coordinator, Arm B expresses the SAME pipeline as one Workflow —
// parallel([conformance, correctness]) with a findings SCHEMA (StructuredOutput
// forces typed findings — no JSON.parse dance, F2 win)
// → plain-JS dedup by (file,line,rule_key) [inlined; scripts have no imports]
// → agent(review-coordinator) with a verdict SCHEMA
// → return { verdict, findings } structured (Phase 7 rendering stays shared/prose).
//
// Phases 14 (parse/triage) + 78 (render/validate/stats) stay prose in BOTH
// arms; the bake-off pins their output and passes it in via `args`. The reviewer
// agents carry Read, so paths are passed (not inlined content):
// args = { briefPath: <path to brief.md>, diffPath: <path to unified diff>,
// triage: <"path → treatment" lines>, quick?: bool }
//
// Run via the Workflow tool: Workflow({ scriptPath: this file, args }).
// The structured result is logged as a single JSON line (prefix RESULT_JSON:)
// AND returned, so the bake-off harness can recover {verdict, findings}.
export const meta = {
name: 'trekreview-armB',
description: 'NW2 bake-off Arm B — /trekreview Phase 56 as a Workflow (schema-validated parallel reviewers, JS triplet-dedup, coordinator verdict)',
phases: [
{ title: 'Review', detail: 'parallel conformance + correctness reviewers, findings schema-forced' },
{ title: 'Coordinate', detail: 'JS dedup-by-triplet, then review-coordinator verdict' },
],
};
const RULE_KEYS = [
'MISSING_BRIEF_REF', 'UNIMPLEMENTED_CRITERION', 'SCOPE_CREEP_BUILT', 'NON_GOAL_VIOLATED',
'MISSING_TEST', 'SECURITY_INJECTION', 'PLACEHOLDER_IN_CODE', 'MISSING_ERROR_HANDLING',
'UNDECLARED_DEPENDENCY', 'PLAN_EXECUTE_DRIFT', 'BROKEN_SUCCESS_CRITERION', 'COVERAGE_SILENT_SKIP',
];
const SEVERITIES = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'];
// Findings schema — mirrors lib/review/findings-schema.mjs (NW1). The enum on
// rule_key enforces the catalogue at the StructuredOutput layer (stronger than
// NW1's post-hoc check: the agent cannot emit an out-of-catalogue key at all).
const FINDINGS_SCHEMA = {
type: 'object',
additionalProperties: true,
required: ['reviewer', 'findings'],
properties: {
reviewer: { type: 'string' },
findings: {
type: 'array',
items: {
type: 'object',
additionalProperties: true,
required: ['severity', 'rule_key', 'file', 'line', 'title', 'detail'],
properties: {
severity: { type: 'string', enum: SEVERITIES },
rule_key: { type: 'string', enum: RULE_KEYS },
file: { type: 'string', minLength: 1 },
line: { type: 'integer', minimum: 0 },
brief_ref: { type: 'string' },
title: { type: 'string', minLength: 1 },
detail: { type: 'string' },
recommended_action: { type: 'string' },
},
},
},
},
};
const VERDICT_SCHEMA = {
type: 'object',
additionalProperties: true,
required: ['verdict', 'findings'],
properties: {
verdict: { type: 'string', enum: ['BLOCK', 'WARN', 'ALLOW'] },
rationale: { type: 'string' },
findings: {
type: 'array',
items: {
type: 'object',
additionalProperties: true,
required: ['severity', 'rule_key', 'file', 'line'],
properties: {
severity: { type: 'string', enum: SEVERITIES },
rule_key: { type: 'string', enum: RULE_KEYS },
file: { type: 'string', minLength: 1 },
line: { type: 'integer', minimum: 0 },
title: { type: 'string' },
detail: { type: 'string' },
},
},
},
},
};
// Inlined dedup by (file,line,rule_key) — the JS-side replacement for the
// coordinator's pass-1 triplet dedup (T2 §5 / lib/review/plan-review-dedup.mjs
// pattern; scripts cannot import, so the triplet key is inlined).
function dedupByTriplet(findings) {
const seen = new Map();
for (const f of findings) {
const key = `${f.file}${f.line}${f.rule_key}`;
if (!seen.has(key)) seen.set(key, f);
}
return [...seen.values()];
}
function reviewerPrompt(kind, a) {
return [
`You are reviewing delivered code for /trekreview (bake-off Arm B, ${kind}).`,
`Read the brief (the contract) at: ${a.briefPath}`,
`Read the unified diff (the delivered change under review) at: ${a.diffPath}`,
'Emit findings via the StructuredOutput tool, conforming to the provided schema.',
'Only emit findings whose rule_key is in the catalogue. Cite file:line from the diff.',
'',
'=== TRIAGE MAP (respect skip decisions) ===',
a.triage || '(no triage map supplied — treat all changed files as summary-only)',
].join('\n');
}
function coordinatorPrompt(merged, a) {
return [
'You are the review-coordinator (Judge Agent) for /trekreview, Arm B.',
'The reviewers have already run and their findings are PRE-DEDUPLICATED by',
'(file,line,rule_key) in JS. Do NOT re-dedup. Apply ONLY bounded operations:',
'HubSpot Judge filters (succinctness / accuracy / actionability), Cloudflare',
'reasonableness (drop speculative or catalogue-violating findings), then compute',
'the verdict (BLOCK if any BLOCKER; WARN if any MAJOR; else ALLOW).',
'Return {verdict, findings} via StructuredOutput. Synthesis across files is forbidden.',
`The brief (the contract) is at: ${a.briefPath} — Read it if needed for context.`,
'',
'=== PRE-DEDUPED FINDINGS (JSON) ===',
JSON.stringify(merged, null, 2),
].join('\n');
}
let a = args || {};
if (typeof a === 'string') {
try { a = JSON.parse(a); } catch { a = {}; }
}
log(`Arm B args: type=${typeof args}; keys=${Object.keys(a || {}).join(',') || '(none)'}`);
if (!a || !a.briefPath || !a.diffPath) {
throw new Error('trekreview-armB: args must include { briefPath, diffPath } (pinned Phase 1-4 output)');
}
phase('Review');
const reviewerThunks = [
() => agent(reviewerPrompt('brief-conformance', a), {
agentType: 'voyage:brief-conformance-reviewer', schema: FINDINGS_SCHEMA, label: 'conformance', phase: 'Review',
}),
() => agent(reviewerPrompt('code-correctness', a), {
agentType: 'voyage:code-correctness-reviewer', schema: FINDINGS_SCHEMA, label: 'correctness', phase: 'Review',
}),
];
const toRun = a.quick ? [reviewerThunks[1]] : reviewerThunks;
const reviewerResults = (await parallel(toRun)).filter(Boolean);
const allFindings = reviewerResults.flatMap((r) => (Array.isArray(r.findings) ? r.findings : []));
const merged = dedupByTriplet(allFindings);
log(`Arm B: ${reviewerResults.length} reviewer(s); ${allFindings.length} raw → ${merged.length} after triplet-dedup`);
phase('Coordinate');
let coord = null;
if (merged.length === 0) {
coord = { verdict: 'ALLOW', findings: [] };
log('Arm B: zero findings → verdict ALLOW (coordinator skipped)');
} else {
coord = await agent(coordinatorPrompt(merged, a), {
agentType: 'voyage:review-coordinator', schema: VERDICT_SCHEMA, label: 'coordinator', phase: 'Coordinate',
});
}
const result = {
verdict: coord?.verdict ?? null,
findings: Array.isArray(coord?.findings) ? coord.findings : merged,
raw_reviewer_count: reviewerResults.length,
raw_finding_count: allFindings.length,
deduped_count: merged.length,
};
log(`RESULT_JSON:${JSON.stringify(result)}`);
return result;