Build the prose-vs-Workflow bake-off machinery for /trekreview Phase 5-6 and
run a 1-run/arm smoke to de-risk before the full measurement (operator posture:
build + smoke, then pause for go/no-go on the full >=3-runs/arm run).
New:
- lib/review/fidelity-diff.mjs (+ tests) — the PRIMARY metric: parse two
review.md (or two structured arm outputs) and compare verdict + jaccard over
(file,line,rule_key)-IDs + per-finding severity/rule_key. Reuses jaccard +
frontmatter + NW1 findings-schema + finding-id. fidelityDiffStructured avoids
rendering review.md per run.
- scripts/trekreview-armB.workflow.mjs — Arm B: Phase 5-6 as a Workflow
(parallel([conformance, correctness]) schema-forced -> JS dedup-by-triplet ->
agent(review-coordinator) verdict schema). Path-based input via args (reviewers
carry Read). Inlines dedup + the 12-key rule_key enum (scripts have no imports).
- tests/fixtures/bakeoff/ — committable fixture: real diff of b149538 (NW1) +
brief reconstructed from plan S9. Both arms review the same pinned input.
- docs/T2-bakeoff-results.md — smoke results + verdict + go/no-go recommendation.
Smoke result: SMOKE PASS. Arm B runs the full pipeline (3 agents) with ZERO
classifier interference; fidelity EQUIVALENT to Arm A at the verdict level
(both ALLOW; jaccard 1.0). Caveat: the clean TDD'd fixture yielded ~0 findings,
so finding-SET fidelity was not stressed (only verdict fidelity proven). A
reviewer-level divergence appeared (Arm B raised 1 raw finding, coordinator
filtered it; Arm A raised 0) — to be quantified in the full run on a
richer-finding-surface fixture. NOT the T2 §5 POSITIVE/NEGATIVE verdict.
Suite 647 -> 662 (660 pass / 2 skip / 0 fail; +15 fidelity-diff). claude plugin
validate clean (known root-CLAUDE.md warning only). Plan: docs/W1-narrow-wins-plan.md S10.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB
179 lines
7.4 KiB
JavaScript
179 lines
7.4 KiB
JavaScript
// scripts/trekreview-armB.workflow.mjs
|
||
// NW2 bake-off — Arm B: /trekreview Phase 5–6 reimplemented as a Workflow.
|
||
//
|
||
// This is the T2 §5 Arm B substrate: where the prose path (Arm A) has main
|
||
// spawn two reviewers, hand-parse their trailing JSON, JS-dedup, then spawn the
|
||
// coordinator, Arm B expresses the SAME pipeline as one Workflow —
|
||
// parallel([conformance, correctness]) with a findings SCHEMA (StructuredOutput
|
||
// forces typed findings — no JSON.parse dance, F2 win)
|
||
// → plain-JS dedup by (file,line,rule_key) [inlined; scripts have no imports]
|
||
// → agent(review-coordinator) with a verdict SCHEMA
|
||
// → return { verdict, findings } structured (Phase 7 rendering stays shared/prose).
|
||
//
|
||
// Phases 1–4 (parse/triage) + 7–8 (render/validate/stats) stay prose in BOTH
|
||
// arms; the bake-off pins their output and passes it in via `args`. The reviewer
|
||
// agents carry Read, so paths are passed (not inlined content):
|
||
// args = { briefPath: <path to brief.md>, diffPath: <path to unified diff>,
|
||
// triage: <"path → treatment" lines>, quick?: bool }
|
||
//
|
||
// Run via the Workflow tool: Workflow({ scriptPath: this file, args }).
|
||
// The structured result is logged as a single JSON line (prefix RESULT_JSON:)
|
||
// AND returned, so the bake-off harness can recover {verdict, findings}.
|
||
|
||
export const meta = {
|
||
name: 'trekreview-armB',
|
||
description: 'NW2 bake-off Arm B — /trekreview Phase 5–6 as a Workflow (schema-validated parallel reviewers, JS triplet-dedup, coordinator verdict)',
|
||
phases: [
|
||
{ title: 'Review', detail: 'parallel conformance + correctness reviewers, findings schema-forced' },
|
||
{ title: 'Coordinate', detail: 'JS dedup-by-triplet, then review-coordinator verdict' },
|
||
],
|
||
};
|
||
|
||
const RULE_KEYS = [
|
||
'MISSING_BRIEF_REF', 'UNIMPLEMENTED_CRITERION', 'SCOPE_CREEP_BUILT', 'NON_GOAL_VIOLATED',
|
||
'MISSING_TEST', 'SECURITY_INJECTION', 'PLACEHOLDER_IN_CODE', 'MISSING_ERROR_HANDLING',
|
||
'UNDECLARED_DEPENDENCY', 'PLAN_EXECUTE_DRIFT', 'BROKEN_SUCCESS_CRITERION', 'COVERAGE_SILENT_SKIP',
|
||
];
|
||
const SEVERITIES = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'];
|
||
|
||
// Findings schema — mirrors lib/review/findings-schema.mjs (NW1). The enum on
|
||
// rule_key enforces the catalogue at the StructuredOutput layer (stronger than
|
||
// NW1's post-hoc check: the agent cannot emit an out-of-catalogue key at all).
|
||
const FINDINGS_SCHEMA = {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['reviewer', 'findings'],
|
||
properties: {
|
||
reviewer: { type: 'string' },
|
||
findings: {
|
||
type: 'array',
|
||
items: {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['severity', 'rule_key', 'file', 'line', 'title', 'detail'],
|
||
properties: {
|
||
severity: { type: 'string', enum: SEVERITIES },
|
||
rule_key: { type: 'string', enum: RULE_KEYS },
|
||
file: { type: 'string', minLength: 1 },
|
||
line: { type: 'integer', minimum: 0 },
|
||
brief_ref: { type: 'string' },
|
||
title: { type: 'string', minLength: 1 },
|
||
detail: { type: 'string' },
|
||
recommended_action: { type: 'string' },
|
||
},
|
||
},
|
||
},
|
||
},
|
||
};
|
||
|
||
const VERDICT_SCHEMA = {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['verdict', 'findings'],
|
||
properties: {
|
||
verdict: { type: 'string', enum: ['BLOCK', 'WARN', 'ALLOW'] },
|
||
rationale: { type: 'string' },
|
||
findings: {
|
||
type: 'array',
|
||
items: {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['severity', 'rule_key', 'file', 'line'],
|
||
properties: {
|
||
severity: { type: 'string', enum: SEVERITIES },
|
||
rule_key: { type: 'string', enum: RULE_KEYS },
|
||
file: { type: 'string', minLength: 1 },
|
||
line: { type: 'integer', minimum: 0 },
|
||
title: { type: 'string' },
|
||
detail: { type: 'string' },
|
||
},
|
||
},
|
||
},
|
||
},
|
||
};
|
||
|
||
// Inlined dedup by (file,line,rule_key) — the JS-side replacement for the
|
||
// coordinator's pass-1 triplet dedup (T2 §5 / lib/review/plan-review-dedup.mjs
|
||
// pattern; scripts cannot import, so the triplet key is inlined).
|
||
function dedupByTriplet(findings) {
|
||
const seen = new Map();
|
||
for (const f of findings) {
|
||
const key = `${f.file} |