An engine's brief-conflict mark could lower a reviewer agent's MAJOR: ingest kept an engine field a reviewer wrote on its own finding, and on a severity tie the survivor depended on payload order (engine first gave ALLOW, reviewer first WARN). ingest now takes engine from the payload source; Pass 1 and the workflow dedup keep the reviewer-agent finding on a tie. Prose (coordinator Pass 1, Handover 6) follows. Decision 13, operator 12.10. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
285 lines
14 KiB
JavaScript
285 lines
14 KiB
JavaScript
// scripts/trekreview-armB.workflow.mjs
|
||
// NW2 bake-off — Arm B: /trekreview Phase 5–6 reimplemented as a Workflow.
|
||
//
|
||
// This is the T2 §5 Arm B substrate: where the prose path (Arm A) has main
|
||
// spawn two reviewers, hand-parse their trailing JSON, JS-dedup, then spawn the
|
||
// coordinator, Arm B expresses the SAME pipeline as one Workflow —
|
||
// parallel([conformance, correctness]) with a findings SCHEMA (StructuredOutput
|
||
// forces typed findings — no JSON.parse dance, F2 win)
|
||
// → plain-JS dedup by (file,line,rule_key) [inlined; scripts have no imports]
|
||
// → agent(review-coordinator) with a verdict SCHEMA
|
||
// → return { verdict, findings } structured (Phase 7 rendering stays shared/prose).
|
||
//
|
||
// Phases 1–4 (parse/triage) + 7–8 (render/validate/stats) stay prose in BOTH
|
||
// arms; the bake-off pins their output and passes it in via `args`. The reviewer
|
||
// agents carry Read, so paths are passed (not inlined content):
|
||
// args = { briefPath: <path to brief.md>, diffPath: <path to unified diff>,
|
||
// triage: <"path → treatment" lines>, quick?: bool,
|
||
// enginePayloads?: [<lib/review/engine-findings.mjs payload>, ...],
|
||
// engineNotRun?: [<reviewer name>, ...] }
|
||
//
|
||
// Ordre 5: the Claude Code engine payloads (Phase 4.7, run in prose before this
|
||
// script) are merged with the reviewers' findings; a reviewer or engine that
|
||
// returned nothing forbids ALLOW (fail-closed, as on the prose path); and no
|
||
// finding is struck for length.
|
||
//
|
||
// Run via the Workflow tool: Workflow({ scriptPath: this file, args }).
|
||
// The structured result is logged as a single JSON line (prefix RESULT_JSON:)
|
||
// AND returned, so the bake-off harness can recover {verdict, findings}.
|
||
|
||
export const meta = {
|
||
name: 'trekreview-armB',
|
||
description: 'NW2 bake-off Arm B — /trekreview Phase 5–6 as a Workflow (schema-validated parallel reviewers, JS triplet-dedup, coordinator verdict)',
|
||
phases: [
|
||
{ title: 'Review', detail: 'parallel conformance + correctness reviewers, findings schema-forced' },
|
||
{ title: 'Coordinate', detail: 'JS dedup-by-triplet, then review-coordinator verdict' },
|
||
],
|
||
};
|
||
|
||
const RULE_KEYS = [
|
||
'MISSING_BRIEF_REF', 'UNIMPLEMENTED_CRITERION', 'SCOPE_CREEP_BUILT', 'NON_GOAL_VIOLATED',
|
||
'MISSING_TEST', 'SECURITY_INJECTION', 'PLACEHOLDER_IN_CODE', 'MISSING_ERROR_HANDLING',
|
||
'UNDECLARED_DEPENDENCY', 'PLAN_EXECUTE_DRIFT', 'BROKEN_SUCCESS_CRITERION', 'COVERAGE_SILENT_SKIP',
|
||
'CODE_REVIEW_DEFECT', 'CODE_REVIEW_IMPROVEMENT', 'SECURITY_REVIEW_VULNERABILITY',
|
||
];
|
||
const SEVERITIES = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'];
|
||
// Every catalogue rule_key -> its tier (lib/review/rule-catalogue.mjs; scripts
|
||
// cannot import, so it is inlined and pinned equal by the workflow test).
|
||
const RULE_TIERS = {
|
||
MISSING_BRIEF_REF: 'MAJOR', UNIMPLEMENTED_CRITERION: 'BLOCKER', SCOPE_CREEP_BUILT: 'MAJOR',
|
||
NON_GOAL_VIOLATED: 'BLOCKER', MISSING_TEST: 'MAJOR', SECURITY_INJECTION: 'BLOCKER',
|
||
PLACEHOLDER_IN_CODE: 'MAJOR', MISSING_ERROR_HANDLING: 'MINOR', UNDECLARED_DEPENDENCY: 'MAJOR',
|
||
PLAN_EXECUTE_DRIFT: 'MAJOR', BROKEN_SUCCESS_CRITERION: 'BLOCKER', COVERAGE_SILENT_SKIP: 'MAJOR',
|
||
CODE_REVIEW_DEFECT: 'MAJOR', CODE_REVIEW_IMPROVEMENT: 'MINOR', SECURITY_REVIEW_VULNERABILITY: 'BLOCKER',
|
||
};
|
||
// Lower number = higher severity; an unknown severity ranks last (as
|
||
// lib/review/coordinator-contract.mjs severityRank does).
|
||
const sevRank = (s) => (SEVERITIES.includes(s) ? SEVERITIES.indexOf(s) : SEVERITIES.length);
|
||
// The catalogue tier of a rule_key, or null when the key is missing or unknown.
|
||
const tierOf = (key) => (typeof key === 'string' && Object.prototype.hasOwnProperty.call(RULE_TIERS, key) ? RULE_TIERS[key] : null);
|
||
|
||
// Findings schema — mirrors lib/review/findings-schema.mjs (NW1). The enum on
|
||
// rule_key enforces the catalogue at the StructuredOutput layer (stronger than
|
||
// NW1's post-hoc check: the agent cannot emit an out-of-catalogue key at all).
|
||
const FINDINGS_SCHEMA = {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['reviewer', 'findings'],
|
||
properties: {
|
||
reviewer: { type: 'string' },
|
||
findings: {
|
||
type: 'array',
|
||
items: {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['severity', 'rule_key', 'file', 'line', 'title', 'detail'],
|
||
properties: {
|
||
severity: { type: 'string', enum: SEVERITIES },
|
||
rule_key: { type: 'string', enum: RULE_KEYS },
|
||
file: { type: 'string', minLength: 1 },
|
||
line: { type: 'integer', minimum: 0 },
|
||
brief_ref: { type: 'string' },
|
||
title: { type: 'string', minLength: 1 },
|
||
detail: { type: 'string' },
|
||
recommended_action: { type: 'string' },
|
||
},
|
||
},
|
||
},
|
||
},
|
||
};
|
||
|
||
const VERDICT_SCHEMA = {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['verdict', 'findings'],
|
||
properties: {
|
||
verdict: { type: 'string', enum: ['BLOCK', 'WARN', 'ALLOW'] },
|
||
rationale: { type: 'string' },
|
||
findings: {
|
||
type: 'array',
|
||
items: {
|
||
type: 'object',
|
||
additionalProperties: true,
|
||
required: ['severity', 'rule_key', 'file', 'line'],
|
||
properties: {
|
||
severity: { type: 'string', enum: SEVERITIES },
|
||
rule_key: { type: 'string', enum: RULE_KEYS },
|
||
file: { type: 'string', minLength: 1 },
|
||
line: { type: 'integer', minimum: 0 },
|
||
title: { type: 'string' },
|
||
detail: { type: 'string' },
|
||
engine: { type: 'string', enum: ['code-review', 'security-review'] },
|
||
contradicts_brief: { type: 'string' },
|
||
brief_lowered_from: { type: 'string', enum: ['MAJOR'] },
|
||
},
|
||
},
|
||
},
|
||
},
|
||
};
|
||
|
||
// Inlined dedup by (file,line,rule_key) — the JS-side replacement for the
|
||
// coordinator's pass-1 triplet dedup (T2 §5 / lib/review/plan-review-dedup.mjs
|
||
// pattern; scripts cannot import, so the triplet key is inlined). On a
|
||
// repeated triplet the higher severity stays; at equal severity a reviewer
|
||
// finding (no `engine`) stays over an engine finding, so an engine's mark never
|
||
// lowers what a reviewer agent asserted (decision 13, operator 12.10, as
|
||
// coordinator-contract Pass 1); otherwise the first-seen one stays.
|
||
function dedupByTriplet(findings) {
|
||
const seen = new Map();
|
||
for (const f of findings) {
|
||
const key = `${f.file}\u0000${f.line}\u0000${f.rule_key}`;
|
||
if (!seen.has(key)) { seen.set(key, f); continue; }
|
||
const kept = seen.get(key);
|
||
const higher = sevRank(f.severity) < sevRank(kept.severity);
|
||
const tieToReviewer = sevRank(f.severity) === sevRank(kept.severity) && !f.engine && Boolean(kept.engine);
|
||
if (higher || tieToReviewer) seen.set(key, f);
|
||
}
|
||
return [...seen.values()];
|
||
}
|
||
|
||
function reviewerPrompt(kind, a) {
|
||
return [
|
||
`You are reviewing delivered code for /trekreview (bake-off Arm B, ${kind}).`,
|
||
`Read the brief (the contract) at: ${a.briefPath}`,
|
||
`Read the unified diff (the delivered change under review) at: ${a.diffPath}`,
|
||
'Emit findings via the StructuredOutput tool, conforming to the provided schema.',
|
||
'Only emit findings whose rule_key is in the catalogue. Cite file:line from the diff.',
|
||
'',
|
||
'=== TRIAGE MAP (respect skip decisions) ===',
|
||
a.triage || '(no triage map supplied — treat all changed files as summary-only)',
|
||
].join('\n');
|
||
}
|
||
|
||
function coordinatorPrompt(merged, a) {
|
||
return [
|
||
'You are the review-coordinator (Judge Agent) for /trekreview, Arm B.',
|
||
'The reviewers have already run and their findings are PRE-DEDUPLICATED by',
|
||
'(file,line,rule_key) in JS. Do NOT re-dedup. Apply ONLY bounded operations:',
|
||
'HubSpot Judge filters (accuracy / actionability; succinctness NEVER removes a',
|
||
'finding — trim only MINOR/SUGGESTION text, keep BLOCKER/MAJOR whole), Cloudflare',
|
||
'reasonableness (drop speculative or catalogue-violating findings), then compute',
|
||
'the verdict (BLOCK if any BLOCKER; WARN if any MAJOR; else ALLOW).',
|
||
'Return {verdict, findings} via StructuredOutput. Synthesis across files is forbidden.',
|
||
`The brief (the contract) is at: ${a.briefPath} — Read it if needed for context.`,
|
||
'',
|
||
'=== BRIEF CHOICES (the brief-conflict rule) ===',
|
||
`Read the brief's choices at ${a.briefPath}: \`## Non-Goals\`, \`## Constraints\`, \`## Preferences\`,`,
|
||
'and the operator decisions in `## Open Questions / Assumptions`.',
|
||
'Only an engine finding (`engine`: code-review or security-review) whose recommended action',
|
||
'contradicts one of those choices is marked, with `contradicts_brief: "<the Non-Goal or choice, verbatim>"`.',
|
||
'Only a /code-review finding at the MAJOR tier and at severity MAJOR then drops to MINOR, with',
|
||
"`brief_lowered_from: 'MAJOR'`. Every other marked finding keeps its severity. A reviewer-agent finding",
|
||
'(no `engine`) and a finding whose rule_key is not in the catalogue are never marked. A brief-conflicting',
|
||
'finding is kept and marked, never removed. Each returned finding keeps its `engine` and `rule_key`.',
|
||
'',
|
||
'=== PRE-DEDUPED FINDINGS (JSON) ===',
|
||
JSON.stringify(merged, null, 2),
|
||
].join('\n');
|
||
}
|
||
|
||
let a = args || {};
|
||
if (typeof a === 'string') {
|
||
try { a = JSON.parse(a); } catch { a = {}; }
|
||
}
|
||
log(`Arm B args: type=${typeof args}; keys=${Object.keys(a || {}).join(',') || '(none)'}`);
|
||
if (!a || !a.briefPath || !a.diffPath) {
|
||
throw new Error('trekreview-armB: args must include { briefPath, diffPath } (pinned Phase 1-4 output)');
|
||
}
|
||
|
||
phase('Review');
|
||
const reviewerThunks = [
|
||
() => agent(reviewerPrompt('brief-conformance', a), {
|
||
agentType: 'voyage:brief-conformance-reviewer', schema: FINDINGS_SCHEMA, label: 'conformance', phase: 'Review',
|
||
}),
|
||
() => agent(reviewerPrompt('code-correctness', a), {
|
||
agentType: 'voyage:code-correctness-reviewer', schema: FINDINGS_SCHEMA, label: 'correctness', phase: 'Review',
|
||
}),
|
||
];
|
||
const toRun = a.quick ? [reviewerThunks[1]] : reviewerThunks;
|
||
const rawResults = await parallel(toRun);
|
||
const reviewerResults = rawResults.filter(Boolean);
|
||
// A reviewer that returned nothing is an absent one, not a clean one. The
|
||
// engines that were due are counted from what actually arrived, never from
|
||
// what the caller remembered to list as not run.
|
||
const enginePayloads = Array.isArray(a.enginePayloads) ? a.enginePayloads.filter(Boolean) : [];
|
||
const expectedEngines = a.quick ? ['claude-code-review'] : ['claude-code-review', 'claude-security-review'];
|
||
const arrivedEngines = new Set(enginePayloads.map((p) => p.reviewer));
|
||
const missingEngines = expectedEngines.filter((name) => !arrivedEngines.has(name));
|
||
const missingReviewers = (rawResults.length - reviewerResults.length) + missingEngines.length;
|
||
// Provenance, before dedup: "came from an engine" is read from the source,
|
||
// never from an `engine` field. A reviewer finding loses any `engine` it
|
||
// carries; an engine payload's findings get `engine` from its `reviewer`
|
||
// (the names lib/review/engine-findings.mjs writes); any other payload gives
|
||
// its findings no `engine`. So an `engine` in `merged` means an engine payload.
|
||
const engineOf = (reviewer) => (reviewer === 'claude-code-review' ? 'code-review'
|
||
: reviewer === 'claude-security-review' ? 'security-review' : null);
|
||
const withEngine = (f, eng) => {
|
||
if (!f || typeof f !== 'object') return f;
|
||
const { engine: _dropped, ...rest } = f;
|
||
return eng ? { ...rest, engine: eng } : rest;
|
||
};
|
||
const allFindings = [
|
||
...reviewerResults.flatMap((r) => (Array.isArray(r?.findings) ? r.findings : []).map((f) => withEngine(f, null))),
|
||
...enginePayloads.flatMap((p) => (Array.isArray(p?.findings) ? p.findings : []).map((f) => withEngine(f, engineOf(p?.reviewer)))),
|
||
];
|
||
const merged = dedupByTriplet(allFindings);
|
||
log(`Arm B: ${reviewerResults.length} reviewer(s); ${allFindings.length} raw → ${merged.length} after triplet-dedup`);
|
||
|
||
phase('Coordinate');
|
||
let coord = null;
|
||
if (merged.length === 0) {
|
||
coord = { verdict: missingReviewers > 0 ? 'WARN' : 'ALLOW', findings: [] };
|
||
log(`Arm B: zero findings → verdict ${coord.verdict} (coordinator skipped; ${missingReviewers} reviewer(s) missing)`);
|
||
} else {
|
||
coord = await agent(coordinatorPrompt(merged, a), {
|
||
agentType: 'voyage:review-coordinator', schema: VERDICT_SCHEMA, label: 'coordinator', phase: 'Coordinate',
|
||
});
|
||
}
|
||
|
||
// The verdict is mechanical: recompute its floor from the findings the
|
||
// coordinator returned, so an LLM-written ALLOW over a BLOCKER cannot pass.
|
||
const returned = Array.isArray(coord?.findings) ? coord.findings : merged;
|
||
// The brief-conflict floor: every returned finding is judged from its
|
||
// counterpart in the coordinator's input (`merged`, same file/line/rule_key),
|
||
// never from the labels the coordinator wrote. The raise is written into
|
||
// `returned`, the mark kept, so the rendered review.md carries it.
|
||
const counterparts = new Map(merged
|
||
.filter((m) => m && typeof m === 'object')
|
||
.map((m) => [`${m.file}\u0000${m.line}\u0000${m.rule_key}`, m]));
|
||
const present = (f, field) => f[field] !== undefined && f[field] !== null;
|
||
const raiseTo = (f, sev) => { if (sevRank(sev) < sevRank(f.severity)) f.severity = sev; };
|
||
for (const f of returned) {
|
||
if (!f || typeof f !== 'object') continue;
|
||
const c = counterparts.get(`${f.file}\u0000${f.line}\u0000${f.rule_key}`);
|
||
const labelled = present(f, 'engine') || present(f, 'contradicts_brief') || present(f, 'brief_lowered_from');
|
||
if (c && c.engine) {
|
||
// (1) an engine counterpart: raised to its severity, unless the one legal lowering.
|
||
const legal = c.engine === 'code-review' && c.severity === 'MAJOR' && tierOf(c.rule_key) === 'MAJOR' &&
|
||
f.severity === 'MINOR' && typeof f.contradicts_brief === 'string' && f.contradicts_brief.trim().length > 0 &&
|
||
f.brief_lowered_from === 'MAJOR';
|
||
if (!legal) raiseTo(f, c.severity);
|
||
f.engine = c.engine;
|
||
} else if (labelled) {
|
||
// (2) a reviewer counterpart, (3) no counterpart: an unknown key raises to BLOCKER.
|
||
raiseTo(f, tierOf(f.rule_key) ?? 'BLOCKER');
|
||
if (c) raiseTo(f, c.severity);
|
||
delete f.engine;
|
||
}
|
||
}
|
||
const has = (sev) => returned.some((f) => f && f.severity === sev);
|
||
let verdict = coord?.verdict ?? null;
|
||
if (has('BLOCKER')) verdict = 'BLOCK';
|
||
else if (has('MAJOR') && verdict !== 'BLOCK') verdict = 'WARN';
|
||
if (verdict === 'ALLOW' && missingReviewers > 0) verdict = 'WARN';
|
||
|
||
const result = {
|
||
verdict,
|
||
missing_reviewers: missingReviewers,
|
||
missing_engines: missingEngines,
|
||
findings: returned,
|
||
raw_reviewer_count: reviewerResults.length,
|
||
raw_finding_count: allFindings.length,
|
||
deduped_count: merged.length,
|
||
};
|
||
log(`RESULT_JSON:${JSON.stringify(result)}`);
|
||
return result;
|