fix(review): the payload decides the engine, and a tie keeps the reviewer finding
An engine's brief-conflict mark could lower a reviewer agent's MAJOR: ingest kept an engine field a reviewer wrote on its own finding, and on a severity tie the survivor depended on payload order (engine first gave ALLOW, reviewer first WARN). ingest now takes engine from the payload source; Pass 1 and the workflow dedup keep the reviewer-agent finding on a tie. Prose (coordinator Pass 1, Handover 6) follows. Decision 13, operator 12.10. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
11315d04eb
commit
305ac9284f
6 changed files with 140 additions and 12 deletions
|
|
@ -90,7 +90,10 @@ Two findings collide when their `(file, line, rule_key)` triplets are
|
|||
identical. When findings collide:
|
||||
- Keep the finding with the highest catalogue severity (BLOCKER >
|
||||
MAJOR > MINOR > SUGGESTION).
|
||||
- If the severity tie, prefer the finding from
|
||||
- If the severity ties, prefer a reviewer-agent finding over an engine
|
||||
finding (one from `claude-code-review` / `claude-security-review`), so an
|
||||
engine's brief-conflict mark never lowers what a reviewer agent asserted;
|
||||
among reviewer-agent findings, prefer the finding from
|
||||
`brief-conformance-reviewer` (its findings are anchored to the brief).
|
||||
- Concatenate the kept finding's `detail` with a one-line note: "Also
|
||||
flagged by {other reviewer}: {their title}." This preserves
|
||||
|
|
|
|||
|
|
@ -475,6 +475,8 @@ any other `brief_lowered_from`, a mark on a reviewer-agent finding, a marked fin
|
|||
and an engine finding below its tier, marked or not (a BLOCK review exits 1 as before).
|
||||
The gate cannot check an `engine` label on the prose path: there the coordinator writes the label, so a finding wrongly labelled `engine` is judged as an engine finding;
|
||||
the workflow path (`--workflow`) takes `engine` from the payload source and writes it into the returned finding.
|
||||
`runContract` (`lib/review/coordinator-contract.mjs`) likewise takes `engine` from the payload a finding arrived in, never from the finding's own field,
|
||||
and on a severity tie in dedup a reviewer-agent finding survives over an engine finding (both paths), so an engine's mark never lowers what a reviewer agent asserted.
|
||||
A plan made from the review skips a lowered finding: it is MINOR, and `/trekplan` takes BLOCKER and MAJOR findings only.
|
||||
|
||||
**Validation strategy:**
|
||||
|
|
|
|||
|
|
@ -98,6 +98,15 @@ export const UNVERIFIED_REASONS = Object.freeze([
|
|||
|
||||
const ENGINE_NAMES = Object.freeze(['code-review', 'security-review']);
|
||||
|
||||
// The payload `reviewer` names lib/review/engine-findings.mjs writes, and the
|
||||
// engine each one stands for.
|
||||
const ENGINE_OF_PAYLOAD = Object.freeze({
|
||||
'claude-code-review': 'code-review',
|
||||
'claude-security-review': 'security-review',
|
||||
});
|
||||
|
||||
const isEngineFinding = (f) => ENGINE_NAMES.includes(f.engine);
|
||||
|
||||
function carries(f, field) {
|
||||
return f?.[field] !== undefined && f?.[field] !== null;
|
||||
}
|
||||
|
|
@ -214,8 +223,18 @@ export function ingest(reviewerPayloads) {
|
|||
skipped.push({ reviewer: payload?.reviewer ?? null, error_codes: r.errors.map((e) => e.code) });
|
||||
continue;
|
||||
}
|
||||
// "Came from an engine" is read from the payload, never from an `engine`
|
||||
// field a reviewer wrote on its own finding (decision 13, operator 12.10):
|
||||
// an engine payload's findings get `engine` from its `reviewer`, every other
|
||||
// payload's findings lose any `engine` they carry.
|
||||
const eng = ENGINE_OF_PAYLOAD[payload.reviewer];
|
||||
for (const f of payload.findings) {
|
||||
findings.push({ ...f, reviewer: f.reviewer ?? payload.reviewer ?? f.owner_reviewer ?? null });
|
||||
const { engine: _claimed, ...rest } = f;
|
||||
findings.push({
|
||||
...rest,
|
||||
...(eng ? { engine: eng } : {}),
|
||||
reviewer: f.reviewer ?? payload.reviewer ?? f.owner_reviewer ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { findings, skipped };
|
||||
|
|
@ -223,7 +242,8 @@ export function ingest(reviewerPayloads) {
|
|||
|
||||
/**
|
||||
* Pass 1 — dedup by (file, line, rule_key) triplet. Survivor = highest
|
||||
* catalogue severity; severity tie → prefer the conformance reviewer; carries
|
||||
* catalogue severity; severity tie → prefer a reviewer-agent finding over an
|
||||
* engine finding, then the conformance reviewer; carries
|
||||
* raised_by provenance, concatenates other reviewers' attribution into detail,
|
||||
* and recomputes the id over the triplet.
|
||||
* @param {object[]} findings
|
||||
|
|
@ -240,15 +260,20 @@ export function dedupByTriplet(findings) {
|
|||
for (const group of groups.values()) {
|
||||
// A brief-conflict mark is not carried over from a group member to the
|
||||
// survivor: when the higher-severity survivor is unmarked, the group stays
|
||||
// unmarked and is judged at its full severity. Conservative on purpose —
|
||||
// a mark can only lower, so dropping it never makes a verdict greener.
|
||||
// unmarked and is judged at its full severity. On a severity tie a
|
||||
// reviewer-agent finding survives over an engine finding (decision 13,
|
||||
// operator 12.10), so an engine's mark never lowers what a reviewer agent
|
||||
// asserted, whatever the payload order; among findings of one kind the
|
||||
// conformance reviewer wins the tie, else the first-seen one stays.
|
||||
let survivor = group[0];
|
||||
for (const f of group.slice(1)) {
|
||||
const higher = severityRank(f.severity) < severityRank(survivor.severity);
|
||||
const rf = severityRank(f.severity);
|
||||
const rs = severityRank(survivor.severity);
|
||||
const tieToReviewer = rf === rs && !isEngineFinding(f) && isEngineFinding(survivor);
|
||||
const tieToConformance =
|
||||
severityRank(f.severity) === severityRank(survivor.severity) &&
|
||||
rf === rs && isEngineFinding(f) === isEngineFinding(survivor) &&
|
||||
isConformance(f.reviewer) && !isConformance(survivor.reviewer);
|
||||
if (higher || tieToConformance) survivor = f;
|
||||
if (rf < rs || tieToReviewer || tieToConformance) survivor = f;
|
||||
}
|
||||
const raised_by = [...new Set(group.map((f) => f.reviewer).filter(Boolean))];
|
||||
const others = group.filter((f) => f !== survivor);
|
||||
|
|
|
|||
|
|
@ -120,8 +120,10 @@ const VERDICT_SCHEMA = {
|
|||
// Inlined dedup by (file,line,rule_key) — the JS-side replacement for the
|
||||
// coordinator's pass-1 triplet dedup (T2 §5 / lib/review/plan-review-dedup.mjs
|
||||
// pattern; scripts cannot import, so the triplet key is inlined). On a
|
||||
// repeated triplet the higher severity stays; at equal severity the finding
|
||||
// with an `engine` stays; otherwise the first-seen one stays.
|
||||
// repeated triplet the higher severity stays; at equal severity a reviewer
|
||||
// finding (no `engine`) stays over an engine finding, so an engine's mark never
|
||||
// lowers what a reviewer agent asserted (decision 13, operator 12.10, as
|
||||
// coordinator-contract Pass 1); otherwise the first-seen one stays.
|
||||
function dedupByTriplet(findings) {
|
||||
const seen = new Map();
|
||||
for (const f of findings) {
|
||||
|
|
@ -129,8 +131,8 @@ function dedupByTriplet(findings) {
|
|||
if (!seen.has(key)) { seen.set(key, f); continue; }
|
||||
const kept = seen.get(key);
|
||||
const higher = sevRank(f.severity) < sevRank(kept.severity);
|
||||
const tieToEngine = sevRank(f.severity) === sevRank(kept.severity) && Boolean(f.engine) && !kept.engine;
|
||||
if (higher || tieToEngine) seen.set(key, f);
|
||||
const tieToReviewer = sevRank(f.severity) === sevRank(kept.severity) && !f.engine && Boolean(kept.engine);
|
||||
if (higher || tieToReviewer) seen.set(key, f);
|
||||
}
|
||||
return [...seen.values()];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -457,3 +457,80 @@ test('Handover 6 — names the fields, engine findings only, the one lowering an
|
|||
assert.match(h6, /cannot check an `engine` label on the prose path/);
|
||||
assert.match(h6, /skips a lowered finding/, 'a plan made from the review skips a lowered (MINOR) finding');
|
||||
});
|
||||
|
||||
// ---- Decision 13 (operator 12.10): the source decides, a tie keeps the reviewer --
|
||||
//
|
||||
// "Came from an engine" is read from the payload a finding arrived in, never
|
||||
// from an `engine` field a reviewer wrote on its own finding; and on a severity
|
||||
// tie a reviewer-agent finding survives over an engine finding, so an engine's
|
||||
// mark can never lower what a reviewer agent asserted, whatever the order.
|
||||
|
||||
function reviewerCopy(extra = {}) {
|
||||
const { engine: _e, ...rest } = codeReviewDefect({ title: 'Reviewer: no retry', ...extra });
|
||||
return rest;
|
||||
}
|
||||
|
||||
test('decision 13 — a reviewer payload whose finding labels itself engine code-review is not lowered', () => {
|
||||
const f = codeReviewDefect({ contradicts_brief: NON_GOAL });
|
||||
const result = m.runContract(payload('code-correctness-reviewer', [f]));
|
||||
const out = result.findings[0];
|
||||
assert.equal(out.severity, 'MAJOR', 'a reviewer MAJOR is never lowered by the brief rule');
|
||||
assert.equal('brief_lowered_from' in out, false);
|
||||
assert.equal('engine' in out, false, 'engine comes from the payload source, not the field');
|
||||
assert.equal('contradicts_brief' in out, false, 'only an engine finding is ever marked');
|
||||
assert.equal(result.verdict, 'WARN');
|
||||
assertAllOutputLegal(result);
|
||||
});
|
||||
|
||||
test('decision 13 — an engine payload decides the engine: a /security-review payload finding labelled code-review keeps its severity', () => {
|
||||
const f = codeReviewDefect({ contradicts_brief: NON_GOAL });
|
||||
const result = m.runContract(payload('claude-security-review', [f]));
|
||||
const out = result.findings[0];
|
||||
assert.equal(out.engine, 'security-review');
|
||||
assert.equal(out.severity, 'MAJOR');
|
||||
assert.equal('brief_lowered_from' in out, false);
|
||||
assert.equal(out.contradicts_brief, NON_GOAL, 'the mark is kept on an engine finding');
|
||||
assertAllOutputLegal(result);
|
||||
});
|
||||
|
||||
for (const order of ['reviewer first', 'engine first']) {
|
||||
test(`decision 13 — a tie between a marked /code-review MAJOR and a reviewer MAJOR keeps the reviewer (${order}) → WARN`, () => {
|
||||
const reviewerPayload = { reviewer: 'code-correctness-reviewer', findings: [reviewerCopy({ rule_key: 'MISSING_TEST' })] };
|
||||
const enginePayload = { reviewer: 'claude-code-review', findings: [codeReviewDefect({ rule_key: 'MISSING_TEST', contradicts_brief: NON_GOAL })] };
|
||||
const payloads = order === 'reviewer first' ? [reviewerPayload, enginePayload] : [enginePayload, reviewerPayload];
|
||||
const result = m.runContract(payloads);
|
||||
assert.equal(result.findings.length, 1);
|
||||
const out = result.findings[0];
|
||||
assert.equal(out.severity, 'MAJOR', 'the reviewer agent asserted MAJOR; the engine mark cannot lower it');
|
||||
assert.equal('brief_lowered_from' in out, false);
|
||||
assert.equal('engine' in out, false, 'the survivor is the reviewer finding');
|
||||
assert.equal('contradicts_brief' in out, false);
|
||||
assert.deepEqual([...out.raised_by].sort(), ['claude-code-review', 'code-correctness-reviewer']);
|
||||
assert.equal(result.verdict, 'WARN');
|
||||
assertAllOutputLegal(result);
|
||||
});
|
||||
}
|
||||
|
||||
test('decision 13 — an engine MAJOR over a reviewer MINOR on the same triplet still drops to MINOR (the engine wins on severity)', () => {
|
||||
const result = m.runContract([
|
||||
{ reviewer: 'code-correctness-reviewer', findings: [reviewerCopy({ rule_key: 'MISSING_TEST', severity: 'MINOR' })] },
|
||||
{ reviewer: 'claude-code-review', findings: [codeReviewDefect({ rule_key: 'MISSING_TEST', contradicts_brief: NON_GOAL })] },
|
||||
]);
|
||||
const out = result.findings[0];
|
||||
assert.equal(out.engine, 'code-review');
|
||||
assert.equal(out.severity, 'MINOR');
|
||||
assert.equal(out.brief_lowered_from, 'MAJOR');
|
||||
assert.equal(result.verdict, 'ALLOW');
|
||||
assertAllOutputLegal(result);
|
||||
});
|
||||
|
||||
test('decision 13 — the prose states the tie rule (coordinator Pass 1, Handover 6)', () => {
|
||||
const coordinator = readDoc('agents', 'review-coordinator.md');
|
||||
const pass1 = cut(coordinator, '### Pass 1', '### Pass 2', 'coordinator Pass 1');
|
||||
assert.match(pass1, /prefer\s+a\s+reviewer-agent\s+finding\s+over\s+an\s+engine\s+finding/,
|
||||
'Pass 1: a tie keeps the reviewer-agent finding');
|
||||
assert.match(pass1, /never\s+lowers\s+what\s+a\s+reviewer\s+agent\s+asserted/);
|
||||
const h6 = cut(readDoc('docs', 'HANDOVER-CONTRACTS.md'), '## Handover 6 — `review.md` → plan', '\n## Handover 7', 'Handover 6');
|
||||
assert.match(h6, /takes? `engine` from the payload a finding arrived in/);
|
||||
assert.match(h6, /on a severity tie[^.]*a reviewer-agent finding survives[^.]*over an engine finding/i);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -321,3 +321,22 @@ test('armB brief rule — VERDICT_SCHEMA declares engine, contradicts_brief and
|
|||
assert.ok(/only an engine finding/i.test(prompt), 'engine findings only');
|
||||
assert.ok(/kept and marked, never removed/i.test(prompt));
|
||||
});
|
||||
|
||||
// Decision 13 (operator 12.10): on a severity tie a reviewer-agent finding
|
||||
// survives dedup over an engine finding, so the engine's mark can never lower
|
||||
// what a reviewer agent asserted (the same rule as coordinator-contract Pass 1).
|
||||
test('armB brief rule (15) — a tie between a reviewer MAJOR and a /code-review MAJOR keeps the reviewer: the legal shape returned with ALLOW → WARN, no engine', async () => {
|
||||
const c = await floorCase({
|
||||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { contradicts_brief: MARK, brief_lowered_from: 'MAJOR', engine: 'code-review' })],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.merged.length, 1);
|
||||
assert.equal(c.merged[0].severity, 'MAJOR');
|
||||
assert.equal('engine' in c.merged[0], false, 'the tie keeps the reviewer finding');
|
||||
assert.equal(c.r.verdict, 'WARN');
|
||||
assert.equal(c.r.findings[0].severity, 'MAJOR');
|
||||
assert.equal('engine' in c.r.findings[0], false);
|
||||
checkRaise(c, true);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue