fix(review): an engine finding raised again outside findings stays counted (decision 222)

A Claude Security finding counted at confidence high in round N that Claude
Security raises again at a lower confidence in round N+1 landed in that
payload's below_threshold, never in findings. Finalize passed only findings
to carry, the engine was in ran.engines, so the round-N finding was marked
resolved and the verdict fell from BLOCK to WARN (part 3 verifier, F1).

carry takes a new option, raised: the engine entries a round raised outside
findings. A previous engine finding that no current finding matches stays
carried, never resolved, when an entry of its own engine matches it (the
same file, then the same rule_key or content overlap); each entry holds at
most one previous finding. Without the option carry behaves as before.
Finalize's carry pass builds raised from every engine payload's
below_threshold and out_of_scope, the engine taken from the payload's
reviewer, never from the entry.

The carry rule is restated in the review-rounds and review-finalize headers,
/trekreview's round prose and Handover 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-11-08 18:22:49 +01:00
commit 3be2bd3386
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 179 additions and 14 deletions

View file

@ -58,9 +58,13 @@
// a top-level `reviewer` that maps back through `ENGINES`
// (lib/review/engine-findings.mjs), and a reviewer ran when its regular
// `.review-out-<reviewer>.md` passes the findings schema; that output's
// `previous: [{id, resolved}]` are its marks. The previous review's `ran`
// counts only when it is `finalized: true`. Writes `resolved` and a
// `## Resolved since round <n>` section just before `## Declined to Judge`.
// `previous: [{id, resolved}]` are its marks. A previous engine finding
// its own engine raised again this round outside `findings` (an entry of
// that payload's `below_threshold` or `out_of_scope`, the engine taken
// from the payload's `reviewer`) stays carried, never resolved (decision
// 222). The previous review's `ran` counts only when it is
// `finalized: true`. Writes `resolved` and a `## Resolved since round <n>`
// section just before `## Declined to Judge`.
// - pre-existing: `markFindings` (lib/review/pre-existing.mjs) sets
// `pre_existing` on every JSON finding from git over
// `scope_sha_start..scope_sha_end` (read raw, as `reviewedSha` reads; a
@ -421,6 +425,27 @@ function resolvedSection(n, list) {
return `## Resolved since round ${n}\n\nPrevious findings this round no longer raises: ${list.length}.\n\n${items.join('\n')}\n\n`;
}
/** The engine a payload's top-level `reviewer` maps back to through ENGINES, or null. */
const payloadEngine = (p) => (isPlainObject(p) && typeof p.reviewer === 'string' && Object.hasOwn(ENGINE_OF_REVIEWER, p.reviewer)
? ENGINE_OF_REVIEWER[p.reviewer] : null);
/**
* The entries each engine raised this round outside its `findings` (decision 222):
* `below_threshold` and `out_of_scope`, plain objects only, each with the
* engine of its payload (never the entry's own label).
*/
function raisedOutside(payloads) {
const out = [];
for (const p of Array.isArray(payloads) ? payloads : []) {
const engine = payloadEngine(p);
if (!engine) continue;
for (const list of [p.below_threshold, p.out_of_scope]) {
for (const e of Array.isArray(list) ? list : []) if (isPlainObject(e)) out.push({ ...e, engine });
}
}
return out;
}
/** Pass `carry`: only with a previous review. */
function passCarry(payload, ctx) {
if (!ctx.previous) return {};
@ -428,6 +453,7 @@ function passCarry(payload, ctx) {
const current = payload.findings;
const r = carry(ctx.previous.findings, current, {
marks: ctx.evidence.marks, ran: ctx.evidence.ran, previousRan: ctx.previous.ran,
raised: raisedOutside(ctx.evidence.payloads),
});
// carry returns a copy of each current finding at its index: a finding an
// earlier pass added (an engine put-back) stays added under its copy.
@ -887,8 +913,7 @@ export function roundEvidence(dir) {
if (PAYLOAD_RE.test(name)) {
let payload;
try { payload = JSON.parse(readFileSync(p, 'utf8')); } catch { continue; }
const engine = isPlainObject(payload) && typeof payload.reviewer === 'string' && Object.hasOwn(ENGINE_OF_REVIEWER, payload.reviewer)
? ENGINE_OF_REVIEWER[payload.reviewer] : null;
const engine = payloadEngine(payload);
if (engine && Array.isArray(payload.findings)) {
engines.add(engine);
payloads.push(payload);

View file

@ -45,7 +45,7 @@
// - `dryRun` computes the same result and writes, copies and deletes nothing.
// Nothing is written before every check has passed.
//
// carry(previous, current, {marks, ran, previousRan}): a current finding
// carry(previous, current, {marks, ran, previousRan, raised}): a current finding
// matches a previous one in the same file with the same rule_key, or with
// `contentOverlap` >= DEFAULT_THRESHOLD (lib/review/plan-review-dedup.mjs); ids
// are never compared, and each finding matches at most one other. A previous
@ -53,8 +53,18 @@
// rule of lib/review/critic-diff.mjs, with "every reviewer that ran then"
// standing in for the raisers):
// - a matched previous finding is carried;
// - an engine finding no current finding matches is carried when its own
// engine raised it again this round outside `findings` (decision 222):
// `raised` holds that round's engine entries from each payload's
// `below_threshold` and `out_of_scope`, each with the `engine` of the
// payload it came from; an entry matches by the same rule (its `engine`
// equal to the finding's) and holds at most one previous finding. A
// Claude Security finding counted at confidence `high` that comes back
// below it is still raised, so it stays counted. Without `raised`,
// nothing is held this way;
// - an engine finding is resolved only when its engine is in `ran.engines`,
// it was not raised again, and no reviewer marks it `resolved: false`;
// it was not raised again (in `findings` or, through `raised`, outside
// them), and no reviewer marks it `resolved: false`;
// - a reviewer-only finding is resolved only when `previousRan.reviewers` is
// non-empty, every reviewer in it is in `ran.reviewers`, each marked it
// `resolved: true`, and none marked it `resolved: false`;
@ -309,12 +319,15 @@ function before(a, b) {
return false;
}
/** The current finding `p` folds into: same rule_key first, then the nearest line, then the larger overlap, then order. */
function bestMatch(p, current, taken) {
/**
* The current finding `p` folds into: same rule_key first, then the nearest line, then the larger overlap, then order.
* Only entries `eligible` accepts are considered.
*/
function bestMatch(p, current, taken, eligible = () => true) {
let best = -1;
let key = null;
current.forEach((c, i) => {
if (taken.has(i) || !sameFinding(p, c)) return;
if (taken.has(i) || !eligible(c) || !sameFinding(p, c)) return;
const k = [
c.rule_key === p.rule_key ? 0 : 1,
Number.isInteger(c.line) && Number.isInteger(p.line) ? Math.abs(c.line - p.line) : Number.MAX_SAFE_INTEGER,
@ -371,14 +384,18 @@ function resolvedNow(p, { marks, ran, previousRan }) {
* Carry the previous round's findings into this round's (see the header).
* @param {object[]} previous - the previous review's JSON findings
* @param {object[]} current - this round's JSON findings
* @param {{ marks?: Array<{id, resolved, reviewer}>, ran?: {engines, reviewers}, previousRan?: {engines, reviewers}|null }} ctx
* @param {{ marks?: Array<{id, resolved, reviewer}>, ran?: {engines, reviewers}, previousRan?: {engines, reviewers}|null,
* raised?: object[] }} ctx - `raised`: the engine entries this round raised outside `findings`
* (`below_threshold`, `out_of_scope`), each with the `engine` of the payload it came from
* @returns {{ findings: object[], resolved: string[], folded: number }}
*/
export function carry(previous, current, { marks = [], ran = { engines: [], reviewers: [] }, previousRan = null } = {}) {
export function carry(previous, current, { marks = [], ran = { engines: [], reviewers: [] }, previousRan = null, raised = [] } = {}) {
const prev = (Array.isArray(previous) ? previous : []).filter((p) => p && typeof p === 'object');
const cur = (Array.isArray(current) ? current : []).filter((c) => c && typeof c === 'object');
const again = (Array.isArray(raised) ? raised : []).filter((e) => e && typeof e === 'object' && !Array.isArray(e));
const findings = cur.map((c) => ({ ...c }));
const taken = new Set();
const raisedTaken = new Set();
const appended = [];
const resolved = [];
for (const p of prev) {
@ -388,6 +405,13 @@ export function carry(previous, current, { marks = [], ran = { engines: [], revi
findings[i] = fold(p, cur[i]);
continue;
}
// Raised again by its own engine outside `findings`: no evidence it is resolved (decision 222).
const j = typeof p.engine === 'string' && p.engine !== '' ? bestMatch(p, again, raisedTaken, (e) => e.engine === p.engine) : -1;
if (j >= 0) {
raisedTaken.add(j);
appended.push(carried(p));
continue;
}
if (resolvedNow(p, { marks, ran, previousRan })) {
if (typeof p.id === 'string') resolved.push(p.id);
continue;