fix(review): an engine finding raised again outside findings stays counted (decision 222)
A Claude Security finding counted at confidence high in round N that Claude Security raises again at a lower confidence in round N+1 landed in that payload's below_threshold, never in findings. Finalize passed only findings to carry, the engine was in ran.engines, so the round-N finding was marked resolved and the verdict fell from BLOCK to WARN (part 3 verifier, F1). carry takes a new option, raised: the engine entries a round raised outside findings. A previous engine finding that no current finding matches stays carried, never resolved, when an entry of its own engine matches it (the same file, then the same rule_key or content overlap); each entry holds at most one previous finding. Without the option carry behaves as before. Finalize's carry pass builds raised from every engine payload's below_threshold and out_of_scope, the engine taken from the payload's reviewer, never from the entry. The carry rule is restated in the review-rounds and review-finalize headers, /trekreview's round prose and Handover 6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d97e9b5b19
commit
3be2bd3386
5 changed files with 179 additions and 14 deletions
|
|
@ -27,7 +27,9 @@ import { tmpdir } from 'node:os';
|
|||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs';
|
||||
import { convertReport } from '../../lib/review/security-engine.mjs';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { record, writeReport } from '../helpers/claude-security-report.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const ROUNDS = join(ROOT, 'lib', 'review', 'review-rounds.mjs');
|
||||
|
|
@ -35,6 +37,7 @@ const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs');
|
|||
const ANSWERS = join(ROOT, 'lib', 'review', 'review-answers.mjs');
|
||||
const ENGINE = join(ROOT, 'lib', 'review', 'engine-findings.mjs');
|
||||
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
||||
const SECURITY = join(ROOT, 'lib', 'review', 'security-engine.mjs');
|
||||
const VALIDATOR = join(ROOT, 'lib', 'validators', 'review-validator.mjs');
|
||||
const PROJECT_REL = '.claude/projects/2026-11-02-r';
|
||||
const TRAILER = 'Co-Authored-By: Claude Test <noreply@example.invalid>';
|
||||
|
|
@ -1086,3 +1089,113 @@ test('[E3] isSecurityFinding: a finding from Claude Security is a security findi
|
|||
assert.equal(rounds.isSecurityFinding({ engine: 'security-review', rule_key: 'CODE_REVIEW_DEFECT' }), true);
|
||||
assert.equal(rounds.isSecurityFinding({ engine: 'code-review', rule_key: 'CODE_REVIEW_DEFECT' }), false, 'control: /code-review is not a security engine');
|
||||
});
|
||||
|
||||
// ---- E 3, decision 222: an engine finding its engine raises again outside `findings` stays counted ----
|
||||
|
||||
// A Claude Security BLOCKER counted in round 1, at b.mjs:3.
|
||||
const CS_PREV = { severity: 'BLOCKER', file: 'b.mjs', line: 3, rule_key: 'SECURITY_REVIEW_VULNERABILITY', engine: 'claude-security',
|
||||
title: 'Shell command built from the branch name', detail: 'checkout() interpolates the branch name into a shell string.' };
|
||||
const CS_RAN = { engines: ['claude-security', 'code-review'], reviewers: [CCR, BCR] };
|
||||
|
||||
/** Claude Security's payload of `records` (b.mjs:3 unless changed), converted by the real convertReport. */
|
||||
function csPayload(records, scopeFiles) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'review-rounds-cs-'));
|
||||
made.push(dir);
|
||||
const base = 'a'.repeat(40);
|
||||
const head = 'b'.repeat(40);
|
||||
writeReport(dir, { head, base, records: records.map((over) => record({ file: 'b.mjs', line: 3, title: CS_PREV.title, ...over })) });
|
||||
const r = convertReport({ reportDir: dir, base, head, scopeFiles });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
return r.payload;
|
||||
}
|
||||
/** A payload list's entries as finalize passes them to carry: plain objects, `engine` from the payload's reviewer. */
|
||||
const raisedBy = (engine, list) => list.map((e) => ({ ...e, engine }));
|
||||
|
||||
test('[E3] decision 222: a counted Claude Security finding its engine raises again below confidence high (below_threshold) stays counted', () => {
|
||||
const p = P(CS_PREV);
|
||||
const pl = csPayload([{ confidence: 'medium' }]);
|
||||
assert.deepEqual([pl.findings.length, pl.below_threshold.length], [0, 1], 'Control: the raise is in below_threshold only');
|
||||
const ctx = { marks: [], ran: CS_RAN, previousRan: CS_RAN };
|
||||
// Control: without `raised`, carry behaves as before: the engine ran and its findings do not hold it.
|
||||
const before = rounds.carry([p], pl.findings, ctx);
|
||||
assert.deepEqual([before.findings.length, before.resolved], [0, [p.id]], 'Control: without raised it resolves');
|
||||
const held = rounds.carry([p], pl.findings, { ...ctx, raised: raisedBy('claude-security', pl.below_threshold) });
|
||||
assert.deepEqual(held.resolved, [], 'a below_threshold raise is no evidence that it is resolved');
|
||||
assert.deepEqual(held.findings.map((x) => [x.severity, x.carried_from, x.engine]), [['BLOCKER', p.id, 'claude-security']],
|
||||
'it is carried as it was, at its own severity');
|
||||
// Control: another engine's entry does not hold it.
|
||||
const other = rounds.carry([p], pl.findings, { ...ctx, raised: raisedBy('security-review', pl.below_threshold) });
|
||||
assert.deepEqual([other.findings.length, other.resolved], [0, [p.id]], 'Control: an entry of another engine holds nothing');
|
||||
});
|
||||
|
||||
test('[E3] decision 222: a counted engine finding its engine raises again outside the review scope (out_of_scope) stays counted', () => {
|
||||
const p = P(CS_PREV);
|
||||
const pl = csPayload([{}], ['a.mjs']);
|
||||
assert.deepEqual([pl.findings.length, pl.out_of_scope.length], [0, 1], 'Control: the raise is in out_of_scope only');
|
||||
const held = rounds.carry([p], pl.findings, { marks: [], ran: CS_RAN, previousRan: CS_RAN, raised: raisedBy('claude-security', pl.out_of_scope) });
|
||||
assert.deepEqual(held.resolved, [], 'an out_of_scope raise is no evidence that it is resolved');
|
||||
assert.deepEqual(held.findings.map((x) => [x.severity, x.carried_from]), [['BLOCKER', p.id]]);
|
||||
});
|
||||
|
||||
test('[E3] decision 222: each raised entry holds at most one previous finding', () => {
|
||||
const p3 = P(CS_PREV);
|
||||
const p8 = P({ ...CS_PREV, line: 8 });
|
||||
const pl = csPayload([{ confidence: 'low' }]);
|
||||
const r = rounds.carry([p3, p8], pl.findings, { marks: [], ran: CS_RAN, previousRan: CS_RAN, raised: raisedBy('claude-security', pl.below_threshold) });
|
||||
assert.deepEqual(r.findings.map((x) => x.carried_from), [p3.id], 'the nearer previous finding is held');
|
||||
assert.deepEqual(r.resolved, [p8.id], 'the one entry holds no second finding');
|
||||
});
|
||||
|
||||
test('[E3] decision 222: round 2 whose Claude Security payload holds the round-1 BLOCKER only in below_threshold keeps it counted: BLOCK, never under Resolved since round 1', () => {
|
||||
const fx = rndRepo();
|
||||
const { prev } = roundOne(fx, [CS_PREV], { engines: ['code-review', 'claude-security'] });
|
||||
const [id] = prev.findings.map((x) => x.id);
|
||||
/** Round 2's Claude Security report of start..c2, converted by the real CLI and saved as Phase 4.7 saves it. */
|
||||
const claudeSecurity = (records) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'review-rounds-cs-report-'));
|
||||
made.push(dir);
|
||||
writeReport(dir, { head: fx.c2, base: fx.start, records });
|
||||
const scope = join(dir, 'scope.txt');
|
||||
writeFileSync(scope, 'a.mjs\nb.mjs\n');
|
||||
const r = run([SECURITY, 'convert', '--report-dir', dir, '--base', fx.start, '--head', fx.c2, '--scope', scope]);
|
||||
assert.equal(r.status, 0, `security-engine convert: ${r.stdout}${r.stderr}`);
|
||||
writeFileSync(join(fx.project, '.review-engine-payload-claude-security.json'), r.stdout);
|
||||
return JSON.parse(r.stdout);
|
||||
};
|
||||
const resolvedPart = (text) => text.slice(text.indexOf('## Resolved since round 1'), text.indexOf('## Declined to Judge'));
|
||||
// The coordinator declined the below-threshold raise: no finding, WARN.
|
||||
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
||||
const pl = claudeSecurity([record({ file: 'b.mjs', line: 3, title: CS_PREV.title, confidence: 'medium' })]);
|
||||
assert.deepEqual([pl.findings.length, pl.below_threshold.length], [0, 1], 'Control: Claude Security raised it again, below high only');
|
||||
const f = fx.finalize2();
|
||||
assert.equal(f.status, 0, f.stdout + f.stderr);
|
||||
const text = fx.read();
|
||||
const payload = jsonOf(text);
|
||||
assert.deepEqual(payload.ran.engines, ['claude-security', 'code-review'], 'Claude Security ran this round');
|
||||
assert.deepEqual(payload.resolved, [], 'the round-1 BLOCKER is not resolved');
|
||||
assert.deepEqual(payload.findings.map((x) => [x.severity, x.carried_from]), [['BLOCKER', id]], 'it stays counted, carried');
|
||||
assert.equal(payload.verdict, 'BLOCK');
|
||||
assert.match(fmOf(text), /^verdict: BLOCK$/m);
|
||||
assert.match(text, /^## Resolved since round 1$/m, text);
|
||||
assert.ok(!resolvedPart(text).includes(CS_PREV.title) && !resolvedPart(text).includes(id), `not under Resolved since round 1:\n${resolvedPart(text)}`);
|
||||
const gate = fx.gate();
|
||||
assert.equal(gate.status, 1, gate.stdout + gate.stderr);
|
||||
assert.equal(parse(gate).reason, 'verdict BLOCK', gate.stdout);
|
||||
// Control: Claude Security ran and did not raise it at all: resolved.
|
||||
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
||||
claudeSecurity([]);
|
||||
const c = fx.finalize2();
|
||||
assert.equal(c.status, 0, c.stdout + c.stderr);
|
||||
assert.deepEqual(jsonOf(fx.read()).resolved, [id], 'Control: a clean Claude Security run resolves it');
|
||||
assert.ok(resolvedPart(fx.read()).includes(CS_PREV.title), 'Control: listed under Resolved since round 1');
|
||||
// Control: an entry in another engine's payload never holds it, whatever engine the entry names.
|
||||
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: [], verdict: 'WARN' });
|
||||
claudeSecurity([]);
|
||||
writeFileSync(join(fx.project, '.review-engine-payload-code-review.json'), JSON.stringify({
|
||||
reviewer: 'claude-code-review', findings: [], out_of_scope: [], below_threshold: pl.below_threshold,
|
||||
}));
|
||||
assert.equal(pl.below_threshold[0].engine, 'claude-security', 'Control: the planted entry names claude-security');
|
||||
const o = fx.finalize2();
|
||||
assert.equal(o.status, 0, o.stdout + o.stderr);
|
||||
assert.deepEqual(jsonOf(fx.read()).resolved, [id], 'Control: the engine is the payload\'s, never the entry\'s');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue