fix(review): the workflow review ranks by tier and puts back every engine finding
Operator decision 23 (13.10); review findings ab1b5d9f and bc7f4e23 (workflow side). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f04fecea7f
commit
05f6b67be9
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 257 additions and 37 deletions
|
|
@ -114,7 +114,9 @@ async function floorCase({ reviewerFindings = [], codeFindings = [], securityFin
|
|||
const prompt = s.calls.coordinatorPrompt;
|
||||
assert.ok(typeof prompt === 'string', 'the coordinator ran');
|
||||
const merged = JSON.parse(prompt.slice(prompt.indexOf(PROMPT_JSON_MARK) + PROMPT_JSON_MARK.length));
|
||||
const raised = r.findings.some((f, i) => rank(f.severity) < rank(before[i].severity));
|
||||
// Only indexes present in `before` are compared: the floor may append engine
|
||||
// findings it puts back (decision 23), so `returned` can be longer.
|
||||
const raised = r.findings.some((f, i) => i < before.length && rank(f.severity) < rank(before[i].severity));
|
||||
return { r, merged, before, raised, s };
|
||||
}
|
||||
|
||||
|
|
@ -243,7 +245,7 @@ test('armB brief rule (10) — a marked BLOCKER-tier /code-review finding return
|
|||
checkRaise(c, true);
|
||||
});
|
||||
|
||||
test('armB brief rule (11) — dedup keeps the /code-review MAJOR over a first-seen reviewer MINOR on the same triplet → WARN', async () => {
|
||||
test('armB brief rule (11) — bc7f4e23: dedup ranks by tier and keeps the reviewer over a /code-review MAJOR on the same triplet, at the group\'s MAJOR → WARN', async () => {
|
||||
const c = await floorCase({
|
||||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MINOR')],
|
||||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
|
|
@ -251,12 +253,12 @@ test('armB brief rule (11) — dedup keeps the /code-review MAJOR over a first-s
|
|||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.merged.length, 1);
|
||||
assert.equal(c.merged[0].severity, 'MAJOR');
|
||||
assert.equal(c.merged[0].engine, 'code-review');
|
||||
assert.equal(c.merged[0].severity, 'MAJOR', 'the survivor carries the highest severity its group asserted');
|
||||
assert.equal('engine' in c.merged[0], false, 'the reviewer finding survives the tier tie');
|
||||
assert.equal(c.r.deduped_count, 1);
|
||||
assert.equal(c.r.raw_finding_count, 2);
|
||||
assert.equal(c.r.verdict, 'WARN');
|
||||
checkRaise(c, true);
|
||||
checkRaise(c, false, true);
|
||||
});
|
||||
|
||||
test('armB brief rule (12) — an unlabelled reviewer MISSING_TEST asserted BLOCKER, returned at MAJOR (catalogue reset) with WARN stays WARN', async () => {
|
||||
|
|
@ -330,9 +332,11 @@ test('armB brief rule — VERDICT_SCHEMA declares engine, contradicts_brief and
|
|||
assert.ok(/kept and marked, never removed/i.test(prompt));
|
||||
});
|
||||
|
||||
// Decision 13 (operator 12.10): on a severity tie a reviewer-agent finding
|
||||
// survives dedup over an engine finding, so the engine's mark can never lower
|
||||
// what a reviewer agent asserted (the same rule as coordinator-contract Pass 1).
|
||||
// Decision 13 (operator 12.10), bc7f4e23: dedup ranks by catalogue tier, which
|
||||
// always ties inside a catalogue triplet, so a reviewer-agent finding survives
|
||||
// over an engine finding and carries the group's highest asserted severity; the
|
||||
// engine's mark can never lower what a reviewer agent asserted (the same order
|
||||
// as coordinator-contract Pass 1).
|
||||
test('armB brief rule (15) — a tie between a reviewer MAJOR and a /code-review MAJOR keeps the reviewer: the legal shape returned with ALLOW → WARN, no engine', async () => {
|
||||
const c = await floorCase({
|
||||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
|
|
@ -348,3 +352,149 @@ test('armB brief rule (15) — a tie between a reviewer MAJOR and a /code-review
|
|||
assert.equal('engine' in c.r.findings[0], false);
|
||||
checkRaise(c, true);
|
||||
});
|
||||
|
||||
// ---- Decision 23 (operator 13.10, review ab1b5d9f): no engine finding leaves ----
|
||||
// the workflow review unreturned. Every engine finding of the input (before
|
||||
// dedup) with no returned finding at its exact (file, line, rule_key) is put
|
||||
// back at its own severity; a returned finding is never below the lower of its
|
||||
// reviewer counterpart's severity and its catalogue tier.
|
||||
|
||||
const at = (findings, file, line, rule_key) =>
|
||||
findings.filter((f) => f && f.file === file && f.line === line && f.rule_key === rule_key);
|
||||
|
||||
test('decision 23 — a /security-review BLOCKER returned as a CODE_REVIEW_IMPROVEMENT MINOR on the same line → BLOCK, the BLOCKER is put back', async () => {
|
||||
const c = await floorCase({
|
||||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||||
returned: [F('CODE_REVIEW_IMPROVEMENT', 'MINOR')],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'BLOCK');
|
||||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1, 'the BLOCKER is in the findings');
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(back[0].engine, 'security-review');
|
||||
assert.equal('brief_lowered_from' in back[0], false);
|
||||
});
|
||||
|
||||
test('decision 23 — the same BLOCKER returned one line lower at MINOR → BLOCK, the BLOCKER is back at its own line', async () => {
|
||||
const c = await floorCase({
|
||||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||||
returned: [F('SECURITY_REVIEW_VULNERABILITY', 'MINOR', { line: LINE + 1 })],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'BLOCK');
|
||||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1, 'the BLOCKER at its own line is in the findings');
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(back[0].engine, 'security-review');
|
||||
});
|
||||
|
||||
test('decision 23 — the same BLOCKER dropped (returned empty with ALLOW) → BLOCK', async () => {
|
||||
const c = await floorCase({
|
||||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||||
returned: [],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'BLOCK');
|
||||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1);
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(back[0].engine, 'security-review');
|
||||
});
|
||||
|
||||
test('decision 23 — a /security-review BLOCKER shadowed in dedup by a reviewer MINOR on the same triplet, returned empty → BLOCK', async () => {
|
||||
const c = await floorCase({
|
||||
reviewerFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'MINOR')],
|
||||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||||
returned: [],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'BLOCK');
|
||||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1);
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(back[0].engine, 'security-review');
|
||||
});
|
||||
|
||||
test('decision 23 — a /code-review MAJOR dropped → WARN', async () => {
|
||||
const c = await floorCase({
|
||||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
returned: [],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'WARN');
|
||||
const back = at(c.r.findings, FILE, LINE, 'CODE_REVIEW_DEFECT');
|
||||
assert.equal(back.length, 1);
|
||||
assert.equal(back[0].severity, 'MAJOR');
|
||||
assert.equal(back[0].engine, 'code-review');
|
||||
});
|
||||
|
||||
for (const order of ['MAJOR first', 'BLOCKER first']) {
|
||||
test(`decision 23 — two /code-review findings on one triplet (${order}), returned empty → BLOCK, one copy, at BLOCKER`, async () => {
|
||||
const pair = [F('CODE_REVIEW_DEFECT', 'MAJOR', { title: 'major' }), F('CODE_REVIEW_DEFECT', 'BLOCKER', { title: 'blocker' })];
|
||||
const c = await floorCase({
|
||||
codeFindings: order === 'MAJOR first' ? pair : [pair[1], pair[0]],
|
||||
returned: [],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'BLOCK');
|
||||
const back = at(c.r.findings, FILE, LINE, 'CODE_REVIEW_DEFECT');
|
||||
assert.equal(back.length, 1, 'exactly one finding at the triplet');
|
||||
assert.equal(back[0].severity, 'BLOCKER', 'the higher-severity copy is put back');
|
||||
assert.equal(back[0].title, 'blocker');
|
||||
assert.equal(back[0].engine, 'code-review');
|
||||
});
|
||||
}
|
||||
|
||||
test('decision 23 floor (b) — a reviewer finding with a rule_key outside the catalogue asserted MAJOR, returned at MINOR → raised to MAJOR, WARN', async () => {
|
||||
const c = await floorCase({
|
||||
reviewerFindings: [F('NOT_IN_THE_CATALOGUE', 'MAJOR')],
|
||||
returned: [F('NOT_IN_THE_CATALOGUE', 'MINOR')],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'WARN');
|
||||
assert.equal(c.r.findings.length, 1);
|
||||
assert.equal(c.r.findings[0].severity, 'MAJOR', "the bound for an unknown key is the counterpart's asserted severity");
|
||||
});
|
||||
|
||||
test('decision 23 — the one legal lowering (marked /code-review MAJOR returned as MINOR with the mark) still gives ALLOW', async () => {
|
||||
const c = await floorCase({
|
||||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { contradicts_brief: MARK, brief_lowered_from: 'MAJOR' })],
|
||||
verdict: 'ALLOW',
|
||||
});
|
||||
assert.equal(c.r.verdict, 'ALLOW');
|
||||
assert.equal(c.r.findings.length, 1, 'nothing is put back over a returned triplet');
|
||||
assert.equal(c.r.findings[0].severity, 'MINOR');
|
||||
assert.equal(c.r.findings[0].engine, 'code-review');
|
||||
});
|
||||
|
||||
// ---- the docs state the workflow floor ------------------------------------------
|
||||
|
||||
// Slice from the first `start` to the next `end` after it; fail when either is missing.
|
||||
function section(text, start, end) {
|
||||
const i = text.indexOf(start);
|
||||
assert.ok(i !== -1, `section start not found: ${start}`);
|
||||
const j = text.indexOf(end, i + start.length);
|
||||
assert.ok(j !== -1, `section end not found after ${start}: ${JSON.stringify(end)}`);
|
||||
return text.slice(i, j);
|
||||
}
|
||||
|
||||
const CMD_TEXT = readFileSync(join(ROOT, 'commands', 'trekreview.md'), 'utf-8');
|
||||
const HANDOVER_TEXT = readFileSync(join(ROOT, 'docs', 'HANDOVER-CONTRACTS.md'), 'utf-8');
|
||||
const PUT_BACK = /put\s+back\s+at\s+its\s+own\s+severity/;
|
||||
const FLOOR_B = /lower\s+of\s+its\s+reviewer\s+counterpart's\s+severity\s+and\s+its\s+catalogue\s+tier/;
|
||||
|
||||
test('decision 23 — commands/trekreview.md workflow section states the put-back and floor (b); the port is no longer the SAME pipeline', () => {
|
||||
const wf = section(CMD_TEXT, '## Phase 5–6 via the Workflow substrate', '## Phase 7');
|
||||
assert.match(wf, PUT_BACK);
|
||||
assert.match(wf, FLOOR_B);
|
||||
assert.doesNotMatch(CMD_TEXT, /the\s+SAME\s+Phase/);
|
||||
});
|
||||
|
||||
test('decision 23 — Handover 6 states the dedup rank on both paths, the put-back and floor (b)', () => {
|
||||
const h6 = section(HANDOVER_TEXT, '## Handover 6 — `review.md` → plan', '\n## Handover 7');
|
||||
assert.match(h6, /dedup\s+ranks\s+as\s+Pass\s+1\s+does\s+\(both\s+paths\)/);
|
||||
assert.match(h6, PUT_BACK);
|
||||
assert.match(h6, FLOOR_B);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue