fix(review): group a triplet's findings title first, so arrival order never changes the rows

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-23 18:30:52 +02:00
commit a5fcb5e8b4
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
6 changed files with 339 additions and 37 deletions

View file

@ -455,3 +455,126 @@ test('runContract — the #k id counts rows of one triplet in the final list, so
assert.equal(result.findings[0].id, computeFindingId('bin/slug.mjs', 46, 'MISSING_ERROR_HANDLING'), 'what fill-finding-ids computes for a review.md holding this row alone');
assert.equal(result.trimmed_count, 1, 'the trimmed row that Pass 2 set aside is not counted');
});
// ---- ORD — the rows a finding joins do not depend on arrival order ---------
/** Every arrival order of `xs`. */
function orders(xs) {
if (xs.length <= 1) return [xs];
return xs.flatMap((x, i) => orders([...xs.slice(0, i), ...xs.slice(i + 1)]).map((rest) => [x, ...rest]));
}
/** A row's members, rebuilt from its survivor plus each "Also flagged by" line, as sorted `reviewer|title` strings. */
function members(row) {
const out = [`${row.reviewer}|${row.title}`];
for (const m of String(row.detail ?? '').matchAll(/^Also flagged by (.+?): (.+)\.$/gm)) out.push(`${m[1]}|${m[2]}`);
return out.sort();
}
const ORD_C = 'code-correctness-reviewer';
const ORD_F = 'brief-conformance-reviewer';
test('[ORD] correctness A, conformance B, correctness B give the rows {correctness A} and {conformance B, correctness B} in every arrival order', () => {
const at = { file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_TEST', detail: 'd' };
const cA = { ...at, severity: 'MAJOR', title: 'A', reviewer: ORD_C };
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
const cB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_C };
const expected = [[`${ORD_C}|A`], [`${ORD_F}|B`, `${ORD_C}|B`]];
const all = orders([['cA', cA], ['fB', fB], ['cB', cB]]);
assert.equal(all.length, 6);
for (const order of all) {
const name = order.map(([n]) => n).join(', ');
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
assert.equal(rows.length, 2, `${name}: ${JSON.stringify(rows.map(members))}`);
const got = rows.map(members).sort((x, y) => x.length - y.length || x.join().localeCompare(y.join()));
assert.deepEqual(got, expected, `order ${name}`);
}
});
test('[ORD] with an off-catalogue rule_key and cA MINOR, fB MAJOR, cB MINOR, every arrival order gives {cA} MINOR, {fB, cB} MAJOR and one verdict', () => {
const at = { file: 'bin/slug.mjs', line: 46, rule_key: 'NOT_IN_THE_CATALOGUE', detail: 'd' };
const cA = { ...at, severity: 'MINOR', title: 'A', reviewer: ORD_C };
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
const cB = { ...at, severity: 'MINOR', title: 'B', reviewer: ORD_C };
const expected = { [`${ORD_C}|A`]: 'MINOR', [`${ORD_F}|B+${ORD_C}|B`]: 'MAJOR' };
const verdicts = new Set();
for (const order of orders([['cA', cA], ['fB', fB], ['cB', cB]])) {
const name = order.map(([n]) => n).join(', ');
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
const bySet = Object.fromEntries(rows.map((r) => [members(r).join('+'), r.severity]));
assert.deepEqual(bySet, expected, `order ${name}`);
verdicts.add(JSON.stringify(computeVerdict(rows)));
}
assert.equal(verdicts.size, 1, [...verdicts].join(' | '));
assert.equal(JSON.parse([...verdicts][0]).verdict, 'WARN');
});
/** The kept and unverified rows of a run, as sorted `reviewer|title|severity` strings, so two runs can be compared. */
function split(result) {
const key = (f) => `${f.reviewer}|${String(f.title ?? '').trim().toLowerCase()}|${f.severity}`;
return { kept: result.findings.map(key).sort(), unverified: result.unverified.map(key).sort() };
}
test('[ORD] a full tie on one row: one reviewer\'s two same-title BLOCKERs, one with an empty recommended_action, give one verdict in both arrival orders', () => {
const at = { file: 'x.mjs', line: 3, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' };
const empty = { ...at, title: 'Injection', recommended_action: '' };
const full = { ...at, title: 'injection ', recommended_action: 'Quote the argument.' };
const a = runContract([{ reviewer: ORD_C, findings: [structuredClone(empty), structuredClone(full)] }]);
const b = runContract([{ reviewer: ORD_C, findings: [structuredClone(full), structuredClone(empty)] }]);
assert.equal(a.verdict, b.verdict, `order (empty, full) gives ${a.verdict}, order (full, empty) gives ${b.verdict}`);
assert.deepEqual(split(a), split(b), 'the kept/unverified split');
});
test('[ORD] a full tie on one row: two /code-review MAJORs titled T, one marked as contradicting the brief, give one verdict in both arrival orders', () => {
const at = { file: 'x.mjs', line: 3, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'T' };
const marked = { ...at, contradicts_brief: 'Non-Goal: no new runtime dependency' };
const plain = { ...at };
const a = runContract([{ reviewer: 'claude-code-review', findings: [structuredClone(marked), structuredClone(plain)] }]);
const b = runContract([{ reviewer: 'claude-code-review', findings: [structuredClone(plain), structuredClone(marked)] }]);
assert.equal(a.verdict, b.verdict, `marked first gives ${a.verdict}, unmarked first gives ${b.verdict}`);
assert.deepEqual(split(a), split(b), 'the kept/unverified split');
});
/** A row's member set, rebuilt from its survivor plus each "Also flagged by" line; an untitled member is named by its rule_key, as the line names it. */
function memberSet(row) {
const out = [`${row.reviewer}|${row.title ?? row.rule_key}`];
for (const m of String(row.detail ?? '').matchAll(/^Also flagged by (.+?): (.+)\.$/gm)) out.push(`${m[1]}|${m[2]}`);
return out.sort().join('+');
}
test('[ORD] the title classes are placed in alphabetical order: cA, fB, cB and an untitled correctness BLOCKER give {cA, cU} BLOCKER and {fB, cB} MAJOR and one verdict in all 24 orders', () => {
const K = 'NOT_IN_THE_CATALOGUE';
const at = { file: 'bin/slug.mjs', line: 46, rule_key: K, detail: 'd' };
// cA and cU tie on every survivor rule, so the row's first member in the
// group's order (class A before the untitled findings) is kept: cA, whose
// empty recommended_action sets the row aside as unverified.
const cA = { ...at, severity: 'BLOCKER', title: 'A', reviewer: ORD_C, recommended_action: '' };
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
const cB = { ...at, severity: 'MINOR', title: 'B', reviewer: ORD_C };
const cU = { ...at, severity: 'BLOCKER', reviewer: ORD_C };
const expected = {
[`${ORD_C}|A+${ORD_C}|${K}`]: 'BLOCKER',
[`${ORD_F}|B+${ORD_C}|B`]: 'MAJOR',
};
const all = orders([['cA', cA], ['fB', fB], ['cB', cB], ['cU', cU]]);
assert.equal(all.length, 24);
const verdicts = new Set();
for (const order of all) {
const name = order.map(([n]) => n).join(', ');
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
const bySet = Object.fromEntries(rows.map((r) => [memberSet(r), r.severity]));
assert.deepEqual(bySet, expected, `order ${name}`);
const j = judgeFilter(rows);
verdicts.add(JSON.stringify({
kept: j.kept.map(memberSet).sort(),
unverified: j.unverified.map(memberSet).sort(),
verdict: computeVerdict(j.kept, { unverified: j.unverified }).verdict,
}));
}
assert.equal(verdicts.size, 1, [...verdicts].join(' | '));
assert.deepEqual(JSON.parse([...verdicts][0]), {
kept: [`${ORD_F}|B+${ORD_C}|B`],
unverified: [`${ORD_C}|A+${ORD_C}|${K}`],
verdict: 'WARN',
});
});