fix(review): the review keeps the findings that are right

voyage-lekerepo end-to-end trial (2026-10-06), report points 1, 5, 11:

- engine-findings: recommended_action is "Fix <file:line> so this no longer
  happens: <failure scenario>", not "Resolve: <title>" (8 of 8 engine
  findings were set aside as actionability:empty); the coordinator never
  sets an engine finding aside for Actionability.
- findings-schema --repo-root: a cited line past the end of an existing
  file is FINDING_LINE_OUT_OF_RANGE, so Phase 5's bounded re-ask fixes it
  before the coordinator sees it. The reviewers' instruction said "the
  actual line number from the diff"; it now says the file's own line,
  never the patch's.
- fill-finding-ids.mjs: the coordinator (no shell) writes @@ID_<n>@@ and
  trekreview Phase 7 computes them over (file, line, rule_key); an
  unresolved placeholder or a wrong 40-hex id fails. The 4-field id with
  `title` the docs named is gone; the code always used three.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-08 06:48:13 +02:00
commit 609eb35015
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
10 changed files with 391 additions and 29 deletions

View file

@ -11,10 +11,10 @@
// uses LLM judgement here; a verb-list approximation would DIVERGE from the
// contract being mirrored, so only the deterministic "recommended_action is
// present-and-non-empty when supplied" half is kept.
// - The doc's 4-tuple `(file,line,rule_key,title)` id recompute — the shipped
// `computeFindingId` is 3-arg `(file,line,rule_key)`; this module follows
// the shipped code and flags the doc divergence (the 4-tuple id is not
// producible by the current helper).
// - Nothing about ids beyond the triplet: `computeFindingId(file, line,
// rule_key)` is the one id. The LLM coordinator writes `@@ID_<n>@@`
// placeholders and lib/review/fill-finding-ids.mjs computes them (the
// old doc named a 4-tuple with `title` the code never computed).
//
// What IS implemented, purely: Pass 1 (triplet dedup → highest-severity-wins
// survivor + conformance tiebreak + detail concat + raised_by provenance),

View file

@ -97,6 +97,8 @@ export function toReviewerPayload(input, opts = {}) {
const narrower = typeof f.rule_key === 'string' && RULE_KEYS.has(f.rule_key)
&& tier(f.rule_key) <= tier(spec.rule_key);
const rule_key = narrower ? f.rule_key : spec.rule_key;
const scenario = String(f.failure_scenario ?? '').trim();
const loc = line > 0 ? `${file}:${line}` : file;
const out = {
id: computeFindingId(file, line, rule_key),
severity: getRule(rule_key).severity,
@ -107,7 +109,12 @@ export function toReviewerPayload(input, opts = {}) {
title: summary,
detail: [String(f.failure_scenario ?? '').trim(), f.category ? `(engine category: ${f.category})` : '']
.filter(Boolean).join(' ') || summary,
recommended_action: `Resolve: ${summary}`,
// An action, not the title again: `Resolve: <title>` read as "restating
// the title" to the coordinator's Actionability filter, and 8 of 8
// engine findings were set aside (voyage-lekerepo trial, 2026-10-06).
recommended_action: scenario
? `Fix ${loc} so this no longer happens: ${scenario}`
: `Fix the defect at ${loc} that ${input.engine === 'security-review' ? '/security-review' : '/code-review'} reported`,
engine: input.engine,
};
if (scope && !scope.has(file)) out_of_scope.push(out);

View file

@ -0,0 +1,101 @@
// lib/review/fill-finding-ids.mjs
// Finding ids are computed by code, never by the coordinator.
//
// The review-coordinator agent has Read/Glob/Grep and no shell, so it cannot
// compute the SHA1 a finding id is. In the voyage-lekerepo trial (2026-10-06)
// it wrote `@@ID_1@@` placeholders and the orchestrator filled them by hand, a
// step no document named. Now the contract says it: the coordinator writes
// `@@ID_<n>@@` wherever finding n's id belongs (n = 1-based position in the
// trailing JSON block's `findings` array), and this tool replaces every
// placeholder with computeFindingId(file, line, rule_key) of that finding.
//
// Fail-closed: a placeholder with no JSON finding behind it, or a 40-hex id
// in the JSON block that is not the computed one, is an error -- never a guess.
// A review that is already correct passes unchanged (idempotent).
//
// CLI: node lib/review/fill-finding-ids.mjs <review.md>
// exit 0 = filled (file rewritten in place) or already correct;
// 1 = unresolved placeholder / wrong id (errors on stderr, file untouched);
// 2 = could not run (unreadable file, no JSON block).
import { readFileSync, writeFileSync, renameSync, realpathSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { computeFindingId } from '../parsers/finding-id.mjs';
import { extractFindingsBlock } from './findings-schema.mjs';
const PLACEHOLDER = /@@ID_(\d+)@@/g;
const HEX40 = /^[0-9a-f]{40}$/;
/**
* @param {string} text - review.md content
* @returns {{ ok: boolean, text: string, errors: string[], fatal?: boolean }}
*/
export function fillFindingIds(text) {
const block = extractFindingsBlock(text);
if (block === null) return { ok: false, fatal: true, text, errors: ['no trailing ```json block: nothing to compute ids from'] };
let parsed;
try {
parsed = JSON.parse(block);
} catch (e) {
return { ok: false, fatal: true, text, errors: [`the JSON block did not parse: ${e.message}`] };
}
const findings = Array.isArray(parsed?.findings) ? parsed.findings : [];
const errors = [];
const ids = findings.map((f, i) => {
try {
return computeFindingId(f.file, f.line, f.rule_key);
} catch (e) {
errors.push(`findings[${i}]: cannot compute an id (${e.message})`);
return null;
}
});
findings.forEach((f, i) => {
if (typeof f.id === 'string' && HEX40.test(f.id) && ids[i] && f.id !== ids[i]) {
errors.push(`findings[${i}].id ${f.id} does not match the id computed over (file, line, rule_key): ${ids[i]}`);
}
});
const out = text.replace(PLACEHOLDER, (m, n) => {
const id = ids[Number(n) - 1];
if (!id) {
errors.push(`${m} has no JSON finding ${n} behind it (${findings.length} findings)`);
return m;
}
return id;
});
if (errors.length > 0) return { ok: false, text, errors };
return { ok: true, text: out, errors: [] };
}
function isMain() {
try {
return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
} catch {
return false;
}
}
if (isMain()) {
const path = process.argv[2];
if (!path) {
process.stderr.write('Usage: fill-finding-ids.mjs <review.md>\n');
process.exit(2);
}
let text;
try {
text = readFileSync(path, 'utf-8');
} catch (e) {
process.stderr.write(`fill-finding-ids: cannot read ${path}: ${e.message}\n`);
process.exit(2);
}
const r = fillFindingIds(text);
if (!r.ok) {
for (const e of r.errors) process.stderr.write(`fill-finding-ids: ${e}\n`);
process.exit(r.fatal ? 2 : 1);
}
if (r.text !== text) {
const tmp = `${path}.tmp-${process.pid}`;
writeFileSync(tmp, r.text);
renameSync(tmp, path);
}
process.stdout.write(`fill-finding-ids: ok ${path}\n`);
}

View file

@ -21,8 +21,9 @@
//
// 3-layer pattern (Content → Raw-text → CLI shim) mirroring the other validators.
import { readFileSync, existsSync, realpathSync } from 'node:fs';
import { readFileSync, existsSync, realpathSync, statSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { resolve, sep } from 'node:path';
import { issue, fail } from '../util/result.mjs';
import { RULE_KEYS, SEVERITY_VALUES } from './rule-catalogue.mjs';
@ -102,7 +103,42 @@ function validateFinding(finding, index, errors) {
* @param {unknown} payload
* @returns {import('../util/result.mjs').Result}
*/
export function validateFindings(payload) {
// A reviewer that reads the diff cites the line in the PATCH, not in the file:
// the voyage-lekerepo trial (2026-10-06) cited tests/cli.test.mjs:194 for a
// 124-line file, the coordinator then set the only brief-anchored MAJOR aside,
// and the claim itself was right. With a repo root, a line past the end of a
// file that exists is an error, so Phase 5's bounded re-ask fixes it before the
// coordinator ever sees it. A file absent from the tree is not judged here.
function lineCount(repoRoot, file) {
const root = resolve(repoRoot);
const abs = resolve(root, file);
if (abs !== root && !abs.startsWith(root + sep)) return null;
try {
if (!statSync(abs).isFile()) return null;
const text = readFileSync(abs, 'utf-8');
if (text === '') return 0;
return text.endsWith('\n') ? text.split('\n').length - 1 : text.split('\n').length;
} catch {
return null;
}
}
function checkLineRange(finding, index, repoRoot, errors) {
if (!finding || typeof finding.file !== 'string' || !Number.isInteger(finding.line) || finding.line <= 0) return;
const n = lineCount(repoRoot, finding.file);
if (n === null || finding.line <= n) return;
errors.push(issue(
'FINDING_LINE_OUT_OF_RANGE',
`findings[${index}] cites ${finding.file}:${finding.line}, but the file has ${n} lines`,
'Cite the line number in the file itself (Read it), not the line in the diff or patch.',
));
}
/**
* @param {unknown} payload
* @param {{ repoRoot?: string }} [opts] - with a repo root, cited lines are checked against the files
*/
export function validateFindings(payload, opts = {}) {
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) {
return fail(issue('FINDINGS_NOT_OBJECT', `Reviewer output must be a JSON object, got ${Array.isArray(payload) ? 'array' : typeof payload}`));
}
@ -121,6 +157,7 @@ export function validateFindings(payload) {
for (let i = 0; i < payload.findings.length; i++) {
validateFinding(payload.findings[i], i, errors);
if (opts.repoRoot) checkLineRange(payload.findings[i], i, opts.repoRoot, errors);
}
return { valid: errors.length === 0, errors, warnings, parsed: payload };
@ -133,7 +170,7 @@ export function validateFindings(payload) {
* @param {string} rawText
* @returns {import('../util/result.mjs').Result}
*/
export function validateReviewerOutput(rawText) {
export function validateReviewerOutput(rawText, opts = {}) {
const block = extractFindingsBlock(rawText);
if (block === null) {
return fail(issue(
@ -148,7 +185,7 @@ export function validateReviewerOutput(rawText) {
} catch (e) {
return fail(issue('FINDINGS_PARSE_ERROR', `Reviewer JSON block did not parse: ${e.message}`));
}
return validateFindings(parsed);
return validateFindings(parsed, opts);
}
// ---- CLI shim ----------------------------------------------------------------
@ -163,16 +200,20 @@ function isMain() {
if (isMain()) {
const args = process.argv.slice(2);
const filePath = args.find((a) => !a.startsWith('--'));
if (!filePath) {
process.stderr.write('Usage: findings-schema.mjs [--json] <reviewer-output.txt|.md>\n');
const rootIdx = args.indexOf('--repo-root');
const repoRoot = rootIdx >= 0 ? args[rootIdx + 1] : undefined;
// Without the flag nothing is skipped (rootIdx + 1 would be 0: the file).
const skipIdx = rootIdx >= 0 ? rootIdx + 1 : -1;
const filePath = args.find((a, i) => !a.startsWith('--') && i !== skipIdx);
if (!filePath || (rootIdx >= 0 && !repoRoot)) {
process.stderr.write('Usage: findings-schema.mjs [--json] [--repo-root <dir>] <reviewer-output.txt|.md>\n');
process.exit(2);
}
if (!existsSync(filePath)) {
process.stderr.write(`findings-schema: file not found: ${filePath}\n`);
process.exit(2);
}
const r = validateReviewerOutput(readFileSync(filePath, 'utf-8'));
const r = validateReviewerOutput(readFileSync(filePath, 'utf-8'), { repoRoot });
if (args.includes('--json')) {
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
} else {