fix(review): the review keeps the findings that are right
voyage-lekerepo end-to-end trial (2026-10-06), report points 1, 5, 11: - engine-findings: recommended_action is "Fix <file:line> so this no longer happens: <failure scenario>", not "Resolve: <title>" (8 of 8 engine findings were set aside as actionability:empty); the coordinator never sets an engine finding aside for Actionability. - findings-schema --repo-root: a cited line past the end of an existing file is FINDING_LINE_OUT_OF_RANGE, so Phase 5's bounded re-ask fixes it before the coordinator sees it. The reviewers' instruction said "the actual line number from the diff"; it now says the file's own line, never the patch's. - fill-finding-ids.mjs: the coordinator (no shell) writes @@ID_<n>@@ and trekreview Phase 7 computes them over (file, line, rule_key); an unresolved placeholder or a wrong 40-hex id fails. The 4-field id with `title` the docs named is gone; the code always used three. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
c56f852ab6
commit
609eb35015
10 changed files with 391 additions and 29 deletions
|
|
@ -11,10 +11,10 @@
|
|||
// uses LLM judgement here; a verb-list approximation would DIVERGE from the
|
||||
// contract being mirrored, so only the deterministic "recommended_action is
|
||||
// present-and-non-empty when supplied" half is kept.
|
||||
// - The doc's 4-tuple `(file,line,rule_key,title)` id recompute — the shipped
|
||||
// `computeFindingId` is 3-arg `(file,line,rule_key)`; this module follows
|
||||
// the shipped code and flags the doc divergence (the 4-tuple id is not
|
||||
// producible by the current helper).
|
||||
// - Nothing about ids beyond the triplet: `computeFindingId(file, line,
|
||||
// rule_key)` is the one id. The LLM coordinator writes `@@ID_<n>@@`
|
||||
// placeholders and lib/review/fill-finding-ids.mjs computes them (the
|
||||
// old doc named a 4-tuple with `title` the code never computed).
|
||||
//
|
||||
// What IS implemented, purely: Pass 1 (triplet dedup → highest-severity-wins
|
||||
// survivor + conformance tiebreak + detail concat + raised_by provenance),
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ export function toReviewerPayload(input, opts = {}) {
|
|||
const narrower = typeof f.rule_key === 'string' && RULE_KEYS.has(f.rule_key)
|
||||
&& tier(f.rule_key) <= tier(spec.rule_key);
|
||||
const rule_key = narrower ? f.rule_key : spec.rule_key;
|
||||
const scenario = String(f.failure_scenario ?? '').trim();
|
||||
const loc = line > 0 ? `${file}:${line}` : file;
|
||||
const out = {
|
||||
id: computeFindingId(file, line, rule_key),
|
||||
severity: getRule(rule_key).severity,
|
||||
|
|
@ -107,7 +109,12 @@ export function toReviewerPayload(input, opts = {}) {
|
|||
title: summary,
|
||||
detail: [String(f.failure_scenario ?? '').trim(), f.category ? `(engine category: ${f.category})` : '']
|
||||
.filter(Boolean).join(' ') || summary,
|
||||
recommended_action: `Resolve: ${summary}`,
|
||||
// An action, not the title again: `Resolve: <title>` read as "restating
|
||||
// the title" to the coordinator's Actionability filter, and 8 of 8
|
||||
// engine findings were set aside (voyage-lekerepo trial, 2026-10-06).
|
||||
recommended_action: scenario
|
||||
? `Fix ${loc} so this no longer happens: ${scenario}`
|
||||
: `Fix the defect at ${loc} that ${input.engine === 'security-review' ? '/security-review' : '/code-review'} reported`,
|
||||
engine: input.engine,
|
||||
};
|
||||
if (scope && !scope.has(file)) out_of_scope.push(out);
|
||||
|
|
|
|||
101
lib/review/fill-finding-ids.mjs
Normal file
101
lib/review/fill-finding-ids.mjs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
// lib/review/fill-finding-ids.mjs
|
||||
// Finding ids are computed by code, never by the coordinator.
|
||||
//
|
||||
// The review-coordinator agent has Read/Glob/Grep and no shell, so it cannot
|
||||
// compute the SHA1 a finding id is. In the voyage-lekerepo trial (2026-10-06)
|
||||
// it wrote `@@ID_1@@` placeholders and the orchestrator filled them by hand, a
|
||||
// step no document named. Now the contract says it: the coordinator writes
|
||||
// `@@ID_<n>@@` wherever finding n's id belongs (n = 1-based position in the
|
||||
// trailing JSON block's `findings` array), and this tool replaces every
|
||||
// placeholder with computeFindingId(file, line, rule_key) of that finding.
|
||||
//
|
||||
// Fail-closed: a placeholder with no JSON finding behind it, or a 40-hex id
|
||||
// in the JSON block that is not the computed one, is an error -- never a guess.
|
||||
// A review that is already correct passes unchanged (idempotent).
|
||||
//
|
||||
// CLI: node lib/review/fill-finding-ids.mjs <review.md>
|
||||
// exit 0 = filled (file rewritten in place) or already correct;
|
||||
// 1 = unresolved placeholder / wrong id (errors on stderr, file untouched);
|
||||
// 2 = could not run (unreadable file, no JSON block).
|
||||
|
||||
import { readFileSync, writeFileSync, renameSync, realpathSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { computeFindingId } from '../parsers/finding-id.mjs';
|
||||
import { extractFindingsBlock } from './findings-schema.mjs';
|
||||
|
||||
const PLACEHOLDER = /@@ID_(\d+)@@/g;
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
|
||||
/**
|
||||
* @param {string} text - review.md content
|
||||
* @returns {{ ok: boolean, text: string, errors: string[], fatal?: boolean }}
|
||||
*/
|
||||
export function fillFindingIds(text) {
|
||||
const block = extractFindingsBlock(text);
|
||||
if (block === null) return { ok: false, fatal: true, text, errors: ['no trailing ```json block: nothing to compute ids from'] };
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(block);
|
||||
} catch (e) {
|
||||
return { ok: false, fatal: true, text, errors: [`the JSON block did not parse: ${e.message}`] };
|
||||
}
|
||||
const findings = Array.isArray(parsed?.findings) ? parsed.findings : [];
|
||||
const errors = [];
|
||||
const ids = findings.map((f, i) => {
|
||||
try {
|
||||
return computeFindingId(f.file, f.line, f.rule_key);
|
||||
} catch (e) {
|
||||
errors.push(`findings[${i}]: cannot compute an id (${e.message})`);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
findings.forEach((f, i) => {
|
||||
if (typeof f.id === 'string' && HEX40.test(f.id) && ids[i] && f.id !== ids[i]) {
|
||||
errors.push(`findings[${i}].id ${f.id} does not match the id computed over (file, line, rule_key): ${ids[i]}`);
|
||||
}
|
||||
});
|
||||
const out = text.replace(PLACEHOLDER, (m, n) => {
|
||||
const id = ids[Number(n) - 1];
|
||||
if (!id) {
|
||||
errors.push(`${m} has no JSON finding ${n} behind it (${findings.length} findings)`);
|
||||
return m;
|
||||
}
|
||||
return id;
|
||||
});
|
||||
if (errors.length > 0) return { ok: false, text, errors };
|
||||
return { ok: true, text: out, errors: [] };
|
||||
}
|
||||
|
||||
function isMain() {
|
||||
try {
|
||||
return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (isMain()) {
|
||||
const path = process.argv[2];
|
||||
if (!path) {
|
||||
process.stderr.write('Usage: fill-finding-ids.mjs <review.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
let text;
|
||||
try {
|
||||
text = readFileSync(path, 'utf-8');
|
||||
} catch (e) {
|
||||
process.stderr.write(`fill-finding-ids: cannot read ${path}: ${e.message}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = fillFindingIds(text);
|
||||
if (!r.ok) {
|
||||
for (const e of r.errors) process.stderr.write(`fill-finding-ids: ${e}\n`);
|
||||
process.exit(r.fatal ? 2 : 1);
|
||||
}
|
||||
if (r.text !== text) {
|
||||
const tmp = `${path}.tmp-${process.pid}`;
|
||||
writeFileSync(tmp, r.text);
|
||||
renameSync(tmp, path);
|
||||
}
|
||||
process.stdout.write(`fill-finding-ids: ok ${path}\n`);
|
||||
}
|
||||
|
|
@ -21,8 +21,9 @@
|
|||
//
|
||||
// 3-layer pattern (Content → Raw-text → CLI shim) mirroring the other validators.
|
||||
|
||||
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
||||
import { readFileSync, existsSync, realpathSync, statSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { resolve, sep } from 'node:path';
|
||||
import { issue, fail } from '../util/result.mjs';
|
||||
import { RULE_KEYS, SEVERITY_VALUES } from './rule-catalogue.mjs';
|
||||
|
||||
|
|
@ -102,7 +103,42 @@ function validateFinding(finding, index, errors) {
|
|||
* @param {unknown} payload
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateFindings(payload) {
|
||||
// A reviewer that reads the diff cites the line in the PATCH, not in the file:
|
||||
// the voyage-lekerepo trial (2026-10-06) cited tests/cli.test.mjs:194 for a
|
||||
// 124-line file, the coordinator then set the only brief-anchored MAJOR aside,
|
||||
// and the claim itself was right. With a repo root, a line past the end of a
|
||||
// file that exists is an error, so Phase 5's bounded re-ask fixes it before the
|
||||
// coordinator ever sees it. A file absent from the tree is not judged here.
|
||||
function lineCount(repoRoot, file) {
|
||||
const root = resolve(repoRoot);
|
||||
const abs = resolve(root, file);
|
||||
if (abs !== root && !abs.startsWith(root + sep)) return null;
|
||||
try {
|
||||
if (!statSync(abs).isFile()) return null;
|
||||
const text = readFileSync(abs, 'utf-8');
|
||||
if (text === '') return 0;
|
||||
return text.endsWith('\n') ? text.split('\n').length - 1 : text.split('\n').length;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function checkLineRange(finding, index, repoRoot, errors) {
|
||||
if (!finding || typeof finding.file !== 'string' || !Number.isInteger(finding.line) || finding.line <= 0) return;
|
||||
const n = lineCount(repoRoot, finding.file);
|
||||
if (n === null || finding.line <= n) return;
|
||||
errors.push(issue(
|
||||
'FINDING_LINE_OUT_OF_RANGE',
|
||||
`findings[${index}] cites ${finding.file}:${finding.line}, but the file has ${n} lines`,
|
||||
'Cite the line number in the file itself (Read it), not the line in the diff or patch.',
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown} payload
|
||||
* @param {{ repoRoot?: string }} [opts] - with a repo root, cited lines are checked against the files
|
||||
*/
|
||||
export function validateFindings(payload, opts = {}) {
|
||||
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
return fail(issue('FINDINGS_NOT_OBJECT', `Reviewer output must be a JSON object, got ${Array.isArray(payload) ? 'array' : typeof payload}`));
|
||||
}
|
||||
|
|
@ -121,6 +157,7 @@ export function validateFindings(payload) {
|
|||
|
||||
for (let i = 0; i < payload.findings.length; i++) {
|
||||
validateFinding(payload.findings[i], i, errors);
|
||||
if (opts.repoRoot) checkLineRange(payload.findings[i], i, opts.repoRoot, errors);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors, warnings, parsed: payload };
|
||||
|
|
@ -133,7 +170,7 @@ export function validateFindings(payload) {
|
|||
* @param {string} rawText
|
||||
* @returns {import('../util/result.mjs').Result}
|
||||
*/
|
||||
export function validateReviewerOutput(rawText) {
|
||||
export function validateReviewerOutput(rawText, opts = {}) {
|
||||
const block = extractFindingsBlock(rawText);
|
||||
if (block === null) {
|
||||
return fail(issue(
|
||||
|
|
@ -148,7 +185,7 @@ export function validateReviewerOutput(rawText) {
|
|||
} catch (e) {
|
||||
return fail(issue('FINDINGS_PARSE_ERROR', `Reviewer JSON block did not parse: ${e.message}`));
|
||||
}
|
||||
return validateFindings(parsed);
|
||||
return validateFindings(parsed, opts);
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
|
@ -163,16 +200,20 @@ function isMain() {
|
|||
|
||||
if (isMain()) {
|
||||
const args = process.argv.slice(2);
|
||||
const filePath = args.find((a) => !a.startsWith('--'));
|
||||
if (!filePath) {
|
||||
process.stderr.write('Usage: findings-schema.mjs [--json] <reviewer-output.txt|.md>\n');
|
||||
const rootIdx = args.indexOf('--repo-root');
|
||||
const repoRoot = rootIdx >= 0 ? args[rootIdx + 1] : undefined;
|
||||
// Without the flag nothing is skipped (rootIdx + 1 would be 0: the file).
|
||||
const skipIdx = rootIdx >= 0 ? rootIdx + 1 : -1;
|
||||
const filePath = args.find((a, i) => !a.startsWith('--') && i !== skipIdx);
|
||||
if (!filePath || (rootIdx >= 0 && !repoRoot)) {
|
||||
process.stderr.write('Usage: findings-schema.mjs [--json] [--repo-root <dir>] <reviewer-output.txt|.md>\n');
|
||||
process.exit(2);
|
||||
}
|
||||
if (!existsSync(filePath)) {
|
||||
process.stderr.write(`findings-schema: file not found: ${filePath}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
const r = validateReviewerOutput(readFileSync(filePath, 'utf-8'));
|
||||
const r = validateReviewerOutput(readFileSync(filePath, 'utf-8'), { repoRoot });
|
||||
if (args.includes('--json')) {
|
||||
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue