voyage/lib/validators/review-validator.mjs
Kjell Tore Guttormsen 29b4ac91b3
fix: round 7 of the release review of 6.3.0, for 6.3.1
Round 7 (516dffa..45aa9e1, WARN: 12 MAJOR, 0 BLOCKER) and the fence defect
found while writing its review.md. Each fix began with a test that failed on
0f400d3 (v6.3.0); suite 2208 (2206/0/2), 14 tests more.

- Tests-only (operator 2026-10-11): the plan's Files: alone decides, and the
  text says so (red-proof, trekexecute Sub-step B, step-implementer,
  HANDOVER-CONTRACTS). red-proof no longer reads the step's diff, so a change
  already in the tree when the step began no longer counts against it, and a
  plan placed outside .claude/ reads the same. The round-6 test that had the
  diff overrule the plan is rewritten to the new rule.
- The Files: reader keeps a plain file beside a backticked path
  (", Dockerfile", "the CLI entry src/cli.mjs", "and Makefile"). Measured on
  the 520 Files: fields in 182 plans: no field reads differently.
- Red proof reports a command its timeout killed as killed after the timeout,
  not "could not start"; the release check tells a timeout from output past
  its buffer and from another signal. One reading of a stop
  (criteria-runner stoppedBy) for red proof, the criteria runner and the
  release check.
- A trailing `# comment` is dropped in a backticked check too, and in a
  command passed as written, so the words screened are the words run.
- The release check reads a relative --review in --repo.
- The 15-minute timeout is defined once (criteria-runner.mjs).
- The findings check reads each cited file once per validation.
- One isMain helper (lib/util/is-main.mjs) for every CLI: the 31 copies of
  the realpath guard and the 20 modules still on the file:// guard, which
  exited 0 having done nothing under a symlink or a path with a space.
- The Bash guard writes its no-one-to-ask refusal once, on a direct condition.
- The findings check ends a ```json block only at a fence on its own line, so
  three backticks inside a JSON string no longer end it (also in 6.2.0).
- Brief-conformance notes: a test for test-lock green on exit 127 (it
  passed at once; it fails when 127 is read as could-not-run); the prose now
  names the git revision `@~1` as the one leading-@ exception.

Not changed: the example plan's steps 1-2 keep their tests-only warning
(operator decision 2026-10-09; the r5 test records it). Brief note 4 (a
Regression: whose runner is not installed goes to the fix rounds) is a
routing question for the operator.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 09:11:57 +02:00

200 lines
7.7 KiB
JavaScript

// lib/validators/review-validator.mjs
// Validate trekreview frontmatter + body invariants.
// 3-layer pattern (Content → File → CLI shim) mirroring brief-validator.
//
// Schema is forward-compatible: unknown top-level frontmatter keys are
// tolerated silently. Strict-key checks are intentionally avoided so new
// optional fields (jf. the source_findings precedent) can be added without
// a review_version bump.
//
// K7 (ordre 5, review_version 1.1): `verdict: ALLOW | WARN | BLOCK` is REQUIRED.
// Strict (write-time) mode rejects a review without it; soft (read-time) mode
// warns, so a 1.0 review can still seed a plan (N-1 window, Handover 6). The
// gate that decides on the verdict is lib/review/verdict-gate.mjs, which never
// reads a missing verdict as ALLOW.
import { readFileSync, existsSync } from 'node:fs';
import { parseDocument } from '../util/frontmatter.mjs';
import { issue, ok, fail } from '../util/result.mjs';
import { isMain } from '../util/is-main.mjs';
export const REVIEW_REQUIRED_FRONTMATTER = [
'type',
'review_version',
'task',
'slug',
'project_dir',
'brief_path',
'scope_sha_end',
'reviewed_files_count',
'findings',
];
export const REVIEW_BODY_SECTIONS = ['Executive Summary', 'Coverage', 'Remediation Summary'];
export const VERDICT_VALUES = Object.freeze(['ALLOW', 'WARN', 'BLOCK']);
const HEX_ID_RE = /^[0-9a-f]{40}$/;
const JSON_FENCE_GLOBAL = /```json[ \t]*\r?\n([\s\S]*?)```/gi;
/** IDs of BLOCKER findings in the trailing JSON block ([] when absent/unparseable). */
function blockerIdsFromBody(body) {
let last = null;
let m;
JSON_FENCE_GLOBAL.lastIndex = 0;
while ((m = JSON_FENCE_GLOBAL.exec(body)) !== null) last = m[1];
if (last === null) return [];
try {
const payload = JSON.parse(last);
return (Array.isArray(payload?.findings) ? payload.findings : [])
.filter((f) => f?.severity === 'BLOCKER' && typeof f.id === 'string')
.map((f) => f.id);
} catch {
return [];
}
}
/** Finding IDs named by `Source finding:` lines under `## Remediation Steps` step headings. */
function remediationStepSources(body) {
const start = body.search(/^##\s+Remediation Steps\b/m);
if (start === -1) return new Set();
const rest = body.slice(start).replace(/^##\s+Remediation Steps[^\n]*\n/, '');
const next = rest.search(/^##\s+/m);
const section = next === -1 ? rest : rest.slice(0, next);
const ids = new Set();
for (const step of section.split(/^###\s+Step\s+R\d+\b/m).slice(1)) {
const m = step.match(/Source finding:\**\s*`?([0-9a-f]{40})`?/);
if (m) ids.add(m[1]);
}
return ids;
}
export function validateReviewContent(text, opts = {}) {
const strict = opts.strict !== false;
const doc = parseDocument(text);
if (!doc.valid) return doc;
const fm = doc.parsed.frontmatter || {};
const body = doc.parsed.body || '';
const errors = [];
const warnings = [];
for (const k of REVIEW_REQUIRED_FRONTMATTER) {
if (!(k in fm)) {
errors.push(issue('REVIEW_MISSING_FIELD', `Required frontmatter field missing: ${k}`));
}
}
if (fm.type !== undefined && fm.type !== 'trekreview') {
errors.push(issue('REVIEW_WRONG_TYPE', `frontmatter.type must be "trekreview", got "${fm.type}"`));
}
if (fm.findings !== undefined) {
if (!Array.isArray(fm.findings)) {
errors.push(issue(
'REVIEW_BAD_FINDINGS_TYPE',
`Field "findings" must be an array of finding-IDs, got ${typeof fm.findings}`,
'Use block-style YAML: `findings:\\n - <id1>\\n - <id2>`',
));
} else {
for (let i = 0; i < fm.findings.length; i++) {
const id = fm.findings[i];
if (typeof id !== 'string' || !HEX_ID_RE.test(id)) {
errors.push(issue(
'REVIEW_BAD_FINDING_ID',
`findings[${i}] is not a 40-char hex ID: ${JSON.stringify(id)}`,
));
}
}
}
}
if (!('verdict' in fm) || fm.verdict === null || fm.verdict === '') {
const issueObj = issue(
'REVIEW_MISSING_VERDICT',
'Required frontmatter field missing: verdict (ALLOW | WARN | BLOCK)',
'Copy the coordinator verdict from the trailing JSON block into the frontmatter.',
);
if (strict) errors.push(issueObj);
else warnings.push(issueObj);
} else if (!VERDICT_VALUES.includes(fm.verdict)) {
errors.push(issue(
'REVIEW_BAD_VERDICT',
`frontmatter.verdict must be one of ${VERDICT_VALUES.join('|')}, got ${JSON.stringify(fm.verdict)}`,
));
}
// Every BLOCKER is written as a numbered remediation step that Handover 6 can
// append to a plan unchanged: `### Step R<n>: …` with `Source finding: <id>`.
// Borrowed: review gaps become appended, numbered tasks with a source-ref,
// never rewriting earlier ones (Spec Kit `templates/commands/converge.md:156-163`, MIT).
// 1.1+ only (N-1 window): a 1.0 review never carried the section.
const v = typeof fm.review_version === 'string' ? fm.review_version.match(/^(\d+)\.(\d+)$/) : null;
const atLeast11 = v && (Number(v[1]) > 1 || (Number(v[1]) === 1 && Number(v[2]) >= 1));
const blockerIds = atLeast11 ? blockerIdsFromBody(body) : [];
if (blockerIds.length > 0) {
const steps = remediationStepSources(body);
for (const id of blockerIds) {
if (!steps.has(id)) {
const issueObj = issue(
'REVIEW_MISSING_REMEDIATION_STEP',
`BLOCKER ${id} has no "### Step R<n>" with "Source finding: ${id}" under ## Remediation Steps`,
'Write one numbered remediation step per BLOCKER (Handover 6).',
);
if (strict) errors.push(issueObj);
else warnings.push(issueObj);
}
}
}
// 1.1+: what the coordinator set aside is listed, never left out (ordre 5).
if (atLeast11 && !/^##\s+Declined to Judge\b/m.test(body)) {
const issueObj = issue('REVIEW_MISSING_SECTION', 'Required body section missing: ## Declined to Judge (review_version ≥ 1.1)');
if (strict) errors.push(issueObj);
else warnings.push(issueObj);
}
for (const section of REVIEW_BODY_SECTIONS) {
const re = new RegExp(`^##\\s+${section}\\b`, 'm');
if (!re.test(body)) {
const issueObj = issue('REVIEW_MISSING_SECTION', `Required body section missing: ## ${section}`);
if (strict) errors.push(issueObj);
else warnings.push(issueObj);
}
}
if (typeof fm.review_version === 'string') {
const m = fm.review_version.match(/^(\d+)\.(\d+)$/);
if (!m) {
warnings.push(issue('REVIEW_VERSION_FORMAT', `review_version "${fm.review_version}" not in N.M form`));
}
}
return { valid: errors.length === 0, errors, warnings, parsed: { frontmatter: fm, body } };
}
export function validateReview(filePath, opts = {}) {
if (!existsSync(filePath)) return fail(issue('REVIEW_NOT_FOUND', `File not found: ${filePath}`));
let text;
try { text = readFileSync(filePath, 'utf-8'); }
catch (e) { return fail(issue('REVIEW_READ_ERROR', `Cannot read ${filePath}: ${e.message}`)); }
const r = validateReviewContent(text, opts);
return { ...r, parsed: { ...r.parsed, filePath } };
}
if (isMain(import.meta.url)) {
const args = process.argv.slice(2);
const strict = !args.includes('--soft');
const filePath = args.find(a => !a.startsWith('--'));
if (!filePath) {
process.stderr.write('Usage: review-validator.mjs [--soft] [--json] <review.md>\n');
process.exit(2);
}
const r = validateReview(filePath, { strict });
if (args.includes('--json')) {
process.stdout.write(JSON.stringify({ valid: r.valid, errors: r.errors, warnings: r.warnings }, null, 2) + '\n');
} else {
process.stdout.write(`review-validator: ${r.valid ? 'PASS' : 'FAIL'} ${filePath}\n`);
for (const e of r.errors) process.stderr.write(` ERROR [${e.code}] ${e.message}\n`);
for (const w of r.warnings) process.stderr.write(` WARN [${w.code}] ${w.message}\n`);
}
process.exit(r.valid ? 0 : 1);
}