328 lines
17 KiB
JavaScript
328 lines
17 KiB
JavaScript
// tests/lib/review-finalized.test.mjs
|
|
// [FN1] fix round 4.5b2 part B, Step 15: an engine finding at MAJOR or BLOCKER
|
|
// that review.md does not hold keeps the verdict gate at exit 2, not measured.
|
|
// /trekreview Phase 4.7 saves each converted engine payload beside the review
|
|
// (`{project_dir}/.review-engine-payload-{engine}.json`);
|
|
// lib/review/review-finalize.mjs puts a dropped one back. When finalize did
|
|
// not run (round 1 used to continue on its exit 1 or 2) or `finalized: true`
|
|
// was written by hand, the gate compares the payloads with the review itself
|
|
// (lib/review/verdict-gate.mjs `droppedEngineFindings`, run by
|
|
// `decideOpenFindings`) and reads a missing one as
|
|
// `review-not-measured: engine finding not in review: <severity> <rule_key> at <file>:<line>`.
|
|
// Every case runs the real CLIs (engine-findings, finalize, gate) in a
|
|
// throwaway git repo with no hooks: main holds the start commit, the run sits
|
|
// on branch `feat`, so `scope_sha_start` is the fork point from main.
|
|
// Pattern: tests/lib/engine-findings-compare.test.mjs.
|
|
|
|
import { after, test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs';
|
|
import { decideOpenFindings } from '../../lib/review/verdict-gate.mjs';
|
|
import { childEnv } from '../helpers/child-env.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
const ENGINE_CLI = join(ROOT, 'lib', 'review', 'engine-findings.mjs');
|
|
const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs');
|
|
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
|
const REVIEW_CMD = join(ROOT, 'commands', 'trekreview.md');
|
|
const CONTRACTS = join(ROOT, 'docs', 'HANDOVER-CONTRACTS.md');
|
|
const REVIEW_REL = '.claude/projects/2026-11-03-fn1/review.md';
|
|
const DROPPED = 'review-not-measured: engine finding not in review';
|
|
|
|
const made = [];
|
|
after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); });
|
|
|
|
function g(cwd, ...args) {
|
|
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv() });
|
|
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
|
return r.stdout.trim();
|
|
}
|
|
|
|
function putIn(dir, rel, text) {
|
|
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
|
writeFileSync(join(dir, rel), text);
|
|
}
|
|
|
|
const run = (args) => spawnSync(process.execPath, args, { encoding: 'utf8', env: childEnv() });
|
|
const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } };
|
|
const ten = (tag) => Array.from({ length: 10 }, (_, i) => `export const ${tag}${i + 1} = ${i + 1};`).join('\n') + '\n';
|
|
|
|
/**
|
|
* A throwaway repo: main holds src/a.mjs and src/b.mjs (the start commit);
|
|
* branch `feat` changes line 1 of src/a.mjs and line 3 of src/b.mjs (the end
|
|
* commit, HEAD). No REVIEW.md, so no policy.
|
|
*/
|
|
function fnRepo() {
|
|
const base = mkdtempSync(join(tmpdir(), 'review-finalized-fn1-'));
|
|
made.push(base);
|
|
const repo = join(base, 'repo');
|
|
mkdirSync(repo, { recursive: true });
|
|
mkdirSync(join(base, 'hooks-none'));
|
|
g(repo, 'init', '-q', '-b', 'main');
|
|
appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
|
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
|
appendFileSync(join(repo, '.git', 'info', 'exclude'), '.claude/projects/\n');
|
|
putIn(repo, 'src/a.mjs', ten('a'));
|
|
putIn(repo, 'src/b.mjs', ten('b'));
|
|
g(repo, 'add', '-A');
|
|
g(repo, 'commit', '-q', '-m', 'chore: start');
|
|
const start = g(repo, 'rev-parse', 'HEAD');
|
|
g(repo, 'checkout', '-q', '-b', 'feat');
|
|
putIn(repo, 'src/a.mjs', ten('a').replace('export const a1 = 1;', 'export const a1 = 11;'));
|
|
putIn(repo, 'src/b.mjs', ten('b').replace('export const b3 = 3;', 'export const b3 = 33;'));
|
|
g(repo, 'add', '-A');
|
|
g(repo, 'commit', '-q', '-m', 'feat: the work');
|
|
const end = g(repo, 'rev-parse', 'HEAD');
|
|
const reviewPath = join(repo, ...REVIEW_REL.split('/'));
|
|
const project = dirname(reviewPath);
|
|
mkdirSync(project, { recursive: true });
|
|
const scope = join(base, 'scope.txt');
|
|
writeFileSync(scope, 'src/a.mjs\nsrc/b.mjs\n');
|
|
return {
|
|
base, repo, start, end, reviewPath, project,
|
|
write: (text) => writeFileSync(reviewPath, text),
|
|
read: () => readFileSync(reviewPath, 'utf8'),
|
|
/** An engine's transcription converted by the real engine-findings.mjs CLI; saved beside the review as Phase 4.7 saves it unless `to` names another path. */
|
|
engine: (engine, findings, to) => {
|
|
const input = join(base, `.review-engine-${engine}.json`);
|
|
writeFileSync(input, JSON.stringify({ engine, findings }));
|
|
const r = run([ENGINE_CLI, '--scope', scope, input]);
|
|
assert.equal(r.status, 0, `engine-findings: ${r.stdout}${r.stderr}`);
|
|
const path = to ?? join(project, `.review-engine-payload-${engine}.json`);
|
|
writeFileSync(path, r.stdout);
|
|
return { payload: JSON.parse(r.stdout), path };
|
|
},
|
|
finalize: () => run([FINALIZE, '--review', reviewPath, '--repo', repo, '--project', project, '--round', '1']),
|
|
gate: () => {
|
|
const r = run([GATE, '--json', reviewPath, '--repo', repo]);
|
|
return { status: r.status, out: parse(r), raw: `${r.stdout}${r.stderr}` };
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* A review_version 1.2 review of `start..end` as the coordinator wrote it and
|
|
* fill-finding-ids filled it. `findings` are [{severity, file, line, rule_key,
|
|
* engine?, title, pre_existing?, carried_from?}]; ids computed over (file,
|
|
* line, rule_key). With `finalized`, the frontmatter says `finalized: true`.
|
|
*/
|
|
function fnReview({ start, end, findings, finalized = false }) {
|
|
const list = findings.map((f, i) => ({
|
|
severity: f.severity, rule_key: f.rule_key, ...(f.engine ? { engine: f.engine } : {}),
|
|
file: f.file, line: f.line, title: f.title ?? `t${i + 1}`, detail: `d${i + 1}`, recommended_action: 'Add a test.',
|
|
...(f.pre_existing !== undefined ? { pre_existing: f.pre_existing } : {}),
|
|
...(f.carried_from !== undefined ? { carried_from: f.carried_from } : {}),
|
|
}));
|
|
const ids = computeFindingIds(list);
|
|
const ordered = list.map((f, i) => ({ id: ids[i], ...f }));
|
|
const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 };
|
|
for (const f of ordered) counts[f.severity] += 1;
|
|
const v = counts.BLOCKER > 0 ? 'BLOCK' : counts.MAJOR > 0 ? 'WARN' : 'ALLOW';
|
|
const json = JSON.stringify({ verdict: v, counts, allow_blocked_by: [], declined_to_judge: [], findings: ordered }, null, 2);
|
|
const sections = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'].map((sev) => {
|
|
const of = ordered.filter((f) => f.severity === sev);
|
|
if (of.length === 0) return '';
|
|
return `## Findings (${sev})\n\n${of.map((f) => `### ${f.id}\n\n- **Rule:** \`${f.rule_key}\`\n- **Location:** \`${f.file}:${f.line}\`\n- **Title:** ${f.title}\n`).join('\n')}\n`;
|
|
}).join('');
|
|
return `---
|
|
type: trekreview
|
|
review_version: "1.2"
|
|
task: "t"
|
|
slug: fn1
|
|
project_dir: .claude/projects/2026-11-03-fn1/
|
|
brief_path: .claude/projects/2026-11-03-fn1/brief.md
|
|
scope_sha_start: ${start}
|
|
scope_sha_end: ${end}
|
|
reviewed_files_count: 2
|
|
verdict: ${v}
|
|
${finalized ? 'finalized: true\n' : ''}findings:
|
|
${ordered.map((f) => ` - ${f.id}`).join('\n')}
|
|
---
|
|
|
|
# Review: t
|
|
|
|
## Executive Summary
|
|
|
|
x
|
|
|
|
## Coverage
|
|
|
|
| File | Treatment | Reason |
|
|
|------|-----------|--------|
|
|
| src/a.mjs | deep-review | default |
|
|
| src/b.mjs | deep-review | default |
|
|
|
|
${sections}## Declined to Judge
|
|
|
|
None.
|
|
|
|
Trimmed: 0
|
|
|
|
## Remediation Summary
|
|
|
|
x
|
|
|
|
## Remediation Steps
|
|
|
|
None.
|
|
|
|
\`\`\`json
|
|
${json}
|
|
\`\`\`
|
|
`;
|
|
}
|
|
|
|
/** The text from `heading` up to the next `## ` heading ('' when absent). */
|
|
function sectionOf(text, heading) {
|
|
const start = text.indexOf(`\n${heading}`);
|
|
if (start === -1) return '';
|
|
const next = text.indexOf('\n## ', start + heading.length + 1);
|
|
return text.slice(start + 1, next === -1 ? undefined : next);
|
|
}
|
|
|
|
/** A reviewer MAJOR on a line the range did not change, marked pre-existing: it leaves the answer set, so the review is WARN and the gate passes. */
|
|
const PRE_MAJOR = { severity: 'MAJOR', file: 'src/a.mjs', line: 5, rule_key: 'MISSING_TEST', title: 'An old helper has no test', pre_existing: true };
|
|
/** A reviewer MINOR: a review holding only it is ALLOW. */
|
|
const REVIEWER_MINOR = { severity: 'MINOR', file: 'src/b.mjs', line: 3, rule_key: 'MISSING_ERROR_HANDLING', title: 'unchecked return' };
|
|
/** The /code-review finding the coordinator dropped: CODE_REVIEW_DEFECT, MAJOR, on a changed line. */
|
|
const ENGINE_MAJOR = { file: 'src/a.mjs', line: 1, summary: 'The counter overflows past 2^31', failure_scenario: 'a long run wraps the count' };
|
|
|
|
// --- a dropped engine finding is not measured -----------------------------------
|
|
|
|
test('[FN1] a WARN review with an engine payload beside it whose MAJOR the review does not hold -> exit 2 review-not-measured: engine finding not in review, naming its file and line', () => {
|
|
const fx = fnRepo();
|
|
fx.write(fnReview({ start: fx.start, end: fx.end, findings: [PRE_MAJOR] }));
|
|
// Control: without the payload the WARN passes (its one MAJOR is pre-existing).
|
|
const before = fx.gate();
|
|
assert.equal(before.status, 0, `the WARN review alone passes the gate: ${before.raw}`);
|
|
assert.equal(before.out.verdict, 'WARN');
|
|
fx.engine('code-review', [ENGINE_MAJOR]);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, `a dropped engine MAJOR is not measured: ${r.raw}`);
|
|
const reason = String(r.out.reason);
|
|
assert.ok(reason.startsWith(`${DROPPED}: `), `the reason names the dropped engine finding: ${reason}`);
|
|
assert.ok(reason.includes('MAJOR CODE_REVIEW_DEFECT at src/a.mjs:1'), `the reason names its severity, rule_key, file and line: ${reason}`);
|
|
// Control: after finalize the MAJOR is put back, so the reason is no longer that one: it is unanswered.
|
|
const fin = fx.finalize();
|
|
assert.equal(fin.status, 0, `finalize: ${fin.stdout}${fin.stderr}`);
|
|
const after = fx.gate();
|
|
assert.equal(after.status, 2, after.raw);
|
|
assert.ok(!String(after.out.reason).startsWith(DROPPED), `a finalized review with the put-back is not read as dropped: ${after.out.reason}`);
|
|
assert.match(String(after.out.reason), /^review-unanswered:/, `the put-back MAJOR counts as unanswered: ${after.raw}`);
|
|
});
|
|
|
|
test('[FN1] the same review with finalized: true written by hand -> the same exit 2', () => {
|
|
const fx = fnRepo();
|
|
fx.write(fnReview({ start: fx.start, end: fx.end, findings: [PRE_MAJOR], finalized: true }));
|
|
const before = fx.gate();
|
|
assert.equal(before.status, 0, `control: the hand-finalized WARN alone passes the gate: ${before.raw}`);
|
|
fx.engine('code-review', [ENGINE_MAJOR]);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, `finalized: true by hand does not hide a dropped engine MAJOR: ${r.raw}`);
|
|
const reason = String(r.out.reason);
|
|
assert.ok(reason.startsWith(`${DROPPED}: `), reason);
|
|
assert.ok(reason.includes('MAJOR CODE_REVIEW_DEFECT at src/a.mjs:1'), reason);
|
|
});
|
|
|
|
test('[FN1] an ALLOW review with a dropped engine BLOCKER -> exit 2 review-not-measured: engine finding not in review', () => {
|
|
const fx = fnRepo();
|
|
fx.write(fnReview({ start: fx.start, end: fx.end, findings: [REVIEWER_MINOR] }));
|
|
const before = fx.gate();
|
|
assert.equal(before.status, 0, `control: the ALLOW review alone passes the gate: ${before.raw}`);
|
|
assert.equal(before.out.verdict, 'ALLOW');
|
|
fx.engine('security-review', [{ file: 'src/a.mjs', line: 1, summary: 'A shell command is built from request input', failure_scenario: 'a crafted name runs a command' }]);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, `a dropped engine BLOCKER is not measured: ${r.raw}`);
|
|
const reason = String(r.out.reason);
|
|
assert.ok(reason.startsWith(`${DROPPED}: `), reason);
|
|
assert.ok(reason.includes('BLOCKER SECURITY_REVIEW_VULNERABILITY at src/a.mjs:1'), reason);
|
|
});
|
|
|
|
// --- the text -----------------------------------------------------------------------
|
|
|
|
test('[FN1] /trekreview Phase 7 stops round 1 on a finalize exit 1 or 2 with "not measured", and the continue sentence is gone', () => {
|
|
const phase7 = sectionOf(readFileSync(REVIEW_CMD, 'utf8'), '## Phase 7 — Write review.md');
|
|
assert.ok(phase7 !== '', 'commands/trekreview.md has a ## Phase 7 — Write review.md section');
|
|
const flat = phase7.replace(/\s+/g, ' ');
|
|
assert.ok(flat.includes('In round 1 too, finalize exit 1 or 2 stops the review: print the reason and "not measured", and stop.'),
|
|
'round 1 stops on a finalize exit 1 or 2');
|
|
assert.ok(flat.includes('`review-not-measured: engine finding not in review`'),
|
|
'the sentence says the verdict gate reads a missing engine MAJOR or BLOCKER as exit 2');
|
|
assert.ok(!flat.includes('In round 1, on exit 1 or 2 the file stays as `fill-finding-ids` wrote it; print the reason in one line and continue.'),
|
|
'the round-1 continue sentence is gone');
|
|
// Control: the round-2 sentence after it stays.
|
|
assert.ok(flat.includes('From round 2, finalize exit 1 or 2 stops the review: print the reason and "not measured", and stop.'),
|
|
'the round-2 sentence stays');
|
|
});
|
|
|
|
test('[FN1] docs/HANDOVER-CONTRACTS.md names the exit 2 engine finding not in review and says the verdict gate reads the engine payloads', () => {
|
|
const h6 = sectionOf(readFileSync(CONTRACTS, 'utf8'), '## Handover 6');
|
|
assert.ok(h6 !== '', 'docs/HANDOVER-CONTRACTS.md has a ## Handover 6 section');
|
|
const gatePara = h6.split('\n').find((l) => l.startsWith('**Second consumer — the verdict gate'));
|
|
assert.ok(gatePara, 'Handover 6 has the verdict gate paragraph');
|
|
assert.ok(gatePara.includes('`review-not-measured: engine finding not in review`'),
|
|
'the verdict gate paragraph names the new exit 2');
|
|
const working = h6.split('\n').find((l) => l.startsWith('- Working files of one review run'));
|
|
assert.ok(working, 'Handover 6 has the working-files bullet');
|
|
assert.match(working, /verdict gate[^\n]*\breads the payloads too\b/,
|
|
'the working-files bullet says the verdict gate reads the engine payloads too');
|
|
});
|
|
|
|
// --- kept behaviour -------------------------------------------------------------------
|
|
|
|
test('keep: no payload beside the review, an engine MINOR missing, an engine finding only in out_of_scope, or a symlinked payload -> the gate\'s result is unchanged', () => {
|
|
const fx = fnRepo();
|
|
fx.write(fnReview({ start: fx.start, end: fx.end, findings: [PRE_MAJOR] }));
|
|
const bare = fx.gate();
|
|
assert.equal(bare.status, 0, `no payload beside the review: the gate passes: ${bare.raw}`);
|
|
const same = (label) => {
|
|
const r = fx.gate();
|
|
assert.deepEqual([r.status, r.out.verdict, r.out.reason], [bare.status, bare.out.verdict, bare.out.reason], `${label}: ${r.raw}`);
|
|
};
|
|
// An engine MINOR (a /code-review improvement) the review does not hold.
|
|
const minor = fx.engine('code-review', [{ ...ENGINE_MAJOR, summary: 'The loop could be a map', category: 'simplification' }]);
|
|
assert.equal(minor.payload.findings[0].severity, 'MINOR', 'the CLI made it a MINOR');
|
|
same('an engine MINOR missing');
|
|
rmSync(minor.path);
|
|
// An engine MAJOR on a file outside the review scope: only in out_of_scope.
|
|
const out = fx.engine('code-review', [{ file: 'lib/outside.mjs', line: 4, summary: 'An unchanged helper leaks a handle', failure_scenario: 'many calls exhaust handles' }]);
|
|
assert.equal(out.payload.findings.length, 0, 'no in-scope finding');
|
|
assert.equal(out.payload.out_of_scope.length, 1, 'the finding sits in out_of_scope');
|
|
same('an engine finding only in out_of_scope');
|
|
rmSync(out.path);
|
|
// A symlinked payload holding a dropped MAJOR: finalize does not read it, nor does the gate.
|
|
const real = fx.engine('code-review', [ENGINE_MAJOR], join(fx.base, 'elsewhere.json'));
|
|
symlinkSync(real.path, join(fx.project, '.review-engine-payload-code-review.json'));
|
|
same('a symlinked payload');
|
|
});
|
|
|
|
test('keep: a current engine finding folded into a carried security finding is not reported as dropped', () => {
|
|
const fx = fnRepo();
|
|
// Round 2, as finalize's carry writes it (lib/review/review-rounds.mjs `fold`): the engine's
|
|
// CODE_REVIEW_DEFECT folded into the previous round's SECURITY_INJECTION BLOCKER keeps the
|
|
// engine's place, title and `engine`, but the previous rule_key and the higher severity, so
|
|
// its (file, line, rule_key) row is not the payload's.
|
|
const prevId = computeFindingId('src/a.mjs', 1, 'SECURITY_INJECTION');
|
|
const text = fnReview({
|
|
start: fx.start, end: fx.end, finalized: true,
|
|
findings: [{ severity: 'BLOCKER', file: 'src/a.mjs', line: 1, rule_key: 'SECURITY_INJECTION', engine: 'code-review', title: ENGINE_MAJOR.summary, carried_from: prevId }],
|
|
});
|
|
fx.write(text);
|
|
const bare = fx.gate();
|
|
const bareOpen = decideOpenFindings(text, { repo: fx.repo, reviewPath: fx.reviewPath });
|
|
assert.equal(bare.status, 1, `the carried security BLOCKER blocks: ${bare.raw}`);
|
|
fx.engine('code-review', [ENGINE_MAJOR]);
|
|
const r = fx.gate();
|
|
assert.deepEqual([r.status, r.out.verdict, r.out.reason], [bare.status, bare.out.verdict, bare.out.reason],
|
|
`the gate's result is unchanged: ${r.raw}`);
|
|
// The open-findings check itself (also read by scripts/release-check.mjs and merge-evidence.mjs)
|
|
// does not report the folded engine finding as dropped.
|
|
const open = decideOpenFindings(text, { repo: fx.repo, reviewPath: fx.reviewPath });
|
|
assert.deepEqual([open.exit, open.reason], [bareOpen.exit, bareOpen.reason],
|
|
`decideOpenFindings is unchanged by the payload: ${JSON.stringify(open)}`);
|
|
});
|