Kjell Tore Guttormsen
b94b060035
- tests/lib/review-rounds.test.mjs: round 2 whose Claude Security payload holds the round-1 BLOCKER only in out_of_scope keeps it counted through the real finalize and verdict gate (BLOCK, gate exit 1, not under Resolved since round 1); a clean run resolves it (F-V4; red under the mutation that drops out_of_scope from raisedOutside). - README.md, SECURITY.md: "blocks only at confidence high" names the exception of decision 222 (counted at high in an earlier round, it stays counted when it comes back below high or outside the scope) (F-V5). - commands/trekreview.md Phase 5: an engine finding is held by a matching finding in review.md (engine MAJOR/BLOCKER put back first) or a matching below_threshold / out_of_scope entry; an engine MINOR or SUGGESTION the coordinator left out holds nothing, a known issue (decision 226). - tests/lib/review-prose-pins.test.mjs: both sentences pinned; red on the old prose. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1247 lines
71 KiB
JavaScript
1247 lines
71 KiB
JavaScript
// tests/lib/review-rounds.test.mjs
|
|
// [RND] [CAP] [CRY] fix round 4.5b2 part B, Step 6: /trekreview rounds converge.
|
|
// RND `lib/review/review-rounds.mjs prepare` derives the round from the
|
|
// run's archives (`review-round-<run>-<n>.md`, <run> = the first 12 hex
|
|
// of scope_sha_start), checks the round base, archives the previous
|
|
// review and clears the previous round's reviewer and engine files;
|
|
// `carry` keeps a previous finding until this round gives positive
|
|
// evidence that it is resolved; review-finalize.mjs runs the carry pass
|
|
// with `--previous`, writes `ran`, `resolved`, `round`, `round_base`,
|
|
// `finalized` and an R-step for a carried BLOCKER.
|
|
// CAP one cap of three rounds (`MAX_REVIEW_ROUNDS`).
|
|
// CRY an accepted answer is carried to the next round only by a link to the
|
|
// same run's earlier archive, checked by the gate.
|
|
// FXL (Step 7) a MAJOR answered "fixed" needs its `### Step R<n>` first:
|
|
// `record --answer fixed` refuses without it; /trekreview Phase 8.8
|
|
// offers only "Fix it" for a finding `pending` marks `security: true`.
|
|
// Tests run the CLIs (prepare, finalize, review-answers, validator, gate) in
|
|
// throwaway git repos with no hooks; `carry` is also called directly. Each
|
|
// tagged test's main assertion is red at the step's start; a case that
|
|
// already holds there is a Control inside it.
|
|
|
|
import { after, test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { appendFileSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs';
|
|
import { convertReport } from '../../lib/review/security-engine.mjs';
|
|
import { childEnv } from '../helpers/child-env.mjs';
|
|
import { record, writeReport } from '../helpers/claude-security-report.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
const ROUNDS = join(ROOT, 'lib', 'review', 'review-rounds.mjs');
|
|
const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs');
|
|
const ANSWERS = join(ROOT, 'lib', 'review', 'review-answers.mjs');
|
|
const ENGINE = join(ROOT, 'lib', 'review', 'engine-findings.mjs');
|
|
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
|
const SECURITY = join(ROOT, 'lib', 'review', 'security-engine.mjs');
|
|
const VALIDATOR = join(ROOT, 'lib', 'validators', 'review-validator.mjs');
|
|
const PROJECT_REL = '.claude/projects/2026-11-02-r';
|
|
const TRAILER = 'Co-Authored-By: Claude Test <noreply@example.invalid>';
|
|
const CCR = 'code-correctness-reviewer';
|
|
const BCR = 'brief-conformance-reviewer';
|
|
|
|
// Loaded so that each new assertion fails on its own while the module is absent.
|
|
let rounds = {};
|
|
try {
|
|
rounds = await import(pathToFileURL(ROUNDS).href);
|
|
} catch (e) {
|
|
if (existsSync(ROUNDS)) throw e;
|
|
}
|
|
|
|
const made = [];
|
|
after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); });
|
|
|
|
function g(cwd, ...args) {
|
|
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv() });
|
|
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
|
return r.stdout.trim();
|
|
}
|
|
|
|
function putIn(dir, rel, text) {
|
|
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
|
writeFileSync(join(dir, rel), text);
|
|
}
|
|
|
|
const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } };
|
|
const run = (args, opts = {}) => spawnSync(process.execPath, args, { encoding: 'utf8', env: childEnv(), ...opts });
|
|
// Ten lines `export const <tag><n> = <n>;`; `changes` maps a line number to new text.
|
|
const ten = (tag, changes = {}) => Array.from({ length: 10 }, (_, i) => (i + 1 in changes ? changes[i + 1] : `export const ${tag}${i + 1} = ${i + 1};`)).join('\n') + '\n';
|
|
const jsonOf = (text) => JSON.parse(text.slice(text.lastIndexOf('```json\n') + 8, text.lastIndexOf('```')));
|
|
const fmOf = (text) => text.slice(0, text.indexOf('\n---\n', 4) + 5);
|
|
|
|
/**
|
|
* A finding as a reviewer writes it. `f` holds severity, file, line and the
|
|
* optional rule_key, engine, title, detail, id and carried_from.
|
|
*/
|
|
function finding(f, i) {
|
|
return {
|
|
...(f.id ? { id: f.id } : {}),
|
|
severity: f.severity, rule_key: f.rule_key ?? 'MISSING_TEST', ...(f.engine ? { engine: f.engine } : {}),
|
|
file: f.file, line: f.line, brief_ref: 'SC1', title: f.title ?? `title ${f.file} ${f.line} ${i + 1}`,
|
|
detail: f.detail ?? `detail ${f.file} ${f.line} ${i + 1}`, recommended_action: f.action ?? 'Add a test.',
|
|
...(f.carried_from ? { carried_from: f.carried_from } : {}),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* A review_version 1.2 review of `start..end`, ids filled as fill-finding-ids
|
|
* fills them (computed over the array). One `### Step R<n>` per BLOCKER
|
|
* unless `steps` names them ([{title, source}]).
|
|
*/
|
|
function rndReview({ start, end, findings, verdict, steps, extraSteps = '' }) {
|
|
const list = findings.map(finding);
|
|
const ids = computeFindingIds(list);
|
|
list.forEach((f, i) => { if (!f.id) f.id = ids[i]; });
|
|
const ordered = list.map(({ id, ...rest }) => ({ id, ...rest }));
|
|
const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 };
|
|
for (const f of ordered) counts[f.severity] += 1;
|
|
const v = verdict ?? (counts.BLOCKER > 0 ? 'BLOCK' : counts.MAJOR > 0 ? 'WARN' : 'ALLOW');
|
|
const json = JSON.stringify({ verdict: v, counts, allow_blocked_by: [], findings: ordered }, null, 2);
|
|
const sections = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'].map((sev) => {
|
|
const of = ordered.filter((f) => f.severity === sev);
|
|
if (of.length === 0) return '';
|
|
return `## Findings (${sev})\n\n${of.map((f) => `### ${f.id}\n\n- file: ${f.file}\n- line: ${f.line}\n- rule_key: ${f.rule_key}\n- title: ${f.title}\n`).join('\n')}\n`;
|
|
}).join('');
|
|
const stepList = steps ?? ordered.filter((f) => f.severity === 'BLOCKER').map((f) => ({ title: `fix ${f.title}`, source: f.id, file: f.file }));
|
|
const stepText = stepList.map((s, i) => `### Step R${i + 1}: ${s.title}\n\n- **Files:** \`${s.file ?? 'a.mjs'}\`\n- **Source finding:** \`${s.source}\`\n- **Test first:** a test that fails today\n- **Verify:** re-run /trekreview\n- **On failure:** escalate\n`).join('\n');
|
|
return {
|
|
ids: ordered.map((f) => f.id),
|
|
text: `---
|
|
type: trekreview
|
|
review_version: "1.2"
|
|
task: "t"
|
|
slug: r
|
|
project_dir: ${PROJECT_REL}/
|
|
brief_path: ${PROJECT_REL}/brief.md
|
|
scope_sha_start: ${start}
|
|
scope_sha_end: ${end}
|
|
reviewed_files_count: 2
|
|
verdict: ${v}
|
|
findings:
|
|
${ordered.length ? ordered.map((f) => ` - ${f.id}`).join('\n') : ' []'}
|
|
---
|
|
|
|
# Review: t
|
|
|
|
## Executive Summary
|
|
|
|
x
|
|
|
|
## Coverage
|
|
|
|
| File | Treatment | Reason |
|
|
|------|-----------|--------|
|
|
| a.mjs | deep-review | default |
|
|
|
|
${sections}## Declined to Judge
|
|
|
|
None.
|
|
|
|
Trimmed: 0
|
|
|
|
## Remediation Steps
|
|
|
|
${stepText || 'None.\n'}${extraSteps}
|
|
## Remediation Summary
|
|
|
|
x
|
|
|
|
\`\`\`json
|
|
${json}
|
|
\`\`\`
|
|
`.replace('findings:\n []', 'findings: []'),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* A throwaway repo: start (a.mjs, b.mjs, ten lines each), c1 (a.mjs and b.mjs
|
|
* line 3 changed: round 1's work), c2 (a.mjs line 5 changed: round 2's work).
|
|
* HEAD is c2. The project folder is excluded from git, as a local artifact.
|
|
*/
|
|
function rndRepo() {
|
|
const base = mkdtempSync(join(tmpdir(), 'review-rounds-rnd-'));
|
|
made.push(base);
|
|
const repo = join(base, 'repo');
|
|
mkdirSync(repo, { recursive: true });
|
|
mkdirSync(join(base, 'hooks-none'));
|
|
g(repo, 'init', '-q', '-b', 'main');
|
|
appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
|
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
|
appendFileSync(join(repo, '.git', 'info', 'exclude'), '.claude/projects/\n');
|
|
const commit = (files, msg) => {
|
|
for (const [rel, text] of Object.entries(files)) putIn(repo, rel, text);
|
|
g(repo, 'add', '-A');
|
|
g(repo, 'commit', '-q', '-m', msg);
|
|
return g(repo, 'rev-parse', 'HEAD');
|
|
};
|
|
const start = commit({ 'a.mjs': ten('a'), 'b.mjs': ten('b') }, 'chore: start');
|
|
const c1 = commit({ 'a.mjs': ten('a', { 3: 'export const a3 = 33;' }), 'b.mjs': ten('b', { 3: 'export const b3 = 33;' }) }, 'feat: round 1');
|
|
const c2 = commit({ 'a.mjs': ten('a', { 3: 'export const a3 = 33;', 5: 'export const a5 = 55;' }) }, 'feat: round 2');
|
|
const project = join(repo, ...PROJECT_REL.split('/'));
|
|
mkdirSync(project, { recursive: true });
|
|
const reviewPath = join(project, 'review.md');
|
|
const fx = {
|
|
base, repo, start, c1, c2, project, reviewPath, run: start.slice(0, 12), commit,
|
|
write: (text, p = reviewPath) => writeFileSync(p, text),
|
|
read: (p = reviewPath) => readFileSync(p, 'utf8'),
|
|
archive: (n, runId = fx.run) => join(project, `review-round-${runId}-${n}.md`),
|
|
/** Every file name in the project folder with its bytes. */
|
|
snapshot: () => Object.fromEntries(readdirSync(project).sort().map((n) => [n, readFileSync(join(project, n), 'utf8')])),
|
|
prepare: (extra = [], since = start) => run([ROUNDS, 'prepare', '--project', project, '--since', since, '--json', ...extra], { cwd: repo }),
|
|
finalize: (round, extra = []) => run([FINALIZE, '--review', reviewPath, '--repo', repo, '--project', project,
|
|
'--round', String(round), ...extra], { cwd: repo }),
|
|
finalize2: () => fx.finalize(2, ['--round-base', c1, '--previous', fx.archive(1)]),
|
|
gate: () => run([GATE, '--json', reviewPath, '--repo', repo]),
|
|
validate: () => run([VALIDATOR, '--strict', reviewPath]),
|
|
/** A reviewer's whole answer, saved as Phase 5 saves it; `previous` marks previous findings by id. */
|
|
out: (reviewer, previous = [], findings = []) => writeFileSync(join(project, `.review-out-${reviewer}.md`),
|
|
`Reviewer prose.\n\n\`\`\`json\n${JSON.stringify({ reviewer, findings, previous }, null, 2)}\n\`\`\`\n`),
|
|
/** An engine payload, converted by the real engine-findings.mjs CLI. */
|
|
engine: (engine, findings = []) => {
|
|
const input = join(base, `engine-${engine}.json`);
|
|
writeFileSync(input, JSON.stringify({ engine, findings }));
|
|
const r = run([ENGINE, input]);
|
|
assert.equal(r.status, 0, `engine-findings: ${r.stdout}${r.stderr}`);
|
|
writeFileSync(join(project, `.review-engine-payload-${engine}.json`), r.stdout);
|
|
},
|
|
};
|
|
return fx;
|
|
}
|
|
|
|
/** Round 1 written, finalized and archived by prepare: the folder at the start of round 2. */
|
|
function roundOne(fx, findings, { outs = [CCR, BCR], engines = ['code-review', 'security-review'], finalize = true } = {}) {
|
|
const r1 = rndReview({ start: fx.start, end: fx.c1, findings });
|
|
fx.write(r1.text);
|
|
for (const name of outs) fx.out(name);
|
|
for (const e of engines) fx.engine(e);
|
|
if (finalize) {
|
|
const f = fx.finalize(1);
|
|
assert.equal(f.status, 0, `round-1 finalize: ${f.stdout}${f.stderr}`);
|
|
}
|
|
const p = fx.prepare();
|
|
assert.equal(p.status, 0, `prepare for round 2: ${p.stdout}${p.stderr}`);
|
|
return { ...r1, prev: jsonOf(fx.read(fx.archive(1))) };
|
|
}
|
|
|
|
/** Round 2's coordinator output, with fresh reviewer and engine files. */
|
|
function roundTwo(fx, findings, { outs = {}, engines = ['code-review', 'security-review'], ...opts } = {}) {
|
|
const r2 = rndReview({ start: fx.start, end: fx.c2, findings, ...opts });
|
|
fx.write(r2.text);
|
|
for (const [name, marks] of Object.entries(outs)) fx.out(name, marks);
|
|
for (const e of engines) fx.engine(e);
|
|
return r2;
|
|
}
|
|
|
|
const MAJ_A3 = { severity: 'MAJOR', file: 'a.mjs', line: 3, title: 'Parser drops the last token', detail: 'parse() stops one token early' };
|
|
const BLK_B3 = { severity: 'BLOCKER', file: 'b.mjs', line: 3, rule_key: 'BROKEN_SUCCESS_CRITERION', title: 'Build output is never written', detail: 'write() returns before the flush' };
|
|
/** A second BLOCKER of BLK_B3's class, for archives the stop rule must read but not fire on. */
|
|
const BLK_A5_RND = { severity: 'BLOCKER', file: 'a.mjs', line: 5, rule_key: 'BROKEN_SUCCESS_CRITERION', title: 'Second build path never flushes either', detail: 'emit() skips the final flush call' };
|
|
|
|
// ---- [RND] range and the fail-closed base ----------------------------------------
|
|
|
|
test('[RND] from round 2, prepare returns the previous scope_sha_end as round_base', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
const r1 = rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] });
|
|
fx.write(r1.text);
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
const out = parse(r);
|
|
assert.equal(out.round, 2, r.stdout);
|
|
assert.equal(out.round_base, fx.c1, r.stdout);
|
|
assert.equal(out.previous, `review-round-${fx.run}-1.md`, r.stdout);
|
|
assert.equal(fx.read(fx.archive(1)), r1.text, 'the previous review is archived byte for byte');
|
|
});
|
|
|
|
test('[RND] prepare exits 2 when the previous scope_sha_end is not an ancestor of HEAD', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
// A side branch from c1, as in [R6] verdict-gate.test.mjs.
|
|
g(fx.repo, 'checkout', '-q', '-b', 'side', fx.c1);
|
|
const side = fx.commit({ 'b.mjs': ten('b', { 3: 'export const b3 = 33;', 9: 'export const b9 = 99;' }) }, 'feat: side');
|
|
g(fx.repo, 'checkout', '-q', 'main');
|
|
fx.write(rndReview({ start: fx.start, end: side, findings: [MAJ_A3] }).text);
|
|
const before = fx.snapshot();
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /round base is not an ancestor of HEAD — not measured/, r.stdout);
|
|
assert.deepEqual(fx.snapshot(), before, 'a refused prepare archives nothing');
|
|
// A short sha is no round base either.
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1.slice(0, 12), findings: [MAJ_A3] }).text);
|
|
const s = fx.prepare();
|
|
assert.equal(s.status, 2, s.stdout + s.stderr);
|
|
// Control: the round-1 end on main is an ancestor of HEAD.
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
const c = fx.prepare(['--dry-run']);
|
|
assert.equal(c.status, 0, `Control: ${c.stdout}${c.stderr}`);
|
|
});
|
|
|
|
// ---- [RND] carry -------------------------------------------------------------------
|
|
|
|
const P = (f, i = 0) => ({ id: computeFindingId(f.file, f.line, f.rule_key ?? 'MISSING_TEST'), ...finding(f, i) });
|
|
|
|
test('[RND] a previous finding is carried until marked resolved by every reviewer', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const p = P(MAJ_A3);
|
|
const both = { engines: [], reviewers: [CCR, BCR] };
|
|
const one = rounds.carry([p], [], { marks: [{ id: p.id, resolved: true, reviewer: CCR }], ran: both, previousRan: both });
|
|
assert.equal(one.findings.length, 1, JSON.stringify(one));
|
|
assert.equal(one.findings[0].carried_from, p.id);
|
|
assert.equal(one.findings[0].severity, 'MAJOR');
|
|
assert.deepEqual(one.resolved, []);
|
|
const all = rounds.carry([p], [], {
|
|
marks: [{ id: p.id, resolved: true, reviewer: CCR }, { id: p.id, resolved: true, reviewer: BCR }], ran: both, previousRan: both,
|
|
});
|
|
assert.deepEqual(all.findings, []);
|
|
assert.deepEqual(all.resolved, [p.id]);
|
|
});
|
|
|
|
test('[RND] a repeated finding is folded and not counted as new', () => {
|
|
const fx = rndRepo();
|
|
roundOne(fx, [MAJ_A3]);
|
|
// The same finding raised again with other words: same file, same rule_key.
|
|
roundTwo(fx, [{ ...MAJ_A3, title: 'Last token lost by the parser' }], { outs: { [CCR]: [] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const payload = jsonOf(fx.read());
|
|
assert.equal(payload.findings.length, 1, JSON.stringify(payload.findings));
|
|
assert.equal(payload.findings[0].carried_from, computeFindingId('a.mjs', 3, 'MISSING_TEST'), 'the re-raised finding is the carried one');
|
|
assert.deepEqual(payload.counts, { BLOCKER: 0, MAJOR: 1, MINOR: 0, SUGGESTION: 0 }, 'one MAJOR, not two');
|
|
});
|
|
|
|
test('[RND] a new MAJOR outside the round diff keeps MAJOR', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const fresh = P({ severity: 'MAJOR', file: 'b.mjs', line: 3, title: 'New thing' });
|
|
const r = rounds.carry([], [fresh], { marks: [], ran: { engines: [], reviewers: [CCR] }, previousRan: { engines: [], reviewers: [CCR] } });
|
|
assert.deepEqual(r.findings.map((x) => [x.severity, x.carried_from]), [['MAJOR', undefined]]);
|
|
// In a finalized round 2: b.mjs:3 changed in round 1 only (outside c1..c2), and it still counts as MAJOR.
|
|
const fx = rndRepo();
|
|
roundOne(fx, [MAJ_A3]);
|
|
roundTwo(fx, [MAJ_A3, { severity: 'MAJOR', file: 'b.mjs', line: 3, title: 'New thing' }], { outs: { [CCR]: [] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const fresh2 = jsonOf(fx.read()).findings.find((x) => x.file === 'b.mjs');
|
|
assert.equal(fresh2.severity, 'MAJOR');
|
|
assert.equal(fresh2.carried_from, undefined);
|
|
const gate = fx.gate();
|
|
assert.equal(gate.status, 2, gate.stdout);
|
|
assert.ok(parse(gate).unanswered.includes(fresh2.id), `the new MAJOR counts: ${gate.stdout}`);
|
|
});
|
|
|
|
test('[RND] two occurrences of one class stay two findings', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const ran = { engines: [], reviewers: [CCR] };
|
|
const p1 = P({ severity: 'MAJOR', file: 'a.mjs', line: 3 });
|
|
const p2 = P({ severity: 'MAJOR', file: 'a.mjs', line: 8 });
|
|
const both = rounds.carry([p1, p2], [P({ severity: 'MAJOR', file: 'a.mjs', line: 8 }), P({ severity: 'MAJOR', file: 'a.mjs', line: 3 })],
|
|
{ marks: [], ran, previousRan: ran });
|
|
assert.deepEqual(both.findings.map((x) => `${x.line}<-${x.carried_from}`).sort(), [`3<-${p1.id}`, `8<-${p2.id}`]);
|
|
// One previous occurrence, two current ones on the same line: one carried, one new, two ids (rule_key#2).
|
|
const fx = rndRepo();
|
|
roundOne(fx, [{ severity: 'MAJOR', file: 'a.mjs', line: 3 }]);
|
|
roundTwo(fx, [{ severity: 'MAJOR', file: 'a.mjs', line: 3 }, { severity: 'MAJOR', file: 'a.mjs', line: 3, title: 'second' }], { outs: { [CCR]: [] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const list = jsonOf(fx.read()).findings;
|
|
assert.equal(list.length, 2, JSON.stringify(list));
|
|
assert.equal(list.filter((x) => x.carried_from).length, 1, JSON.stringify(list));
|
|
assert.deepEqual(list.map((x) => x.id), computeFindingIds(list));
|
|
assert.notEqual(list[0].id, list[1].id);
|
|
});
|
|
|
|
test('[RND] a review.md from another run starts at round 1', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.c1, end: fx.c2, findings: [MAJ_A3] }).text);
|
|
const before = fx.snapshot();
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
assert.deepEqual([parse(r).round, parse(r).round_base, parse(r).previous], [1, null, null], r.stdout);
|
|
assert.deepEqual(fx.snapshot(), before, 'nothing is archived for another run');
|
|
});
|
|
|
|
// ---- [RND] engine findings ----------------------------------------------------------
|
|
|
|
const ENG_MAJ = { severity: 'MAJOR', file: 'b.mjs', line: 3, rule_key: 'CODE_REVIEW_DEFECT', engine: 'code-review', title: 'Off by one in loop' };
|
|
|
|
test('[RND] an engine MAJOR is resolved when its engine ran this round and did not raise it again', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const p = P(ENG_MAJ);
|
|
const prevRan = { engines: ['code-review'], reviewers: [CCR] };
|
|
const r = rounds.carry([p], [], { marks: [], ran: { engines: ['code-review'], reviewers: [CCR] }, previousRan: prevRan });
|
|
assert.deepEqual([r.findings.length, r.resolved], [0, [p.id]]);
|
|
});
|
|
|
|
test('[RND] an engine MAJOR is carried when its engine did not run, or raised it again, whatever a reviewer marks', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const p = P(ENG_MAJ);
|
|
const prevRan = { engines: ['code-review'], reviewers: [CCR] };
|
|
const marks = [{ id: p.id, resolved: true, reviewer: CCR }];
|
|
const notRun = rounds.carry([p], [], { marks, ran: { engines: [], reviewers: [CCR] }, previousRan: prevRan });
|
|
assert.deepEqual(notRun.findings.map((x) => x.carried_from), [p.id]);
|
|
assert.deepEqual(notRun.resolved, []);
|
|
const again = rounds.carry([p], [P({ ...ENG_MAJ, line: 4 })], { marks, ran: { engines: ['code-review'], reviewers: [CCR] }, previousRan: prevRan });
|
|
assert.deepEqual(again.findings.map((x) => x.carried_from), [p.id]);
|
|
assert.deepEqual(again.resolved, []);
|
|
});
|
|
|
|
test('[RND] a --quick round 2 (no security payload) carries every previous security-review finding', () => {
|
|
const fx = rndRepo();
|
|
const sec = { severity: 'BLOCKER', file: 'b.mjs', line: 3, rule_key: 'SECURITY_REVIEW_VULNERABILITY', engine: 'security-review', title: 'Path escapes the root' };
|
|
const { prev } = roundOne(fx, [sec, ENG_MAJ], { outs: [CCR] });
|
|
const [secId, majId] = prev.findings.map((x) => x.id);
|
|
// --quick: only /code-review ran and wrote its payload.
|
|
roundTwo(fx, [], { outs: { [CCR]: [{ id: secId, resolved: true }, { id: majId, resolved: true }] }, engines: ['code-review'] });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const payload = jsonOf(fx.read());
|
|
assert.deepEqual(payload.ran, { engines: ['code-review'], reviewers: [CCR] });
|
|
assert.deepEqual(payload.findings.map((x) => x.carried_from), [secId], 'the security finding is carried');
|
|
assert.deepEqual(payload.resolved, [majId], 'Control: the code-review MAJOR is resolved by a clean code-review run');
|
|
});
|
|
|
|
test('[RND] an engine that ran clean while a reviewer marks the finding resolved: false carries it', () => {
|
|
assert.equal(typeof rounds.carry, 'function', 'review-rounds.mjs exports carry');
|
|
const p = P(ENG_MAJ);
|
|
const ran = { engines: ['code-review'], reviewers: [CCR] };
|
|
const r = rounds.carry([p], [], { marks: [{ id: p.id, resolved: false, reviewer: CCR }], ran, previousRan: ran });
|
|
assert.deepEqual([r.findings.map((x) => x.carried_from), r.resolved], [[p.id], []]);
|
|
});
|
|
|
|
// ---- [RND] reviewer-only findings ---------------------------------------------------
|
|
|
|
test('[RND] a reviewer-only finding: one reviewer\'s true while the other round-1 reviewer did not run carries it; both true resolves it', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [MAJ_A3]);
|
|
const [id] = prev.findings.map((x) => x.id);
|
|
roundTwo(fx, [], { outs: { [CCR]: [{ id, resolved: true }] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const one = jsonOf(fx.read());
|
|
assert.deepEqual(one.findings.map((x) => x.carried_from), [id], 'carried: brief-conformance-reviewer ran in round 1, not in round 2');
|
|
assert.deepEqual(one.resolved, []);
|
|
roundTwo(fx, [], { outs: { [CCR]: [{ id, resolved: true }], [BCR]: [{ id, resolved: true }] } });
|
|
const f2 = fx.finalize2();
|
|
assert.equal(f2.status, 0, f2.stdout + f2.stderr);
|
|
const both = jsonOf(fx.read());
|
|
assert.deepEqual(both.findings, []);
|
|
assert.deepEqual(both.resolved, [id]);
|
|
const text = fx.read();
|
|
assert.match(text, /^## Resolved since round 1$/m, text);
|
|
assert.ok(text.slice(text.indexOf('## Resolved since round 1')).includes('Parser drops the last token'), text);
|
|
});
|
|
|
|
// ---- [RND] round count ------------------------------------------------------------------
|
|
|
|
test('[RND] a round-1 review with no round field, then prepare, returns round 2 and writes review-round-<run>-1.md; a second prepare after a stop returns round 2 again', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
const r1 = rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] });
|
|
assert.doesNotMatch(fmOf(r1.text), /^round:/m, 'the round-1 review has no round field (finalize failed)');
|
|
fx.write(r1.text);
|
|
const a = fx.prepare();
|
|
assert.equal(a.status, 0, a.stdout + a.stderr);
|
|
assert.equal(parse(a).round, 2, a.stdout);
|
|
assert.equal(fx.read(fx.archive(1)), r1.text);
|
|
// A stop at this point: review.md is still round 1, and the archive equals it.
|
|
const b = fx.prepare();
|
|
assert.equal(b.status, 0, b.stdout + b.stderr);
|
|
assert.deepEqual([parse(b).round, parse(b).previous], [2, `review-round-${fx.run}-1.md`], b.stdout);
|
|
assert.equal(fx.read(fx.archive(1)), r1.text, 'the archive is kept');
|
|
assert.ok(!existsSync(fx.archive(2)), 'no second archive');
|
|
});
|
|
|
|
test('[RND] an archive with other bytes is exit 2; a round field that disagrees with the archives is exit 2', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
// Another review with a readable JSON block (the stop rule reads it): two
|
|
// BLOCKERs of another class, so the rule does not fire on the pair.
|
|
const another = rndReview({ start: fx.start, end: fx.c1, findings: [BLK_B3, BLK_A5_RND] }).text;
|
|
fx.write(another, fx.archive(2));
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), new RegExp(`archive review-round-${fx.run}-2\\.md differs — not measured`), r.stdout);
|
|
assert.equal(fx.read(fx.archive(2)), another, 'the differing archive is left alone');
|
|
rmSync(fx.archive(2));
|
|
// round: 2 with no archive of round 1.
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text.replace('\nfindings:', '\nround: 2\nfindings:'));
|
|
const m = fx.prepare();
|
|
assert.equal(m.status, 2, m.stdout + m.stderr);
|
|
assert.match(String(parse(m).reason), /round count does not match the archives — not measured/, m.stdout);
|
|
});
|
|
|
|
test('[RND] prepare --dry-run returns the round and leaves the folder byte for byte unchanged', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
fx.out(CCR);
|
|
fx.engine('code-review');
|
|
const before = fx.snapshot();
|
|
const r = fx.prepare(['--dry-run']);
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
assert.deepEqual([parse(r).round, parse(r).round_base], [2, fx.c1], r.stdout);
|
|
assert.deepEqual(fx.snapshot(), before);
|
|
});
|
|
|
|
test('[RND] a second run in the same folder starts at round 1, and its round 2 archives under its own run', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
assert.equal(fx.prepare().status, 0);
|
|
const first = fx.read(fx.archive(1));
|
|
fx.write(rndReview({ start: fx.start, end: fx.c2, findings: [MAJ_A3] }).text);
|
|
// Run 2 starts at c1.
|
|
const one = fx.prepare([], fx.c1);
|
|
assert.equal(one.status, 0, one.stdout + one.stderr);
|
|
assert.equal(parse(one).round, 1, one.stdout);
|
|
fx.write(rndReview({ start: fx.c1, end: fx.c2, findings: [MAJ_A3] }).text);
|
|
const two = fx.prepare([], fx.c1);
|
|
assert.equal(two.status, 0, two.stdout + two.stderr);
|
|
const run2 = fx.c1.slice(0, 12);
|
|
assert.deepEqual([parse(two).round, parse(two).previous], [2, `review-round-${run2}-1.md`], two.stdout);
|
|
assert.ok(existsSync(fx.archive(1, run2)));
|
|
assert.equal(fx.read(fx.archive(1)), first, 'the first run\'s archive is untouched');
|
|
assert.deepEqual(readdirSync(fx.project).filter((n) => n.startsWith('review-round-')).sort(),
|
|
[`review-round-${fx.run}-1.md`, `review-round-${run2}-1.md`].sort());
|
|
});
|
|
|
|
// ---- [RND] resolution by evidence -----------------------------------------------------
|
|
|
|
test('[RND] a previous review without finalized: true resolves nothing', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [MAJ_A3, ENG_MAJ], { finalize: false });
|
|
assert.doesNotMatch(fmOf(fx.read(fx.archive(1))), /^finalized:/m);
|
|
const marks = prev.findings.map((x) => ({ id: x.id, resolved: true }));
|
|
roundTwo(fx, [], { outs: { [CCR]: marks, [BCR]: marks } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const payload = jsonOf(fx.read());
|
|
assert.deepEqual(payload.findings.map((x) => x.carried_from).sort(), prev.findings.map((x) => x.id).sort());
|
|
assert.deepEqual(payload.resolved, []);
|
|
});
|
|
|
|
test('[RND] a previous review with ran.reviewers: [] resolves no reviewer finding', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [MAJ_A3], { outs: [] });
|
|
assert.deepEqual(prev.ran?.reviewers, [], 'round 1 was finalized with no reviewer output');
|
|
const [id] = prev.findings.map((x) => x.id);
|
|
roundTwo(fx, [], { outs: { [CCR]: [{ id, resolved: true }], [BCR]: [{ id, resolved: true }] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const payload = jsonOf(fx.read());
|
|
assert.deepEqual(payload.findings.map((x) => x.carried_from), [id]);
|
|
assert.deepEqual(payload.resolved, []);
|
|
});
|
|
|
|
test('[RND] a previous code-review MAJOR is resolved by a code-review payload converted from an empty engine run', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [ENG_MAJ], { outs: [CCR] });
|
|
const [id] = prev.findings.map((x) => x.id);
|
|
// Control: with no code-review payload this round, the finding is carried.
|
|
roundTwo(fx, [], { outs: { [CCR]: [] }, engines: [] });
|
|
const c = fx.finalize2();
|
|
assert.equal(c.status, 0, c.stdout + c.stderr);
|
|
assert.deepEqual(jsonOf(fx.read()).findings.map((x) => x.carried_from), [id]);
|
|
// `node lib/review/engine-findings.mjs` on {"engine":"code-review","findings":[]}.
|
|
roundTwo(fx, [], { outs: { [CCR]: [] }, engines: ['code-review'] });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const payload = jsonOf(fx.read());
|
|
assert.deepEqual(payload.ran.engines, ['code-review']);
|
|
assert.deepEqual(payload.findings, []);
|
|
assert.deepEqual(payload.resolved, [id]);
|
|
});
|
|
|
|
test('[RND] prepare deletes old .review-engine-payload-*.json and .review-out-*.md files', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
fx.out(CCR);
|
|
fx.out(BCR);
|
|
fx.engine('code-review');
|
|
fx.engine('security-review');
|
|
writeFileSync(join(fx.project, '.review-engine-code-review.json'), '{}');
|
|
writeFileSync(join(fx.project, '.review-scope.txt'), 'a.mjs\n');
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
const left = readdirSync(fx.project).sort();
|
|
assert.deepEqual(left.filter((n) => n.startsWith('.review-out-') || n.startsWith('.review-engine-payload-')), [], left.join(', '));
|
|
// Control: the engine transcription and the scope list stay.
|
|
assert.ok(left.includes('.review-engine-code-review.json') && left.includes('.review-scope.txt'), left.join(', '));
|
|
});
|
|
|
|
// ---- [RND] ids and R-steps after the carry ----------------------------------------------
|
|
|
|
/** Every `Source finding:` id under ## Remediation Steps. */
|
|
const sources = (text) => [...text.slice(text.indexOf('## Remediation Steps')).matchAll(/Source finding:\**\s*`?([0-9a-f]{40})/g)].map((m) => m[1]);
|
|
|
|
test('[RND] a finalized round-2 review with a fold and a carried BLOCKER has every id computed, the BLOCKER\'s R-step, and passes the validator and the gate', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [MAJ_A3, BLK_B3]);
|
|
const blkPrev = prev.findings.find((x) => x.severity === 'BLOCKER').id;
|
|
// Round 2 re-raises the MAJOR (one line lower: a fold) and not the BLOCKER.
|
|
roundTwo(fx, [{ ...MAJ_A3, line: 4 }], { outs: { [CCR]: [], [BCR]: [] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const text = fx.read();
|
|
const payload = jsonOf(text);
|
|
assert.equal(payload.findings.length, 2, JSON.stringify(payload.findings));
|
|
assert.deepEqual(payload.findings.map((x) => x.id), computeFindingIds(payload.findings), 'every id is the computed one');
|
|
const blk = payload.findings.find((x) => x.severity === 'BLOCKER');
|
|
assert.equal(blk.carried_from, blkPrev);
|
|
assert.equal(payload.verdict, 'BLOCK', 'the verdict is raised by the carried BLOCKER');
|
|
assert.match(fmOf(text), /^verdict: BLOCK$/m);
|
|
assert.deepEqual(payload.counts, { BLOCKER: 1, MAJOR: 1, MINOR: 0, SUGGESTION: 0 });
|
|
assert.match(fmOf(text), /^round: 2$/m);
|
|
assert.match(fmOf(text), new RegExp(`^round_base: ${fx.c1}$`, 'm'));
|
|
assert.match(fmOf(text), /^finalized: true$/m);
|
|
const fm = fmOf(text);
|
|
for (const x of payload.findings) {
|
|
assert.ok(fm.includes(` - ${x.id}\n`), `the frontmatter lists ${x.id}`);
|
|
assert.ok(text.includes(`### ${x.id}\n`), `the body has an entry for ${x.id}`);
|
|
}
|
|
assert.ok(sources(text).includes(blk.id), `the carried BLOCKER has its R-step with the recomputed id:\n${text}`);
|
|
const v = fx.validate();
|
|
assert.equal(v.status, 0, `review-validator --strict: ${v.stdout}${v.stderr}`);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
assert.equal(parse(r).reason, 'verdict BLOCK', r.stdout);
|
|
});
|
|
|
|
test('[RND] a BLOCKER re-raised with the coordinator\'s own R-step keeps exactly that one R-step; an orphan R-step makes finalize exit 1', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [BLK_B3]);
|
|
const blkPrev = prev.findings[0].id;
|
|
const r2 = roundTwo(fx, [{ ...BLK_B3, line: 4, title: 'Build output never reaches disk' }], { outs: { [CCR]: [], [BCR]: [] } });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const text = fx.read();
|
|
const payload = jsonOf(text);
|
|
const carried = payload.findings.find((x) => x.carried_from === blkPrev);
|
|
assert.ok(carried, `the re-raised BLOCKER is the carried one: ${JSON.stringify(payload.findings)}`);
|
|
assert.equal((text.match(/^### Step R\d+/gm) ?? []).length, 1, text);
|
|
assert.ok(text.includes('### Step R1: fix Build output never reaches disk'), 'the one R-step is the coordinator\'s');
|
|
assert.deepEqual(sources(text), [carried.id]);
|
|
const ids = new Set(payload.findings.map((x) => x.id));
|
|
assert.deepEqual(sources(text).filter((s) => !ids.has(s)), [], 'no Source finding names an absent id');
|
|
assert.equal(fx.validate().status, 0);
|
|
// A planted orphan R-step.
|
|
const orphan = '\n### Step R9: fix nothing\n\n- **Files:** `a.mjs`\n- **Source finding:** `' + 'e'.repeat(40) + '`\n- **On failure:** escalate\n';
|
|
roundTwo(fx, [{ ...BLK_B3, line: 4, title: 'Build output never reaches disk' }], { outs: { [CCR]: [], [BCR]: [] }, extraSteps: orphan });
|
|
const planted = fx.read();
|
|
assert.equal(r2.ids.length, 1);
|
|
const o = fx.finalize2();
|
|
assert.equal(o.status, 1, o.stdout + o.stderr);
|
|
assert.equal(fx.read(), planted, 'a refused finalize leaves the file as it was');
|
|
});
|
|
|
|
test('[RND] a round-2 review that finalize did not finalize is not measured: the gate exits 2', () => {
|
|
const fx = rndRepo();
|
|
roundOne(fx, [BLK_B3, MAJ_A3]);
|
|
// Round 2 re-raises nothing; the coordinator's one R-step names an id the review does not hold.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, steps: [{ title: 'fix an old finding', source: 'f'.repeat(40) }] });
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 1, f.stdout + f.stderr);
|
|
assert.doesNotMatch(fmOf(fx.read()), /^finalized: true$/m, 'a refused finalize leaves the file unfinalized');
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.equal(parse(r).reason, 'review-not-measured: round 2+ review not finalized', r.stdout);
|
|
// An unfinalized round-2 review with no findings and no R-step: exit 2 as well.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] } });
|
|
const n = fx.gate();
|
|
assert.equal(n.status, 2, n.stdout + n.stderr);
|
|
assert.equal(parse(n).reason, 'review-not-measured: round 2+ review not finalized', n.stdout);
|
|
// Control: the same review finalized is decided on (the carried BLOCKER blocks).
|
|
const ok = fx.finalize2();
|
|
assert.equal(ok.status, 0, `Control: ${ok.stdout}${ok.stderr}`);
|
|
const b = fx.gate();
|
|
assert.equal(b.status, 1, `Control: ${b.stdout}${b.stderr}`);
|
|
// Control: an unfinalized round-1 review with only another run's archive beside it is decided on.
|
|
const other = rndRepo();
|
|
other.write(rndReview({ start: other.start, end: other.c2, findings: [] }).text);
|
|
other.write('another run\n', other.archive(1, 'f'.repeat(12)));
|
|
const one = other.gate();
|
|
assert.equal(one.status, 0, `Control: ${one.stdout}${one.stderr}`);
|
|
});
|
|
|
|
test('[RND] a carried MAJOR whose line moved under the round\'s fix is never pre-existing: the gate keeps exit 2', () => {
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
fx.out(CCR);
|
|
fx.out(BCR);
|
|
const f1 = fx.finalize(1);
|
|
assert.equal(f1.status, 0, f1.stdout + f1.stderr);
|
|
// The round's fix puts two lines on top of a.mjs: the old line 3 is now line 5.
|
|
const c3 = fx.commit({ 'a.mjs': `// header 1\n// header 2\n${readFileSync(join(fx.repo, 'a.mjs'), 'utf8')}` }, 'fix: round 2 work');
|
|
const p = fx.prepare();
|
|
assert.equal(p.status, 0, p.stdout + p.stderr);
|
|
fx.write(rndReview({ start: fx.start, end: c3, findings: [] }).text);
|
|
fx.out(CCR);
|
|
fx.out(BCR);
|
|
const f2 = fx.finalize(2, ['--round-base', fx.c1, '--previous', fx.archive(1)]);
|
|
assert.equal(f2.status, 0, f2.stdout + f2.stderr);
|
|
const text = fx.read();
|
|
const carried = jsonOf(text).findings.find((x) => x.carried_from);
|
|
assert.ok(carried, `the MAJOR is carried: ${JSON.stringify(jsonOf(text).findings)}`);
|
|
assert.equal(carried.line, 3, 'a carried finding keeps its line');
|
|
assert.equal(carried.pre_existing, false, 'finalize never marks a carried finding pre-existing');
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /^review-unanswered: 1 MAJOR/, r.stdout);
|
|
// A hand-set mark does not take it out either.
|
|
fx.write(editFindings(text, (x) => (x.carried_from ? { ...x, pre_existing: true } : x)));
|
|
const h = fx.gate();
|
|
assert.equal(h.status, 2, h.stdout + h.stderr);
|
|
assert.match(String(parse(h).reason), /^review-unanswered: 1 MAJOR/, h.stdout);
|
|
});
|
|
|
|
// ---- [CAP] the cap ----------------------------------------------------------------------
|
|
|
|
test('[CAP] MAX_REVIEW_ROUNDS is 3, and --cap prints it', () => {
|
|
assert.equal(rounds.MAX_REVIEW_ROUNDS, 3);
|
|
const r = run([ROUNDS, '--cap']);
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
assert.equal(r.stdout.trim(), '3');
|
|
});
|
|
|
|
test('[CAP] prepare refuses round 4 with exit 1', () => {
|
|
assert.ok(existsSync(ROUNDS), 'lib/review/review-rounds.mjs must exist');
|
|
const fx = rndRepo();
|
|
const r3 = rndReview({ start: fx.start, end: fx.c2, findings: [MAJ_A3] }).text.replace('\nfindings:', '\nround: 3\nfindings:');
|
|
fx.write(r3);
|
|
// Earlier rounds with readable JSON blocks (the stop rule reads them): two
|
|
// BLOCKERs of another class, so the rule does not fire before round 3.
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [BLK_B3, BLK_A5_RND] }).text, fx.archive(1));
|
|
fx.write(rndReview({ start: fx.start, end: fx.c2, findings: [BLK_B3, BLK_A5_RND] }).text, fx.archive(2));
|
|
const before = fx.snapshot();
|
|
const r = fx.prepare();
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /review round cap reached \(3\) — the operator re-plans/, r.stdout);
|
|
assert.deepEqual(fx.snapshot(), before, 'a refused round writes nothing');
|
|
// Control: round 3 is allowed.
|
|
rmSync(fx.archive(2));
|
|
fx.write(r3.replace('round: 3', 'round: 2'));
|
|
const c = fx.prepare(['--dry-run']);
|
|
assert.equal(c.status, 0, `Control: ${c.stdout}${c.stderr}`);
|
|
assert.equal(parse(c).round, 3);
|
|
});
|
|
|
|
// ---- [CRY] carried answers ---------------------------------------------------------------
|
|
|
|
/**
|
|
* Round 1 finalized with a MAJOR on a.mjs:3, answered `accepted` through
|
|
* review-answers.mjs record (K2 record committed); prepare; round 2 re-raises
|
|
* the MAJOR; finalize carries it; review-answers.mjs carry writes the row.
|
|
*/
|
|
function cryFlow() {
|
|
const fx = rndRepo();
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [MAJ_A3] }).text);
|
|
fx.out(CCR);
|
|
const f1 = fx.finalize(1);
|
|
assert.equal(f1.status, 0, f1.stdout + f1.stderr);
|
|
const oldId = jsonOf(fx.read()).findings[0].id;
|
|
const rec = run([ANSWERS, 'record', '--review', fx.reviewPath, '--finding', oldId, '--answer', 'accepted', '--repo', fx.repo,
|
|
'--round', '1', '--store', 'local', '--trailer', TRAILER], { input: 'the parser keeps the old contract on purpose' });
|
|
assert.equal(rec.status, 0, `record: ${rec.stdout}${rec.stderr}`);
|
|
const record = parse(rec).record;
|
|
const p = fx.prepare();
|
|
assert.equal(p.status, 0, p.stdout + p.stderr);
|
|
roundTwo(fx, [MAJ_A3], { outs: { [CCR]: [] } });
|
|
const f2 = fx.finalize2();
|
|
assert.equal(f2.status, 0, f2.stdout + f2.stderr);
|
|
const carry = run([ANSWERS, 'carry', '--review', fx.reviewPath, '--previous', fx.archive(1), '--repo', fx.repo]);
|
|
return { fx, oldId, record, carry, link: `review-round-${fx.run}-1.md#${oldId}` };
|
|
}
|
|
|
|
/** The review with its one `## Answers` row's cells replaced by `fn(cells)`. */
|
|
function editRow(text, fn) {
|
|
return text.split('\n').map((l) => {
|
|
if (!/^\| [0-9a-f]{40} \| /.test(l)) return l;
|
|
return `| ${fn(l.slice(1, -1).split('|').map((c) => c.trim())).join(' | ')} |`;
|
|
}).join('\n');
|
|
}
|
|
|
|
/** The review with its JSON block's findings changed by `fn` (ids kept). */
|
|
function editFindings(text, fn) {
|
|
const at = text.lastIndexOf('```json\n') + 8;
|
|
const end = text.lastIndexOf('```');
|
|
const payload = JSON.parse(text.slice(at, end));
|
|
payload.findings = payload.findings.map(fn);
|
|
return `${text.slice(0, at)}${JSON.stringify(payload, null, 2)}\n${text.slice(end)}`;
|
|
}
|
|
|
|
test('[CRY] a round-2 review that carries an accepted MAJOR with the link to its earlier answer exits 0', () => {
|
|
const { fx, oldId, record, carry, link } = cryFlow();
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const text = fx.read();
|
|
const newId = jsonOf(text).findings[0].id;
|
|
assert.equal(jsonOf(text).findings[0].carried_from, oldId);
|
|
assert.ok(text.includes(`| ${newId} | accepted | the parser keeps the old contract on purpose | ${record} | ${link} |`), text);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
test('[CRY] the same answer without a carried finding and link keeps exit 2', () => {
|
|
const { fx, carry } = cryFlow();
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const good = fx.read();
|
|
assert.equal(fx.gate().status, 0, 'the carried row opens the gate');
|
|
// Control: the row without its link.
|
|
fx.write(editRow(good, (c) => [...c.slice(0, 4), '-']));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout);
|
|
assert.match(String(parse(r).reason), /^review-unanswered:/, r.stdout);
|
|
// Control: the link kept, the finding not carried.
|
|
fx.write(editFindings(good, ({ carried_from: _c, ...rest }) => rest));
|
|
const n = fx.gate();
|
|
assert.equal(n.status, 2, n.stdout);
|
|
assert.match(String(parse(n).reason), /^review-unanswered:/, n.stdout);
|
|
});
|
|
|
|
test('[CRY] a carried row whose finding no longer matches is unanswered', () => {
|
|
const { fx, carry } = cryFlow();
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const good = fx.read();
|
|
assert.equal(fx.gate().status, 0, 'the carried row opens the gate');
|
|
// Control: the finding (same id, same carried_from) now says something else under another key.
|
|
fx.write(editFindings(good, (f) => ({ ...f, rule_key: 'MISSING_ERROR_HANDLING', title: 'Unbounded retry storm', detail: 'renderer loops forever on timeout' })));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout);
|
|
assert.match(String(parse(r).reason), /^review-unanswered:/, r.stdout);
|
|
});
|
|
|
|
test('[CRY] a row linking to another run\'s archive keeps exit 2, even with a valid K2 record and a content match', () => {
|
|
const { fx, oldId, carry } = cryFlow();
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const good = fx.read();
|
|
assert.equal(fx.gate().status, 0, 'the carried row opens the gate');
|
|
// Control: the same archive bytes under another run's name.
|
|
const other = 'f'.repeat(12);
|
|
copyFileSync(fx.archive(1), fx.archive(1, other));
|
|
fx.write(editRow(good, (c) => [...c.slice(0, 4), `review-round-${other}-1.md#${oldId}`]));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout);
|
|
assert.match(String(parse(r).reason), /^review-unanswered:/, r.stdout);
|
|
});
|
|
|
|
test('[CRY] a row linking to an archive whose <n> is not lower than the review\'s round keeps exit 2', () => {
|
|
const { fx, oldId, carry } = cryFlow();
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const good = fx.read();
|
|
assert.equal(fx.gate().status, 0, 'the carried row opens the gate');
|
|
// Control: the same archive bytes as round 2 of this run, in a round-2 review.
|
|
copyFileSync(fx.archive(1), fx.archive(2));
|
|
fx.write(editRow(good, (c) => [...c.slice(0, 4), `review-round-${fx.run}-2.md#${oldId}`]));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout);
|
|
assert.match(String(parse(r).reason), /^review-unanswered:/, r.stdout);
|
|
// Control: a review with no round field accepts no carried link.
|
|
fx.write(good.replace(/^round: 2\n/m, ''));
|
|
const n = fx.gate();
|
|
assert.equal(n.status, 2, n.stdout);
|
|
assert.match(String(parse(n).reason), /^review-unanswered:/, n.stdout);
|
|
});
|
|
|
|
test('[CRY] an answer to one carried occurrence, linked from another carried occurrence of the same class, keeps exit 2', () => {
|
|
const fx = rndRepo();
|
|
const Y1 = { severity: 'MAJOR', file: 'a.mjs', line: 3, title: 'Parser drops the last token', detail: 'parse() stops one token early' };
|
|
const Y2 = { severity: 'MAJOR', file: 'a.mjs', line: 5, title: 'Cache never expires entries', detail: 'ttl ignored in get()' };
|
|
// Y3 is resolved in round 2, so the blocking count falls (3 -> 2) and the stop
|
|
// rule (lib/review/stop-rule.mjs, "not fewer") leaves round 3 open for the Control below.
|
|
const Y3 = { severity: 'MAJOR', file: 'b.mjs', line: 3, rule_key: 'SCOPE_CREEP_BUILT', title: 'Adds a flag nobody asked for', detail: 'a --fast switch the brief never named' };
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [Y1, Y2, Y3] }).text);
|
|
fx.out(CCR);
|
|
const f1 = fx.finalize(1);
|
|
assert.equal(f1.status, 0, f1.stdout + f1.stderr);
|
|
const [id1, id2] = jsonOf(fx.read()).findings.filter((x) => x.file === 'a.mjs').map((x) => x.id);
|
|
const id3 = jsonOf(fx.read()).findings.find((x) => x.file === 'b.mjs').id;
|
|
const rec = run([ANSWERS, 'record', '--review', fx.reviewPath, '--finding', id1, '--answer', 'accepted', '--repo', fx.repo,
|
|
'--round', '1', '--store', 'local', '--trailer', TRAILER], { input: 'the parser keeps the old contract on purpose' });
|
|
assert.equal(rec.status, 0, `record: ${rec.stdout}${rec.stderr}`);
|
|
const p = fx.prepare();
|
|
assert.equal(p.status, 0, p.stdout + p.stderr);
|
|
roundTwo(fx, [Y1, Y2], { outs: { [CCR]: [{ id: id3, resolved: true }] } });
|
|
const f2 = fx.finalize2();
|
|
assert.equal(f2.status, 0, f2.stdout + f2.stderr);
|
|
const carry = run([ANSWERS, 'carry', '--review', fx.reviewPath, '--previous', fx.archive(1), '--repo', fx.repo]);
|
|
assert.equal(carry.status, 0, `review-answers carry: ${carry.stdout}${carry.stderr}`);
|
|
const good = fx.read();
|
|
assert.deepEqual(jsonOf(good).resolved, [id3], 'Y3 is resolved in round 2');
|
|
const g1 = jsonOf(good).findings.find((x) => x.carried_from === id1);
|
|
const g2 = jsonOf(good).findings.find((x) => x.carried_from === id2);
|
|
assert.ok(g1 && g2, `both MAJORs are carried: ${JSON.stringify(jsonOf(good).findings)}`);
|
|
const row = good.split('\n').find((l) => l.startsWith(`| ${g1.id} | accepted |`));
|
|
assert.ok(row && row.endsWith(`| review-round-${fx.run}-1.md#${id1} |`), `the carried row of the answered MAJOR:\n${good}`);
|
|
// Control: only the answered occurrence is answered.
|
|
const c = fx.gate();
|
|
assert.equal(c.status, 2, c.stdout);
|
|
assert.deepEqual(parse(c).unanswered, [g2.id], c.stdout);
|
|
// The same row copied to the other occurrence: its link names id1, the finding was carried from id2.
|
|
fx.write(good.replace(row, `${row}\n${row.replace(`| ${g1.id} |`, `| ${g2.id} |`)}`));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout);
|
|
assert.match(String(parse(r).reason), /^review-unanswered:/, r.stdout);
|
|
assert.deepEqual(parse(r).unanswered, [g2.id], r.stdout);
|
|
// Control: round 3 carries the answer through the round-2 archive; a copy for the other occurrence does not.
|
|
fx.write(good);
|
|
const p3 = fx.prepare();
|
|
assert.equal(p3.status, 0, `Control: ${p3.stdout}${p3.stderr}`);
|
|
assert.equal(parse(p3).round, 3, p3.stdout);
|
|
fx.write(rndReview({ start: fx.start, end: fx.c2, findings: [Y1, Y2] }).text);
|
|
fx.out(CCR, []);
|
|
const f3 = fx.finalize(3, ['--round-base', fx.c2, '--previous', fx.archive(2)]);
|
|
assert.equal(f3.status, 0, `Control: ${f3.stdout}${f3.stderr}`);
|
|
const carry3 = run([ANSWERS, 'carry', '--review', fx.reviewPath, '--previous', fx.archive(2), '--repo', fx.repo]);
|
|
assert.equal(carry3.status, 0, `Control: ${carry3.stdout}${carry3.stderr}`);
|
|
const third = fx.read();
|
|
const h1 = jsonOf(third).findings.find((x) => x.carried_from === g1.id);
|
|
const h2 = jsonOf(third).findings.find((x) => x.carried_from === g2.id);
|
|
assert.ok(h1 && h2, `Control: both MAJORs are carried into round 3: ${JSON.stringify(jsonOf(third).findings)}`);
|
|
const row3 = third.split('\n').find((l) => l.startsWith(`| ${h1.id} | accepted |`));
|
|
assert.ok(row3 && row3.endsWith(`| review-round-${fx.run}-1.md#${id1} |`), `Control: the round-3 row links where the answer was given:\n${third}`);
|
|
const c3 = fx.gate();
|
|
assert.deepEqual(parse(c3).unanswered, [h2.id], `Control: ${c3.stdout}`);
|
|
fx.write(third.replace(row3, `${row3}\n${row3.replace(`| ${h1.id} |`, `| ${h2.id} |`)}`));
|
|
const r3 = fx.gate();
|
|
assert.equal(r3.status, 2, r3.stdout);
|
|
assert.deepEqual(parse(r3).unanswered, [h2.id], r3.stdout);
|
|
});
|
|
|
|
// ---- [FXL] answers that ask for a fix (Step 7) ----------------------------------------
|
|
|
|
/** The review with one `### Step R1` for `id` under `## Remediation Steps`, in place of its "None.". */
|
|
function withRStep(text, id, title = 'fix the parser') {
|
|
const step = `### Step R1: ${title}\n\n- **Files:** \`a.mjs\`\n- **Source finding:** \`${id}\`\n- **Test first:** a test that fails today\n- **Verify:** re-run /trekreview\n- **On failure:** escalate\n`;
|
|
assert.ok(text.includes('## Remediation Steps\n\nNone.\n'), 'the review has no R-step yet');
|
|
return text.replace('## Remediation Steps\n\nNone.\n', `## Remediation Steps\n\n${step}`);
|
|
}
|
|
|
|
test('[FXL] a MAJOR answered fixed gives a ### Step R<n>: record --answer fixed refuses without the R-step and records with it, and the validator then passes', () => {
|
|
const fx = rndRepo();
|
|
// A review of HEAD (c2), so the gate decides on the answers, not on staleness.
|
|
fx.write(rndReview({ start: fx.start, end: fx.c2, findings: [MAJ_A3] }).text);
|
|
fx.out(CCR);
|
|
const f1 = fx.finalize(1);
|
|
assert.equal(f1.status, 0, f1.stdout + f1.stderr);
|
|
const id = jsonOf(fx.read()).findings[0].id;
|
|
const logFile = join(fx.repo, '.claude', 'voyage', 'approvals.jsonl');
|
|
const record = () => run([ANSWERS, 'record', '--review', fx.reviewPath, '--finding', id, '--answer', 'fixed', '--repo', fx.repo,
|
|
'--round', '1', '--store', 'local', '--trailer', TRAILER], { input: 'the parser gets the missing token back' });
|
|
const before = fx.read();
|
|
const refused = record();
|
|
assert.equal(refused.status, 1, `a fixed answer without its R-step is refused: ${refused.stdout}${refused.stderr}`);
|
|
assert.match(String(parse(refused).reason), /write its R-step first/, refused.stdout);
|
|
assert.equal(fx.read(), before, 'a refused answer writes no row');
|
|
assert.ok(!existsSync(logFile), 'a refused answer appends no K2 record');
|
|
// An R-step for another finding is not this finding's R-step.
|
|
fx.write(withRStep(before, 'e'.repeat(40)));
|
|
const other = record();
|
|
assert.equal(other.status, 1, `an R-step for another finding does not count: ${other.stdout}${other.stderr}`);
|
|
assert.ok(!existsSync(logFile), 'a refused answer appends no K2 record');
|
|
// With its R-step: recorded.
|
|
fx.write(withRStep(before, id));
|
|
const ok = record();
|
|
assert.equal(ok.status, 0, `with its R-step the answer is recorded: ${ok.stdout}${ok.stderr}`);
|
|
const text = fx.read();
|
|
assert.ok(text.includes(`| ${id} | fixed | the parser gets the missing token back | ${parse(ok).record} | - |`), text);
|
|
const v = fx.validate();
|
|
assert.equal(v.status, 0, `review-validator --strict: ${v.stdout}${v.stderr}`);
|
|
// Control: a fixed answer keeps the merge closed as fixed-pending.
|
|
const gate = fx.gate();
|
|
assert.equal(gate.status, 2, `Control: ${gate.stdout}${gate.stderr}`);
|
|
assert.match(String(parse(gate).reason), /^review-unanswered: fixed-pending 1/, `Control: ${gate.stdout}`);
|
|
});
|
|
|
|
test('[FXL] Phase 8.8 offers only Fix it for a finding pending marks security: true; pending returns that flag for a MAJOR-tier key from the security-review engine', () => {
|
|
const p88 = (() => {
|
|
const cmd = readFileSync(join(ROOT, 'commands', 'trekreview.md'), 'utf8');
|
|
const at = cmd.indexOf('## Phase 8.8 — Answers');
|
|
assert.ok(at >= 0, '/trekreview has a Phase 8.8 — Answers');
|
|
const next = cmd.indexOf('\n## ', at + 4);
|
|
return cmd.slice(at, next === -1 ? undefined : next).replace(/\s+/g, ' ');
|
|
})();
|
|
assert.match(p88, /Never offered for a finding `pending` marks `security: true`/, 'Accept is never offered for a security finding');
|
|
assert.match(p88, /That finding gets one option, Fix it, and its question says a security finding cannot be accepted/,
|
|
'a security finding gets one option, Fix it, and the question says why');
|
|
// Control: pending marks a MAJOR-tier key from the security-review engine security: true, and record refuses to accept it.
|
|
const fx = rndRepo();
|
|
const SEC = { severity: 'MAJOR', file: 'a.mjs', line: 3, rule_key: 'MISSING_TEST', engine: 'security-review', title: 'Token logged in clear' };
|
|
fx.write(rndReview({ start: fx.start, end: fx.c1, findings: [SEC, { ...MAJ_A3, line: 5 }] }).text);
|
|
const p = run([ANSWERS, 'pending', '--review', fx.reviewPath, '--repo', fx.repo, '--json']);
|
|
assert.equal(p.status, 1, `Control: ${p.stdout}${p.stderr}`);
|
|
const pending = parse(p).pending ?? [];
|
|
const sec = pending.find((x) => x.line === 3);
|
|
const plain = pending.find((x) => x.line === 5);
|
|
assert.ok(sec && plain, `Control: both MAJORs are pending: ${p.stdout}`);
|
|
assert.equal(sec.security, true, `Control: the security-review MAJOR is marked security: ${p.stdout}`);
|
|
assert.equal(plain.security, false, `Control: the other MAJOR is not: ${p.stdout}`);
|
|
const acc = run([ANSWERS, 'record', '--review', fx.reviewPath, '--finding', sec.id, '--answer', 'accepted', '--repo', fx.repo,
|
|
'--round', '1', '--store', 'local', '--trailer', TRAILER], { input: 'the token is a test token' });
|
|
assert.equal(acc.status, 1, `Control: a security finding is never accepted: ${acc.stdout}${acc.stderr}`);
|
|
});
|
|
|
|
// ---- [STP] the stop rule (E plan, Part 1 step 1.3; decisions 217, 220) ----------------
|
|
//
|
|
// prepare judges the previous round's archive and review.md when it would open
|
|
// round 3 (lib/review/stop-rule.mjs): one class with new blocking findings in
|
|
// both rounds (`same-class`), or blocking findings that did not get fewer
|
|
// (`not-fewer`), refuses the round with exit 1 and `review-stop-rule: ...`,
|
|
// --dry-run included, and archives nothing. Blocking = BLOCKER + counting
|
|
// MAJOR, with the verdict gate's pre-existing reading; new = no carried_from.
|
|
|
|
const BLK_A5 = { severity: 'BLOCKER', file: 'a.mjs', line: 5, rule_key: 'BROKEN_SUCCESS_CRITERION', title: 'Second build path never flushes either', detail: 'emit() skips the final flush call' };
|
|
const MAJ_A5 = { severity: 'MAJOR', file: 'a.mjs', line: 5, rule_key: 'SCOPE_CREEP_BUILT', title: 'Adds a cache nobody asked for', detail: 'a memo layer the brief never named' };
|
|
|
|
test('[STP] countingMajors lives in review-rounds and review-answers re-exports the same function', async () => {
|
|
assert.equal(typeof rounds.countingMajors, 'function', 'review-rounds.mjs exports countingMajors');
|
|
const answers = await import(pathToFileURL(ANSWERS).href);
|
|
assert.equal(answers.countingMajors, rounds.countingMajors, 'review-answers re-exports it');
|
|
const payload = { findings: [{ severity: 'major', rule_key: 'MISSING_TEST' }, { severity: 'MINOR' }, { severity: 'MAJOR', rule_key: 'SECURITY_INJECTION', pre_existing: true }, { severity: 'MAJOR', rule_key: 'MISSING_TEST', pre_existing: true }] };
|
|
assert.equal(rounds.countingMajors(payload, { preExisting: (f) => f.pre_existing === true }).length, 2);
|
|
});
|
|
|
|
test('[STP] a new BLOCKER of the same class in rounds 1 and 2: prepare refuses round 3 with review-stop-rule, --dry-run too, nothing archived', () => {
|
|
const fx = rndRepo();
|
|
roundOne(fx, [BLK_B3]);
|
|
roundTwo(fx, [BLK_A5], { engines: [] });
|
|
const before = fx.snapshot();
|
|
for (const extra of [['--dry-run'], []]) {
|
|
const r = fx.prepare(extra);
|
|
assert.equal(r.status, 1, `${extra.join(' ')}: ${r.stdout}${r.stderr}`);
|
|
const reason = String(parse(r).reason);
|
|
assert.match(reason, /^review-stop-rule: stop rule \(same-class\)/, r.stdout);
|
|
assert.match(reason, /class: BROKEN_SUCCESS_CRITERION/, r.stdout);
|
|
assert.match(reason, /round 1: 1 blocking/, r.stdout);
|
|
assert.match(reason, /round 2: 1 blocking/, r.stdout);
|
|
assert.doesNotMatch(reason, /one more round/i);
|
|
assert.deepEqual(fx.snapshot(), before, 'a refused round writes, copies and deletes nothing');
|
|
}
|
|
});
|
|
|
|
test('[STP] a carried MAJOR and nothing fewer: prepare refuses round 3 (not-fewer)', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [MAJ_A3]);
|
|
roundTwo(fx, [{ ...MAJ_A3, carried_from: prev.findings[0].id }], { engines: [] });
|
|
const r = fx.prepare(['--dry-run']);
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /^review-stop-rule: stop rule \(not-fewer\)/, r.stdout);
|
|
});
|
|
|
|
test('[STP] fewer blocking findings of another class: round 3 opens (a MINOR of the old class is not blocking)', () => {
|
|
const fx = rndRepo();
|
|
roundOne(fx, [BLK_B3, MAJ_A3]);
|
|
roundTwo(fx, [MAJ_A5, { ...BLK_A5, severity: 'MINOR' }], { engines: [] });
|
|
const r = fx.prepare(['--dry-run']);
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
assert.equal(parse(r).round, 3);
|
|
});
|
|
|
|
test('[STP] the gate\'s pre-existing reading: a pre-existing MAJOR does not count, a pre-existing security MAJOR does', () => {
|
|
const fx = rndRepo();
|
|
// The work on a branch whose fork point from main is `start`, so a mark can be re-derived.
|
|
g(fx.repo, 'checkout', '-q', '-b', 'feat');
|
|
g(fx.repo, 'branch', '-f', 'main', fx.start);
|
|
roundOne(fx, [BLK_B3, MAJ_A3]);
|
|
const PRE = { severity: 'MAJOR', file: 'a.mjs', line: 8, rule_key: 'PLAN_EXECUTE_DRIFT', title: 'Old helper drifts from the plan', detail: 'an untouched line' };
|
|
const r2 = roundTwo(fx, [MAJ_A5, PRE], { engines: [] });
|
|
const mark = (rule) => fx.write(editFindings(r2.text, (x) => (x.line === 8 ? { ...x, rule_key: rule, pre_existing: true } : x)));
|
|
// Control: a pre-existing MAJOR leaves the count: 1 < 2, round 3 opens.
|
|
mark('PLAN_EXECUTE_DRIFT');
|
|
const quiet = fx.prepare(['--dry-run']);
|
|
assert.equal(quiet.status, 0, `Control: ${quiet.stdout}${quiet.stderr}`);
|
|
// A security finding counts whatever the mark: 2 >= 2.
|
|
mark('SECURITY_INJECTION');
|
|
const r = fx.prepare(['--dry-run']);
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /^review-stop-rule: stop rule \(not-fewer\).*round 2: 2 blocking/, r.stdout);
|
|
});
|
|
|
|
test('[STP] a round with no readable JSON block at P = 2: prepare is not measured (exit 2), --dry-run too, nothing written', () => {
|
|
const fx = rndRepo();
|
|
const r2 = rndReview({ start: fx.start, end: fx.c2, findings: [MAJ_A3] }).text.replace('\nfindings:', '\nround: 2\nfindings:');
|
|
const r1 = rndReview({ start: fx.start, end: fx.c1, findings: [BLK_B3, BLK_A5_RND] }).text;
|
|
const noBlock = (t) => t.slice(0, t.lastIndexOf('```json\n'));
|
|
for (const [label, archived, current] of [
|
|
['the previous archive', 'round 1\n', r2],
|
|
['review.md', r1, noBlock(r2)],
|
|
]) {
|
|
fx.write(current);
|
|
fx.write(archived, fx.archive(1));
|
|
const before = fx.snapshot();
|
|
for (const extra of [['--dry-run'], []]) {
|
|
const r = fx.prepare(extra);
|
|
assert.equal(r.status, 2, `${label} ${extra.join(' ')}: ${r.stdout}${r.stderr}`);
|
|
assert.match(String(parse(r).reason), /^stop rule: a round has no readable JSON block — not measured/, r.stdout);
|
|
assert.deepEqual(fx.snapshot(), before, `${label}: nothing written`);
|
|
}
|
|
}
|
|
});
|
|
|
|
// ---- E 3.1: a finding from Claude Security is a security finding ------------------
|
|
|
|
test('[E3] isSecurityFinding: a finding from Claude Security is a security finding whatever its key, as one from /security-review is', () => {
|
|
assert.equal(rounds.isSecurityFinding({ engine: 'claude-security', rule_key: 'CODE_REVIEW_DEFECT' }), true);
|
|
assert.equal(rounds.isSecurityFinding({ engine: 'security-review', rule_key: 'CODE_REVIEW_DEFECT' }), true);
|
|
assert.equal(rounds.isSecurityFinding({ engine: 'code-review', rule_key: 'CODE_REVIEW_DEFECT' }), false, 'control: /code-review is not a security engine');
|
|
});
|
|
|
|
// ---- E 3, decision 222: an engine finding its engine raises again outside `findings` stays counted ----
|
|
|
|
// A Claude Security BLOCKER counted in round 1, at b.mjs:3.
|
|
const CS_PREV = { severity: 'BLOCKER', file: 'b.mjs', line: 3, rule_key: 'SECURITY_REVIEW_VULNERABILITY', engine: 'claude-security',
|
|
title: 'Shell command built from the branch name', detail: 'checkout() interpolates the branch name into a shell string.' };
|
|
const CS_RAN = { engines: ['claude-security', 'code-review'], reviewers: [CCR, BCR] };
|
|
|
|
/** Claude Security's payload of `records` (b.mjs:3 unless changed), converted by the real convertReport. */
|
|
function csPayload(records, scopeFiles) {
|
|
const dir = mkdtempSync(join(tmpdir(), 'review-rounds-cs-'));
|
|
made.push(dir);
|
|
const base = 'a'.repeat(40);
|
|
const head = 'b'.repeat(40);
|
|
writeReport(dir, { head, base, records: records.map((over) => record({ file: 'b.mjs', line: 3, title: CS_PREV.title, ...over })) });
|
|
const r = convertReport({ reportDir: dir, base, head, scopeFiles });
|
|
assert.equal(r.exit, 0, r.reason);
|
|
return r.payload;
|
|
}
|
|
/** A payload list's entries as finalize passes them to carry: plain objects, `engine` from the payload's reviewer. */
|
|
const raisedBy = (engine, list) => list.map((e) => ({ ...e, engine }));
|
|
|
|
test('[E3] decision 222: a counted Claude Security finding its engine raises again below confidence high (below_threshold) stays counted', () => {
|
|
const p = P(CS_PREV);
|
|
const pl = csPayload([{ confidence: 'medium' }]);
|
|
assert.deepEqual([pl.findings.length, pl.below_threshold.length], [0, 1], 'Control: the raise is in below_threshold only');
|
|
const ctx = { marks: [], ran: CS_RAN, previousRan: CS_RAN };
|
|
// Control: without `raised`, carry behaves as before: the engine ran and its findings do not hold it.
|
|
const before = rounds.carry([p], pl.findings, ctx);
|
|
assert.deepEqual([before.findings.length, before.resolved], [0, [p.id]], 'Control: without raised it resolves');
|
|
const held = rounds.carry([p], pl.findings, { ...ctx, raised: raisedBy('claude-security', pl.below_threshold) });
|
|
assert.deepEqual(held.resolved, [], 'a below_threshold raise is no evidence that it is resolved');
|
|
assert.deepEqual(held.findings.map((x) => [x.severity, x.carried_from, x.engine]), [['BLOCKER', p.id, 'claude-security']],
|
|
'it is carried as it was, at its own severity');
|
|
// Control: another engine's entry does not hold it.
|
|
const other = rounds.carry([p], pl.findings, { ...ctx, raised: raisedBy('security-review', pl.below_threshold) });
|
|
assert.deepEqual([other.findings.length, other.resolved], [0, [p.id]], 'Control: an entry of another engine holds nothing');
|
|
});
|
|
|
|
test('[E3] decision 222: a counted engine finding its engine raises again outside the review scope (out_of_scope) stays counted', () => {
|
|
const p = P(CS_PREV);
|
|
const pl = csPayload([{}], ['a.mjs']);
|
|
assert.deepEqual([pl.findings.length, pl.out_of_scope.length], [0, 1], 'Control: the raise is in out_of_scope only');
|
|
const held = rounds.carry([p], pl.findings, { marks: [], ran: CS_RAN, previousRan: CS_RAN, raised: raisedBy('claude-security', pl.out_of_scope) });
|
|
assert.deepEqual(held.resolved, [], 'an out_of_scope raise is no evidence that it is resolved');
|
|
assert.deepEqual(held.findings.map((x) => [x.severity, x.carried_from]), [['BLOCKER', p.id]]);
|
|
});
|
|
|
|
test('[E3] decision 222: each raised entry holds at most one previous finding', () => {
|
|
const p3 = P(CS_PREV);
|
|
const p8 = P({ ...CS_PREV, line: 8 });
|
|
const pl = csPayload([{ confidence: 'low' }]);
|
|
const r = rounds.carry([p3, p8], pl.findings, { marks: [], ran: CS_RAN, previousRan: CS_RAN, raised: raisedBy('claude-security', pl.below_threshold) });
|
|
assert.deepEqual(r.findings.map((x) => x.carried_from), [p3.id], 'the nearer previous finding is held');
|
|
assert.deepEqual(r.resolved, [p8.id], 'the one entry holds no second finding');
|
|
});
|
|
|
|
test('[E3] decision 222: round 2 whose Claude Security payload holds the round-1 BLOCKER only in below_threshold keeps it counted: BLOCK, never under Resolved since round 1', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [CS_PREV], { engines: ['code-review', 'claude-security'] });
|
|
const [id] = prev.findings.map((x) => x.id);
|
|
/** Round 2's Claude Security report of start..c2, converted by the real CLI and saved as Phase 4.7 saves it. */
|
|
const claudeSecurity = (records) => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'review-rounds-cs-report-'));
|
|
made.push(dir);
|
|
writeReport(dir, { head: fx.c2, base: fx.start, records });
|
|
const scope = join(dir, 'scope.txt');
|
|
writeFileSync(scope, 'a.mjs\nb.mjs\n');
|
|
const r = run([SECURITY, 'convert', '--report-dir', dir, '--base', fx.start, '--head', fx.c2, '--scope', scope]);
|
|
assert.equal(r.status, 0, `security-engine convert: ${r.stdout}${r.stderr}`);
|
|
writeFileSync(join(fx.project, '.review-engine-payload-claude-security.json'), r.stdout);
|
|
return JSON.parse(r.stdout);
|
|
};
|
|
const resolvedPart = (text) => text.slice(text.indexOf('## Resolved since round 1'), text.indexOf('## Declined to Judge'));
|
|
// The coordinator declined the below-threshold raise: no finding, WARN.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
|
const pl = claudeSecurity([record({ file: 'b.mjs', line: 3, title: CS_PREV.title, confidence: 'medium' })]);
|
|
assert.deepEqual([pl.findings.length, pl.below_threshold.length], [0, 1], 'Control: Claude Security raised it again, below high only');
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const text = fx.read();
|
|
const payload = jsonOf(text);
|
|
assert.deepEqual(payload.ran.engines, ['claude-security', 'code-review'], 'Claude Security ran this round');
|
|
assert.deepEqual(payload.resolved, [], 'the round-1 BLOCKER is not resolved');
|
|
assert.deepEqual(payload.findings.map((x) => [x.severity, x.carried_from]), [['BLOCKER', id]], 'it stays counted, carried');
|
|
assert.equal(payload.verdict, 'BLOCK');
|
|
assert.match(fmOf(text), /^verdict: BLOCK$/m);
|
|
assert.match(text, /^## Resolved since round 1$/m, text);
|
|
assert.ok(!resolvedPart(text).includes(CS_PREV.title) && !resolvedPart(text).includes(id), `not under Resolved since round 1:\n${resolvedPart(text)}`);
|
|
const gate = fx.gate();
|
|
assert.equal(gate.status, 1, gate.stdout + gate.stderr);
|
|
assert.equal(parse(gate).reason, 'verdict BLOCK', gate.stdout);
|
|
// Control: Claude Security ran and did not raise it at all: resolved.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
|
claudeSecurity([]);
|
|
const c = fx.finalize2();
|
|
assert.equal(c.status, 0, c.stdout + c.stderr);
|
|
assert.deepEqual(jsonOf(fx.read()).resolved, [id], 'Control: a clean Claude Security run resolves it');
|
|
assert.ok(resolvedPart(fx.read()).includes(CS_PREV.title), 'Control: listed under Resolved since round 1');
|
|
// Control: an entry in another engine's payload never holds it, whatever engine the entry names.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: [], verdict: 'WARN' });
|
|
claudeSecurity([]);
|
|
writeFileSync(join(fx.project, '.review-engine-payload-code-review.json'), JSON.stringify({
|
|
reviewer: 'claude-code-review', findings: [], out_of_scope: [], below_threshold: pl.below_threshold,
|
|
}));
|
|
assert.equal(pl.below_threshold[0].engine, 'claude-security', 'Control: the planted entry names claude-security');
|
|
const o = fx.finalize2();
|
|
assert.equal(o.status, 0, o.stdout + o.stderr);
|
|
assert.deepEqual(jsonOf(fx.read()).resolved, [id], 'Control: the engine is the payload\'s, never the entry\'s');
|
|
});
|
|
|
|
test('[E3] decision 222: round 2 whose Claude Security payload holds the round-1 BLOCKER only in out_of_scope keeps it counted through finalize: BLOCK, never under Resolved since round 1', () => {
|
|
const fx = rndRepo();
|
|
const { prev } = roundOne(fx, [CS_PREV], { engines: ['code-review', 'claude-security'] });
|
|
const [id] = prev.findings.map((x) => x.id);
|
|
/** Round 2's Claude Security report of start..c2 at confidence high, converted with `scope` as the review's scope. */
|
|
const claudeSecurity = (records, scope) => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'review-rounds-cs-oos-'));
|
|
made.push(dir);
|
|
writeReport(dir, { head: fx.c2, base: fx.start, records });
|
|
const scopeFile = join(dir, 'scope.txt');
|
|
writeFileSync(scopeFile, scope);
|
|
const r = run([SECURITY, 'convert', '--report-dir', dir, '--base', fx.start, '--head', fx.c2, '--scope', scopeFile]);
|
|
assert.equal(r.status, 0, `security-engine convert: ${r.stdout}${r.stderr}`);
|
|
writeFileSync(join(fx.project, '.review-engine-payload-claude-security.json'), r.stdout);
|
|
return JSON.parse(r.stdout);
|
|
};
|
|
const resolvedPart = (text) => text.slice(text.indexOf('## Resolved since round 1'), text.indexOf('## Declined to Judge'));
|
|
const raise = [record({ file: 'b.mjs', line: 3, title: CS_PREV.title })];
|
|
// b.mjs left the review's scope: the coordinator declined the raise as engine:out-of-scope, no finding, WARN.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
|
const pl = claudeSecurity(raise, 'a.mjs\n');
|
|
assert.deepEqual([pl.findings.length, pl.below_threshold.length, pl.out_of_scope.length], [0, 0, 1],
|
|
'Control: Claude Security raised it again, at high, outside the review scope only');
|
|
const f = fx.finalize2();
|
|
assert.equal(f.status, 0, f.stdout + f.stderr);
|
|
const text = fx.read();
|
|
const payload = jsonOf(text);
|
|
assert.deepEqual(payload.ran.engines, ['claude-security', 'code-review'], 'Claude Security ran this round');
|
|
assert.deepEqual(payload.resolved, [], 'the round-1 BLOCKER is not resolved');
|
|
assert.deepEqual(payload.findings.map((x) => [x.severity, x.carried_from]), [['BLOCKER', id]], 'it stays counted, carried');
|
|
assert.equal(payload.verdict, 'BLOCK');
|
|
assert.match(fmOf(text), /^verdict: BLOCK$/m);
|
|
assert.match(text, /^## Resolved since round 1$/m, text);
|
|
assert.ok(!resolvedPart(text).includes(CS_PREV.title) && !resolvedPart(text).includes(id), `not under Resolved since round 1:\n${resolvedPart(text)}`);
|
|
const gate = fx.gate();
|
|
assert.equal(gate.status, 1, gate.stdout + gate.stderr);
|
|
assert.equal(parse(gate).reason, 'verdict BLOCK', gate.stdout);
|
|
// Control: Claude Security ran and did not raise it at all: resolved.
|
|
roundTwo(fx, [], { outs: { [CCR]: [], [BCR]: [] }, engines: ['code-review'], verdict: 'WARN' });
|
|
claudeSecurity([], 'a.mjs\n');
|
|
const c = fx.finalize2();
|
|
assert.equal(c.status, 0, c.stdout + c.stderr);
|
|
assert.deepEqual(jsonOf(fx.read()).resolved, [id], 'Control: a clean Claude Security run resolves it');
|
|
assert.ok(resolvedPart(fx.read()).includes(CS_PREV.title), 'Control: listed under Resolved since round 1');
|
|
});
|