// tests/lib/engine-findings-compare.test.mjs // [ENG] fix round 4.5b2 part B, Step 9: on the standard path an engine finding // the coordinator dropped is put back at its severity. /trekreview Phase 4.7 // saves each converted engine payload to // `{project_dir}/.review-engine-payload-{engine}.json`; // lib/review/review-finalize.mjs runs its `engine` pass first: each engine // finding at MAJOR or BLOCKER whose row (lib/review/engine-compare.mjs // `rowOf`, the armB workflow's row rule, or the row of its trimmed title) no // finding of review.md carries is appended at its own severity with `engine` // kept, a `### ` entry with the `**Source:**` line, the JSON, the frontmatter // list and counts, and a verdict that is only raised. A put-back BLOCKER gets // its `### Step R`. An engine finding in `out_of_scope` is not compared. // Tests run the real CLIs (engine-findings, finalize, fill-finding-ids, // validator, gate) in throwaway git repos with no hooks; engine-compare.mjs // is also read directly and pinned equal to the armB workflow's copy. import { after, test } from 'node:test'; import { strict as assert } from 'node:assert'; import { spawnSync } from 'node:child_process'; import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs'; import { childEnv } from '../helpers/child-env.mjs'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); const COMPARE = join(ROOT, 'lib', 'review', 'engine-compare.mjs'); const ENGINE_CLI = join(ROOT, 'lib', 'review', 'engine-findings.mjs'); const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs'); const FILL = join(ROOT, 'lib', 'review', 'fill-finding-ids.mjs'); const VALIDATOR = join(ROOT, 'lib', 'validators', 'review-validator.mjs'); const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs'); const ARMB = join(ROOT, 'scripts', 'trekreview-armB.workflow.mjs'); const REVIEW_REL = '.claude/projects/2026-11-02-eng/review.md'; const SOURCE_LINE = (engine) => `**Source:** engine ${engine}, put back by the standard-path comparison`; // Loaded so that each new assertion fails on its own while the module is absent. let compareMod = {}; try { compareMod = await import(pathToFileURL(COMPARE).href); } catch (e) { if (existsSync(COMPARE)) throw e; } const made = []; after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); }); function g(cwd, ...args) { const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv() }); if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); return r.stdout.trim(); } function putIn(dir, rel, text) { mkdirSync(dirname(join(dir, rel)), { recursive: true }); writeFileSync(join(dir, rel), text); } const run = (args) => spawnSync(process.execPath, args, { encoding: 'utf8', env: childEnv() }); const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } }; const ten = (tag) => Array.from({ length: 10 }, (_, i) => `export const ${tag}${i + 1} = ${i + 1};`).join('\n') + '\n'; /** * A throwaway repo: the start commit holds src/a.mjs, src/b.mjs and * docs/guide/x.md; the end commit changes line 1 of src/a.mjs and line 3 of * src/b.mjs. No REVIEW.md, so no policy. */ function engRepo() { const base = mkdtempSync(join(tmpdir(), 'engine-compare-eng-')); made.push(base); const repo = join(base, 'repo'); mkdirSync(repo, { recursive: true }); mkdirSync(join(base, 'hooks-none')); g(repo, 'init', '-q', '-b', 'main'); appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n' + `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`); appendFileSync(join(repo, '.git', 'info', 'exclude'), '.claude/projects/\n'); putIn(repo, 'src/a.mjs', ten('a')); putIn(repo, 'src/b.mjs', ten('b')); putIn(repo, 'docs/guide/x.md', '# Guide\n\none\ntwo\nthree\n'); g(repo, 'add', '-A'); g(repo, 'commit', '-q', '-m', 'chore: start'); const start = g(repo, 'rev-parse', 'HEAD'); putIn(repo, 'src/a.mjs', ten('a').replace('export const a1 = 1;', 'export const a1 = 11;')); putIn(repo, 'src/b.mjs', ten('b').replace('export const b3 = 3;', 'export const b3 = 33;')); g(repo, 'add', '-A'); g(repo, 'commit', '-q', '-m', 'feat: the work'); const end = g(repo, 'rev-parse', 'HEAD'); const reviewPath = join(repo, ...REVIEW_REL.split('/')); const project = dirname(reviewPath); mkdirSync(project, { recursive: true }); const scope = join(base, 'scope.txt'); writeFileSync(scope, 'src/a.mjs\nsrc/b.mjs\ndocs/guide/x.md\n'); return { repo, start, end, reviewPath, project, write: (text) => writeFileSync(reviewPath, text), read: () => readFileSync(reviewPath, 'utf8'), /** An engine's transcription converted by the real engine-findings.mjs CLI and saved as Phase 4.7 saves it. */ engine: (engine, findings) => { const input = join(base, `.review-engine-${engine}.json`); writeFileSync(input, JSON.stringify({ engine, findings })); const r = run([ENGINE_CLI, '--scope', scope, input]); assert.equal(r.status, 0, `engine-findings: ${r.stdout}${r.stderr}`); writeFileSync(join(project, `.review-engine-payload-${engine}.json`), r.stdout); return JSON.parse(r.stdout); }, finalize: () => run([FINALIZE, '--review', reviewPath, '--repo', repo, '--project', project, '--round', '1']), fill: () => run([FILL, reviewPath]), validate: () => run([VALIDATOR, '--strict', reviewPath]), gate: () => run([GATE, '--json', reviewPath, '--repo', repo]), }; } /** * A review_version 1.2 review of `start..end` as the coordinator wrote it and * fill-finding-ids filled it. `findings` are [{severity, file, line, rule_key, * engine?, title}]; ids computed over (file, line, rule_key). `declined` goes * into the JSON `declined_to_judge`. */ function engReview({ start, end, findings, declined = [] }) { const list = findings.map((f, i) => ({ severity: f.severity, rule_key: f.rule_key, ...(f.engine ? { engine: f.engine } : {}), file: f.file, line: f.line, title: f.title ?? `t${i + 1}`, detail: `d${i + 1}`, recommended_action: 'Add a test.', })); const ids = computeFindingIds(list); const ordered = list.map((f, i) => ({ id: ids[i], ...f })); const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }; for (const f of ordered) counts[f.severity] += 1; const v = counts.BLOCKER > 0 ? 'BLOCK' : counts.MAJOR > 0 ? 'WARN' : 'ALLOW'; const json = JSON.stringify({ verdict: v, counts, allow_blocked_by: [], declined_to_judge: declined, findings: ordered }, null, 2); const sections = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'].map((sev) => { const of = ordered.filter((f) => f.severity === sev); if (of.length === 0) return ''; return `## Findings (${sev})\n\n${of.map((f) => `### ${f.id}\n\n- **Rule:** \`${f.rule_key}\`\n- **Location:** \`${f.file}:${f.line}\`\n- **Title:** ${f.title}\n`).join('\n')}\n`; }).join(''); const steps = ordered.filter((f) => f.severity === 'BLOCKER') .map((f, i) => `### Step R${i + 1}: fix ${f.title}\n\n- **Files:** \`${f.file}\`\n- **Source finding:** \`${f.id}\`\n- **Test first:** a test that fails today\n- **Verify:** re-run /trekreview\n- **On failure:** escalate\n`).join('\n'); const declinedLines = declined.length === 0 ? 'None.' : declined.map((d) => `- ${d.file}:${d.line} ${d.rule_key} — ${d.reason} (${d.title})`).join('\n'); return `--- type: trekreview review_version: "1.2" task: "t" slug: eng project_dir: .claude/projects/2026-11-02-eng/ brief_path: .claude/projects/2026-11-02-eng/brief.md scope_sha_start: ${start} scope_sha_end: ${end} reviewed_files_count: 3 verdict: ${v} findings: ${ordered.map((f) => ` - ${f.id}`).join('\n')} --- # Review: t ## Executive Summary x ## Coverage | File | Treatment | Reason | |------|-----------|--------| | src/a.mjs | deep-review | default | | src/b.mjs | deep-review | default | | docs/guide/x.md | deep-review | default | ${sections}## Declined to Judge ${declinedLines} Trimmed: 0 ## Remediation Summary x ## Remediation Steps ${steps || 'None.\n'} \`\`\`json ${json} \`\`\` `; } const jsonOf = (text) => JSON.parse(text.slice(text.lastIndexOf('```json\n') + 8, text.lastIndexOf('```'))); const frontmatter = (text) => /^---\n([\s\S]*?)\n---\n/.exec(text)?.[1] ?? ''; const fmField = (text, key) => { const m = new RegExp(`^${key}:[ \\t]*(.*)$`, 'm').exec(frontmatter(text)); return m ? m[1].replace(/^"(.*)"$/, '$1') : undefined; }; const fmIds = (text) => frontmatter(text).split('\n').filter((l) => /^ {2}- [0-9a-f]{40}$/.test(l)).map((l) => l.trim().slice(2)); /** The text from `heading` up to the next `## ` heading ('' when absent). */ function sectionOf(text, heading) { const start = text.indexOf(`\n${heading}`); if (start === -1) return ''; const next = text.indexOf('\n## ', start + heading.length + 1); return text.slice(start + 1, next === -1 ? undefined : next); } /** The `### ` entry of `id` in `section`, up to the next `### ` or end ('' when absent). */ function entryOf(section, id) { const at = section.indexOf(`### ${id}\n`); if (at === -1) return ''; const next = section.indexOf('\n### ', at + 4); return section.slice(at, next === -1 ? undefined : next); } const on = (findings, file, line, rule_key) => findings.filter((f) => f.file === file && f.line === line && f.rule_key === rule_key); /** A MINOR the reviewers raised: the coordinator's review holds only it, so the review is ALLOW. */ const REVIEWER_MINOR = { severity: 'MINOR', file: 'src/b.mjs', line: 3, rule_key: 'MISSING_ERROR_HANDLING', title: 'unchecked return' }; // --- put back -------------------------------------------------------------------- test('[ENG] an engine MAJOR the coordinator\'s review.md does not hold is put back at its severity and the review is at least WARN', () => { const fx = engRepo(); fx.write(engReview({ start: fx.start, end: fx.end, findings: [REVIEWER_MINOR] })); fx.engine('code-review', [{ file: 'src/a.mjs', line: 1, summary: 'The counter overflows past 2^31', failure_scenario: 'a long run wraps the count' }]); const r = fx.finalize(); assert.equal(r.status, 0, r.stdout + r.stderr); const text = fx.read(); const json = jsonOf(text); const back = on(json.findings, 'src/a.mjs', 1, 'CODE_REVIEW_DEFECT'); assert.equal(back.length, 1, `the dropped engine MAJOR is back in the JSON findings: ${JSON.stringify(json.findings.map((f) => f.title))}`); assert.equal(back[0].severity, 'MAJOR', 'at its own severity'); assert.equal(back[0].engine, 'code-review', 'engine is kept'); assert.equal(back[0].title, 'The counter overflows past 2^31'); assert.equal(json.verdict, 'WARN', 'the review is at least WARN'); assert.equal(fmField(text, 'verdict'), 'WARN', 'the frontmatter verdict is raised with it'); assert.deepEqual(json.counts, { BLOCKER: 0, MAJOR: 1, MINOR: 1, SUGGESTION: 0 }, 'counts hold the put-back'); assert.deepEqual(fmIds(text), json.findings.map((f) => f.id), 'the frontmatter list is the JSON ids'); const entry = entryOf(sectionOf(text, '## Findings (MAJOR)'), back[0].id); assert.ok(entry !== '', `a ### entry under ## Findings (MAJOR):\n${sectionOf(text, '## Findings (MAJOR)')}`); assert.ok(entry.includes(SOURCE_LINE('code-review')), `the entry names its source:\n${entry}`); // Control: the reviewer MINOR stays as it was. assert.equal(on(json.findings, 'src/b.mjs', 3, 'MISSING_ERROR_HANDLING').length, 1); }); test('[ENG] a dropped engine BLOCKER is put back and the verdict is BLOCK', () => { const fx = engRepo(); fx.write(engReview({ start: fx.start, end: fx.end, findings: [REVIEWER_MINOR] })); fx.engine('security-review', [{ file: 'src/a.mjs', line: 1, summary: 'A shell command is built from request input', failure_scenario: 'a crafted name runs a command' }]); const r = fx.finalize(); assert.equal(r.status, 0, r.stdout + r.stderr); const text = fx.read(); const json = jsonOf(text); const back = on(json.findings, 'src/a.mjs', 1, 'SECURITY_REVIEW_VULNERABILITY'); assert.equal(back.length, 1, `the dropped engine BLOCKER is back: ${JSON.stringify(json.findings.map((f) => f.title))}`); assert.equal(back[0].severity, 'BLOCKER'); assert.equal(back[0].engine, 'security-review'); assert.equal(json.verdict, 'BLOCK'); assert.equal(fmField(text, 'verdict'), 'BLOCK'); assert.ok(entryOf(sectionOf(text, '## Findings (BLOCKER)'), back[0].id).includes(SOURCE_LINE('security-review')), `a ### entry under ## Findings (BLOCKER) names its source:\n${sectionOf(text, '## Findings (BLOCKER)')}`); const gate = fx.gate(); assert.equal(gate.status, 1, `the gate reads BLOCK: ${gate.stdout}${gate.stderr}`); }); // --- not duplicated -------------------------------------------------------------- test('[ENG] a finding returned under a trimmed title is not put back twice; an engine finding in out_of_scope is left to the existing declined rule', () => { const fx = engRepo(); const long = `The retry loop never stops when the server answers 503 ${'and keeps the connection open '.repeat(4)}forever`; assert.ok(long.length > 100, 'the title is over the coordinator\'s 100-character limit'); // The coordinator's trim (lib/review/coordinator-contract.mjs judgeFilter): cut to leave room for the marker, then the marker. const trimmed = `${long.slice(0, 100 - ' […trimmed]'.length)} […trimmed]`; const payload = fx.engine('code-review', [ { file: 'src/a.mjs', line: 1, summary: long, failure_scenario: 'a 503 hangs the run' }, { file: 'src/b.mjs', line: 3, summary: 'The parser drops the last line', failure_scenario: 'no newline at end of file' }, { file: 'lib/outside.mjs', line: 4, summary: 'An unchanged helper leaks a handle', failure_scenario: 'many calls exhaust handles' }, ]); assert.equal(payload.out_of_scope.length, 1, 'the CLI placed the lib/outside.mjs finding in out_of_scope'); const out = payload.out_of_scope[0]; fx.write(engReview({ start: fx.start, end: fx.end, findings: [{ severity: 'MAJOR', file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', engine: 'code-review', title: trimmed }], declined: [{ file: out.file, line: out.line, rule_key: out.rule_key, severity: out.severity, title: out.title, engine: 'code-review', reason: 'engine:out-of-scope' }], })); const r = fx.finalize(); assert.equal(r.status, 0, r.stdout + r.stderr); const json = jsonOf(fx.read()); const engineTitles = json.findings.filter((f) => f.engine === 'code-review').map((f) => f.title).sort(); assert.deepEqual(engineTitles, ['The parser drops the last line', trimmed].sort(), 'only the dropped finding is put back: the trimmed one is not put back twice'); assert.equal(on(json.findings, 'src/a.mjs', 1, 'CODE_REVIEW_DEFECT').length, 1, 'one row on the trimmed finding\'s triplet'); assert.equal(json.findings.filter((f) => f.file === 'lib/outside.mjs').length, 0, 'the out_of_scope finding is not put back'); assert.deepEqual(json.declined_to_judge.map((d) => [d.file, d.reason]), [['lib/outside.mjs', 'engine:out-of-scope']], 'it stays declined by the existing rule'); }); // --- the lib row rule equals armB's ------------------------------------------------ test('[ENG] lib rowOf and trimmedTitle equal the armB workflow\'s on a table of titles', () => { assert.equal(typeof compareMod.rowOf, 'function', 'engine-compare.mjs exports rowOf'); assert.equal(typeof compareMod.trimmedTitle, 'function', 'engine-compare.mjs exports trimmedTitle'); // The armB definitions, read and evaluated as text (scripts/trekreview-armB.workflow.mjs). const src = readFileSync(ARMB, 'utf8'); const from = src.indexOf('const tripletOf = '); const endMark = 'const trimmedTitle = '; const to = src.indexOf('\n', src.indexOf(endMark)); assert.ok(from !== -1 && src.indexOf(endMark) > from && to > from, 'the armB workflow carries tripletOf .. trimmedTitle in one block'); const armb = new Function(`${src.slice(from, to)}\nreturn { rowOf, trimmedTitle };`)(); const titles = ['', 'x', ' Padded Title ', 'MiXeD Case', 'æøå … ünïcode', 'a'.repeat(99), 'b'.repeat(100), 'c'.repeat(101), 'd'.repeat(250), `${'e'.repeat(95)} […trimmed]`, 'tab\there', 'line\nbreak']; for (const t of titles) { assert.equal(compareMod.trimmedTitle(t), armb.trimmedTitle(t), `trimmedTitle(${JSON.stringify(t.slice(0, 20))}…, ${t.length})`); for (const f of [ { file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', title: t }, { file: 'src/a.mjs', line: '1', rule_key: 'SECURITY_REVIEW_VULNERABILITY', title: t }, { file: 'docs/x.md', line: 0, rule_key: undefined, title: t }, ]) { assert.equal(compareMod.rowOf(f), armb.rowOf(f), `rowOf on ${JSON.stringify(f).slice(0, 80)}`); } } const untitled = { file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT' }; assert.equal(compareMod.rowOf(untitled), armb.rowOf(untitled), 'rowOf of an untitled finding'); assert.notEqual(compareMod.rowOf(untitled), compareMod.rowOf({ ...untitled, title: '' }), 'an untitled finding gets a key no titled row has'); }); // --- the gate reads the finalized file ------------------------------------------ test('[ENG] a review finalized with a put-back has every id computed, no placeholder left, and the gate reads it', () => { // A MAJOR: the engine's finding shares its triplet with a reviewer finding under another title, so it is a row of its own. const fx = engRepo(); const reviewerMajor = { severity: 'MAJOR', file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', title: 'The reviewer saw a wrong bound' }; fx.write(engReview({ start: fx.start, end: fx.end, findings: [reviewerMajor, REVIEWER_MINOR] })); fx.engine('code-review', [{ file: 'src/a.mjs', line: 1, summary: 'The counter overflows past 2^31', failure_scenario: 'a long run wraps the count' }]); const r = fx.finalize(); assert.equal(r.status, 0, r.stdout + r.stderr); const text = fx.read(); const json = jsonOf(text); const back = json.findings.filter((f) => f.title === 'The counter overflows past 2^31'); assert.equal(back.length, 1, `the engine MAJOR is put back: ${JSON.stringify(json.findings.map((f) => f.title))}`); // The second finding on one triplet gets the id over `rule_key#2`; the reviewer's keeps the plain one. assert.equal(back[0].id, computeFindingId('src/a.mjs', 1, 'CODE_REVIEW_DEFECT#2'), 'the put-back has its computed id'); const kept = json.findings.find((f) => f.title === reviewerMajor.title); assert.equal(kept.id, computeFindingId('src/a.mjs', 1, 'CODE_REVIEW_DEFECT'), 'the reviewer finding keeps its id'); assert.doesNotMatch(text, /@@ID_\d+@@/, 'no placeholder is left'); assert.deepEqual(json.findings.map((f) => f.id), computeFindingIds(json.findings), 'every JSON id is the computed one'); for (const f of json.findings) { assert.equal(text.split('\n').filter((l) => l === `### ${f.id}`).length, 1, `one ### entry for ${f.title}`); } assert.ok(entryOf(sectionOf(text, '## Findings (MAJOR)'), kept.id).includes(reviewerMajor.title), 'the reviewer entry keeps its own text'); assert.deepEqual(fmIds(text), json.findings.map((f) => f.id), 'the frontmatter list is the JSON ids'); const fill = fx.fill(); assert.equal(fill.status, 0, `fill-finding-ids finds nothing to fill: ${fill.stdout}${fill.stderr}`); const v = fx.validate(); assert.equal(v.status, 0, `review-validator --strict: ${v.stdout}${v.stderr}`); // The gate passes its frontmatter-vs-JSON id check and reaches the answers: the put-back MAJOR is unanswered. const gate = fx.gate(); const verdict = parse(gate); assert.equal(gate.status, 2, gate.stdout + gate.stderr); assert.match(String(verdict.reason), /^review-unanswered:/, `the gate reads the file up to the answers: ${gate.stdout}`); assert.ok(Array.isArray(verdict.unanswered) && verdict.unanswered.includes(back[0].id), `the put-back MAJOR counts: ${gate.stdout}`); // A BLOCKER: the put-back gets its ### Step R, and the strict validator finds it. const bx = engRepo(); bx.write(engReview({ start: bx.start, end: bx.end, findings: [REVIEWER_MINOR] })); bx.engine('security-review', [{ file: 'src/a.mjs', line: 1, summary: 'A shell command is built from request input', failure_scenario: 'a crafted name runs a command' }]); const rb = bx.finalize(); assert.equal(rb.status, 0, rb.stdout + rb.stderr); const btext = bx.read(); const bjson = jsonOf(btext); const blocker = bjson.findings.find((f) => f.severity === 'BLOCKER'); assert.ok(blocker, `the engine BLOCKER is put back: ${JSON.stringify(bjson.findings.map((f) => f.title))}`); assert.equal(blocker.id, computeFindingId('src/a.mjs', 1, 'SECURITY_REVIEW_VULNERABILITY'), 'it has its computed id'); assert.doesNotMatch(btext, /@@ID_\d+@@/, 'no placeholder is left'); const steps = sectionOf(btext, '## Remediation Steps'); assert.match(steps, /^### Step R1\b/m, `a ### Step R for the put-back BLOCKER:\n${steps}`); assert.ok(steps.includes(`**Source finding:** \`${blocker.id}\``), `the step names the put-back:\n${steps}`); assert.deepEqual(fmIds(btext), bjson.findings.map((f) => f.id), 'the frontmatter list is the JSON ids'); const bfill = bx.fill(); assert.equal(bfill.status, 0, `fill-finding-ids finds nothing to fill: ${bfill.stdout}${bfill.stderr}`); const bv = bx.validate(); assert.equal(bv.status, 0, `review-validator --strict finds the R-step: ${bv.stdout}${bv.stderr}`); const bgate = bx.gate(); assert.equal(bgate.status, 1, `the gate reads BLOCK: ${bgate.stdout}${bgate.stderr}`); });