feat(review): the standard path puts back engine findings the coordinator dropped
This commit is contained in:
parent
8d399fddfb
commit
349a4fc789
6 changed files with 559 additions and 21 deletions
2
tests/fixtures/jsonl-schemas.md
vendored
2
tests/fixtures/jsonl-schemas.md
vendored
|
|
@ -25,7 +25,7 @@
|
|||
| trekexecute-stats (Phase 9 record) | ts, plan, plan_type, mode, result, steps_total, steps_passed, steps_failed, steps_skipped, failed_at_step | lib/stats/stats-append.mjs (trekexecute.md Phase 9) | trekexecute.md:3005-3027 | profile, phase_models, profile_source | none |
|
||||
| trekexecute-stats (autonomy events) | ts, event, known_event, payload | lib/stats/event-emit.mjs `emit()` | event-emit.mjs:78-99 | payload.profile, payload.phase_models, payload.profile_source | none |
|
||||
| trekexecute-stats (PostToolUse Bash) | ts, session_id, command_excerpt, duration_ms, success | hooks/scripts/post-bash-stats.mjs (Bash PostToolUse) | post-bash-stats.mjs:42-54 | none (hook is plugin-level, not profile-aware) | command_excerpt (CWE-212) |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:723-776 | profile, phase_models, profile_source | none |
|
||||
| trekreview-stats | ts, slug, verdict, counts (BLOCKER/MAJOR/MINOR/SUGGESTION), reviewed_files_count, mode, duration_ms | lib/stats/stats-append.mjs (trekreview.md Phase 8) | trekreview.md:734-787 | profile, phase_models, profile_source | none |
|
||||
| trekcontinue-stats | ts, project, next_session_label, status | lib/stats/stats-append.mjs (trekcontinue.md Phase 5) | trekcontinue.md:304-308 | profile, profile_source | none |
|
||||
| token-usage | ts, session_id, scope, model, tokens_input, tokens_output, tokens_cache_creation, tokens_cache_read, cost_usd, is_estimate, price_table_version | lib/stats/token-usage.mjs `captureTokenUsage` (via hooks/scripts/otel-export.mjs Stop hook, opt-in `VOYAGE_TOKEN_METER`) | token-usage.mjs:buildRecord | n/a (SKAL-2 schema) | session_id (stripped at export) |
|
||||
|
||||
|
|
|
|||
380
tests/lib/engine-findings-compare.test.mjs
Normal file
380
tests/lib/engine-findings-compare.test.mjs
Normal file
|
|
@ -0,0 +1,380 @@
|
|||
// tests/lib/engine-findings-compare.test.mjs
|
||||
// [ENG] fix round 4.5b2 part B, Step 9: on the standard path an engine finding
|
||||
// the coordinator dropped is put back at its severity. /trekreview Phase 4.7
|
||||
// saves each converted engine payload to
|
||||
// `{project_dir}/.review-engine-payload-{engine}.json`;
|
||||
// lib/review/review-finalize.mjs runs its `engine` pass first: each engine
|
||||
// finding at MAJOR or BLOCKER whose row (lib/review/engine-compare.mjs
|
||||
// `rowOf`, the armB workflow's row rule, or the row of its trimmed title) no
|
||||
// finding of review.md carries is appended at its own severity with `engine`
|
||||
// kept, a `### ` entry with the `**Source:**` line, the JSON, the frontmatter
|
||||
// list and counts, and a verdict that is only raised. A put-back BLOCKER gets
|
||||
// its `### Step R<n>`. An engine finding in `out_of_scope` is not compared.
|
||||
// Tests run the real CLIs (engine-findings, finalize, fill-finding-ids,
|
||||
// validator, gate) in throwaway git repos with no hooks; engine-compare.mjs
|
||||
// is also read directly and pinned equal to the armB workflow's copy.
|
||||
|
||||
import { after, test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const COMPARE = join(ROOT, 'lib', 'review', 'engine-compare.mjs');
|
||||
const ENGINE_CLI = join(ROOT, 'lib', 'review', 'engine-findings.mjs');
|
||||
const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs');
|
||||
const FILL = join(ROOT, 'lib', 'review', 'fill-finding-ids.mjs');
|
||||
const VALIDATOR = join(ROOT, 'lib', 'validators', 'review-validator.mjs');
|
||||
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
||||
const ARMB = join(ROOT, 'scripts', 'trekreview-armB.workflow.mjs');
|
||||
const REVIEW_REL = '.claude/projects/2026-11-02-eng/review.md';
|
||||
const SOURCE_LINE = (engine) => `**Source:** engine ${engine}, put back by the standard-path comparison`;
|
||||
|
||||
// Loaded so that each new assertion fails on its own while the module is absent.
|
||||
let compareMod = {};
|
||||
try {
|
||||
compareMod = await import(pathToFileURL(COMPARE).href);
|
||||
} catch (e) {
|
||||
if (existsSync(COMPARE)) throw e;
|
||||
}
|
||||
|
||||
const made = [];
|
||||
after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); });
|
||||
|
||||
function g(cwd, ...args) {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv() });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
function putIn(dir, rel, text) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), text);
|
||||
}
|
||||
|
||||
const run = (args) => spawnSync(process.execPath, args, { encoding: 'utf8', env: childEnv() });
|
||||
const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } };
|
||||
const ten = (tag) => Array.from({ length: 10 }, (_, i) => `export const ${tag}${i + 1} = ${i + 1};`).join('\n') + '\n';
|
||||
|
||||
/**
|
||||
* A throwaway repo: the start commit holds src/a.mjs, src/b.mjs and
|
||||
* docs/guide/x.md; the end commit changes line 1 of src/a.mjs and line 3 of
|
||||
* src/b.mjs. No REVIEW.md, so no policy.
|
||||
*/
|
||||
function engRepo() {
|
||||
const base = mkdtempSync(join(tmpdir(), 'engine-compare-eng-'));
|
||||
made.push(base);
|
||||
const repo = join(base, 'repo');
|
||||
mkdirSync(repo, { recursive: true });
|
||||
mkdirSync(join(base, 'hooks-none'));
|
||||
g(repo, 'init', '-q', '-b', 'main');
|
||||
appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
||||
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
||||
appendFileSync(join(repo, '.git', 'info', 'exclude'), '.claude/projects/\n');
|
||||
putIn(repo, 'src/a.mjs', ten('a'));
|
||||
putIn(repo, 'src/b.mjs', ten('b'));
|
||||
putIn(repo, 'docs/guide/x.md', '# Guide\n\none\ntwo\nthree\n');
|
||||
g(repo, 'add', '-A');
|
||||
g(repo, 'commit', '-q', '-m', 'chore: start');
|
||||
const start = g(repo, 'rev-parse', 'HEAD');
|
||||
putIn(repo, 'src/a.mjs', ten('a').replace('export const a1 = 1;', 'export const a1 = 11;'));
|
||||
putIn(repo, 'src/b.mjs', ten('b').replace('export const b3 = 3;', 'export const b3 = 33;'));
|
||||
g(repo, 'add', '-A');
|
||||
g(repo, 'commit', '-q', '-m', 'feat: the work');
|
||||
const end = g(repo, 'rev-parse', 'HEAD');
|
||||
const reviewPath = join(repo, ...REVIEW_REL.split('/'));
|
||||
const project = dirname(reviewPath);
|
||||
mkdirSync(project, { recursive: true });
|
||||
const scope = join(base, 'scope.txt');
|
||||
writeFileSync(scope, 'src/a.mjs\nsrc/b.mjs\ndocs/guide/x.md\n');
|
||||
return {
|
||||
repo, start, end, reviewPath, project,
|
||||
write: (text) => writeFileSync(reviewPath, text),
|
||||
read: () => readFileSync(reviewPath, 'utf8'),
|
||||
/** An engine's transcription converted by the real engine-findings.mjs CLI and saved as Phase 4.7 saves it. */
|
||||
engine: (engine, findings) => {
|
||||
const input = join(base, `.review-engine-${engine}.json`);
|
||||
writeFileSync(input, JSON.stringify({ engine, findings }));
|
||||
const r = run([ENGINE_CLI, '--scope', scope, input]);
|
||||
assert.equal(r.status, 0, `engine-findings: ${r.stdout}${r.stderr}`);
|
||||
writeFileSync(join(project, `.review-engine-payload-${engine}.json`), r.stdout);
|
||||
return JSON.parse(r.stdout);
|
||||
},
|
||||
finalize: () => run([FINALIZE, '--review', reviewPath, '--repo', repo, '--project', project, '--round', '1']),
|
||||
fill: () => run([FILL, reviewPath]),
|
||||
validate: () => run([VALIDATOR, '--strict', reviewPath]),
|
||||
gate: () => run([GATE, '--json', reviewPath, '--repo', repo]),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A review_version 1.2 review of `start..end` as the coordinator wrote it and
|
||||
* fill-finding-ids filled it. `findings` are [{severity, file, line, rule_key,
|
||||
* engine?, title}]; ids computed over (file, line, rule_key). `declined` goes
|
||||
* into the JSON `declined_to_judge`.
|
||||
*/
|
||||
function engReview({ start, end, findings, declined = [] }) {
|
||||
const list = findings.map((f, i) => ({
|
||||
severity: f.severity, rule_key: f.rule_key, ...(f.engine ? { engine: f.engine } : {}),
|
||||
file: f.file, line: f.line, title: f.title ?? `t${i + 1}`, detail: `d${i + 1}`, recommended_action: 'Add a test.',
|
||||
}));
|
||||
const ids = computeFindingIds(list);
|
||||
const ordered = list.map((f, i) => ({ id: ids[i], ...f }));
|
||||
const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 };
|
||||
for (const f of ordered) counts[f.severity] += 1;
|
||||
const v = counts.BLOCKER > 0 ? 'BLOCK' : counts.MAJOR > 0 ? 'WARN' : 'ALLOW';
|
||||
const json = JSON.stringify({ verdict: v, counts, allow_blocked_by: [], declined_to_judge: declined, findings: ordered }, null, 2);
|
||||
const sections = ['BLOCKER', 'MAJOR', 'MINOR', 'SUGGESTION'].map((sev) => {
|
||||
const of = ordered.filter((f) => f.severity === sev);
|
||||
if (of.length === 0) return '';
|
||||
return `## Findings (${sev})\n\n${of.map((f) => `### ${f.id}\n\n- **Rule:** \`${f.rule_key}\`\n- **Location:** \`${f.file}:${f.line}\`\n- **Title:** ${f.title}\n`).join('\n')}\n`;
|
||||
}).join('');
|
||||
const steps = ordered.filter((f) => f.severity === 'BLOCKER')
|
||||
.map((f, i) => `### Step R${i + 1}: fix ${f.title}\n\n- **Files:** \`${f.file}\`\n- **Source finding:** \`${f.id}\`\n- **Test first:** a test that fails today\n- **Verify:** re-run /trekreview\n- **On failure:** escalate\n`).join('\n');
|
||||
const declinedLines = declined.length === 0 ? 'None.'
|
||||
: declined.map((d) => `- ${d.file}:${d.line} ${d.rule_key} — ${d.reason} (${d.title})`).join('\n');
|
||||
return `---
|
||||
type: trekreview
|
||||
review_version: "1.2"
|
||||
task: "t"
|
||||
slug: eng
|
||||
project_dir: .claude/projects/2026-11-02-eng/
|
||||
brief_path: .claude/projects/2026-11-02-eng/brief.md
|
||||
scope_sha_start: ${start}
|
||||
scope_sha_end: ${end}
|
||||
reviewed_files_count: 3
|
||||
verdict: ${v}
|
||||
findings:
|
||||
${ordered.map((f) => ` - ${f.id}`).join('\n')}
|
||||
---
|
||||
|
||||
# Review: t
|
||||
|
||||
## Executive Summary
|
||||
|
||||
x
|
||||
|
||||
## Coverage
|
||||
|
||||
| File | Treatment | Reason |
|
||||
|------|-----------|--------|
|
||||
| src/a.mjs | deep-review | default |
|
||||
| src/b.mjs | deep-review | default |
|
||||
| docs/guide/x.md | deep-review | default |
|
||||
|
||||
${sections}## Declined to Judge
|
||||
|
||||
${declinedLines}
|
||||
|
||||
Trimmed: 0
|
||||
|
||||
## Remediation Summary
|
||||
|
||||
x
|
||||
|
||||
## Remediation Steps
|
||||
|
||||
${steps || 'None.\n'}
|
||||
\`\`\`json
|
||||
${json}
|
||||
\`\`\`
|
||||
`;
|
||||
}
|
||||
|
||||
const jsonOf = (text) => JSON.parse(text.slice(text.lastIndexOf('```json\n') + 8, text.lastIndexOf('```')));
|
||||
const frontmatter = (text) => /^---\n([\s\S]*?)\n---\n/.exec(text)?.[1] ?? '';
|
||||
const fmField = (text, key) => {
|
||||
const m = new RegExp(`^${key}:[ \\t]*(.*)$`, 'm').exec(frontmatter(text));
|
||||
return m ? m[1].replace(/^"(.*)"$/, '$1') : undefined;
|
||||
};
|
||||
const fmIds = (text) => frontmatter(text).split('\n').filter((l) => /^ {2}- [0-9a-f]{40}$/.test(l)).map((l) => l.trim().slice(2));
|
||||
/** The text from `heading` up to the next `## ` heading ('' when absent). */
|
||||
function sectionOf(text, heading) {
|
||||
const start = text.indexOf(`\n${heading}`);
|
||||
if (start === -1) return '';
|
||||
const next = text.indexOf('\n## ', start + heading.length + 1);
|
||||
return text.slice(start + 1, next === -1 ? undefined : next);
|
||||
}
|
||||
/** The `### <id>` entry of `id` in `section`, up to the next `### ` or end ('' when absent). */
|
||||
function entryOf(section, id) {
|
||||
const at = section.indexOf(`### ${id}\n`);
|
||||
if (at === -1) return '';
|
||||
const next = section.indexOf('\n### ', at + 4);
|
||||
return section.slice(at, next === -1 ? undefined : next);
|
||||
}
|
||||
const on = (findings, file, line, rule_key) => findings.filter((f) => f.file === file && f.line === line && f.rule_key === rule_key);
|
||||
|
||||
/** A MINOR the reviewers raised: the coordinator's review holds only it, so the review is ALLOW. */
|
||||
const REVIEWER_MINOR = { severity: 'MINOR', file: 'src/b.mjs', line: 3, rule_key: 'MISSING_ERROR_HANDLING', title: 'unchecked return' };
|
||||
|
||||
// --- put back --------------------------------------------------------------------
|
||||
|
||||
test('[ENG] an engine MAJOR the coordinator\'s review.md does not hold is put back at its severity and the review is at least WARN', () => {
|
||||
const fx = engRepo();
|
||||
fx.write(engReview({ start: fx.start, end: fx.end, findings: [REVIEWER_MINOR] }));
|
||||
fx.engine('code-review', [{ file: 'src/a.mjs', line: 1, summary: 'The counter overflows past 2^31', failure_scenario: 'a long run wraps the count' }]);
|
||||
const r = fx.finalize();
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const text = fx.read();
|
||||
const json = jsonOf(text);
|
||||
const back = on(json.findings, 'src/a.mjs', 1, 'CODE_REVIEW_DEFECT');
|
||||
assert.equal(back.length, 1, `the dropped engine MAJOR is back in the JSON findings: ${JSON.stringify(json.findings.map((f) => f.title))}`);
|
||||
assert.equal(back[0].severity, 'MAJOR', 'at its own severity');
|
||||
assert.equal(back[0].engine, 'code-review', 'engine is kept');
|
||||
assert.equal(back[0].title, 'The counter overflows past 2^31');
|
||||
assert.equal(json.verdict, 'WARN', 'the review is at least WARN');
|
||||
assert.equal(fmField(text, 'verdict'), 'WARN', 'the frontmatter verdict is raised with it');
|
||||
assert.deepEqual(json.counts, { BLOCKER: 0, MAJOR: 1, MINOR: 1, SUGGESTION: 0 }, 'counts hold the put-back');
|
||||
assert.deepEqual(fmIds(text), json.findings.map((f) => f.id), 'the frontmatter list is the JSON ids');
|
||||
const entry = entryOf(sectionOf(text, '## Findings (MAJOR)'), back[0].id);
|
||||
assert.ok(entry !== '', `a ### entry under ## Findings (MAJOR):\n${sectionOf(text, '## Findings (MAJOR)')}`);
|
||||
assert.ok(entry.includes(SOURCE_LINE('code-review')), `the entry names its source:\n${entry}`);
|
||||
// Control: the reviewer MINOR stays as it was.
|
||||
assert.equal(on(json.findings, 'src/b.mjs', 3, 'MISSING_ERROR_HANDLING').length, 1);
|
||||
});
|
||||
|
||||
test('[ENG] a dropped engine BLOCKER is put back and the verdict is BLOCK', () => {
|
||||
const fx = engRepo();
|
||||
fx.write(engReview({ start: fx.start, end: fx.end, findings: [REVIEWER_MINOR] }));
|
||||
fx.engine('security-review', [{ file: 'src/a.mjs', line: 1, summary: 'A shell command is built from request input', failure_scenario: 'a crafted name runs a command' }]);
|
||||
const r = fx.finalize();
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const text = fx.read();
|
||||
const json = jsonOf(text);
|
||||
const back = on(json.findings, 'src/a.mjs', 1, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1, `the dropped engine BLOCKER is back: ${JSON.stringify(json.findings.map((f) => f.title))}`);
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(back[0].engine, 'security-review');
|
||||
assert.equal(json.verdict, 'BLOCK');
|
||||
assert.equal(fmField(text, 'verdict'), 'BLOCK');
|
||||
assert.ok(entryOf(sectionOf(text, '## Findings (BLOCKER)'), back[0].id).includes(SOURCE_LINE('security-review')),
|
||||
`a ### entry under ## Findings (BLOCKER) names its source:\n${sectionOf(text, '## Findings (BLOCKER)')}`);
|
||||
const gate = fx.gate();
|
||||
assert.equal(gate.status, 1, `the gate reads BLOCK: ${gate.stdout}${gate.stderr}`);
|
||||
});
|
||||
|
||||
// --- not duplicated --------------------------------------------------------------
|
||||
|
||||
test('[ENG] a finding returned under a trimmed title is not put back twice; an engine finding in out_of_scope is left to the existing declined rule', () => {
|
||||
const fx = engRepo();
|
||||
const long = `The retry loop never stops when the server answers 503 ${'and keeps the connection open '.repeat(4)}forever`;
|
||||
assert.ok(long.length > 100, 'the title is over the coordinator\'s 100-character limit');
|
||||
// The coordinator's trim (lib/review/coordinator-contract.mjs judgeFilter): cut to leave room for the marker, then the marker.
|
||||
const trimmed = `${long.slice(0, 100 - ' […trimmed]'.length)} […trimmed]`;
|
||||
const payload = fx.engine('code-review', [
|
||||
{ file: 'src/a.mjs', line: 1, summary: long, failure_scenario: 'a 503 hangs the run' },
|
||||
{ file: 'src/b.mjs', line: 3, summary: 'The parser drops the last line', failure_scenario: 'no newline at end of file' },
|
||||
{ file: 'lib/outside.mjs', line: 4, summary: 'An unchanged helper leaks a handle', failure_scenario: 'many calls exhaust handles' },
|
||||
]);
|
||||
assert.equal(payload.out_of_scope.length, 1, 'the CLI placed the lib/outside.mjs finding in out_of_scope');
|
||||
const out = payload.out_of_scope[0];
|
||||
fx.write(engReview({
|
||||
start: fx.start, end: fx.end,
|
||||
findings: [{ severity: 'MAJOR', file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', engine: 'code-review', title: trimmed }],
|
||||
declined: [{ file: out.file, line: out.line, rule_key: out.rule_key, severity: out.severity, title: out.title, engine: 'code-review', reason: 'engine:out-of-scope' }],
|
||||
}));
|
||||
const r = fx.finalize();
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const json = jsonOf(fx.read());
|
||||
const engineTitles = json.findings.filter((f) => f.engine === 'code-review').map((f) => f.title).sort();
|
||||
assert.deepEqual(engineTitles, ['The parser drops the last line', trimmed].sort(),
|
||||
'only the dropped finding is put back: the trimmed one is not put back twice');
|
||||
assert.equal(on(json.findings, 'src/a.mjs', 1, 'CODE_REVIEW_DEFECT').length, 1, 'one row on the trimmed finding\'s triplet');
|
||||
assert.equal(json.findings.filter((f) => f.file === 'lib/outside.mjs').length, 0, 'the out_of_scope finding is not put back');
|
||||
assert.deepEqual(json.declined_to_judge.map((d) => [d.file, d.reason]), [['lib/outside.mjs', 'engine:out-of-scope']],
|
||||
'it stays declined by the existing rule');
|
||||
});
|
||||
|
||||
// --- the lib row rule equals armB's ------------------------------------------------
|
||||
|
||||
test('[ENG] lib rowOf and trimmedTitle equal the armB workflow\'s on a table of titles', () => {
|
||||
assert.equal(typeof compareMod.rowOf, 'function', 'engine-compare.mjs exports rowOf');
|
||||
assert.equal(typeof compareMod.trimmedTitle, 'function', 'engine-compare.mjs exports trimmedTitle');
|
||||
// The armB definitions, read and evaluated as text (scripts/trekreview-armB.workflow.mjs).
|
||||
const src = readFileSync(ARMB, 'utf8');
|
||||
const from = src.indexOf('const tripletOf = ');
|
||||
const endMark = 'const trimmedTitle = ';
|
||||
const to = src.indexOf('\n', src.indexOf(endMark));
|
||||
assert.ok(from !== -1 && src.indexOf(endMark) > from && to > from, 'the armB workflow carries tripletOf .. trimmedTitle in one block');
|
||||
const armb = new Function(`${src.slice(from, to)}\nreturn { rowOf, trimmedTitle };`)();
|
||||
const titles = ['', 'x', ' Padded Title ', 'MiXeD Case', 'æøå … ünïcode', 'a'.repeat(99), 'b'.repeat(100), 'c'.repeat(101),
|
||||
'd'.repeat(250), `${'e'.repeat(95)} […trimmed]`, 'tab\there', 'line\nbreak'];
|
||||
for (const t of titles) {
|
||||
assert.equal(compareMod.trimmedTitle(t), armb.trimmedTitle(t), `trimmedTitle(${JSON.stringify(t.slice(0, 20))}…, ${t.length})`);
|
||||
for (const f of [
|
||||
{ file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', title: t },
|
||||
{ file: 'src/a.mjs', line: '1', rule_key: 'SECURITY_REVIEW_VULNERABILITY', title: t },
|
||||
{ file: 'docs/x.md', line: 0, rule_key: undefined, title: t },
|
||||
]) {
|
||||
assert.equal(compareMod.rowOf(f), armb.rowOf(f), `rowOf on ${JSON.stringify(f).slice(0, 80)}`);
|
||||
}
|
||||
}
|
||||
const untitled = { file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT' };
|
||||
assert.equal(compareMod.rowOf(untitled), armb.rowOf(untitled), 'rowOf of an untitled finding');
|
||||
assert.notEqual(compareMod.rowOf(untitled), compareMod.rowOf({ ...untitled, title: '' }), 'an untitled finding gets a key no titled row has');
|
||||
});
|
||||
|
||||
// --- the gate reads the finalized file ------------------------------------------
|
||||
|
||||
test('[ENG] a review finalized with a put-back has every id computed, no placeholder left, and the gate reads it', () => {
|
||||
// A MAJOR: the engine's finding shares its triplet with a reviewer finding under another title, so it is a row of its own.
|
||||
const fx = engRepo();
|
||||
const reviewerMajor = { severity: 'MAJOR', file: 'src/a.mjs', line: 1, rule_key: 'CODE_REVIEW_DEFECT', title: 'The reviewer saw a wrong bound' };
|
||||
fx.write(engReview({ start: fx.start, end: fx.end, findings: [reviewerMajor, REVIEWER_MINOR] }));
|
||||
fx.engine('code-review', [{ file: 'src/a.mjs', line: 1, summary: 'The counter overflows past 2^31', failure_scenario: 'a long run wraps the count' }]);
|
||||
const r = fx.finalize();
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const text = fx.read();
|
||||
const json = jsonOf(text);
|
||||
const back = json.findings.filter((f) => f.title === 'The counter overflows past 2^31');
|
||||
assert.equal(back.length, 1, `the engine MAJOR is put back: ${JSON.stringify(json.findings.map((f) => f.title))}`);
|
||||
// The second finding on one triplet gets the id over `rule_key#2`; the reviewer's keeps the plain one.
|
||||
assert.equal(back[0].id, computeFindingId('src/a.mjs', 1, 'CODE_REVIEW_DEFECT#2'), 'the put-back has its computed id');
|
||||
const kept = json.findings.find((f) => f.title === reviewerMajor.title);
|
||||
assert.equal(kept.id, computeFindingId('src/a.mjs', 1, 'CODE_REVIEW_DEFECT'), 'the reviewer finding keeps its id');
|
||||
assert.doesNotMatch(text, /@@ID_\d+@@/, 'no placeholder is left');
|
||||
assert.deepEqual(json.findings.map((f) => f.id), computeFindingIds(json.findings), 'every JSON id is the computed one');
|
||||
for (const f of json.findings) {
|
||||
assert.equal(text.split('\n').filter((l) => l === `### ${f.id}`).length, 1, `one ### entry for ${f.title}`);
|
||||
}
|
||||
assert.ok(entryOf(sectionOf(text, '## Findings (MAJOR)'), kept.id).includes(reviewerMajor.title), 'the reviewer entry keeps its own text');
|
||||
assert.deepEqual(fmIds(text), json.findings.map((f) => f.id), 'the frontmatter list is the JSON ids');
|
||||
const fill = fx.fill();
|
||||
assert.equal(fill.status, 0, `fill-finding-ids finds nothing to fill: ${fill.stdout}${fill.stderr}`);
|
||||
const v = fx.validate();
|
||||
assert.equal(v.status, 0, `review-validator --strict: ${v.stdout}${v.stderr}`);
|
||||
// The gate passes its frontmatter-vs-JSON id check and reaches the answers: the put-back MAJOR is unanswered.
|
||||
const gate = fx.gate();
|
||||
const verdict = parse(gate);
|
||||
assert.equal(gate.status, 2, gate.stdout + gate.stderr);
|
||||
assert.match(String(verdict.reason), /^review-unanswered:/, `the gate reads the file up to the answers: ${gate.stdout}`);
|
||||
assert.ok(Array.isArray(verdict.unanswered) && verdict.unanswered.includes(back[0].id), `the put-back MAJOR counts: ${gate.stdout}`);
|
||||
|
||||
// A BLOCKER: the put-back gets its ### Step R<n>, and the strict validator finds it.
|
||||
const bx = engRepo();
|
||||
bx.write(engReview({ start: bx.start, end: bx.end, findings: [REVIEWER_MINOR] }));
|
||||
bx.engine('security-review', [{ file: 'src/a.mjs', line: 1, summary: 'A shell command is built from request input', failure_scenario: 'a crafted name runs a command' }]);
|
||||
const rb = bx.finalize();
|
||||
assert.equal(rb.status, 0, rb.stdout + rb.stderr);
|
||||
const btext = bx.read();
|
||||
const bjson = jsonOf(btext);
|
||||
const blocker = bjson.findings.find((f) => f.severity === 'BLOCKER');
|
||||
assert.ok(blocker, `the engine BLOCKER is put back: ${JSON.stringify(bjson.findings.map((f) => f.title))}`);
|
||||
assert.equal(blocker.id, computeFindingId('src/a.mjs', 1, 'SECURITY_REVIEW_VULNERABILITY'), 'it has its computed id');
|
||||
assert.doesNotMatch(btext, /@@ID_\d+@@/, 'no placeholder is left');
|
||||
const steps = sectionOf(btext, '## Remediation Steps');
|
||||
assert.match(steps, /^### Step R1\b/m, `a ### Step R<n> for the put-back BLOCKER:\n${steps}`);
|
||||
assert.ok(steps.includes(`**Source finding:** \`${blocker.id}\``), `the step names the put-back:\n${steps}`);
|
||||
assert.deepEqual(fmIds(btext), bjson.findings.map((f) => f.id), 'the frontmatter list is the JSON ids');
|
||||
const bfill = bx.fill();
|
||||
assert.equal(bfill.status, 0, `fill-finding-ids finds nothing to fill: ${bfill.stdout}${bfill.stderr}`);
|
||||
const bv = bx.validate();
|
||||
assert.equal(bv.status, 0, `review-validator --strict finds the R-step: ${bv.stdout}${bv.stderr}`);
|
||||
const bgate = bx.gate();
|
||||
assert.equal(bgate.status, 1, `the gate reads BLOCK: ${bgate.stdout}${bgate.stderr}`);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue