An adversarial read of d88245c reproduced 15 problems; all fixed here, each
with a test that fails on the old code.
- verdict-gate: a greener verdict must be backed by the LAST fence being a
`json` block of the right shape, with no BLOCKER in the JSON (any case),
in `## Findings (BLOCKER)`, or a differing finding-id list; two verdict
lines and more than one file argument are exit 2. ({}, null, lowercase
"blocker", a trailing ```jsonc and a later json fence all gave exit 0.)
- engine-findings: a transcribed "narrower" key can no longer lower an
engine tier; absolute paths under --repo-root match the scope; "12" lines
parse; an empty scope is an error; out-of-scope engine findings are
unverified (`engine:out-of-scope`) and forbid ALLOW.
- recurrence: the catch-all engine keys and keys normalised from unknown
ones are never proposed; the same unfixed finding in two iterations is not
a recurrence; proposals are validated before they are offered; \r/U+2028
and damaged markers are refused; writes go through a symlinked CLAUDE.md.
Prose points --repo at the session directory, not the git toplevel.
- Workflow substrate: engines are counted from what arrived, and the
verdict's floor is recomputed from the coordinator's findings; now tested
by running the script with stubbed Workflow globals. Two literal NUL bytes
written as \u0000 (git saw the file as binary).
- Trimming keeps Pass 1's "Also flagged by" provenance; `## Declined to
Judge` and remediation steps are required from review_version 1.1 only
(N-1); the zero-diff review now carries the JSON block the gate needs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
111 lines
6.1 KiB
JavaScript
111 lines
6.1 KiB
JavaScript
// tests/lib/engine-findings.test.mjs
|
|
// Claude Code's /code-review and /security-review as the engine (ordre 5, P11):
|
|
// their transcribed findings become a coordinator payload, and meet Voyage's
|
|
// brief control in the same dedup + verdict.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { toReviewerPayload, ENGINES } from '../../lib/review/engine-findings.mjs';
|
|
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
|
import { validateFindings } from '../../lib/review/findings-schema.mjs';
|
|
|
|
test('engine-findings — a /code-review finding becomes a MAJOR CODE_REVIEW_DEFECT', () => {
|
|
const r = toReviewerPayload({ engine: 'code-review', findings: [
|
|
{ file: './lib/a.mjs', line: 12, summary: 'Off-by-one drops the last page', failure_scenario: '10 items, page size 5 → page 2 empty', category: 'correctness' },
|
|
] });
|
|
assert.equal(r.ok, true, r.errors.join('\n'));
|
|
assert.equal(r.payload.reviewer, ENGINES['code-review'].reviewer);
|
|
const [f] = r.payload.findings;
|
|
assert.deepEqual([f.rule_key, f.severity, f.file, f.line], ['CODE_REVIEW_DEFECT', 'MAJOR', 'lib/a.mjs', 12]);
|
|
assert.match(f.detail, /page 2 empty/);
|
|
assert.equal(validateFindings(r.payload).valid, true);
|
|
});
|
|
|
|
test('engine-findings — a /security-review finding becomes a BLOCKER SECURITY_REVIEW_VULNERABILITY', () => {
|
|
const r = toReviewerPayload({ engine: 'security-review', findings: [
|
|
{ file: 'lib/q.mjs', line: 4, summary: 'SQL built from request body' },
|
|
] });
|
|
assert.equal(r.ok, true);
|
|
assert.equal(r.payload.findings[0].severity, 'BLOCKER');
|
|
});
|
|
|
|
test('engine-findings — a narrower catalogue key given by the orchestrator is kept; an unknown one falls back', () => {
|
|
const r = toReviewerPayload({ engine: 'code-review', findings: [
|
|
{ file: 'a.mjs', line: 1, summary: 's', rule_key: 'MISSING_TEST' },
|
|
{ file: 'b.mjs', line: 1, summary: 's', rule_key: 'MADE_UP' },
|
|
] });
|
|
assert.deepEqual(r.payload.findings.map((f) => f.rule_key), ['MISSING_TEST', 'CODE_REVIEW_DEFECT']);
|
|
});
|
|
|
|
test('engine-findings — a file outside the review scope is listed, never dropped silently', () => {
|
|
const r = toReviewerPayload({ engine: 'security-review', findings: [
|
|
{ file: 'in.mjs', line: 1, summary: 'a' },
|
|
{ file: 'old/out.mjs', line: 1, summary: 'b' },
|
|
] }, { scopeFiles: ['in.mjs'] });
|
|
assert.equal(r.payload.findings.length, 1);
|
|
assert.equal(r.out_of_scope.length, 1);
|
|
assert.equal(r.out_of_scope[0].file, 'old/out.mjs');
|
|
});
|
|
|
|
test('engine-findings — empty file or summary is an input error, unknown engine too', () => {
|
|
assert.equal(toReviewerPayload({ engine: 'code-review', findings: [{ file: '', summary: 'x' }] }).ok, false);
|
|
assert.equal(toReviewerPayload({ engine: 'code-review', findings: [{ file: 'a', summary: '' }] }).ok, false);
|
|
assert.equal(toReviewerPayload({ engine: 'simplify', findings: [] }).ok, false);
|
|
});
|
|
|
|
test('engine-findings — engine and brief control meet in one verdict; engine + Voyage on the same triplet collapse', () => {
|
|
const engine = toReviewerPayload({ engine: 'code-review', findings: [
|
|
{ file: 'lib/a.mjs', line: 12, summary: 'No test covers the new branch', rule_key: 'MISSING_TEST' },
|
|
] }).payload;
|
|
const conformance = { reviewer: 'brief-conformance-reviewer', findings: [
|
|
{ file: 'lib/a.mjs', line: 12, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'SC2 untested' },
|
|
{ file: 'lib/b.mjs', line: 3, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: 'SC3 missing' },
|
|
] };
|
|
const result = runContract([engine, conformance],
|
|
{ expectedReviewers: [engine.reviewer, 'brief-conformance-reviewer'] });
|
|
assert.equal(result.verdict, 'BLOCK', 'the brief control stops');
|
|
assert.equal(result.findings.length, 2);
|
|
const a = result.findings.find((f) => f.file === 'lib/a.mjs');
|
|
assert.deepEqual([...a.raised_by].sort(), ['brief-conformance-reviewer', 'claude-code-review']);
|
|
});
|
|
|
|
test('engine-findings — a "narrower" key can never lower a security BLOCKER', () => {
|
|
const r = toReviewerPayload({ engine: 'security-review', findings: [
|
|
{ file: 'a.mjs', line: 1, summary: 's', rule_key: 'MISSING_ERROR_HANDLING' }, // MINOR → refused
|
|
{ file: 'b.mjs', line: 1, summary: 's', rule_key: 'SECURITY_INJECTION' }, // BLOCKER → kept
|
|
] });
|
|
assert.deepEqual(r.payload.findings.map((f) => [f.rule_key, f.severity]),
|
|
[['SECURITY_REVIEW_VULNERABILITY', 'BLOCKER'], ['SECURITY_INJECTION', 'BLOCKER']]);
|
|
});
|
|
|
|
test('engine-findings — absolute paths under the repo root match the scope; a numeric-string line is kept', () => {
|
|
const r = toReviewerPayload({ engine: 'code-review', findings: [
|
|
{ file: '/abs/repo/lib/a.mjs', line: '12', summary: 's' },
|
|
] }, { scopeFiles: ['lib/a.mjs'], repoRoot: '/abs/repo' });
|
|
assert.equal(r.payload.findings.length, 1);
|
|
assert.equal(r.payload.findings[0].file, 'lib/a.mjs');
|
|
assert.equal(r.payload.findings[0].line, 12);
|
|
});
|
|
|
|
test('engine-findings — an empty scope list is an error, never "everything out of scope"', () => {
|
|
const r = toReviewerPayload({ engine: 'security-review', findings: [{ file: 'a.mjs', summary: 's' }] }, { scopeFiles: [] });
|
|
assert.equal(r.ok, false);
|
|
});
|
|
|
|
test('engine-findings — an out-of-scope engine finding forbids ALLOW and is declined to judge', () => {
|
|
const r = toReviewerPayload({ engine: 'security-review', findings: [{ file: 'old/x.mjs', line: 2, summary: 'leak' }] },
|
|
{ scopeFiles: ['in.mjs'] });
|
|
const result = runContract(
|
|
[r.payload, { reviewer: 'brief-conformance-reviewer', findings: [] }],
|
|
{ expectedReviewers: [r.payload.reviewer, 'brief-conformance-reviewer'], engineOutOfScope: r.out_of_scope },
|
|
);
|
|
assert.equal(result.verdict, 'WARN');
|
|
assert.deepEqual(result.declined_to_judge.map((d) => d.reason), ['engine:out-of-scope']);
|
|
});
|
|
|
|
test('engine-findings — an engine that was expected but never reported forbids ALLOW', () => {
|
|
const result = runContract([{ reviewer: 'brief-conformance-reviewer', findings: [] }],
|
|
{ expectedReviewers: ['brief-conformance-reviewer', ENGINES['security-review'].reviewer] });
|
|
assert.equal(result.verdict, 'WARN');
|
|
assert.deepEqual(result.missing_reviewers, ['claude-security-review']);
|
|
});
|