fix(review): close the gate's fail-open paths an independent review found
An adversarial read of d88245c reproduced 15 problems; all fixed here, each
with a test that fails on the old code.
- verdict-gate: a greener verdict must be backed by the LAST fence being a
`json` block of the right shape, with no BLOCKER in the JSON (any case),
in `## Findings (BLOCKER)`, or a differing finding-id list; two verdict
lines and more than one file argument are exit 2. ({}, null, lowercase
"blocker", a trailing ```jsonc and a later json fence all gave exit 0.)
- engine-findings: a transcribed "narrower" key can no longer lower an
engine tier; absolute paths under --repo-root match the scope; "12" lines
parse; an empty scope is an error; out-of-scope engine findings are
unverified (`engine:out-of-scope`) and forbid ALLOW.
- recurrence: the catch-all engine keys and keys normalised from unknown
ones are never proposed; the same unfixed finding in two iterations is not
a recurrence; proposals are validated before they are offered; \r/U+2028
and damaged markers are refused; writes go through a symlinked CLAUDE.md.
Prose points --repo at the session directory, not the git toplevel.
- Workflow substrate: engines are counted from what arrived, and the
verdict's floor is recomputed from the coordinator's findings; now tested
by running the script with stubbed Workflow globals. Two literal NUL bytes
written as \u0000 (git saw the file as binary).
- Trimming keeps Pass 1's "Also flagged by" provenance; `## Declined to
Judge` and remediation steps are required from review_version 1.1 only
(N-1); the zero-diff review now carries the JSON block the gate needs.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d88245c06e
commit
40f6ae1177
17 changed files with 581 additions and 78 deletions
71
tests/scripts/trekreview-armb-workflow.test.mjs
Normal file
71
tests/scripts/trekreview-armb-workflow.test.mjs
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
// tests/scripts/trekreview-armb-workflow.test.mjs
|
||||
// The opt-in Workflow substrate for /trekreview Phase 5–6, run for real with
|
||||
// stubbed Workflow globals (agent/parallel/phase/log). Ordre 5: a reviewer or
|
||||
// engine that did not report forbids ALLOW, and the verdict's floor is
|
||||
// recomputed from the findings the coordinator returns.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const SRC = readFileSync(join(ROOT, 'scripts', 'trekreview-armB.workflow.mjs'), 'utf-8')
|
||||
.replace('export const meta', 'const meta');
|
||||
const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor;
|
||||
const run = new AsyncFunction('args', 'agent', 'parallel', 'phase', 'log', SRC);
|
||||
|
||||
const BASE = { briefPath: 'b.md', diffPath: 'd.diff', triage: '' };
|
||||
const engine = (reviewer, findings = []) => ({ reviewer, findings });
|
||||
|
||||
function stubs({ reviewers = [], coordinator = null } = {}) {
|
||||
let i = 0;
|
||||
return {
|
||||
agent: async (_prompt, opts) => (opts.label === 'coordinator' ? coordinator : reviewers[i++] ?? null),
|
||||
parallel: async (thunks) => Promise.all(thunks.map((t) => t())),
|
||||
phase: () => {},
|
||||
log: () => {},
|
||||
};
|
||||
}
|
||||
|
||||
async function exec(args, s) {
|
||||
return run(args, s.agent, s.parallel, s.phase, s.log);
|
||||
}
|
||||
|
||||
test('armB — both reviewers and both engines clean → ALLOW (known-positive control)', async () => {
|
||||
const s = stubs({ reviewers: [{ reviewer: 'c', findings: [] }, { reviewer: 'k', findings: [] }] });
|
||||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||||
assert.equal(r.verdict, 'ALLOW');
|
||||
assert.deepEqual(r.missing_engines, []);
|
||||
});
|
||||
|
||||
test('armB — engines the caller never mentioned are missing → not ALLOW', async () => {
|
||||
const s = stubs({ reviewers: [{ reviewer: 'c', findings: [] }, { reviewer: 'k', findings: [] }] });
|
||||
const r = await exec({ ...BASE }, s);
|
||||
assert.equal(r.verdict, 'WARN');
|
||||
assert.deepEqual(r.missing_engines, ['claude-code-review', 'claude-security-review']);
|
||||
});
|
||||
|
||||
test('armB — a reviewer that returned nothing → not ALLOW', async () => {
|
||||
const s = stubs({ reviewers: [null, { reviewer: 'k', findings: [] }] });
|
||||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||||
assert.equal(r.verdict, 'WARN');
|
||||
});
|
||||
|
||||
test('armB — a coordinator ALLOW over a BLOCKER finding is recomputed to BLOCK', async () => {
|
||||
const f = { severity: 'BLOCKER', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'a.mjs', line: 1, title: 't', detail: 'd' };
|
||||
const s = stubs({
|
||||
reviewers: [{ reviewer: 'c', findings: [f] }, { reviewer: 'k', findings: [] }],
|
||||
coordinator: { verdict: 'ALLOW', findings: [f] },
|
||||
});
|
||||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||||
assert.equal(r.verdict, 'BLOCK');
|
||||
});
|
||||
|
||||
test('armB — quick mode expects only the /code-review engine', async () => {
|
||||
const s = stubs({ reviewers: [{ reviewer: 'k', findings: [] }] });
|
||||
const r = await exec({ ...BASE, quick: true, enginePayloads: [engine('claude-code-review')] }, s);
|
||||
assert.equal(r.verdict, 'ALLOW');
|
||||
assert.deepEqual(r.missing_engines, []);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue