700 lines
35 KiB
JavaScript
700 lines
35 KiB
JavaScript
// tests/scripts/trekreview-armb-workflow.test.mjs
|
||
// The opt-in Workflow substrate for /trekreview Phase 5–6, run for real with
|
||
// stubbed Workflow globals (agent/parallel/phase/log). Ordre 5: a reviewer or
|
||
// engine that did not report forbids ALLOW, and the verdict's floor is
|
||
// recomputed from the findings the coordinator returns.
|
||
|
||
import { test } from 'node:test';
|
||
import { strict as assert } from 'node:assert';
|
||
import { readFileSync } from 'node:fs';
|
||
import { join, dirname } from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { illegalBriefLowering, severityRank as rank } from '../../lib/review/coordinator-contract.mjs';
|
||
import { ENGINES } from '../../lib/review/engine-findings.mjs';
|
||
import { RULE_CATALOGUE } from '../../lib/review/rule-catalogue.mjs';
|
||
|
||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||
const SRC = readFileSync(join(ROOT, 'scripts', 'trekreview-armB.workflow.mjs'), 'utf-8')
|
||
.replace('export const meta', 'const meta');
|
||
const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor;
|
||
const run = new AsyncFunction('args', 'agent', 'parallel', 'phase', 'log', SRC);
|
||
|
||
const BASE = { briefPath: 'b.md', diffPath: 'd.diff', triage: '' };
|
||
const engine = (reviewer, findings = []) => ({ reviewer, findings });
|
||
|
||
function stubs({ reviewers = [], coordinator = null } = {}) {
|
||
let i = 0;
|
||
const calls = { coordinatorPrompt: null, coordinatorOpts: null };
|
||
return {
|
||
calls,
|
||
agent: async (prompt, opts) => {
|
||
if (opts.label === 'coordinator') {
|
||
calls.coordinatorPrompt = prompt;
|
||
calls.coordinatorOpts = opts;
|
||
return coordinator;
|
||
}
|
||
return reviewers[i++] ?? null;
|
||
},
|
||
parallel: async (thunks) => Promise.all(thunks.map((t) => t())),
|
||
phase: () => {},
|
||
log: () => {},
|
||
};
|
||
}
|
||
|
||
async function exec(args, s) {
|
||
return run(args, s.agent, s.parallel, s.phase, s.log);
|
||
}
|
||
|
||
test('armB — both reviewers and both engines clean → ALLOW (known-positive control)', async () => {
|
||
const s = stubs({ reviewers: [{ reviewer: 'c', findings: [] }, { reviewer: 'k', findings: [] }] });
|
||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||
assert.equal(r.verdict, 'ALLOW');
|
||
assert.deepEqual(r.missing_engines, []);
|
||
});
|
||
|
||
test('armB — engines the caller never mentioned are missing → not ALLOW', async () => {
|
||
const s = stubs({ reviewers: [{ reviewer: 'c', findings: [] }, { reviewer: 'k', findings: [] }] });
|
||
const r = await exec({ ...BASE }, s);
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.deepEqual(r.missing_engines, ['claude-code-review', 'claude-security-review']);
|
||
});
|
||
|
||
test('armB — a reviewer that returned nothing → not ALLOW', async () => {
|
||
const s = stubs({ reviewers: [null, { reviewer: 'k', findings: [] }] });
|
||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||
assert.equal(r.verdict, 'WARN');
|
||
});
|
||
|
||
test('armB — a coordinator ALLOW over a BLOCKER finding is recomputed to BLOCK', async () => {
|
||
const f = { severity: 'BLOCKER', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'a.mjs', line: 1, title: 't', detail: 'd' };
|
||
const s = stubs({
|
||
reviewers: [{ reviewer: 'c', findings: [f] }, { reviewer: 'k', findings: [] }],
|
||
coordinator: { verdict: 'ALLOW', findings: [f] },
|
||
});
|
||
const r = await exec({ ...BASE, enginePayloads: [engine('claude-code-review'), engine('claude-security-review')] }, s);
|
||
assert.equal(r.verdict, 'BLOCK');
|
||
});
|
||
|
||
test('armB — quick mode expects only the /code-review engine', async () => {
|
||
const s = stubs({ reviewers: [{ reviewer: 'k', findings: [] }] });
|
||
const r = await exec({ ...BASE, quick: true, enginePayloads: [engine('claude-code-review')] }, s);
|
||
assert.equal(r.verdict, 'ALLOW');
|
||
assert.deepEqual(r.missing_engines, []);
|
||
});
|
||
|
||
// ---- Review edf2ec56: a coordinator that returns nothing fails closed ----------------
|
||
//
|
||
// A coordinator that returns no verdict and findings counts as a missing reviewer:
|
||
// its partial list is not trusted (the floors read every merged finding, each raised
|
||
// to its catalogue tier, never lowered), ALLOW is forbidden, a BLOCK is never lowered,
|
||
// and every reviewer, engine and coordinator that did not deliver is named in
|
||
// `allow_blocked_by` as `missing-reviewer:<name>`.
|
||
|
||
const COORD_MISSING = 'missing-reviewer:review-coordinator';
|
||
const MEH_MINOR = () => ({ severity: 'MINOR', rule_key: 'MISSING_ERROR_HANDLING', file: 'src/r.mjs', line: 4, title: 't', detail: 'd' });
|
||
const CRI_MINOR = () => ({ severity: 'MINOR', rule_key: 'CODE_REVIEW_IMPROVEMENT', file: 'src/e.mjs', line: 8, title: 't', detail: 'd' });
|
||
const UC_BLOCKER = () => ({ severity: 'BLOCKER', rule_key: 'UNIMPLEMENTED_CRITERION', file: 'src/r.mjs', line: 2, title: 't', detail: 'd' });
|
||
const BOTH_ENGINES = (code = [], security = []) => [engine('claude-code-review', code), engine('claude-security-review', security)];
|
||
|
||
async function coordCase({ coordinator, reviewerFindings = [MEH_MINOR()], codeFindings = [CRI_MINOR()] }) {
|
||
const s = stubs({
|
||
reviewers: [{ reviewer: 'brief-conformance', findings: reviewerFindings }, { reviewer: 'code-correctness', findings: [] }],
|
||
coordinator,
|
||
});
|
||
const r = await exec({ ...BASE, enginePayloads: BOTH_ENGINES(codeFindings) }, s);
|
||
return { r, s };
|
||
}
|
||
|
||
const failedCoordinators = [
|
||
['returns null', () => null],
|
||
['returns findings (a partial list) and no verdict', () => ({ findings: [CRI_MINOR()] })],
|
||
['returns ALLOW with no findings array', () => ({ verdict: 'ALLOW' })],
|
||
];
|
||
for (const [label, make] of failedCoordinators) {
|
||
test(`edf2ec56 — a coordinator that ${label} → WARN, every merged finding still MINOR, the coordinator named`, async () => {
|
||
const { r, s } = await coordCase({ coordinator: make() });
|
||
assert.ok(typeof s.calls.coordinatorPrompt === 'string', 'the coordinator was called');
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.deepEqual(r.findings.map((f) => f.rule_key).sort(), ['CODE_REVIEW_IMPROVEMENT', 'MISSING_ERROR_HANDLING'],
|
||
'the floors read every merged finding, not a partial list');
|
||
for (const f of r.findings) assert.equal(f.severity, 'MINOR');
|
||
assert.equal(r.missing_reviewers, 1);
|
||
assert.deepEqual(r.allow_blocked_by, [COORD_MISSING]);
|
||
});
|
||
}
|
||
|
||
test('edf2ec56 — a coordinator returning an empty findings array and no verdict over a reviewer BLOCKER → BLOCK, the BLOCKER in findings', async () => {
|
||
const { r } = await coordCase({ coordinator: { findings: [] }, reviewerFindings: [UC_BLOCKER()], codeFindings: [] });
|
||
assert.equal(r.verdict, 'BLOCK');
|
||
assert.equal(r.findings.filter((f) => f.rule_key === 'UNIMPLEMENTED_CRITERION' && f.severity === 'BLOCKER').length, 1);
|
||
assert.deepEqual(r.allow_blocked_by, [COORD_MISSING]);
|
||
});
|
||
|
||
test('edf2ec56 — a coordinator returning BLOCK and no findings array → BLOCK (never lowered)', async () => {
|
||
const { r } = await coordCase({ coordinator: { verdict: 'BLOCK' } });
|
||
assert.equal(r.verdict, 'BLOCK');
|
||
assert.deepEqual(r.allow_blocked_by, [COORD_MISSING]);
|
||
});
|
||
|
||
test('edf2ec56 — a coordinator returning the verdict PASS (outside BLOCK/WARN/ALLOW) → WARN, the coordinator named', async () => {
|
||
const { r } = await coordCase({ coordinator: { verdict: 'PASS', findings: [MEH_MINOR(), CRI_MINOR()] } });
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.ok(r.allow_blocked_by.includes(COORD_MISSING));
|
||
});
|
||
|
||
test('edf2ec56 — no engine payloads: both engines are named in allow_blocked_by, WARN', async () => {
|
||
const s = stubs({
|
||
reviewers: [{ reviewer: 'brief-conformance', findings: [MEH_MINOR()] }, { reviewer: 'code-correctness', findings: [] }],
|
||
coordinator: { verdict: 'ALLOW', findings: [MEH_MINOR()] },
|
||
});
|
||
const r = await exec({ ...BASE }, s);
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.deepEqual(r.allow_blocked_by, ['missing-reviewer:claude-code-review', 'missing-reviewer:claude-security-review']);
|
||
});
|
||
|
||
test('edf2ec56 — a first reviewer that returns null is named missing-reviewer:brief-conformance-reviewer', async () => {
|
||
const s = stubs({
|
||
reviewers: [null, { reviewer: 'code-correctness', findings: [MEH_MINOR()] }],
|
||
coordinator: { verdict: 'ALLOW', findings: [MEH_MINOR()] },
|
||
});
|
||
const r = await exec({ ...BASE, enginePayloads: BOTH_ENGINES() }, s);
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.deepEqual(r.allow_blocked_by, ['missing-reviewer:brief-conformance-reviewer']);
|
||
});
|
||
|
||
test('edf2ec56 — quick mode: a correctness reviewer that returns null is named missing-reviewer:code-correctness-reviewer, no brief-conformance-reviewer', async () => {
|
||
const s = stubs({ reviewers: [null], coordinator: { verdict: 'ALLOW', findings: [CRI_MINOR()] } });
|
||
const r = await exec({ ...BASE, quick: true, enginePayloads: [engine('claude-code-review', [CRI_MINOR()])] }, s);
|
||
assert.equal(r.verdict, 'WARN');
|
||
assert.deepEqual(r.allow_blocked_by, ['missing-reviewer:code-correctness-reviewer']);
|
||
assert.ok(!r.allow_blocked_by.includes('missing-reviewer:brief-conformance-reviewer'));
|
||
});
|
||
|
||
test('edf2ec56 — a null coordinator: a MISSING_TEST asserted MINOR is read at MAJOR (its tier), a MISSING_ERROR_HANDLING asserted BLOCKER stays BLOCKER → BLOCK', async () => {
|
||
const mt = { severity: 'MINOR', rule_key: 'MISSING_TEST', file: 'src/r.mjs', line: 5, title: 't', detail: 'd' };
|
||
const mehBlocker = { severity: 'BLOCKER', rule_key: 'MISSING_ERROR_HANDLING', file: 'src/r.mjs', line: 6, title: 't', detail: 'd' };
|
||
const { r } = await coordCase({ coordinator: null, reviewerFindings: [mt, mehBlocker], codeFindings: [] });
|
||
const sevOf = (key) => r.findings.find((f) => f.rule_key === key)?.severity;
|
||
assert.equal(sevOf('MISSING_TEST'), 'MAJOR', 'raised to its catalogue tier');
|
||
assert.equal(sevOf('MISSING_ERROR_HANDLING'), 'BLOCKER', 'a raise, never a reset');
|
||
assert.equal(r.verdict, 'BLOCK');
|
||
});
|
||
|
||
test('edf2ec56 control — a coordinator returning ALLOW with the MINOR findings and both engines → ALLOW, nothing named', async () => {
|
||
const { r } = await coordCase({ coordinator: { verdict: 'ALLOW', findings: [MEH_MINOR(), CRI_MINOR()] } });
|
||
assert.equal(r.verdict, 'ALLOW');
|
||
assert.equal(r.missing_reviewers, 0);
|
||
assert.deepEqual(r.allow_blocked_by, []);
|
||
});
|
||
|
||
test('edf2ec56 control — no findings at all → ALLOW, the coordinator skipped and not named', async () => {
|
||
const { r, s } = await coordCase({ coordinator: null, reviewerFindings: [], codeFindings: [] });
|
||
assert.equal(s.calls.coordinatorPrompt, null, 'the coordinator was skipped');
|
||
assert.equal(r.verdict, 'ALLOW');
|
||
assert.deepEqual(r.allow_blocked_by, []);
|
||
assert.ok(!r.allow_blocked_by.includes(COORD_MISSING));
|
||
});
|
||
|
||
test('edf2ec56 control — a BLOCKER with a null coordinator → BLOCK', async () => {
|
||
const { r } = await coordCase({ coordinator: null, reviewerFindings: [UC_BLOCKER()], codeFindings: [] });
|
||
assert.equal(r.verdict, 'BLOCK');
|
||
assert.deepEqual(r.allow_blocked_by, [COORD_MISSING]);
|
||
});
|
||
|
||
test('edf2ec56 — the workflow section of commands/trekreview.md and Handover 6 name the coordinator; the section names allow_blocked_by', () => {
|
||
const cmd = readFileSync(join(ROOT, 'commands', 'trekreview.md'), 'utf-8');
|
||
const handover = readFileSync(join(ROOT, 'docs', 'HANDOVER-CONTRACTS.md'), 'utf-8');
|
||
const wf = section(cmd, '## Phase 5–6 via the Workflow substrate', '## Phase 7');
|
||
assert.match(wf, /missing-reviewer:review-coordinator/);
|
||
assert.match(wf, /allow_blocked_by/);
|
||
const h6 = section(handover, '## Handover 6 — `review.md` → plan', '\n## Handover 7');
|
||
assert.match(h6, /missing-reviewer:review-coordinator/);
|
||
});
|
||
|
||
// ---- Step 9: the brief-conflict rule on the Workflow path ---------------------------
|
||
//
|
||
// Each case stubs the reviewers' and engines' payloads (so `merged` holds the
|
||
// input), the coordinator's returned findings and its verdict, then asserts the
|
||
// workflow's verdict. `merged` is read from the prompt the coordinator stub
|
||
// receives. For every case the test states the value of `illegalBriefLowering`
|
||
// on the subject finding as returned, with `engine` set from its counterpart's
|
||
// source (removed when the counterpart is a reviewer finding or absent), and
|
||
// checks the floor raised a finding exactly where that value is true — plus
|
||
// case 7, where a false `engine` label alone raises.
|
||
|
||
const MARK = 'Non-Goal: no new runtime dependency';
|
||
const FILE = 'src/a.mjs';
|
||
const LINE = 10;
|
||
const F = (rule_key, severity, extra = {}) => ({ severity, rule_key, file: FILE, line: LINE, title: 't', detail: 'd', ...extra });
|
||
const PROMPT_JSON_MARK = '=== PRE-DEDUPED FINDINGS (JSON) ===';
|
||
const triplet = (f) => `${f.file}\u0000${f.line}\u0000${f.rule_key}`;
|
||
|
||
async function floorCase({ reviewerFindings = [], codeFindings = [], securityFindings = [], quick = false, returned, verdict }) {
|
||
const before = structuredClone(returned);
|
||
const coordinator = { verdict, findings: structuredClone(returned) };
|
||
const reviewers = quick
|
||
? [{ reviewer: 'code-correctness', findings: structuredClone(reviewerFindings) }]
|
||
: [{ reviewer: 'brief-conformance', findings: structuredClone(reviewerFindings) }, { reviewer: 'code-correctness', findings: [] }];
|
||
const enginePayloads = quick
|
||
? [engine('claude-code-review', structuredClone(codeFindings))]
|
||
: [engine('claude-code-review', structuredClone(codeFindings)), engine('claude-security-review', structuredClone(securityFindings))];
|
||
const s = stubs({ reviewers, coordinator });
|
||
const r = await exec({ ...BASE, quick, enginePayloads }, s);
|
||
const prompt = s.calls.coordinatorPrompt;
|
||
assert.ok(typeof prompt === 'string', 'the coordinator ran');
|
||
const merged = JSON.parse(prompt.slice(prompt.indexOf(PROMPT_JSON_MARK) + PROMPT_JSON_MARK.length));
|
||
// Only indexes present in `before` are compared: the floor may append engine
|
||
// findings it puts back (decision 23), so `returned` can be longer.
|
||
const raised = r.findings.some((f, i) => i < before.length && rank(f.severity) < rank(before[i].severity));
|
||
return { r, merged, before, raised, s };
|
||
}
|
||
|
||
// The subject (index 0 of `returned`) with `engine` from its counterpart's source.
|
||
function subjectIllegal(before, merged, idx = 0) {
|
||
const subject = { ...before[idx] };
|
||
const counterpart = merged.find((m) => triplet(m) === triplet(subject));
|
||
if (counterpart && counterpart.engine) subject.engine = counterpart.engine;
|
||
else delete subject.engine;
|
||
return illegalBriefLowering(subject);
|
||
}
|
||
|
||
function checkRaise({ before, merged, raised }, statedIllegal, extraRaise = false) {
|
||
assert.equal(subjectIllegal(before, merged), statedIllegal, 'illegalBriefLowering on the subject');
|
||
assert.equal(raised, statedIllegal || extraRaise, 'the floor raised a finding exactly where the shape is illegal');
|
||
}
|
||
|
||
test('armB brief rule (1) — an unlabelled SECURITY_REVIEW_VULNERABILITY returned at MINOR over a /security-review BLOCKER → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||
returned: [F('SECURITY_REVIEW_VULNERABILITY', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
assert.equal(c.r.findings[0].engine, 'security-review');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (2) — an unlabelled CODE_REVIEW_DEFECT returned at MINOR over a /code-review MAJOR with ALLOW → WARN', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
for (const withEngine of [true, false]) {
|
||
test(`armB brief rule (3) — the legal lowering keeps ALLOW (returned ${withEngine ? 'with' : 'without'} engine) and carries engine code-review`, async () => {
|
||
const lowered = F('CODE_REVIEW_DEFECT', 'MINOR', { contradicts_brief: MARK, brief_lowered_from: 'MAJOR' });
|
||
if (withEngine) lowered.engine = 'code-review';
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [lowered],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'ALLOW');
|
||
assert.equal(c.r.findings[0].engine, 'code-review');
|
||
assert.equal(c.r.findings[0].severity, 'MINOR');
|
||
checkRaise(c, false);
|
||
});
|
||
}
|
||
|
||
test('armB brief rule (4) — the legal shape over a reviewer CODE_REVIEW_DEFECT asserted BLOCKER (relabelled engine) → BLOCK, no engine', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'BLOCKER')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { engine: 'code-review', contradicts_brief: MARK, brief_lowered_from: 'MAJOR' })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
assert.equal(c.r.findings[0].engine, undefined);
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (5) — a reviewer finding that carries engine code-review is not an engine finding: the source decides → WARN', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR', { engine: 'code-review' })],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { engine: 'code-review', contradicts_brief: MARK, brief_lowered_from: 'MAJOR' })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.merged[0].engine, undefined, 'a reviewer engine field never reaches merged');
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (6) — a marked reviewer MISSING_TEST asserted BLOCKER, returned at MAJOR with WARN → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('MISSING_TEST', 'BLOCKER')],
|
||
returned: [F('MISSING_TEST', 'MAJOR', { contradicts_brief: MARK })],
|
||
verdict: 'WARN',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (7) — a reviewer MISSING_TEST asserted BLOCKER, returned at MAJOR with only engine security-review → BLOCK, no engine', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('MISSING_TEST', 'BLOCKER')],
|
||
returned: [F('MISSING_TEST', 'MAJOR', { engine: 'security-review' })],
|
||
verdict: 'WARN',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
assert.equal(c.r.findings[0].engine, undefined);
|
||
checkRaise(c, false, true);
|
||
});
|
||
|
||
test('armB brief rule (8) — a stray brief_lowered_from on a reviewer finding asserted BLOCKER, returned at MAJOR with WARN → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('MISSING_TEST', 'BLOCKER')],
|
||
returned: [F('MISSING_TEST', 'MAJOR', { brief_lowered_from: 'MAJOR' })],
|
||
verdict: 'WARN',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (9) — a marked finding with no counterpart and an unknown rule_key, returned at MINOR with ALLOW → BLOCK', async () => {
|
||
const other = { severity: 'MINOR', rule_key: 'MISSING_ERROR_HANDLING', file: 'src/b.mjs', line: 3, title: 't', detail: 'd' };
|
||
const c = await floorCase({
|
||
reviewerFindings: [other],
|
||
returned: [{ severity: 'MINOR', rule_key: 'NOT_IN_THE_CATALOGUE', file: 'src/ghost.mjs', line: 7, title: 't', detail: 'd', contradicts_brief: MARK }, other],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (10) — a marked BLOCKER-tier /code-review finding returned at MINOR → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('SECURITY_INJECTION', 'BLOCKER')],
|
||
returned: [F('SECURITY_INJECTION', 'MINOR', { engine: 'code-review', contradicts_brief: MARK })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
test('armB brief rule (11) — bc7f4e23: dedup ranks by tier and keeps the reviewer over a /code-review MAJOR on the same triplet, at the group\'s MAJOR → WARN', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MINOR')],
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.merged.length, 1);
|
||
assert.equal(c.merged[0].severity, 'MAJOR', 'the survivor carries the highest severity its group asserted');
|
||
assert.equal('engine' in c.merged[0], false, 'the reviewer finding survives the tier tie');
|
||
assert.equal(c.r.deduped_count, 1);
|
||
assert.equal(c.r.raw_finding_count, 2);
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
checkRaise(c, false, true);
|
||
});
|
||
|
||
test('armB brief rule (12) — an unlabelled reviewer MISSING_TEST asserted BLOCKER, returned at MAJOR (catalogue reset) with WARN stays WARN', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('MISSING_TEST', 'BLOCKER')],
|
||
returned: [F('MISSING_TEST', 'MAJOR')],
|
||
verdict: 'WARN',
|
||
});
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
checkRaise(c, false);
|
||
});
|
||
|
||
test('armB brief rule (13) — a marked MINOR-tier engine finding at its tier (CODE_REVIEW_IMPROVEMENT) keeps ALLOW', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_IMPROVEMENT', 'MINOR')],
|
||
returned: [F('CODE_REVIEW_IMPROVEMENT', 'MINOR', { engine: 'code-review', contradicts_brief: MARK })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'ALLOW');
|
||
assert.equal(c.r.findings[0].engine, 'code-review');
|
||
checkRaise(c, false);
|
||
});
|
||
|
||
test('armB brief rule (14) — an unlabelled reviewer MISSING_TEST at MINOR (quick mode) keeps the coordinator verdict', async () => {
|
||
const c = await floorCase({
|
||
quick: true,
|
||
reviewerFindings: [F('MISSING_TEST', 'MINOR')],
|
||
returned: [F('MISSING_TEST', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'ALLOW');
|
||
checkRaise(c, false);
|
||
});
|
||
|
||
test('armB brief rule — RULE_TIERS in the script equals the catalogue (same keys, same tiers)', () => {
|
||
const start = SRC.indexOf('const RULE_TIERS = {');
|
||
assert.ok(start !== -1, 'the script carries a RULE_TIERS literal block');
|
||
const block = SRC.slice(start, SRC.indexOf('};', start));
|
||
const tiers = Object.fromEntries([...block.matchAll(/([A-Z_]+):\s*'([A-Z]+)'/g)].map((m) => [m[1], m[2]]));
|
||
const catalogue = Object.fromEntries(RULE_CATALOGUE.map((r) => [r.rule_key, r.severity]));
|
||
assert.deepEqual(tiers, catalogue);
|
||
});
|
||
|
||
test('armB brief rule — ENGINE_OF_PAYLOAD in the script maps each ENGINES reviewer name to its engine', () => {
|
||
const start = SRC.indexOf('const ENGINE_OF_PAYLOAD = {');
|
||
assert.ok(start !== -1, 'the script carries an ENGINE_OF_PAYLOAD literal block');
|
||
const block = SRC.slice(start, SRC.indexOf('};', start));
|
||
const map = Object.fromEntries([...block.matchAll(/'([a-z-]+)':\s*'([a-z-]+)'/g)].map((m) => [m[1], m[2]]));
|
||
const expected = Object.fromEntries(Object.entries(ENGINES).map(([engine, spec]) => [spec.reviewer, engine]));
|
||
assert.deepEqual(map, expected);
|
||
});
|
||
|
||
test('armB brief rule — VERDICT_SCHEMA declares engine, contradicts_brief and brief_lowered_from; the prompt states the rule', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
verdict: 'WARN',
|
||
});
|
||
const props = c.s.calls.coordinatorOpts.schema.properties.findings.items.properties;
|
||
assert.deepEqual(props.engine, { type: 'string', enum: ['code-review', 'security-review'] });
|
||
assert.deepEqual(props.contradicts_brief, { type: 'string' });
|
||
assert.deepEqual(props.brief_lowered_from, { type: 'string', enum: ['MAJOR'] });
|
||
const prompt = c.s.calls.coordinatorPrompt;
|
||
assert.ok(prompt.includes('contradicts_brief'));
|
||
assert.ok(prompt.includes('brief_lowered_from'));
|
||
assert.ok(prompt.includes('## Non-Goals'));
|
||
assert.ok(prompt.includes('## Constraints'));
|
||
assert.ok(prompt.includes('## Preferences'));
|
||
assert.ok(prompt.includes('## Open Questions / Assumptions'));
|
||
assert.ok(/only an engine finding/i.test(prompt), 'engine findings only');
|
||
assert.ok(/kept and marked, never removed/i.test(prompt));
|
||
});
|
||
|
||
// Decision 13 (operator 12.10), bc7f4e23: dedup ranks by catalogue tier, which
|
||
// always ties inside a catalogue triplet, so a reviewer-agent finding survives
|
||
// over an engine finding and carries the group's highest asserted severity; the
|
||
// engine's mark can never lower what a reviewer agent asserted (the same order
|
||
// as coordinator-contract Pass 1).
|
||
test('armB brief rule (15) — a tie between a reviewer MAJOR and a /code-review MAJOR keeps the reviewer: the legal shape returned with ALLOW → WARN, no engine', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { contradicts_brief: MARK, brief_lowered_from: 'MAJOR', engine: 'code-review' })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.merged.length, 1);
|
||
assert.equal(c.merged[0].severity, 'MAJOR');
|
||
assert.equal('engine' in c.merged[0], false, 'the tie keeps the reviewer finding');
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
assert.equal(c.r.findings[0].severity, 'MAJOR');
|
||
assert.equal('engine' in c.r.findings[0], false);
|
||
checkRaise(c, true);
|
||
});
|
||
|
||
// ---- Decision 23 (operator 13.10, review ab1b5d9f): no engine finding leaves ----
|
||
// the workflow review unreturned. Every engine finding of the input (before
|
||
// dedup) with no returned finding at its exact (file, line, rule_key) is put
|
||
// back at its own severity; a returned finding is never below the lower of its
|
||
// reviewer counterpart's severity and its catalogue tier.
|
||
|
||
const at = (findings, file, line, rule_key) =>
|
||
findings.filter((f) => f && f.file === file && f.line === line && f.rule_key === rule_key);
|
||
|
||
test('decision 23 — a /security-review BLOCKER returned as a CODE_REVIEW_IMPROVEMENT MINOR on the same line → BLOCK, the BLOCKER is put back', async () => {
|
||
const c = await floorCase({
|
||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||
returned: [F('CODE_REVIEW_IMPROVEMENT', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||
assert.equal(back.length, 1, 'the BLOCKER is in the findings');
|
||
assert.equal(back[0].severity, 'BLOCKER');
|
||
assert.equal(back[0].engine, 'security-review');
|
||
assert.equal('brief_lowered_from' in back[0], false);
|
||
});
|
||
|
||
test('decision 23 — the same BLOCKER returned one line lower at MINOR → BLOCK, the BLOCKER is back at its own line', async () => {
|
||
const c = await floorCase({
|
||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||
returned: [F('SECURITY_REVIEW_VULNERABILITY', 'MINOR', { line: LINE + 1 })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||
assert.equal(back.length, 1, 'the BLOCKER at its own line is in the findings');
|
||
assert.equal(back[0].severity, 'BLOCKER');
|
||
assert.equal(back[0].engine, 'security-review');
|
||
});
|
||
|
||
test('decision 23 — the same BLOCKER dropped (returned empty with ALLOW) → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||
returned: [],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||
assert.equal(back.length, 1);
|
||
assert.equal(back[0].severity, 'BLOCKER');
|
||
assert.equal(back[0].engine, 'security-review');
|
||
});
|
||
|
||
test('decision 23 — a /security-review BLOCKER shadowed in dedup by a reviewer MINOR on the same triplet, returned empty → BLOCK', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'MINOR')],
|
||
securityFindings: [F('SECURITY_REVIEW_VULNERABILITY', 'BLOCKER')],
|
||
returned: [],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
const back = at(c.r.findings, FILE, LINE, 'SECURITY_REVIEW_VULNERABILITY');
|
||
assert.equal(back.length, 1);
|
||
assert.equal(back[0].severity, 'BLOCKER');
|
||
assert.equal(back[0].engine, 'security-review');
|
||
});
|
||
|
||
test('decision 23 — a /code-review MAJOR dropped → WARN', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
const back = at(c.r.findings, FILE, LINE, 'CODE_REVIEW_DEFECT');
|
||
assert.equal(back.length, 1);
|
||
assert.equal(back[0].severity, 'MAJOR');
|
||
assert.equal(back[0].engine, 'code-review');
|
||
});
|
||
|
||
for (const order of ['MAJOR first', 'BLOCKER first']) {
|
||
test(`decision 23 — two /code-review findings on one triplet (${order}), returned empty → BLOCK, one copy, at BLOCKER`, async () => {
|
||
const pair = [F('CODE_REVIEW_DEFECT', 'MAJOR', { title: 'major' }), F('CODE_REVIEW_DEFECT', 'BLOCKER', { title: 'blocker' })];
|
||
const c = await floorCase({
|
||
codeFindings: order === 'MAJOR first' ? pair : [pair[1], pair[0]],
|
||
returned: [],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'BLOCK');
|
||
const back = at(c.r.findings, FILE, LINE, 'CODE_REVIEW_DEFECT');
|
||
assert.equal(back.length, 1, 'exactly one finding at the triplet');
|
||
assert.equal(back[0].severity, 'BLOCKER', 'the higher-severity copy is put back');
|
||
assert.equal(back[0].title, 'blocker');
|
||
assert.equal(back[0].engine, 'code-review');
|
||
});
|
||
}
|
||
|
||
test('decision 23 floor (b) — a reviewer finding with a rule_key outside the catalogue asserted MAJOR, returned at MINOR → raised to MAJOR, WARN', async () => {
|
||
const c = await floorCase({
|
||
reviewerFindings: [F('NOT_IN_THE_CATALOGUE', 'MAJOR')],
|
||
returned: [F('NOT_IN_THE_CATALOGUE', 'MINOR')],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'WARN');
|
||
assert.equal(c.r.findings.length, 1);
|
||
assert.equal(c.r.findings[0].severity, 'MAJOR', "the bound for an unknown key is the counterpart's asserted severity");
|
||
});
|
||
|
||
test('decision 23 — the one legal lowering (marked /code-review MAJOR returned as MINOR with the mark) still gives ALLOW', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MINOR', { contradicts_brief: MARK, brief_lowered_from: 'MAJOR' })],
|
||
verdict: 'ALLOW',
|
||
});
|
||
assert.equal(c.r.verdict, 'ALLOW');
|
||
assert.equal(c.r.findings.length, 1, 'nothing is put back over a returned triplet');
|
||
assert.equal(c.r.findings[0].severity, 'MINOR');
|
||
assert.equal(c.r.findings[0].engine, 'code-review');
|
||
});
|
||
|
||
// ---- the docs state the workflow floor ------------------------------------------
|
||
|
||
// Slice from the first `start` to the next `end` after it; fail when either is missing.
|
||
function section(text, start, end) {
|
||
const i = text.indexOf(start);
|
||
assert.ok(i !== -1, `section start not found: ${start}`);
|
||
const j = text.indexOf(end, i + start.length);
|
||
assert.ok(j !== -1, `section end not found after ${start}: ${JSON.stringify(end)}`);
|
||
return text.slice(i, j);
|
||
}
|
||
|
||
const CMD_TEXT = readFileSync(join(ROOT, 'commands', 'trekreview.md'), 'utf-8');
|
||
const HANDOVER_TEXT = readFileSync(join(ROOT, 'docs', 'HANDOVER-CONTRACTS.md'), 'utf-8');
|
||
const PUT_BACK = /put\s+back\s+at\s+its\s+own\s+severity/;
|
||
const FLOOR_B = /lower\s+of\s+its\s+reviewer\s+counterpart's\s+severity\s+and\s+its\s+catalogue\s+tier/;
|
||
|
||
test('decision 23 — commands/trekreview.md workflow section states the put-back and floor (b); the port is no longer the SAME pipeline', () => {
|
||
const wf = section(CMD_TEXT, '## Phase 5–6 via the Workflow substrate', '## Phase 7');
|
||
assert.match(wf, PUT_BACK);
|
||
assert.match(wf, FLOOR_B);
|
||
assert.doesNotMatch(CMD_TEXT, /the\s+SAME\s+Phase/);
|
||
});
|
||
|
||
test('decision 23 — Handover 6 states the dedup rank on both paths, the put-back and floor (b)', () => {
|
||
const h6 = section(HANDOVER_TEXT, '## Handover 6 — `review.md` → plan', '\n## Handover 7');
|
||
assert.match(h6, /dedup\s+ranks\s+as\s+Pass\s+1\s+does\s+\(both\s+paths\)/);
|
||
assert.match(h6, PUT_BACK);
|
||
assert.match(h6, FLOOR_B);
|
||
});
|
||
|
||
test('armB — two different /code-review findings on one triplet stay two rows for the coordinator (trial 5, finding 7)', async () => {
|
||
const c = await floorCase({
|
||
codeFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR', { title: 'EAGAIN on stdin' }), F('CODE_REVIEW_DEFECT', 'MAJOR', { title: 'A lone CR does not split lines' })],
|
||
reviewerFindings: [F('CODE_REVIEW_DEFECT', 'MAJOR', { title: 'EAGAIN on stdin' })],
|
||
returned: [F('CODE_REVIEW_DEFECT', 'MAJOR')],
|
||
verdict: 'WARN',
|
||
});
|
||
assert.deepEqual(c.merged.map((f) => f.title).sort(), ['A lone CR does not split lines', 'EAGAIN on stdin']);
|
||
});
|
||
|
||
// ---- ORD sibling: the workflow's inlined dedup does not depend on arrival order ----
|
||
|
||
test('[ORD] sibling: workflow — conformance B with /code-review A and B, in both engine orders, gives one row of A alone and one row of both B findings', async () => {
|
||
const K = 'CODE_REVIEW_DEFECT';
|
||
const engineA = F(K, 'MINOR', { title: 'A' });
|
||
const engineB = F(K, 'MAJOR', { title: 'B' });
|
||
for (const codeFindings of [[engineA, engineB], [engineB, engineA]]) {
|
||
const name = codeFindings.map((f) => f.title).join(', ');
|
||
const c = await floorCase({
|
||
reviewerFindings: [F(K, 'MINOR', { title: 'B' })],
|
||
codeFindings,
|
||
returned: [],
|
||
verdict: 'WARN',
|
||
});
|
||
const rows = c.merged.filter((m) => triplet(m) === triplet(F(K, 'MINOR')));
|
||
assert.equal(rows.length, 2, `engine order ${name}: ${JSON.stringify(rows)}`);
|
||
const got = rows
|
||
.map((m) => ({ title: m.title, engine: m.engine ?? null, severity: m.severity }))
|
||
.sort((x, y) => x.title.localeCompare(y.title));
|
||
// A alone: the /code-review A finding survives with its own MINOR. Both B
|
||
// findings: the conformance B finding survives (a reviewer over an engine)
|
||
// at the MAJOR the /code-review B finding asserted.
|
||
assert.deepEqual(got, [
|
||
{ title: 'A', engine: 'code-review', severity: 'MINOR' },
|
||
{ title: 'B', engine: null, severity: 'MAJOR' },
|
||
], `engine order ${name}`);
|
||
}
|
||
});
|
||
|
||
test('[ORD] sibling: workflow — the title classes are placed in alphabetical order: conformance B with /code-review A, B and an untitled BLOCKER gives {A, untitled} BLOCKER kept as A and {B, B} MAJOR in all 6 engine orders', async () => {
|
||
const K = 'CODE_REVIEW_DEFECT';
|
||
const engineA = F(K, 'BLOCKER', { title: 'A' });
|
||
const engineB = F(K, 'MINOR', { title: 'B' });
|
||
const { title: _t, ...engineU } = F(K, 'BLOCKER');
|
||
// The workflow reads the reviewer findings first, so class B always arrives
|
||
// first: only the alphabetical order puts the untitled finding in A's row.
|
||
// A and the untitled finding tie on every survivor rule, so the first in the
|
||
// group's order (class A before the untitled findings) is kept.
|
||
const engineOrders = [
|
||
[engineA, engineB, engineU], [engineA, engineU, engineB], [engineB, engineA, engineU],
|
||
[engineB, engineU, engineA], [engineU, engineA, engineB], [engineU, engineB, engineA],
|
||
];
|
||
for (const codeFindings of engineOrders) {
|
||
const name = codeFindings.map((f) => f.title ?? 'U').join(', ');
|
||
const c = await floorCase({
|
||
reviewerFindings: [F(K, 'MAJOR', { title: 'B' })],
|
||
codeFindings,
|
||
returned: [],
|
||
verdict: 'WARN',
|
||
});
|
||
const rows = c.merged.filter((m) => triplet(m) === triplet(F(K, 'MINOR')));
|
||
const got = rows
|
||
.map((m) => ({ title: m.title ?? null, engine: m.engine ?? null, severity: m.severity }))
|
||
.sort((x, y) => String(x.title).localeCompare(String(y.title)));
|
||
assert.deepEqual(got, [
|
||
{ title: 'A', engine: 'code-review', severity: 'BLOCKER' },
|
||
{ title: 'B', engine: null, severity: 'MAJOR' },
|
||
], `engine order ${name}`);
|
||
}
|
||
});
|