- park-attempt-files: parks in the git dir (outside the work tree, so Sub-step C.2 never registers a parked file again), reads a path with a space, names a new file under a header it cannot read, parks a symlink, never overwrites a parked file, and exits 1 while a file the attempt made is still in the tree; the upward diff is saved in one form whatever the git config. - brief-validator: BRIEF_SC_NOT_RUNNABLE only when a later backticked span is a command; a criterion that only names a flag or a path stays prose (Handover 1 says so now). - review ids: counted over the final list (runContract), as fill-finding-ids counts review.md; trimmed_count matches rows by id; engine-findings and fidelity-diff use computeFindingIds; the --workflow dedup keeps one source's different findings apart, the decision-23 put-back stays one copy per triplet. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
441 lines
24 KiB
JavaScript
441 lines
24 KiB
JavaScript
// tests/lib/coordinator-contract.test.mjs
|
|
// SKAL-1·4a — deterministic test of the review-coordinator contract subset.
|
|
//
|
|
// Inline reviewer-JSON fixtures (idiom: tests/validators/brief-validator.test.mjs).
|
|
// Hermetic: no LLM, network, time, or randomness.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { computeFindingId } from '../../lib/parsers/finding-id.mjs';
|
|
import {
|
|
severityRank,
|
|
ingest,
|
|
dedupByTriplet,
|
|
judgeFilter,
|
|
reasonablenessFilter,
|
|
computeVerdict,
|
|
classifySuppression,
|
|
REFUTING_REASONS,
|
|
UNVERIFIED_REASONS,
|
|
runContract,
|
|
} from '../../lib/review/coordinator-contract.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
|
|
// ---- Pass 1 — dedup --------------------------------------------------------
|
|
|
|
test('dedupByTriplet — genuine cross-reviewer collapse (identical triplet) → 1, raised_by both', () => {
|
|
// Both reviewers flag the SAME (file,line,rule_key) triplet — this is the
|
|
// real collapse the coordinator performs.
|
|
const findings = [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header', reviewer: 'correctness' },
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1 violated', reviewer: 'conformance' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1, 'identical triplets must collapse to one');
|
|
assert.deepEqual([...out[0].raised_by].sort(), ['conformance', 'correctness']);
|
|
});
|
|
|
|
test('dedupByTriplet — README #2 two DIFFERENT rule_keys at same file:line do NOT collapse', () => {
|
|
// The bakeoff-rich README marks issue #2 dual-flaggable via SECURITY_INJECTION
|
|
// AND NON_GOAL_VIOLATED. Those are DIFFERENT triplets → distinct defects → kept.
|
|
const findings = [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' },
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', reviewer: 'conformance' },
|
|
];
|
|
assert.equal(dedupByTriplet(findings).length, 2);
|
|
});
|
|
|
|
test('dedupByTriplet — survivor is the highest-severity finding in the group', () => {
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'MINOR', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].severity, 'BLOCKER');
|
|
});
|
|
|
|
test('dedupByTriplet — two DIFFERENT findings from one reviewer on one triplet stay two rows with their own ids (trial 5, finding 7)', () => {
|
|
// /code-review gave both the catch-all key at bin/slug.mjs:46; one merged row lost the second finding's action.
|
|
const findings = [
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'EAGAIN on a non-blocking stdin', recommended_action: 'Read stdin with a stream.', reviewer: 'claude-code-review' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'A lone \\r does not split lines', recommended_action: 'Split on \\r too.', reviewer: 'claude-code-review' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'EAGAIN on a non-blocking stdin', recommended_action: 'Read stdin with a stream.', reviewer: 'code-correctness-reviewer' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 2, JSON.stringify(out.map((f) => f.title)));
|
|
assert.deepEqual(out.map((f) => f.recommended_action), ['Read stdin with a stream.', 'Split on \\r too.']);
|
|
assert.equal(out[0].id, computeFindingId('bin/slug.mjs', 46, 'CODE_REVIEW_DEFECT'));
|
|
assert.equal(out[1].id, computeFindingId('bin/slug.mjs', 46, 'CODE_REVIEW_DEFECT#2'));
|
|
assert.deepEqual([...out[0].raised_by].sort(), ['claude-code-review', 'code-correctness-reviewer']);
|
|
});
|
|
|
|
test('dedupByTriplet — severity tie breaks toward the conformance reviewer', () => {
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'corr', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'conf', reviewer: 'conformance' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].reviewer, 'conformance');
|
|
});
|
|
|
|
// ---- severity ranking (4-tier, incl. synthetic SUGGESTION) -----------------
|
|
|
|
test('severityRank — orders the full 4-tier SEVERITY_VALUES incl. SUGGESTION', () => {
|
|
assert.ok(severityRank('BLOCKER') < severityRank('MAJOR'));
|
|
assert.ok(severityRank('MAJOR') < severityRank('MINOR'));
|
|
assert.ok(severityRank('MINOR') < severityRank('SUGGESTION'));
|
|
});
|
|
|
|
test('dedupByTriplet — SUGGESTION vs MINOR (synthetic) keeps the MINOR survivor', () => {
|
|
// gold corpus has no SUGGESTION; exercise the 4th tier synthetically.
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'SUGGESTION', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', reviewer: 'correctness' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].severity, 'MINOR');
|
|
});
|
|
|
|
// ---- Pass 4 — verdict thresholds -------------------------------------------
|
|
|
|
test('computeVerdict — BLOCKER>=1 → BLOCK, MAJOR-only → WARN, else ALLOW', () => {
|
|
assert.equal(computeVerdict([{ severity: 'BLOCKER' }, { severity: 'MAJOR' }]).verdict, 'BLOCK');
|
|
assert.equal(computeVerdict([{ severity: 'MAJOR' }, { severity: 'MINOR' }]).verdict, 'WARN');
|
|
assert.equal(computeVerdict([{ severity: 'MINOR' }]).verdict, 'ALLOW');
|
|
assert.equal(computeVerdict([{ severity: 'SUGGESTION' }]).verdict, 'ALLOW');
|
|
assert.equal(computeVerdict([]).verdict, 'ALLOW');
|
|
});
|
|
|
|
test('computeVerdict — counts each severity tier', () => {
|
|
const { counts } = computeVerdict([
|
|
{ severity: 'BLOCKER' }, { severity: 'BLOCKER' }, { severity: 'MAJOR' }, { severity: 'MINOR' },
|
|
]);
|
|
assert.deepEqual(counts, { BLOCKER: 2, MAJOR: 1, MINOR: 1, SUGGESTION: 0 });
|
|
});
|
|
|
|
// ---- Pass 3 — reasonableness -----------------------------------------------
|
|
|
|
test('reasonablenessFilter — citation-less is REFUTED, unknown rule_key is UNVERIFIED, severity mismatch corrected', () => {
|
|
// Contract change (fail-closed): only `no-citation` refutes. An ad-hoc
|
|
// rule_key is a real defect wearing the wrong label — v5.1.1 high-effort mode
|
|
// already keeps those, normalised to PLAN_EXECUTE_DRIFT.
|
|
const r = reasonablenessFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'BLOCKER' }, // unknown → unverified
|
|
{ file: '', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // no file → drop
|
|
{ file: 'x.mjs', line: -1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // line < 0 → drop
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR' }, // catalogue is MAJOR → correct, keep
|
|
]);
|
|
assert.equal(r.kept.length, 1);
|
|
assert.equal(r.dropped.length, 2);
|
|
assert.deepEqual(r.dropped.map((f) => f.suppressed_reason), ['no-citation', 'no-citation']);
|
|
assert.equal(r.unverified.length, 1);
|
|
assert.equal(r.unverified[0].suppressed_reason, 'unknown-rule_key');
|
|
assert.equal(r.kept[0].severity, 'MAJOR');
|
|
assert.equal(r.kept[0].original_severity, 'MINOR');
|
|
});
|
|
|
|
// ---- Pass 2 — judge --------------------------------------------------------
|
|
|
|
test('judgeFilter — an over-long MAJOR title is kept whole; an empty recommended_action is UNVERIFIED', () => {
|
|
// Length never removes a finding (ordre 5, P11). Actionability reads a
|
|
// `.length` and never examines the claim, so it cannot refute the finding.
|
|
// `dropped` is empty here on purpose — the refuting Pass 2 filter (Accuracy)
|
|
// is the one this deterministic subset excludes.
|
|
const longTitle = 'x'.repeat(101);
|
|
const j = judgeFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: longTitle }, // MAJOR → kept whole
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok', recommended_action: ' ' }, // empty action → unverified
|
|
{ file: 'x.mjs', line: 3, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok' }, // keep (no action field is fine)
|
|
]);
|
|
assert.equal(j.kept.length, 2);
|
|
assert.equal(j.kept[0].title, longTitle, 'a MAJOR keeps every character');
|
|
assert.equal(j.dropped.length, 0);
|
|
assert.equal(j.trimmed.length, 0);
|
|
assert.deepEqual(j.unverified.map((f) => f.suppressed_reason), ['actionability:empty']);
|
|
});
|
|
|
|
test('judgeFilter — only MINOR/SUGGESTION text is trimmed, and every trim is counted', () => {
|
|
const j = judgeFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 't'.repeat(150), detail: 'd'.repeat(1200) },
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'short', detail: 'short' },
|
|
]);
|
|
assert.equal(j.kept.length, 2, 'a trimmed finding is still kept');
|
|
assert.equal(j.trimmed.length, 1);
|
|
assert.ok(j.kept[0].title.length <= 100 && j.kept[0].detail.length <= 800);
|
|
assert.deepEqual(j.kept[0].trimmed_fields, ['title', 'detail']);
|
|
assert.ok(j.kept[0].detail.endsWith('[…trimmed]'), 'the cut is visible in the text');
|
|
});
|
|
|
|
test('judgeFilter — a trimmed MINOR keeps its "Also flagged by" provenance', () => {
|
|
const [f] = dedupByTriplet([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'a', detail: 'q'.repeat(900), reviewer: 'code-correctness-reviewer' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'b', detail: 'r', reviewer: 'claude-code-review' },
|
|
]);
|
|
const j = judgeFilter([f]);
|
|
assert.ok(j.kept[0].detail.length <= 800);
|
|
assert.match(j.kept[0].detail, /Also flagged by claude-code-review: b\.$/);
|
|
});
|
|
|
|
test('judgeFilter — the trim decision reads the catalogue tier, not the reviewer label', () => {
|
|
// MISSING_TEST is MAJOR in the catalogue; a reviewer calling it MINOR must
|
|
// not get its text cut.
|
|
const detail = 'd'.repeat(900);
|
|
const j = judgeFilter([{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR', title: 't', detail }]);
|
|
assert.equal(j.trimmed.length, 0);
|
|
assert.equal(j.kept[0].detail, detail);
|
|
});
|
|
|
|
// ---- ingest ----------------------------------------------------------------
|
|
|
|
test('ingest — collects findings from valid payloads, skips invalid ones', () => {
|
|
const { findings, skipped } = ingest([
|
|
{ reviewer: 'correctness', findings: [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] },
|
|
{ reviewer: 'bad', findings: [{ line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, // missing file → invalid → skipped
|
|
]);
|
|
assert.equal(findings.length, 1);
|
|
assert.equal(findings[0].reviewer, 'correctness');
|
|
assert.equal(skipped.length, 1);
|
|
});
|
|
|
|
// ---- end-to-end ------------------------------------------------------------
|
|
|
|
test('runContract — two reviewers, dual-flag triplet collapses, verdict BLOCK', () => {
|
|
const result = runContract([
|
|
{ reviewer: 'correctness', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header' },
|
|
{ file: 'lib/auth/refresh.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'no concurrent test' },
|
|
] },
|
|
{ reviewer: 'conformance', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1' }, // collapses with correctness's
|
|
{ file: 'lib/handlers/login.mjs', line: 17, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: '200 not 401' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 3, '4 raw findings, jwt:19 SECURITY_INJECTION collapses → 3');
|
|
const jwt = result.findings.find((f) => f.file === 'lib/auth/jwt.mjs');
|
|
assert.deepEqual([...jwt.raised_by].sort(), ['conformance', 'correctness']);
|
|
});
|
|
|
|
test('runContract — deterministic: identical input yields identical output', () => {
|
|
const input = [
|
|
{ reviewer: 'correctness', findings: [{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 't' }] },
|
|
{ reviewer: 'conformance', findings: [{ file: 'b.mjs', line: 2, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: 'u' }] },
|
|
];
|
|
assert.deepEqual(runContract(input), runContract(input));
|
|
});
|
|
|
|
// ---- Fail-closed: the `unverified` bucket (ORDRE 834432937) -----------------
|
|
//
|
|
// The defect: a finding REMOVED by Pass 2/Pass 3, and a reviewer whose payload
|
|
// was thrown away or never arrived, are all arithmetically identical to a
|
|
// finding that never existed -- they push the verdict toward ALLOW. Measured
|
|
// before the fix (probe, 2026-09-01): an over-long-title BLOCKER -> ALLOW; a
|
|
// payload with one ad-hoc rule_key -> the whole payload skipped, its valid
|
|
// BLOCKER sibling gone -> ALLOW.
|
|
//
|
|
// The rule under test: a removal is `dropped` ONLY when the test refutes the
|
|
// finding as a claim about this codebase. Every other removal is `unverified`,
|
|
// and a non-empty `unverified` -- or a reviewer that did not report -- forbids
|
|
// ALLOW.
|
|
|
|
test('classifySuppression — only no-citation refutes; form and taxonomy failures are unverified', () => {
|
|
assert.equal(classifySuppression('no-citation'), 'refuted',
|
|
'a finding that names no location makes no checkable claim');
|
|
assert.equal(classifySuppression('succinctness:title'), 'unverified');
|
|
assert.equal(classifySuppression('succinctness:detail'), 'unverified');
|
|
assert.equal(classifySuppression('actionability:empty'), 'unverified');
|
|
assert.equal(classifySuppression('unknown-rule_key'), 'unverified');
|
|
assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified');
|
|
assert.equal(classifySuppression('something-nobody-declared'), 'unverified',
|
|
'an unclassified reason must fail CLOSED, not open');
|
|
});
|
|
|
|
test('computeVerdict — non-empty unverified forbids ALLOW but never downgrades BLOCK or WARN', () => {
|
|
const u = [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'BLOCKER' }];
|
|
const withUnverified = computeVerdict([], { unverified: u });
|
|
assert.equal(withUnverified.verdict, 'WARN', 'ALLOW is forbidden while anything is unverified');
|
|
assert.deepEqual(withUnverified.counts, { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 },
|
|
'the unverified finding is NOT counted into a severity tier');
|
|
assert.ok(withUnverified.allow_blocked_by.length > 0);
|
|
|
|
assert.equal(computeVerdict([{ severity: 'BLOCKER' }], { unverified: u }).verdict, 'BLOCK',
|
|
'BLOCK stands regardless of the unverified bucket');
|
|
assert.equal(computeVerdict([{ severity: 'MAJOR' }], { unverified: u }).verdict, 'WARN');
|
|
assert.equal(computeVerdict([], { unverified: [] }).verdict, 'ALLOW',
|
|
'known-positive control: an empty unverified bucket still allows ALLOW');
|
|
});
|
|
|
|
test('computeVerdict — a reviewer that did not report forbids ALLOW', () => {
|
|
const r = computeVerdict([], { missingReviewers: ['brief-conformance-reviewer'] });
|
|
assert.equal(r.verdict, 'WARN');
|
|
assert.ok(r.allow_blocked_by.some((x) => x.includes('brief-conformance-reviewer')));
|
|
});
|
|
|
|
test('runContract — a BLOCKER with an over-long title is kept and yields BLOCK', () => {
|
|
// Before ordre 5 this finding went to `unverified` on `succinctness:title`
|
|
// (the fail-closed floor kept it from ALLOW, but it fell out of the list).
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'x'.repeat(101), detail: 'algo taken from the JWT header' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 1);
|
|
assert.equal(result.unverified.length, 0);
|
|
assert.equal(result.suppressed.length, 0);
|
|
});
|
|
|
|
test('runContract — declined_to_judge lists every unverified finding with its reason; trimmed_count counts trims', () => {
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'empty action', recommended_action: '' },
|
|
{ file: 'b.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'long', detail: 'z'.repeat(850), recommended_action: 'Guard it.' },
|
|
] },
|
|
]);
|
|
assert.equal(result.declined_to_judge.length, 1);
|
|
assert.deepEqual(
|
|
{ file: result.declined_to_judge[0].file, reason: result.declined_to_judge[0].reason },
|
|
{ file: 'a.mjs', reason: 'actionability:empty' },
|
|
);
|
|
assert.equal(result.trimmed_count, 1);
|
|
assert.equal(result.counts.MINOR, 1, 'the trimmed MINOR still counts in its tier');
|
|
});
|
|
|
|
test('runContract — a schema-invalid payload cannot yield ALLOW (an unread reviewer is an absent one)', () => {
|
|
// Measured: one ad-hoc rule_key invalidates the WHOLE payload at ingest, so a
|
|
// valid BLOCKER sibling disappears with it. That must not read as "clean".
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'real' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'MINOR', title: 'ad-hoc key' },
|
|
] },
|
|
]);
|
|
assert.equal(result.skipped.length, 1);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
assert.ok(result.allow_blocked_by.some((x) => x.includes('code-correctness-reviewer')));
|
|
});
|
|
|
|
test('runContract — a reviewer named in expectedReviewers that never reported cannot yield ALLOW', () => {
|
|
const result = runContract(
|
|
[{ reviewer: 'code-correctness-reviewer', findings: [] }],
|
|
{ expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] },
|
|
);
|
|
assert.deepEqual(result.missing_reviewers, ['brief-conformance-reviewer']);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
});
|
|
|
|
test('runContract — known-positive control: every reviewer reported, nothing suppressed → ALLOW', () => {
|
|
// Proves ALLOW is still REACHABLE. Without this, "no ALLOW" is not a
|
|
// fail-closed contract, only a broken one.
|
|
const result = runContract(
|
|
[
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'a.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'unguarded await', recommended_action: 'Wrap the await in a try/catch.' },
|
|
] },
|
|
{ reviewer: 'brief-conformance-reviewer', findings: [] },
|
|
],
|
|
{ expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] },
|
|
);
|
|
assert.equal(result.verdict, 'ALLOW');
|
|
assert.equal(result.unverified.length, 0);
|
|
assert.deepEqual(result.missing_reviewers, []);
|
|
assert.deepEqual(result.allow_blocked_by, []);
|
|
});
|
|
|
|
test('classifySuppression — the refuting reasons the LLM coordinator emits are declared here too', () => {
|
|
// agents/review-coordinator.md Pass 2 "Accuracy" and Pass 3 "Non-existent
|
|
// file" DO refute (a citation outside the repo root, a file absent from both
|
|
// tree and diff). Both are fs/judgement branches this deterministic subset
|
|
// excludes, but the vocabulary is owned here so prose and lib cannot drift.
|
|
assert.equal(classifySuppression('accuracy:refuted'), 'refuted');
|
|
assert.equal(classifySuppression('file-existence:refuted'), 'refuted');
|
|
assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified',
|
|
'unresolvable must never collapse into refuted');
|
|
});
|
|
|
|
test('suppression vocabulary — the two sets are disjoint and every reason is documented in the prose', () => {
|
|
const refuting = [...REFUTING_REASONS];
|
|
const overlap = refuting.filter((r) => UNVERIFIED_REASONS.includes(r));
|
|
assert.deepEqual(overlap, [], 'a reason cannot be both refuting and unverified');
|
|
|
|
const prose = readFileSync(join(ROOT, 'agents/review-coordinator.md'), 'utf-8');
|
|
assert.ok(prose.includes('review-coordinator'), 'known-positive control: the prose file loaded');
|
|
for (const reason of [...refuting, ...UNVERIFIED_REASONS]) {
|
|
assert.ok(prose.includes(reason),
|
|
`reason "${reason}" is declared in the lib but never documented in agents/review-coordinator.md`);
|
|
}
|
|
});
|
|
|
|
test('runContract — an anonymous invalid payload is unattributable, not a reviewer named "unnamed reviewer"', () => {
|
|
// `validateFindings` only WARNS on a missing `reviewer`, so a payload can
|
|
// fail schema while carrying no name. Reporting it as a reviewer name
|
|
// invents an agent nobody launched, and double-counts with expectedReviewers
|
|
// when they are in fact the same failure.
|
|
const result = runContract(
|
|
[{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }],
|
|
{ expectedReviewers: ['code-correctness-reviewer'] },
|
|
);
|
|
assert.deepEqual(result.missing_reviewers, ['code-correctness-reviewer'],
|
|
'missing_reviewers carries real names only');
|
|
assert.equal(result.unattributable_payloads, 1);
|
|
assert.ok(result.allow_blocked_by.some((x) => x.startsWith('unattributable-payload')));
|
|
assert.ok(!result.allow_blocked_by.some((x) => x.includes('unnamed reviewer')));
|
|
});
|
|
|
|
test('runContract — an anonymous invalid payload forbids ALLOW on its own, with no expectedReviewers', () => {
|
|
// The fail-closed floor must not depend on the caller passing an expected
|
|
// set: without this, dropping the name from a payload would restore ALLOW.
|
|
const result = runContract([{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }]);
|
|
assert.deepEqual(result.missing_reviewers, []);
|
|
assert.equal(result.unattributable_payloads, 1);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
});
|
|
|
|
// ---- No finding is struck for length (ordre 5, P11) --------------------------
|
|
//
|
|
// The defect (STATE, Åpne funn; S118): a finding whose `detail` ran past 800
|
|
// characters went to `unverified` on `succinctness:detail` alone, and S118's most
|
|
// serious finding fell out of the Handover 6 list that way. Length is packaging,
|
|
// not a claim: only MINOR/SUGGESTION text is trimmed, and every trim is counted.
|
|
|
|
test('runContract — a BLOCKER with a 900-character detail survives and yields BLOCK', () => {
|
|
const detail = 'The refresh token is accepted after logout because the revocation list is never consulted. '.repeat(10).slice(0, 900);
|
|
assert.equal(detail.length, 900);
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/refresh.mjs', line: 31, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER',
|
|
title: 'Revoked refresh token still accepted', detail, recommended_action: 'Check the revocation list before issuing.' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 1, 'the long BLOCKER is a kept finding');
|
|
assert.equal(result.findings[0].detail, detail, 'its detail is not cut');
|
|
assert.equal(result.unverified.length, 0);
|
|
});
|
|
|
|
test('the coordinator prose says one reviewer never duplicates itself, as dedupByTriplet does (trial 5, finding 7)', () => {
|
|
const prose = readFileSync(join(ROOT, 'agents/review-coordinator.md'), 'utf8');
|
|
const pass1 = prose.slice(prose.indexOf('### Pass 1'), prose.indexOf('### Pass 2'));
|
|
assert.match(pass1, /one reviewer never duplicates itself/i);
|
|
assert.match(pass1, /different titles are two\s+findings/);
|
|
assert.match(prose, /k-th row on one triplet\s+gets it over `rule_key#k`/);
|
|
});
|
|
|
|
test('runContract — the #k id counts rows of one triplet in the final list, so a row whose sibling was set aside gets the plain id (review of b47cabd)', () => {
|
|
const long = 'x'.repeat(120);
|
|
const result = runContract([{ reviewer: 'claude-code-review', findings: [
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: `EAGAIN ${long}`, detail: 'd', recommended_action: '' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: `Lone CR ${long}`, detail: 'd', recommended_action: 'Split on CR too.' },
|
|
] }]);
|
|
assert.equal(result.findings.length, 1, JSON.stringify(result.findings.map((f) => f.title)));
|
|
assert.match(result.findings[0].title, /^Lone CR/);
|
|
assert.equal(result.findings[0].id, computeFindingId('bin/slug.mjs', 46, 'MISSING_ERROR_HANDLING'), 'what fill-finding-ids computes for a review.md holding this row alone');
|
|
assert.equal(result.trimmed_count, 1, 'the trimmed row that Pass 2 set aside is not counted');
|
|
});
|