// tests/lib/coordinator-contract.test.mjs // SKAL-1·4a — deterministic test of the review-coordinator contract subset. // // Inline reviewer-JSON fixtures (idiom: tests/validators/brief-validator.test.mjs). // Hermetic: no LLM, network, time, or randomness. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { severityRank, ingest, dedupByTriplet, judgeFilter, reasonablenessFilter, computeVerdict, classifySuppression, REFUTING_REASONS, UNVERIFIED_REASONS, runContract, } from '../../lib/review/coordinator-contract.mjs'; const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); // ---- Pass 1 — dedup -------------------------------------------------------- test('dedupByTriplet — genuine cross-reviewer collapse (identical triplet) → 1, raised_by both', () => { // Both reviewers flag the SAME (file,line,rule_key) triplet — this is the // real collapse the coordinator performs. const findings = [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header', reviewer: 'correctness' }, { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1 violated', reviewer: 'conformance' }, ]; const out = dedupByTriplet(findings); assert.equal(out.length, 1, 'identical triplets must collapse to one'); assert.deepEqual([...out[0].raised_by].sort(), ['conformance', 'correctness']); }); test('dedupByTriplet — README #2 two DIFFERENT rule_keys at same file:line do NOT collapse', () => { // The bakeoff-rich README marks issue #2 dual-flaggable via SECURITY_INJECTION // AND NON_GOAL_VIOLATED. Those are DIFFERENT triplets → distinct defects → kept. const findings = [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' }, { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', reviewer: 'conformance' }, ]; assert.equal(dedupByTriplet(findings).length, 2); }); test('dedupByTriplet — survivor is the highest-severity finding in the group', () => { const findings = [ { file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'MINOR', reviewer: 'correctness' }, { file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' }, ]; const out = dedupByTriplet(findings); assert.equal(out.length, 1); assert.equal(out[0].severity, 'BLOCKER'); }); test('dedupByTriplet — severity tie breaks toward the conformance reviewer', () => { const findings = [ { file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'corr', reviewer: 'correctness' }, { file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'conf', reviewer: 'conformance' }, ]; const out = dedupByTriplet(findings); assert.equal(out.length, 1); assert.equal(out[0].reviewer, 'conformance'); }); // ---- severity ranking (4-tier, incl. synthetic SUGGESTION) ----------------- test('severityRank — orders the full 4-tier SEVERITY_VALUES incl. SUGGESTION', () => { assert.ok(severityRank('BLOCKER') < severityRank('MAJOR')); assert.ok(severityRank('MAJOR') < severityRank('MINOR')); assert.ok(severityRank('MINOR') < severityRank('SUGGESTION')); }); test('dedupByTriplet — SUGGESTION vs MINOR (synthetic) keeps the MINOR survivor', () => { // gold corpus has no SUGGESTION; exercise the 4th tier synthetically. const findings = [ { file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'SUGGESTION', reviewer: 'correctness' }, { file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', reviewer: 'correctness' }, ]; const out = dedupByTriplet(findings); assert.equal(out.length, 1); assert.equal(out[0].severity, 'MINOR'); }); // ---- Pass 4 — verdict thresholds ------------------------------------------- test('computeVerdict — BLOCKER>=1 → BLOCK, MAJOR-only → WARN, else ALLOW', () => { assert.equal(computeVerdict([{ severity: 'BLOCKER' }, { severity: 'MAJOR' }]).verdict, 'BLOCK'); assert.equal(computeVerdict([{ severity: 'MAJOR' }, { severity: 'MINOR' }]).verdict, 'WARN'); assert.equal(computeVerdict([{ severity: 'MINOR' }]).verdict, 'ALLOW'); assert.equal(computeVerdict([{ severity: 'SUGGESTION' }]).verdict, 'ALLOW'); assert.equal(computeVerdict([]).verdict, 'ALLOW'); }); test('computeVerdict — counts each severity tier', () => { const { counts } = computeVerdict([ { severity: 'BLOCKER' }, { severity: 'BLOCKER' }, { severity: 'MAJOR' }, { severity: 'MINOR' }, ]); assert.deepEqual(counts, { BLOCKER: 2, MAJOR: 1, MINOR: 1, SUGGESTION: 0 }); }); // ---- Pass 3 — reasonableness ----------------------------------------------- test('reasonablenessFilter — citation-less is REFUTED, unknown rule_key is UNVERIFIED, severity mismatch corrected', () => { // Contract change (fail-closed): only `no-citation` refutes. An ad-hoc // rule_key is a real defect wearing the wrong label — v5.1.1 high-effort mode // already keeps those, normalised to PLAN_EXECUTE_DRIFT. const r = reasonablenessFilter([ { file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'BLOCKER' }, // unknown → unverified { file: '', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // no file → drop { file: 'x.mjs', line: -1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // line < 0 → drop { file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR' }, // catalogue is MAJOR → correct, keep ]); assert.equal(r.kept.length, 1); assert.equal(r.dropped.length, 2); assert.deepEqual(r.dropped.map((f) => f.suppressed_reason), ['no-citation', 'no-citation']); assert.equal(r.unverified.length, 1); assert.equal(r.unverified[0].suppressed_reason, 'unknown-rule_key'); assert.equal(r.kept[0].severity, 'MAJOR'); assert.equal(r.kept[0].original_severity, 'MINOR'); }); // ---- Pass 2 — judge -------------------------------------------------------- test('judgeFilter — over-long title and empty recommended_action are UNVERIFIED, not dropped', () => { // Contract change (fail-closed): both implemented Pass 2 tests read a // `.length` and never examine the claim, so neither refutes the finding. // `dropped` is empty here on purpose — the refuting Pass 2 filter (Accuracy) // is the one this deterministic subset excludes. const j = judgeFilter([ { file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'x'.repeat(101) }, // too long → unverified { file: 'x.mjs', line: 2, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok', recommended_action: ' ' }, // empty action → unverified { file: 'x.mjs', line: 3, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok' }, // keep (no action field is fine) ]); assert.equal(j.kept.length, 1); assert.equal(j.dropped.length, 0); assert.equal(j.unverified.length, 2); assert.deepEqual(j.unverified.map((f) => f.suppressed_reason), ['succinctness:title', 'actionability:empty']); }); // ---- ingest ---------------------------------------------------------------- test('ingest — collects findings from valid payloads, skips invalid ones', () => { const { findings, skipped } = ingest([ { reviewer: 'correctness', findings: [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, { reviewer: 'bad', findings: [{ line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, // missing file → invalid → skipped ]); assert.equal(findings.length, 1); assert.equal(findings[0].reviewer, 'correctness'); assert.equal(skipped.length, 1); }); // ---- end-to-end ------------------------------------------------------------ test('runContract — two reviewers, dual-flag triplet collapses, verdict BLOCK', () => { const result = runContract([ { reviewer: 'correctness', findings: [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header' }, { file: 'lib/auth/refresh.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'no concurrent test' }, ] }, { reviewer: 'conformance', findings: [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1' }, // collapses with correctness's { file: 'lib/handlers/login.mjs', line: 17, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: '200 not 401' }, ] }, ]); assert.equal(result.verdict, 'BLOCK'); assert.equal(result.findings.length, 3, '4 raw findings, jwt:19 SECURITY_INJECTION collapses → 3'); const jwt = result.findings.find((f) => f.file === 'lib/auth/jwt.mjs'); assert.deepEqual([...jwt.raised_by].sort(), ['conformance', 'correctness']); }); test('runContract — deterministic: identical input yields identical output', () => { const input = [ { reviewer: 'correctness', findings: [{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 't' }] }, { reviewer: 'conformance', findings: [{ file: 'b.mjs', line: 2, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: 'u' }] }, ]; assert.deepEqual(runContract(input), runContract(input)); }); // ---- Fail-closed: the `unverified` bucket (ORDRE 834432937) ----------------- // // The defect: a finding REMOVED by Pass 2/Pass 3, and a reviewer whose payload // was thrown away or never arrived, are all arithmetically identical to a // finding that never existed -- they push the verdict toward ALLOW. Measured // before the fix (probe, 2026-09-01): an over-long-title BLOCKER -> ALLOW; a // payload with one ad-hoc rule_key -> the whole payload skipped, its valid // BLOCKER sibling gone -> ALLOW. // // The rule under test: a removal is `dropped` ONLY when the test refutes the // finding as a claim about this codebase. Every other removal is `unverified`, // and a non-empty `unverified` -- or a reviewer that did not report -- forbids // ALLOW. test('classifySuppression — only no-citation refutes; form and taxonomy failures are unverified', () => { assert.equal(classifySuppression('no-citation'), 'refuted', 'a finding that names no location makes no checkable claim'); assert.equal(classifySuppression('succinctness:title'), 'unverified'); assert.equal(classifySuppression('succinctness:detail'), 'unverified'); assert.equal(classifySuppression('actionability:empty'), 'unverified'); assert.equal(classifySuppression('unknown-rule_key'), 'unverified'); assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified'); assert.equal(classifySuppression('something-nobody-declared'), 'unverified', 'an unclassified reason must fail CLOSED, not open'); }); test('computeVerdict — non-empty unverified forbids ALLOW but never downgrades BLOCK or WARN', () => { const u = [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'BLOCKER' }]; const withUnverified = computeVerdict([], { unverified: u }); assert.equal(withUnverified.verdict, 'WARN', 'ALLOW is forbidden while anything is unverified'); assert.deepEqual(withUnverified.counts, { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }, 'the unverified finding is NOT counted into a severity tier'); assert.ok(withUnverified.allow_blocked_by.length > 0); assert.equal(computeVerdict([{ severity: 'BLOCKER' }], { unverified: u }).verdict, 'BLOCK', 'BLOCK stands regardless of the unverified bucket'); assert.equal(computeVerdict([{ severity: 'MAJOR' }], { unverified: u }).verdict, 'WARN'); assert.equal(computeVerdict([], { unverified: [] }).verdict, 'ALLOW', 'known-positive control: an empty unverified bucket still allows ALLOW'); }); test('computeVerdict — a reviewer that did not report forbids ALLOW', () => { const r = computeVerdict([], { missingReviewers: ['brief-conformance-reviewer'] }); assert.equal(r.verdict, 'WARN'); assert.ok(r.allow_blocked_by.some((x) => x.includes('brief-conformance-reviewer'))); }); test('runContract — a BLOCKER dropped for an over-long title cannot yield ALLOW', () => { // Pass 2 succinctness reads `.length`. It never examines the claim, so it // cannot establish the finding is unreal -- it is unverified, not refuted. const result = runContract([ { reviewer: 'code-correctness-reviewer', findings: [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'x'.repeat(101), detail: 'algo taken from the JWT header' }, ] }, ]); assert.notEqual(result.verdict, 'ALLOW', 'an unsubstantiated BLOCKER must never clear the review'); assert.equal(result.findings.length, 0, 'it is still not a kept finding'); assert.equal(result.unverified.length, 1); assert.equal(result.unverified[0].suppressed_reason, 'succinctness:title'); assert.equal(result.suppressed.length, 1, 'suppressed stays the union of dropped + unverified'); }); test('runContract — a schema-invalid payload cannot yield ALLOW (an unread reviewer is an absent one)', () => { // Measured: one ad-hoc rule_key invalidates the WHOLE payload at ingest, so a // valid BLOCKER sibling disappears with it. That must not read as "clean". const result = runContract([ { reviewer: 'code-correctness-reviewer', findings: [ { file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'real' }, { file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'MINOR', title: 'ad-hoc key' }, ] }, ]); assert.equal(result.skipped.length, 1); assert.notEqual(result.verdict, 'ALLOW'); assert.ok(result.allow_blocked_by.some((x) => x.includes('code-correctness-reviewer'))); }); test('runContract — a reviewer named in expectedReviewers that never reported cannot yield ALLOW', () => { const result = runContract( [{ reviewer: 'code-correctness-reviewer', findings: [] }], { expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] }, ); assert.deepEqual(result.missing_reviewers, ['brief-conformance-reviewer']); assert.notEqual(result.verdict, 'ALLOW'); }); test('runContract — known-positive control: every reviewer reported, nothing suppressed → ALLOW', () => { // Proves ALLOW is still REACHABLE. Without this, "no ALLOW" is not a // fail-closed contract, only a broken one. const result = runContract( [ { reviewer: 'code-correctness-reviewer', findings: [ { file: 'a.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'unguarded await', recommended_action: 'Wrap the await in a try/catch.' }, ] }, { reviewer: 'brief-conformance-reviewer', findings: [] }, ], { expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] }, ); assert.equal(result.verdict, 'ALLOW'); assert.equal(result.unverified.length, 0); assert.deepEqual(result.missing_reviewers, []); assert.deepEqual(result.allow_blocked_by, []); }); test('classifySuppression — the refuting reasons the LLM coordinator emits are declared here too', () => { // agents/review-coordinator.md Pass 2 "Accuracy" and Pass 3 "Non-existent // file" DO refute (a citation outside the repo root, a file absent from both // tree and diff). Both are fs/judgement branches this deterministic subset // excludes, but the vocabulary is owned here so prose and lib cannot drift. assert.equal(classifySuppression('accuracy:refuted'), 'refuted'); assert.equal(classifySuppression('file-existence:refuted'), 'refuted'); assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified', 'unresolvable must never collapse into refuted'); }); test('suppression vocabulary — the two sets are disjoint and every reason is documented in the prose', () => { const refuting = [...REFUTING_REASONS]; const overlap = refuting.filter((r) => UNVERIFIED_REASONS.includes(r)); assert.deepEqual(overlap, [], 'a reason cannot be both refuting and unverified'); const prose = readFileSync(join(ROOT, 'agents/review-coordinator.md'), 'utf-8'); assert.ok(prose.includes('review-coordinator'), 'known-positive control: the prose file loaded'); for (const reason of [...refuting, ...UNVERIFIED_REASONS]) { assert.ok(prose.includes(reason), `reason "${reason}" is declared in the lib but never documented in agents/review-coordinator.md`); } }); test('runContract — an anonymous invalid payload is unattributable, not a reviewer named "unnamed reviewer"', () => { // `validateFindings` only WARNS on a missing `reviewer`, so a payload can // fail schema while carrying no name. Reporting it as a reviewer name // invents an agent nobody launched, and double-counts with expectedReviewers // when they are in fact the same failure. const result = runContract( [{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }], { expectedReviewers: ['code-correctness-reviewer'] }, ); assert.deepEqual(result.missing_reviewers, ['code-correctness-reviewer'], 'missing_reviewers carries real names only'); assert.equal(result.unattributable_payloads, 1); assert.ok(result.allow_blocked_by.some((x) => x.startsWith('unattributable-payload'))); assert.ok(!result.allow_blocked_by.some((x) => x.includes('unnamed reviewer'))); }); test('runContract — an anonymous invalid payload forbids ALLOW on its own, with no expectedReviewers', () => { // The fail-closed floor must not depend on the caller passing an expected // set: without this, dropping the name from a payload would restore ALLOW. const result = runContract([{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }]); assert.deepEqual(result.missing_reviewers, []); assert.equal(result.unattributable_payloads, 1); assert.notEqual(result.verdict, 'ALLOW'); });