580 lines
32 KiB
JavaScript
580 lines
32 KiB
JavaScript
// tests/lib/coordinator-contract.test.mjs
|
|
// SKAL-1·4a — deterministic test of the review-coordinator contract subset.
|
|
//
|
|
// Inline reviewer-JSON fixtures (idiom: tests/validators/brief-validator.test.mjs).
|
|
// Hermetic: no LLM, network, time, or randomness.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { computeFindingId } from '../../lib/parsers/finding-id.mjs';
|
|
import {
|
|
severityRank,
|
|
ingest,
|
|
dedupByTriplet,
|
|
judgeFilter,
|
|
reasonablenessFilter,
|
|
computeVerdict,
|
|
classifySuppression,
|
|
REFUTING_REASONS,
|
|
UNVERIFIED_REASONS,
|
|
runContract,
|
|
} from '../../lib/review/coordinator-contract.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
|
|
// ---- Pass 1 — dedup --------------------------------------------------------
|
|
|
|
test('dedupByTriplet — genuine cross-reviewer collapse (identical triplet) → 1, raised_by both', () => {
|
|
// Both reviewers flag the SAME (file,line,rule_key) triplet — this is the
|
|
// real collapse the coordinator performs.
|
|
const findings = [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header', reviewer: 'correctness' },
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1 violated', reviewer: 'conformance' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1, 'identical triplets must collapse to one');
|
|
assert.deepEqual([...out[0].raised_by].sort(), ['conformance', 'correctness']);
|
|
});
|
|
|
|
test('dedupByTriplet — README #2 two DIFFERENT rule_keys at same file:line do NOT collapse', () => {
|
|
// The bakeoff-rich README marks issue #2 dual-flaggable via SECURITY_INJECTION
|
|
// AND NON_GOAL_VIOLATED. Those are DIFFERENT triplets → distinct defects → kept.
|
|
const findings = [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' },
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', reviewer: 'conformance' },
|
|
];
|
|
assert.equal(dedupByTriplet(findings).length, 2);
|
|
});
|
|
|
|
test('dedupByTriplet — survivor is the highest-severity finding in the group', () => {
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'MINOR', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', reviewer: 'correctness' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].severity, 'BLOCKER');
|
|
});
|
|
|
|
test('dedupByTriplet — two DIFFERENT findings from one reviewer on one triplet stay two rows with their own ids (trial 5, finding 7)', () => {
|
|
// /code-review gave both the catch-all key at bin/slug.mjs:46; one merged row lost the second finding's action.
|
|
const findings = [
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'EAGAIN on a non-blocking stdin', recommended_action: 'Read stdin with a stream.', reviewer: 'claude-code-review' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'A lone \\r does not split lines', recommended_action: 'Split on \\r too.', reviewer: 'claude-code-review' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'EAGAIN on a non-blocking stdin', recommended_action: 'Read stdin with a stream.', reviewer: 'code-correctness-reviewer' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 2, JSON.stringify(out.map((f) => f.title)));
|
|
assert.deepEqual(out.map((f) => f.recommended_action), ['Read stdin with a stream.', 'Split on \\r too.']);
|
|
assert.equal(out[0].id, computeFindingId('bin/slug.mjs', 46, 'CODE_REVIEW_DEFECT'));
|
|
assert.equal(out[1].id, computeFindingId('bin/slug.mjs', 46, 'CODE_REVIEW_DEFECT#2'));
|
|
assert.deepEqual([...out[0].raised_by].sort(), ['claude-code-review', 'code-correctness-reviewer']);
|
|
});
|
|
|
|
test('dedupByTriplet — severity tie breaks toward the conformance reviewer', () => {
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'corr', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NON_GOAL_VIOLATED', severity: 'BLOCKER', title: 'conf', reviewer: 'conformance' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].reviewer, 'conformance');
|
|
});
|
|
|
|
// ---- severity ranking (4-tier, incl. synthetic SUGGESTION) -----------------
|
|
|
|
test('severityRank — orders the full 4-tier SEVERITY_VALUES incl. SUGGESTION', () => {
|
|
assert.ok(severityRank('BLOCKER') < severityRank('MAJOR'));
|
|
assert.ok(severityRank('MAJOR') < severityRank('MINOR'));
|
|
assert.ok(severityRank('MINOR') < severityRank('SUGGESTION'));
|
|
});
|
|
|
|
test('dedupByTriplet — SUGGESTION vs MINOR (synthetic) keeps the MINOR survivor', () => {
|
|
// gold corpus has no SUGGESTION; exercise the 4th tier synthetically.
|
|
const findings = [
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'SUGGESTION', reviewer: 'correctness' },
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', reviewer: 'correctness' },
|
|
];
|
|
const out = dedupByTriplet(findings);
|
|
assert.equal(out.length, 1);
|
|
assert.equal(out[0].severity, 'MINOR');
|
|
});
|
|
|
|
// ---- Pass 4 — verdict thresholds -------------------------------------------
|
|
|
|
test('computeVerdict — BLOCKER>=1 → BLOCK, MAJOR-only → WARN, else ALLOW', () => {
|
|
assert.equal(computeVerdict([{ severity: 'BLOCKER' }, { severity: 'MAJOR' }]).verdict, 'BLOCK');
|
|
assert.equal(computeVerdict([{ severity: 'MAJOR' }, { severity: 'MINOR' }]).verdict, 'WARN');
|
|
assert.equal(computeVerdict([{ severity: 'MINOR' }]).verdict, 'ALLOW');
|
|
assert.equal(computeVerdict([{ severity: 'SUGGESTION' }]).verdict, 'ALLOW');
|
|
assert.equal(computeVerdict([]).verdict, 'ALLOW');
|
|
});
|
|
|
|
test('computeVerdict — counts each severity tier', () => {
|
|
const { counts } = computeVerdict([
|
|
{ severity: 'BLOCKER' }, { severity: 'BLOCKER' }, { severity: 'MAJOR' }, { severity: 'MINOR' },
|
|
]);
|
|
assert.deepEqual(counts, { BLOCKER: 2, MAJOR: 1, MINOR: 1, SUGGESTION: 0 });
|
|
});
|
|
|
|
// ---- Pass 3 — reasonableness -----------------------------------------------
|
|
|
|
test('reasonablenessFilter — citation-less is REFUTED, unknown rule_key is UNVERIFIED, severity mismatch corrected', () => {
|
|
// Contract change (fail-closed): only `no-citation` refutes. An ad-hoc
|
|
// rule_key is a real defect wearing the wrong label — v5.1.1 high-effort mode
|
|
// already keeps those, normalised to PLAN_EXECUTE_DRIFT.
|
|
const r = reasonablenessFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'BLOCKER' }, // unknown → unverified
|
|
{ file: '', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // no file → drop
|
|
{ file: 'x.mjs', line: -1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }, // line < 0 → drop
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR' }, // catalogue is MAJOR → correct, keep
|
|
]);
|
|
assert.equal(r.kept.length, 1);
|
|
assert.equal(r.dropped.length, 2);
|
|
assert.deepEqual(r.dropped.map((f) => f.suppressed_reason), ['no-citation', 'no-citation']);
|
|
assert.equal(r.unverified.length, 1);
|
|
assert.equal(r.unverified[0].suppressed_reason, 'unknown-rule_key');
|
|
assert.equal(r.kept[0].severity, 'MAJOR');
|
|
assert.equal(r.kept[0].original_severity, 'MINOR');
|
|
});
|
|
|
|
// ---- Pass 2 — judge --------------------------------------------------------
|
|
|
|
test('judgeFilter — an over-long MAJOR title is kept whole; an empty recommended_action is UNVERIFIED', () => {
|
|
// Length never removes a finding (ordre 5, P11). Actionability reads a
|
|
// `.length` and never examines the claim, so it cannot refute the finding.
|
|
// `dropped` is empty here on purpose — the refuting Pass 2 filter (Accuracy)
|
|
// is the one this deterministic subset excludes.
|
|
const longTitle = 'x'.repeat(101);
|
|
const j = judgeFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: longTitle }, // MAJOR → kept whole
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok', recommended_action: ' ' }, // empty action → unverified
|
|
{ file: 'x.mjs', line: 3, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'ok' }, // keep (no action field is fine)
|
|
]);
|
|
assert.equal(j.kept.length, 2);
|
|
assert.equal(j.kept[0].title, longTitle, 'a MAJOR keeps every character');
|
|
assert.equal(j.dropped.length, 0);
|
|
assert.equal(j.trimmed.length, 0);
|
|
assert.deepEqual(j.unverified.map((f) => f.suppressed_reason), ['actionability:empty']);
|
|
});
|
|
|
|
test('judgeFilter — only MINOR/SUGGESTION text is trimmed, and every trim is counted', () => {
|
|
const j = judgeFilter([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 't'.repeat(150), detail: 'd'.repeat(1200) },
|
|
{ file: 'x.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'short', detail: 'short' },
|
|
]);
|
|
assert.equal(j.kept.length, 2, 'a trimmed finding is still kept');
|
|
assert.equal(j.trimmed.length, 1);
|
|
assert.ok(j.kept[0].title.length <= 100 && j.kept[0].detail.length <= 800);
|
|
assert.deepEqual(j.kept[0].trimmed_fields, ['title', 'detail']);
|
|
assert.ok(j.kept[0].detail.endsWith('[…trimmed]'), 'the cut is visible in the text');
|
|
});
|
|
|
|
test('judgeFilter — a trimmed MINOR keeps its "Also flagged by" provenance', () => {
|
|
const [f] = dedupByTriplet([
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'a', detail: 'q'.repeat(900), reviewer: 'code-correctness-reviewer' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'b', detail: 'r', reviewer: 'claude-code-review' },
|
|
]);
|
|
const j = judgeFilter([f]);
|
|
assert.ok(j.kept[0].detail.length <= 800);
|
|
assert.match(j.kept[0].detail, /Also flagged by claude-code-review: b\.$/);
|
|
});
|
|
|
|
test('judgeFilter — the trim decision reads the catalogue tier, not the reviewer label', () => {
|
|
// MISSING_TEST is MAJOR in the catalogue; a reviewer calling it MINOR must
|
|
// not get its text cut.
|
|
const detail = 'd'.repeat(900);
|
|
const j = judgeFilter([{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MINOR', title: 't', detail }]);
|
|
assert.equal(j.trimmed.length, 0);
|
|
assert.equal(j.kept[0].detail, detail);
|
|
});
|
|
|
|
// ---- ingest ----------------------------------------------------------------
|
|
|
|
test('ingest — collects findings from valid payloads, skips invalid ones', () => {
|
|
const { findings, skipped } = ingest([
|
|
{ reviewer: 'correctness', findings: [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] },
|
|
{ reviewer: 'bad', findings: [{ line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR' }] }, // missing file → invalid → skipped
|
|
]);
|
|
assert.equal(findings.length, 1);
|
|
assert.equal(findings[0].reviewer, 'correctness');
|
|
assert.equal(skipped.length, 1);
|
|
});
|
|
|
|
// ---- end-to-end ------------------------------------------------------------
|
|
|
|
test('runContract — two reviewers, dual-flag triplet collapses, verdict BLOCK', () => {
|
|
const result = runContract([
|
|
{ reviewer: 'correctness', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'algo from header' },
|
|
{ file: 'lib/auth/refresh.mjs', line: 0, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'no concurrent test' },
|
|
] },
|
|
{ reviewer: 'conformance', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'NG1' }, // collapses with correctness's
|
|
{ file: 'lib/handlers/login.mjs', line: 17, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: '200 not 401' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 3, '4 raw findings, jwt:19 SECURITY_INJECTION collapses → 3');
|
|
const jwt = result.findings.find((f) => f.file === 'lib/auth/jwt.mjs');
|
|
assert.deepEqual([...jwt.raised_by].sort(), ['conformance', 'correctness']);
|
|
});
|
|
|
|
test('runContract — deterministic: identical input yields identical output', () => {
|
|
const input = [
|
|
{ reviewer: 'correctness', findings: [{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 't' }] },
|
|
{ reviewer: 'conformance', findings: [{ file: 'b.mjs', line: 2, rule_key: 'UNIMPLEMENTED_CRITERION', severity: 'BLOCKER', title: 'u' }] },
|
|
];
|
|
assert.deepEqual(runContract(input), runContract(input));
|
|
});
|
|
|
|
// ---- Fail-closed: the `unverified` bucket (ORDRE 834432937) -----------------
|
|
//
|
|
// The defect: a finding REMOVED by Pass 2/Pass 3, and a reviewer whose payload
|
|
// was thrown away or never arrived, are all arithmetically identical to a
|
|
// finding that never existed -- they push the verdict toward ALLOW. Measured
|
|
// before the fix (probe, 2026-09-01): an over-long-title BLOCKER -> ALLOW; a
|
|
// payload with one ad-hoc rule_key -> the whole payload skipped, its valid
|
|
// BLOCKER sibling gone -> ALLOW.
|
|
//
|
|
// The rule under test: a removal is `dropped` ONLY when the test refutes the
|
|
// finding as a claim about this codebase. Every other removal is `unverified`,
|
|
// and a non-empty `unverified` -- or a reviewer that did not report -- forbids
|
|
// ALLOW.
|
|
|
|
test('classifySuppression — only no-citation refutes; form and taxonomy failures are unverified', () => {
|
|
assert.equal(classifySuppression('no-citation'), 'refuted',
|
|
'a finding that names no location makes no checkable claim');
|
|
assert.equal(classifySuppression('succinctness:title'), 'unverified');
|
|
assert.equal(classifySuppression('succinctness:detail'), 'unverified');
|
|
assert.equal(classifySuppression('actionability:empty'), 'unverified');
|
|
assert.equal(classifySuppression('unknown-rule_key'), 'unverified');
|
|
assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified');
|
|
assert.equal(classifySuppression('something-nobody-declared'), 'unverified',
|
|
'an unclassified reason must fail CLOSED, not open');
|
|
});
|
|
|
|
test('computeVerdict — non-empty unverified forbids ALLOW but never downgrades BLOCK or WARN', () => {
|
|
const u = [{ file: 'x.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'BLOCKER' }];
|
|
const withUnverified = computeVerdict([], { unverified: u });
|
|
assert.equal(withUnverified.verdict, 'WARN', 'ALLOW is forbidden while anything is unverified');
|
|
assert.deepEqual(withUnverified.counts, { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 },
|
|
'the unverified finding is NOT counted into a severity tier');
|
|
assert.ok(withUnverified.allow_blocked_by.length > 0);
|
|
|
|
assert.equal(computeVerdict([{ severity: 'BLOCKER' }], { unverified: u }).verdict, 'BLOCK',
|
|
'BLOCK stands regardless of the unverified bucket');
|
|
assert.equal(computeVerdict([{ severity: 'MAJOR' }], { unverified: u }).verdict, 'WARN');
|
|
assert.equal(computeVerdict([], { unverified: [] }).verdict, 'ALLOW',
|
|
'known-positive control: an empty unverified bucket still allows ALLOW');
|
|
});
|
|
|
|
test('computeVerdict — a reviewer that did not report forbids ALLOW', () => {
|
|
const r = computeVerdict([], { missingReviewers: ['brief-conformance-reviewer'] });
|
|
assert.equal(r.verdict, 'WARN');
|
|
assert.ok(r.allow_blocked_by.some((x) => x.includes('brief-conformance-reviewer')));
|
|
});
|
|
|
|
test('runContract — a BLOCKER with an over-long title is kept and yields BLOCK', () => {
|
|
// Before ordre 5 this finding went to `unverified` on `succinctness:title`
|
|
// (the fail-closed floor kept it from ALLOW, but it fell out of the list).
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'x'.repeat(101), detail: 'algo taken from the JWT header' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 1);
|
|
assert.equal(result.unverified.length, 0);
|
|
assert.equal(result.suppressed.length, 0);
|
|
});
|
|
|
|
test('runContract — declined_to_judge lists every unverified finding with its reason; trimmed_count counts trims', () => {
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'a.mjs', line: 1, rule_key: 'MISSING_TEST', severity: 'MAJOR', title: 'empty action', recommended_action: '' },
|
|
{ file: 'b.mjs', line: 2, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'long', detail: 'z'.repeat(850), recommended_action: 'Guard it.' },
|
|
] },
|
|
]);
|
|
assert.equal(result.declined_to_judge.length, 1);
|
|
assert.deepEqual(
|
|
{ file: result.declined_to_judge[0].file, reason: result.declined_to_judge[0].reason },
|
|
{ file: 'a.mjs', reason: 'actionability:empty' },
|
|
);
|
|
assert.equal(result.trimmed_count, 1);
|
|
assert.equal(result.counts.MINOR, 1, 'the trimmed MINOR still counts in its tier');
|
|
});
|
|
|
|
test('runContract — a schema-invalid payload cannot yield ALLOW (an unread reviewer is an absent one)', () => {
|
|
// Measured: one ad-hoc rule_key invalidates the WHOLE payload at ingest, so a
|
|
// valid BLOCKER sibling disappears with it. That must not read as "clean".
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/jwt.mjs', line: 19, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER', title: 'real' },
|
|
{ file: 'x.mjs', line: 1, rule_key: 'NOPE_KEY', severity: 'MINOR', title: 'ad-hoc key' },
|
|
] },
|
|
]);
|
|
assert.equal(result.skipped.length, 1);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
assert.ok(result.allow_blocked_by.some((x) => x.includes('code-correctness-reviewer')));
|
|
});
|
|
|
|
test('runContract — a reviewer named in expectedReviewers that never reported cannot yield ALLOW', () => {
|
|
const result = runContract(
|
|
[{ reviewer: 'code-correctness-reviewer', findings: [] }],
|
|
{ expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] },
|
|
);
|
|
assert.deepEqual(result.missing_reviewers, ['brief-conformance-reviewer']);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
});
|
|
|
|
test('runContract — known-positive control: every reviewer reported, nothing suppressed → ALLOW', () => {
|
|
// Proves ALLOW is still REACHABLE. Without this, "no ALLOW" is not a
|
|
// fail-closed contract, only a broken one.
|
|
const result = runContract(
|
|
[
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'a.mjs', line: 1, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: 'unguarded await', recommended_action: 'Wrap the await in a try/catch.' },
|
|
] },
|
|
{ reviewer: 'brief-conformance-reviewer', findings: [] },
|
|
],
|
|
{ expectedReviewers: ['code-correctness-reviewer', 'brief-conformance-reviewer'] },
|
|
);
|
|
assert.equal(result.verdict, 'ALLOW');
|
|
assert.equal(result.unverified.length, 0);
|
|
assert.deepEqual(result.missing_reviewers, []);
|
|
assert.deepEqual(result.allow_blocked_by, []);
|
|
});
|
|
|
|
test('classifySuppression — the refuting reasons the LLM coordinator emits are declared here too', () => {
|
|
// agents/review-coordinator.md Pass 2 "Accuracy" and Pass 3 "Non-existent
|
|
// file" DO refute (a citation outside the repo root, a file absent from both
|
|
// tree and diff). Both are fs/judgement branches this deterministic subset
|
|
// excludes, but the vocabulary is owned here so prose and lib cannot drift.
|
|
assert.equal(classifySuppression('accuracy:refuted'), 'refuted');
|
|
assert.equal(classifySuppression('file-existence:refuted'), 'refuted');
|
|
assert.equal(classifySuppression('file-existence:indeterminate'), 'unverified',
|
|
'unresolvable must never collapse into refuted');
|
|
});
|
|
|
|
test('classifySuppression — a line past the end of the file is unverified, never refuted', () => {
|
|
// Fail-closed already: an undeclared reason is unverified. Holds before and
|
|
// after the reason is declared (agents/review-coordinator.md Pass 2 Accuracy).
|
|
assert.equal(classifySuppression('accuracy:line-out-of-range'), 'unverified');
|
|
});
|
|
|
|
test('[CL] accuracy:line-out-of-range is a declared unverified reason, not a refuting one', () => {
|
|
// A line number alone never refutes a finding: the citation is wrong, the
|
|
// claim is unchecked. Declaring it here makes the vocabulary test below hold
|
|
// the word in agents/review-coordinator.md.
|
|
assert.ok(UNVERIFIED_REASONS.includes('accuracy:line-out-of-range'),
|
|
'accuracy:line-out-of-range is not in UNVERIFIED_REASONS');
|
|
assert.equal(REFUTING_REASONS.has('accuracy:line-out-of-range'), false,
|
|
'accuracy:line-out-of-range must never refute');
|
|
});
|
|
|
|
test('suppression vocabulary — the two sets are disjoint and every reason is documented in the prose', () => {
|
|
const refuting = [...REFUTING_REASONS];
|
|
const overlap = refuting.filter((r) => UNVERIFIED_REASONS.includes(r));
|
|
assert.deepEqual(overlap, [], 'a reason cannot be both refuting and unverified');
|
|
|
|
const prose = readFileSync(join(ROOT, 'agents/review-coordinator.md'), 'utf-8');
|
|
assert.ok(prose.includes('review-coordinator'), 'known-positive control: the prose file loaded');
|
|
for (const reason of [...refuting, ...UNVERIFIED_REASONS]) {
|
|
assert.ok(prose.includes(reason),
|
|
`reason "${reason}" is declared in the lib but never documented in agents/review-coordinator.md`);
|
|
}
|
|
});
|
|
|
|
test('runContract — an anonymous invalid payload is unattributable, not a reviewer named "unnamed reviewer"', () => {
|
|
// `validateFindings` only WARNS on a missing `reviewer`, so a payload can
|
|
// fail schema while carrying no name. Reporting it as a reviewer name
|
|
// invents an agent nobody launched, and double-counts with expectedReviewers
|
|
// when they are in fact the same failure.
|
|
const result = runContract(
|
|
[{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }],
|
|
{ expectedReviewers: ['code-correctness-reviewer'] },
|
|
);
|
|
assert.deepEqual(result.missing_reviewers, ['code-correctness-reviewer'],
|
|
'missing_reviewers carries real names only');
|
|
assert.equal(result.unattributable_payloads, 1);
|
|
assert.ok(result.allow_blocked_by.some((x) => x.startsWith('unattributable-payload')));
|
|
assert.ok(!result.allow_blocked_by.some((x) => x.includes('unnamed reviewer')));
|
|
});
|
|
|
|
test('runContract — an anonymous invalid payload forbids ALLOW on its own, with no expectedReviewers', () => {
|
|
// The fail-closed floor must not depend on the caller passing an expected
|
|
// set: without this, dropping the name from a payload would restore ALLOW.
|
|
const result = runContract([{ findings: [{ file: 'x.mjs', line: 1, rule_key: 'NOPE', severity: 'MAJOR' }] }]);
|
|
assert.deepEqual(result.missing_reviewers, []);
|
|
assert.equal(result.unattributable_payloads, 1);
|
|
assert.notEqual(result.verdict, 'ALLOW');
|
|
});
|
|
|
|
// ---- No finding is struck for length (ordre 5, P11) --------------------------
|
|
//
|
|
// The defect (STATE, Åpne funn; S118): a finding whose `detail` ran past 800
|
|
// characters went to `unverified` on `succinctness:detail` alone, and S118's most
|
|
// serious finding fell out of the Handover 6 list that way. Length is packaging,
|
|
// not a claim: only MINOR/SUGGESTION text is trimmed, and every trim is counted.
|
|
|
|
test('runContract — a BLOCKER with a 900-character detail survives and yields BLOCK', () => {
|
|
const detail = 'The refresh token is accepted after logout because the revocation list is never consulted. '.repeat(10).slice(0, 900);
|
|
assert.equal(detail.length, 900);
|
|
const result = runContract([
|
|
{ reviewer: 'code-correctness-reviewer', findings: [
|
|
{ file: 'lib/auth/refresh.mjs', line: 31, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER',
|
|
title: 'Revoked refresh token still accepted', detail, recommended_action: 'Check the revocation list before issuing.' },
|
|
] },
|
|
]);
|
|
assert.equal(result.verdict, 'BLOCK');
|
|
assert.equal(result.findings.length, 1, 'the long BLOCKER is a kept finding');
|
|
assert.equal(result.findings[0].detail, detail, 'its detail is not cut');
|
|
assert.equal(result.unverified.length, 0);
|
|
});
|
|
|
|
test('the coordinator prose says one reviewer never duplicates itself, as dedupByTriplet does (trial 5, finding 7)', () => {
|
|
const prose = readFileSync(join(ROOT, 'agents/review-coordinator.md'), 'utf8');
|
|
const pass1 = prose.slice(prose.indexOf('### Pass 1'), prose.indexOf('### Pass 2'));
|
|
assert.match(pass1, /one reviewer never duplicates itself/i);
|
|
assert.match(pass1, /different titles are two\s+findings/);
|
|
assert.match(prose, /k-th row on one triplet\s+gets it over `rule_key#k`/);
|
|
});
|
|
|
|
test('runContract — the #k id counts rows of one triplet in the final list, so a row whose sibling was set aside gets the plain id (review of b47cabd)', () => {
|
|
const long = 'x'.repeat(120);
|
|
const result = runContract([{ reviewer: 'claude-code-review', findings: [
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: `EAGAIN ${long}`, detail: 'd', recommended_action: '' },
|
|
{ file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_ERROR_HANDLING', severity: 'MINOR', title: `Lone CR ${long}`, detail: 'd', recommended_action: 'Split on CR too.' },
|
|
] }]);
|
|
assert.equal(result.findings.length, 1, JSON.stringify(result.findings.map((f) => f.title)));
|
|
assert.match(result.findings[0].title, /^Lone CR/);
|
|
assert.equal(result.findings[0].id, computeFindingId('bin/slug.mjs', 46, 'MISSING_ERROR_HANDLING'), 'what fill-finding-ids computes for a review.md holding this row alone');
|
|
assert.equal(result.trimmed_count, 1, 'the trimmed row that Pass 2 set aside is not counted');
|
|
});
|
|
|
|
// ---- ORD — the rows a finding joins do not depend on arrival order ---------
|
|
|
|
/** Every arrival order of `xs`. */
|
|
function orders(xs) {
|
|
if (xs.length <= 1) return [xs];
|
|
return xs.flatMap((x, i) => orders([...xs.slice(0, i), ...xs.slice(i + 1)]).map((rest) => [x, ...rest]));
|
|
}
|
|
|
|
/** A row's members, rebuilt from its survivor plus each "Also flagged by" line, as sorted `reviewer|title` strings. */
|
|
function members(row) {
|
|
const out = [`${row.reviewer}|${row.title}`];
|
|
for (const m of String(row.detail ?? '').matchAll(/^Also flagged by (.+?): (.+)\.$/gm)) out.push(`${m[1]}|${m[2]}`);
|
|
return out.sort();
|
|
}
|
|
|
|
const ORD_C = 'code-correctness-reviewer';
|
|
const ORD_F = 'brief-conformance-reviewer';
|
|
|
|
test('[ORD] correctness A, conformance B, correctness B give the rows {correctness A} and {conformance B, correctness B} in every arrival order', () => {
|
|
const at = { file: 'bin/slug.mjs', line: 46, rule_key: 'MISSING_TEST', detail: 'd' };
|
|
const cA = { ...at, severity: 'MAJOR', title: 'A', reviewer: ORD_C };
|
|
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
|
|
const cB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_C };
|
|
const expected = [[`${ORD_C}|A`], [`${ORD_F}|B`, `${ORD_C}|B`]];
|
|
const all = orders([['cA', cA], ['fB', fB], ['cB', cB]]);
|
|
assert.equal(all.length, 6);
|
|
for (const order of all) {
|
|
const name = order.map(([n]) => n).join(', ');
|
|
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
|
|
assert.equal(rows.length, 2, `${name}: ${JSON.stringify(rows.map(members))}`);
|
|
const got = rows.map(members).sort((x, y) => x.length - y.length || x.join().localeCompare(y.join()));
|
|
assert.deepEqual(got, expected, `order ${name}`);
|
|
}
|
|
});
|
|
|
|
test('[ORD] with an off-catalogue rule_key and cA MINOR, fB MAJOR, cB MINOR, every arrival order gives {cA} MINOR, {fB, cB} MAJOR and one verdict', () => {
|
|
const at = { file: 'bin/slug.mjs', line: 46, rule_key: 'NOT_IN_THE_CATALOGUE', detail: 'd' };
|
|
const cA = { ...at, severity: 'MINOR', title: 'A', reviewer: ORD_C };
|
|
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
|
|
const cB = { ...at, severity: 'MINOR', title: 'B', reviewer: ORD_C };
|
|
const expected = { [`${ORD_C}|A`]: 'MINOR', [`${ORD_F}|B+${ORD_C}|B`]: 'MAJOR' };
|
|
const verdicts = new Set();
|
|
for (const order of orders([['cA', cA], ['fB', fB], ['cB', cB]])) {
|
|
const name = order.map(([n]) => n).join(', ');
|
|
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
|
|
const bySet = Object.fromEntries(rows.map((r) => [members(r).join('+'), r.severity]));
|
|
assert.deepEqual(bySet, expected, `order ${name}`);
|
|
verdicts.add(JSON.stringify(computeVerdict(rows)));
|
|
}
|
|
assert.equal(verdicts.size, 1, [...verdicts].join(' | '));
|
|
assert.equal(JSON.parse([...verdicts][0]).verdict, 'WARN');
|
|
});
|
|
|
|
/** The kept and unverified rows of a run, as sorted `reviewer|title|severity` strings, so two runs can be compared. */
|
|
function split(result) {
|
|
const key = (f) => `${f.reviewer}|${String(f.title ?? '').trim().toLowerCase()}|${f.severity}`;
|
|
return { kept: result.findings.map(key).sort(), unverified: result.unverified.map(key).sort() };
|
|
}
|
|
|
|
test('[ORD] a full tie on one row: one reviewer\'s two same-title BLOCKERs, one with an empty recommended_action, give one verdict in both arrival orders', () => {
|
|
const at = { file: 'x.mjs', line: 3, rule_key: 'SECURITY_INJECTION', severity: 'BLOCKER' };
|
|
const empty = { ...at, title: 'Injection', recommended_action: '' };
|
|
const full = { ...at, title: 'injection ', recommended_action: 'Quote the argument.' };
|
|
const a = runContract([{ reviewer: ORD_C, findings: [structuredClone(empty), structuredClone(full)] }]);
|
|
const b = runContract([{ reviewer: ORD_C, findings: [structuredClone(full), structuredClone(empty)] }]);
|
|
assert.equal(a.verdict, b.verdict, `order (empty, full) gives ${a.verdict}, order (full, empty) gives ${b.verdict}`);
|
|
assert.deepEqual(split(a), split(b), 'the kept/unverified split');
|
|
});
|
|
|
|
test('[ORD] a full tie on one row: two /code-review MAJORs titled T, one marked as contradicting the brief, give one verdict in both arrival orders', () => {
|
|
const at = { file: 'x.mjs', line: 3, rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR', title: 'T' };
|
|
const marked = { ...at, contradicts_brief: 'Non-Goal: no new runtime dependency' };
|
|
const plain = { ...at };
|
|
const a = runContract([{ reviewer: 'claude-code-review', findings: [structuredClone(marked), structuredClone(plain)] }]);
|
|
const b = runContract([{ reviewer: 'claude-code-review', findings: [structuredClone(plain), structuredClone(marked)] }]);
|
|
assert.equal(a.verdict, b.verdict, `marked first gives ${a.verdict}, unmarked first gives ${b.verdict}`);
|
|
assert.deepEqual(split(a), split(b), 'the kept/unverified split');
|
|
});
|
|
|
|
/** A row's member set, rebuilt from its survivor plus each "Also flagged by" line; an untitled member is named by its rule_key, as the line names it. */
|
|
function memberSet(row) {
|
|
const out = [`${row.reviewer}|${row.title ?? row.rule_key}`];
|
|
for (const m of String(row.detail ?? '').matchAll(/^Also flagged by (.+?): (.+)\.$/gm)) out.push(`${m[1]}|${m[2]}`);
|
|
return out.sort().join('+');
|
|
}
|
|
|
|
test('[ORD] the title classes are placed in alphabetical order: cA, fB, cB and an untitled correctness BLOCKER give {cA, cU} BLOCKER and {fB, cB} MAJOR and one verdict in all 24 orders', () => {
|
|
const K = 'NOT_IN_THE_CATALOGUE';
|
|
const at = { file: 'bin/slug.mjs', line: 46, rule_key: K, detail: 'd' };
|
|
// cA and cU tie on every survivor rule, so the row's first member in the
|
|
// group's order (class A before the untitled findings) is kept: cA, whose
|
|
// empty recommended_action sets the row aside as unverified.
|
|
const cA = { ...at, severity: 'BLOCKER', title: 'A', reviewer: ORD_C, recommended_action: '' };
|
|
const fB = { ...at, severity: 'MAJOR', title: 'B', reviewer: ORD_F };
|
|
const cB = { ...at, severity: 'MINOR', title: 'B', reviewer: ORD_C };
|
|
const cU = { ...at, severity: 'BLOCKER', reviewer: ORD_C };
|
|
const expected = {
|
|
[`${ORD_C}|A+${ORD_C}|${K}`]: 'BLOCKER',
|
|
[`${ORD_F}|B+${ORD_C}|B`]: 'MAJOR',
|
|
};
|
|
const all = orders([['cA', cA], ['fB', fB], ['cB', cB], ['cU', cU]]);
|
|
assert.equal(all.length, 24);
|
|
const verdicts = new Set();
|
|
for (const order of all) {
|
|
const name = order.map(([n]) => n).join(', ');
|
|
const rows = dedupByTriplet(order.map(([, f]) => structuredClone(f)));
|
|
const bySet = Object.fromEntries(rows.map((r) => [memberSet(r), r.severity]));
|
|
assert.deepEqual(bySet, expected, `order ${name}`);
|
|
const j = judgeFilter(rows);
|
|
verdicts.add(JSON.stringify({
|
|
kept: j.kept.map(memberSet).sort(),
|
|
unverified: j.unverified.map(memberSet).sort(),
|
|
verdict: computeVerdict(j.kept, { unverified: j.unverified }).verdict,
|
|
}));
|
|
}
|
|
assert.equal(verdicts.size, 1, [...verdicts].join(' | '));
|
|
assert.deepEqual(JSON.parse([...verdicts][0]), {
|
|
kept: [`${ORD_F}|B+${ORD_C}|B`],
|
|
unverified: [`${ORD_C}|A+${ORD_C}|${K}`],
|
|
verdict: 'WARN',
|
|
});
|
|
});
|