feat(review): the security engine chosen and Claude Security converted (E 3.1)
lib/review/security-engine.mjs (decisions 175, 196, 219): - choose: Claude Security when available and base..head has committed changes; else /security-review when origin/HEAD resolves and origin/HEAD...HEAD is non-empty; else no engine (no-engine, no-origin, empty-diff), which is "not measured", never 0. A git failure or bad usage is exit 2, never a reason. - convert: reads the newest CLAUDE-SECURITY-<ts>/ report whose stamp is a changes scan of exactly base..head; confidence high becomes engine findings through toReviewerPayload, the rest go to below_threshold and never into findings. An unverified scan, missing researcher counts, fewer researchers returned than dispatched, an empty diff or scope, or a malformed record is exit 2. - NO_ENGINE_REVIEWER: with no engine chosen the expected reviewer security-engine never reports, so the verdict gate exits 2. ENGINES gains claude-security (its own reviewer and brief_ref, the engines' key) with a per-engine label for the written action; it is a security engine in isSecurityFinding, RAN_ENGINES, the coordinator contract (out-of-scope and brief-conflict rules) and the coordinator prose; engine:below-threshold is an unverified reason fed like engineOutOfScope. Pins of the changed coordinator prose follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d72d3667f6
commit
cb7c13b408
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
18 changed files with 1045 additions and 44 deletions
|
|
@ -21,6 +21,8 @@ import {
|
|||
REFUTING_REASONS,
|
||||
UNVERIFIED_REASONS,
|
||||
runContract,
|
||||
lowerableBriefConflict,
|
||||
illegalBriefLowering,
|
||||
} from '../../lib/review/coordinator-contract.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
|
|
@ -616,3 +618,46 @@ test('runContract — a MAJOR /code-review defect cited at an unchanged file sti
|
|||
assert.notEqual(c.verdict, 'ALLOW');
|
||||
assert.equal(c.declined_to_judge.length, 1);
|
||||
});
|
||||
|
||||
// ---- E 3.1: Claude Security is a security engine; below its threshold is declined ----
|
||||
|
||||
const csFinding = (over) => ({
|
||||
id: 'c'.repeat(40), severity: 'BLOCKER', rule_key: 'SECURITY_REVIEW_VULNERABILITY',
|
||||
file: 'src/run.mjs', line: 12, brief_ref: 'NFR - security', title: 'Shell command built from the branch name',
|
||||
detail: 'A crafted branch name runs a command', recommended_action: 'Pass the name as an argv element',
|
||||
...over,
|
||||
});
|
||||
|
||||
test('[E3] engine:below-threshold is an unverified reason: declined to judge, not counted, it keeps the verdict from ALLOW', () => {
|
||||
assert.ok(UNVERIFIED_REASONS.includes('engine:below-threshold'));
|
||||
assert.equal(classifySuppression('engine:below-threshold'), 'unverified');
|
||||
const below = csFinding({ engine: 'claude-security', title: 'Token compared with ===' });
|
||||
const c = runContract([{ reviewer: 'claude-security', findings: [] }],
|
||||
{ expectedReviewers: ['claude-security'], engineBelowThreshold: [below] });
|
||||
assert.equal(c.verdict, 'WARN', JSON.stringify(c.allow_blocked_by));
|
||||
assert.equal(c.counts.BLOCKER, 0, 'a below-threshold finding is not counted');
|
||||
assert.deepEqual(c.findings, []);
|
||||
assert.deepEqual(c.declined_to_judge.map((d) => [d.title, d.reason]), [['Token compared with ===', 'engine:below-threshold']]);
|
||||
assert.deepEqual(c.allow_blocked_by, ['unverified:engine:below-threshold (1)']);
|
||||
});
|
||||
|
||||
test('[E3] a Claude Security payload\'s findings carry engine claude-security, read from the payload', () => {
|
||||
const { findings } = ingest([{ reviewer: 'claude-security', findings: [csFinding({ engine: 'code-review' })] }]);
|
||||
assert.equal(findings[0].engine, 'claude-security', 'the engine comes from the payload, never from the finding');
|
||||
});
|
||||
|
||||
test('[E3] a Claude Security finding outside the scope forbids ALLOW, as a /security-review one does', () => {
|
||||
const c = runContract([], { engineOutOfScope: [csFinding({ engine: 'claude-security', file: 'lib/old.mjs' })] });
|
||||
assert.notEqual(c.verdict, 'ALLOW');
|
||||
assert.deepEqual(c.declined_to_judge.map((d) => d.reason), ['engine:out-of-scope']);
|
||||
});
|
||||
|
||||
test('[E3] the brief-conflict rule: a Claude Security finding is only ever marked, never lowered', () => {
|
||||
const marked = csFinding({ engine: 'claude-security', contradicts_brief: 'No shell wrapper (Non-Goal)', rule_key: 'CODE_REVIEW_DEFECT', severity: 'MAJOR' });
|
||||
assert.equal(lowerableBriefConflict(marked), false, 'only a /code-review MAJOR drops');
|
||||
assert.equal(illegalBriefLowering({ ...marked, severity: 'MINOR', brief_lowered_from: 'MAJOR' }), true);
|
||||
assert.equal(illegalBriefLowering(csFinding({ engine: 'claude-security', severity: 'MAJOR' })), true,
|
||||
'a Claude Security finding below its catalogue tier is illegal');
|
||||
assert.equal(illegalBriefLowering(csFinding({ engine: 'claude-security', contradicts_brief: 'No shell wrapper (Non-Goal)' })), false,
|
||||
'marked at its tier is the legal shape');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import { dirname, join } from 'node:path';
|
|||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { computeFindingId, computeFindingIds } from '../../lib/parsers/finding-id.mjs';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { record, writeReport } from '../helpers/claude-security-report.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const COMPARE = join(ROOT, 'lib', 'review', 'engine-compare.mjs');
|
||||
|
|
@ -31,6 +32,7 @@ const FINALIZE = join(ROOT, 'lib', 'review', 'review-finalize.mjs');
|
|||
const FILL = join(ROOT, 'lib', 'review', 'fill-finding-ids.mjs');
|
||||
const VALIDATOR = join(ROOT, 'lib', 'validators', 'review-validator.mjs');
|
||||
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
||||
const SECURITY_CLI = join(ROOT, 'lib', 'review', 'security-engine.mjs');
|
||||
const REVIEW_REL = '.claude/projects/2026-11-02-eng/review.md';
|
||||
const SOURCE_LINE = (engine) => `**Source:** engine ${engine}, put back by the standard-path comparison`;
|
||||
|
||||
|
|
@ -355,3 +357,70 @@ test('[ENG] a review finalized with a put-back has every id computed, no placeho
|
|||
const bgate = bx.gate();
|
||||
assert.equal(bgate.status, 1, `the gate reads BLOCK: ${bgate.stdout}${bgate.stderr}`);
|
||||
});
|
||||
|
||||
// --- E 3.1: Claude Security's payload, converted from its report ------------------
|
||||
|
||||
/** Claude Security's report of `start..end` in the repo root, converted by the real CLI and saved as Phase 4.7 saves it. */
|
||||
function claudeSecurity(fx, records) {
|
||||
writeReport(fx.repo, { head: fx.end, base: fx.start, records });
|
||||
const scope = join(fx.project, '.review-scope.txt');
|
||||
writeFileSync(scope, 'src/a.mjs\nsrc/b.mjs\ndocs/guide/x.md\n');
|
||||
const r = run([SECURITY_CLI, 'convert', '--report-dir', fx.repo, '--base', fx.start, '--head', fx.end, '--scope', scope]);
|
||||
assert.equal(r.status, 0, `security-engine convert: ${r.stdout}${r.stderr}`);
|
||||
writeFileSync(join(fx.project, '.review-engine-payload-claude-security.json'), r.stdout);
|
||||
return JSON.parse(r.stdout);
|
||||
}
|
||||
|
||||
const CS_HIGH = record({ file: 'src/a.mjs', line: 1, title: 'A shell command is built from the branch name' });
|
||||
const CS_MEDIUM = record({ id: 'F2', file: 'src/b.mjs', line: 3, title: 'A path is joined from a query value', confidence: 'medium', cwe_id: 'CWE-22' });
|
||||
|
||||
test('[E3] a Claude Security finding the coordinator dropped is put back; a below-threshold one never is; ran names claude-security', () => {
|
||||
const fx = engRepo();
|
||||
fx.write(engReview({
|
||||
start: fx.start, end: fx.end, findings: [REVIEWER_MINOR],
|
||||
declined: [{ file: 'src/b.mjs', line: 3, rule_key: 'SECURITY_REVIEW_VULNERABILITY', severity: 'BLOCKER', title: CS_MEDIUM.title, reason: 'engine:below-threshold' }],
|
||||
}));
|
||||
const payload = claudeSecurity(fx, [CS_HIGH, CS_MEDIUM]);
|
||||
assert.ok(Array.isArray(payload.below_threshold), `the payload has below_threshold: ${JSON.stringify(payload)}`);
|
||||
assert.deepEqual(payload.below_threshold.map((b) => b.title), [CS_MEDIUM.title]);
|
||||
const r = fx.finalize();
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const text = fx.read();
|
||||
const json = jsonOf(text);
|
||||
const back = on(json.findings, 'src/a.mjs', 1, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(back.length, 1, `the dropped Claude Security finding is back: ${JSON.stringify(json.findings.map((f) => f.title))}`);
|
||||
assert.equal(back[0].engine, 'claude-security');
|
||||
assert.equal(back[0].severity, 'BLOCKER');
|
||||
assert.equal(json.verdict, 'BLOCK');
|
||||
assert.ok(entryOf(sectionOf(text, '## Findings (BLOCKER)'), back[0].id).includes(SOURCE_LINE('claude-security')),
|
||||
`the entry names its source:\n${sectionOf(text, '## Findings (BLOCKER)')}`);
|
||||
assert.equal(json.findings.filter((f) => f.title === CS_MEDIUM.title).length, 0, 'a below-threshold finding is never put back');
|
||||
assert.deepEqual(json.ran.engines, ['claude-security'], 'ran names the engine whose payload is in the project folder');
|
||||
const v = fx.validate();
|
||||
assert.equal(v.status, 0, `review-validator --strict reads ran: ${v.stdout}${v.stderr}`);
|
||||
const gate = fx.gate();
|
||||
assert.equal(gate.status, 1, `the gate reads BLOCK: ${gate.stdout}${gate.stderr}`);
|
||||
});
|
||||
|
||||
test('[E3] the verdict gate reads a Claude Security finding missing from the review as not measured; a below-threshold one alone is not', () => {
|
||||
// Only a below-threshold finding: the review that declines it is read (exit 0, WARN).
|
||||
const declined = [{ file: 'src/b.mjs', line: 3, rule_key: 'SECURITY_REVIEW_VULNERABILITY', severity: 'BLOCKER', title: CS_MEDIUM.title, reason: 'engine:below-threshold' }];
|
||||
// The coordinator's review: the reviewer MINOR, the below-threshold finding declined, so WARN.
|
||||
const warn = (f) => engReview({ start: f.start, end: f.end, findings: [REVIEWER_MINOR], declined })
|
||||
.replace('verdict: ALLOW', 'verdict: WARN')
|
||||
.replace('"verdict": "ALLOW"', '"verdict": "WARN"')
|
||||
.replace('"allow_blocked_by": []', '"allow_blocked_by": ["unverified:engine:below-threshold (1)"]');
|
||||
const fx = engRepo();
|
||||
fx.write(warn(fx));
|
||||
assert.equal(jsonOf(fx.read()).verdict, 'WARN', 'the fixture review is WARN');
|
||||
claudeSecurity(fx, [CS_MEDIUM]);
|
||||
const quiet = fx.gate();
|
||||
assert.equal(quiet.status, 0, `a below-threshold finding is no dropped engine finding: ${quiet.stdout}${quiet.stderr}`);
|
||||
// A high-confidence finding the review does not hold: not measured.
|
||||
const hx = engRepo();
|
||||
hx.write(warn(hx));
|
||||
claudeSecurity(hx, [CS_HIGH, CS_MEDIUM]);
|
||||
const loud = hx.gate();
|
||||
assert.equal(loud.status, 2, loud.stdout + loud.stderr);
|
||||
assert.match(String(parse(loud).reason), /engine finding not in review: BLOCKER SECURITY_REVIEW_VULNERABILITY at src\/a\.mjs:1/, loud.stdout);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1078,3 +1078,11 @@ test('[STP] a round with no readable JSON block at P = 2: prepare is not measure
|
|||
}
|
||||
}
|
||||
});
|
||||
|
||||
// ---- E 3.1: a finding from Claude Security is a security finding ------------------
|
||||
|
||||
test('[E3] isSecurityFinding: a finding from Claude Security is a security finding whatever its key, as one from /security-review is', () => {
|
||||
assert.equal(rounds.isSecurityFinding({ engine: 'claude-security', rule_key: 'CODE_REVIEW_DEFECT' }), true);
|
||||
assert.equal(rounds.isSecurityFinding({ engine: 'security-review', rule_key: 'CODE_REVIEW_DEFECT' }), true);
|
||||
assert.equal(rounds.isSecurityFinding({ engine: 'code-review', rule_key: 'CODE_REVIEW_DEFECT' }), false, 'control: /code-review is not a security engine');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -114,3 +114,16 @@ test('[V12] a malformed round, pre-existing or ran field at 1.2 fails with its c
|
|||
assert.ok(codes(r.errors).includes(code), `${name}: ${code} expected, got ${JSON.stringify(r.errors)}`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- E 3.1: Claude Security is an engine that ran ---------------------------------
|
||||
|
||||
test('[E3] ran.engines may name claude-security; an unknown engine is still refused', () => {
|
||||
const v12 = fixture('review-1-2.md');
|
||||
const from = '"engines": ["code-review", "security-review"]';
|
||||
assert.ok(v12.includes(from), 'the 1.2 fixture names its engines');
|
||||
const r = validateReviewContent(mutate(v12, from, '"engines": ["claude-security", "code-review"]'), { strict: true });
|
||||
assert.ok(!codes(r.errors).includes('REVIEW_ROUND_MALFORMED'), JSON.stringify(r.errors));
|
||||
assert.equal(r.valid, true, JSON.stringify(r.errors));
|
||||
const bad = validateReviewContent(mutate(v12, from, '"engines": ["claude-security-review"]'), { strict: true });
|
||||
assert.ok(codes(bad.errors).includes('REVIEW_ROUND_MALFORMED'), 'control: a reviewer name is not an engine');
|
||||
});
|
||||
|
|
|
|||
394
tests/lib/security-engine.test.mjs
Normal file
394
tests/lib/security-engine.test.mjs
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
// tests/lib/security-engine.test.mjs
|
||||
// E 3.1 (decisions 175, 196, 219): the security engine of /trekreview.
|
||||
// choose - which engine reviews security: Claude Security when available
|
||||
// and base..head has committed changes; else /security-review when
|
||||
// origin/HEAD resolves and origin/HEAD...HEAD is non-empty; else
|
||||
// none (no-engine, no-origin, empty-diff) = "not measured", never 0.
|
||||
// A git failure or bad usage is exit 2, never a reason.
|
||||
// convert - Claude Security's report folder turned into a reviewer payload:
|
||||
// confidence `high` -> engine findings, the rest -> `below_threshold`,
|
||||
// never in `findings`; a scan that cannot be trusted is exit 2.
|
||||
// ENGINES - claude-security is an engine of its own, and the action written
|
||||
// for a finding names the engine that reported it.
|
||||
// none - no engine chosen: the expected reviewer `security-engine` never
|
||||
// reports, so the verdict gate exits 2, never ALLOW.
|
||||
// choose runs the real CLI in throwaway git repos with no hooks and no global
|
||||
// or system git config.
|
||||
|
||||
import { after, test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { appendFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { childEnv } from '../helpers/child-env.mjs';
|
||||
import { record, writeReport } from '../helpers/claude-security-report.mjs';
|
||||
import { convertReport, NO_ENGINE_REVIEWER } from '../../lib/review/security-engine.mjs';
|
||||
import { ENGINES, toReviewerPayload } from '../../lib/review/engine-findings.mjs';
|
||||
import { runContract } from '../../lib/review/coordinator-contract.mjs';
|
||||
import { decideVerdict } from '../../lib/review/verdict-gate.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
||||
const CLI = join(ROOT, 'lib', 'review', 'security-engine.mjs');
|
||||
const GIT_ENV = { GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1' };
|
||||
const HEX40 = /^[0-9a-f]{40}$/;
|
||||
|
||||
const made = [];
|
||||
after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); });
|
||||
|
||||
const run = (args) => spawnSync(process.execPath, [CLI, ...args], { encoding: 'utf8', env: childEnv(GIT_ENV) });
|
||||
const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } };
|
||||
|
||||
function g(cwd, ...args) {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv(GIT_ENV) });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
/** A throwaway repo with two commits (start, end) that change a.mjs; no origin remote. */
|
||||
function repo() {
|
||||
const base = mkdtempSync(join(tmpdir(), 'security-engine-'));
|
||||
made.push(base);
|
||||
const dir = join(base, 'repo');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
mkdirSync(join(base, 'hooks-none'));
|
||||
g(dir, 'init', '-q', '-b', 'main');
|
||||
appendFileSync(join(dir, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
||||
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
||||
writeFileSync(join(dir, 'a.mjs'), 'export const a = 1;\n');
|
||||
g(dir, 'add', 'a.mjs');
|
||||
g(dir, 'commit', '-q', '-m', 'chore: start');
|
||||
const start = g(dir, 'rev-parse', 'HEAD');
|
||||
writeFileSync(join(dir, 'a.mjs'), 'export const a = 2;\n');
|
||||
g(dir, 'add', 'a.mjs');
|
||||
g(dir, 'commit', '-q', '-m', 'feat: the work');
|
||||
const end = g(dir, 'rev-parse', 'HEAD');
|
||||
return {
|
||||
base, dir, start, end,
|
||||
/** An `origin` remote (never fetched) whose HEAD points at `sha`. */
|
||||
originAt(sha) {
|
||||
g(dir, 'remote', 'add', 'origin', join(base, 'nowhere.git'));
|
||||
g(dir, 'update-ref', 'refs/remotes/origin/main', sha);
|
||||
g(dir, 'symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/main');
|
||||
},
|
||||
/** An `origin` remote with no remote-tracking refs at all. */
|
||||
originWithoutHead() {
|
||||
g(dir, 'remote', 'add', 'origin', join(base, 'nowhere.git'));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const choose = (fx, { base = fx.start, head = fx.end, cs = 'missing' } = {}) =>
|
||||
run(['choose', '--repo', fx.dir, '--base', base, '--head', head, '--claude-security', cs]);
|
||||
|
||||
function decided(r) {
|
||||
assert.equal(r.status, 0, `choose decides: ${r.stdout}${r.stderr}`);
|
||||
const j = parse(r);
|
||||
assert.deepEqual(Object.keys(j).sort(), ['engine', 'reason'], `the output is {engine, reason}: ${r.stdout}`);
|
||||
return j;
|
||||
}
|
||||
|
||||
// ---- choose ----------------------------------------------------------------------
|
||||
|
||||
test('[CHS] Claude Security available and base..head has committed changes: claude-security', () => {
|
||||
const fx = repo();
|
||||
assert.deepEqual(decided(choose(fx, { cs: 'available' })), { engine: 'claude-security', reason: 'claude-security-available' });
|
||||
});
|
||||
|
||||
test('[CHS] Claude Security missing, origin/HEAD behind HEAD: security-review', () => {
|
||||
const fx = repo();
|
||||
fx.originAt(fx.start);
|
||||
assert.deepEqual(decided(choose(fx)), { engine: 'security-review', reason: 'claude-security-missing' });
|
||||
});
|
||||
|
||||
test('[CHS] Claude Security available but base..head empty, origin/HEAD behind HEAD: security-review', () => {
|
||||
const fx = repo();
|
||||
fx.originAt(fx.start);
|
||||
assert.deepEqual(decided(choose(fx, { base: fx.end, cs: 'available' })), { engine: 'security-review', reason: 'claude-security-empty-diff' });
|
||||
});
|
||||
|
||||
test('[CHS] origin/HEAD equals HEAD and Claude Security is missing: no engine, empty-diff (a pushed main is not measured, never 0)', () => {
|
||||
const fx = repo();
|
||||
fx.originAt(fx.end);
|
||||
assert.deepEqual(decided(choose(fx)), { engine: null, reason: 'empty-diff' });
|
||||
});
|
||||
|
||||
test('[CHS] Claude Security available, base..head empty, no origin/HEAD: no engine, empty-diff', () => {
|
||||
const fx = repo();
|
||||
assert.deepEqual(decided(choose(fx, { base: fx.end, cs: 'available' })), { engine: null, reason: 'empty-diff' });
|
||||
});
|
||||
|
||||
test('[CHS] only committed changes count: an uncommitted edit on top of an empty range is still empty-diff', () => {
|
||||
const fx = repo();
|
||||
writeFileSync(join(fx.dir, 'a.mjs'), 'export const a = 3;\n');
|
||||
assert.deepEqual(decided(choose(fx, { base: fx.end, cs: 'available' })), { engine: null, reason: 'empty-diff' });
|
||||
});
|
||||
|
||||
test('[CHS] no origin remote and Claude Security missing: no engine, no-engine', () => {
|
||||
const fx = repo();
|
||||
assert.deepEqual(decided(choose(fx)), { engine: null, reason: 'no-engine' });
|
||||
});
|
||||
|
||||
test('[CHS] an origin remote whose HEAD does not resolve, Claude Security missing: no engine, no-origin', () => {
|
||||
const fx = repo();
|
||||
fx.originWithoutHead();
|
||||
assert.deepEqual(decided(choose(fx)), { engine: null, reason: 'no-origin' });
|
||||
});
|
||||
|
||||
test('[CHS] a git failure is exit 2 (could not run), never a reason', () => {
|
||||
const fx = repo();
|
||||
const plain = join(fx.base, 'plain');
|
||||
mkdirSync(plain);
|
||||
const notRepo = run(['choose', '--repo', plain, '--base', fx.start, '--head', fx.end, '--claude-security', 'available']);
|
||||
assert.equal(notRepo.status, 2, `not a git repository: ${notRepo.stdout}${notRepo.stderr}`);
|
||||
assert.equal(notRepo.stdout, '', 'nothing on stdout');
|
||||
const unknown = 'f'.repeat(40);
|
||||
for (const [base, head] of [[unknown, fx.end], [fx.start, unknown]]) {
|
||||
const r = choose(fx, { base, head, cs: 'available' });
|
||||
assert.equal(r.status, 2, `a sha that does not resolve: ${r.stdout}${r.stderr}`);
|
||||
assert.equal(r.stdout, '');
|
||||
}
|
||||
});
|
||||
|
||||
test('[CHS] bad usage is exit 2', () => {
|
||||
const fx = repo();
|
||||
const cases = [
|
||||
['no subcommand', []],
|
||||
['an unknown subcommand', ['pick', '--repo', fx.dir]],
|
||||
['--claude-security missing', ['choose', '--repo', fx.dir, '--base', fx.start, '--head', fx.end]],
|
||||
['--claude-security maybe', ['choose', '--repo', fx.dir, '--base', fx.start, '--head', fx.end, '--claude-security', 'maybe']],
|
||||
['--base a ref name', ['choose', '--repo', fx.dir, '--base', 'HEAD~1', '--head', fx.end, '--claude-security', 'available']],
|
||||
['--base an option', ['choose', '--repo', fx.dir, '--base', '--output=/tmp/x', '--head', fx.end, '--claude-security', 'available']],
|
||||
['--repo missing', ['choose', '--base', fx.start, '--head', fx.end, '--claude-security', 'available']],
|
||||
['an unknown flag', ['choose', '--repo', fx.dir, '--base', fx.start, '--head', fx.end, '--claude-security', 'available', '--force']],
|
||||
];
|
||||
for (const [name, args] of cases) {
|
||||
const r = run(args);
|
||||
assert.equal(r.status, 2, `${name}: exit 2, got ${r.status}: ${r.stdout}${r.stderr}`);
|
||||
assert.equal(r.stdout, '', `${name}: nothing on stdout`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- convert ---------------------------------------------------------------------
|
||||
|
||||
const BASE = 'a'.repeat(40);
|
||||
const HEAD = 'b'.repeat(40);
|
||||
|
||||
function reportDir() {
|
||||
const d = mkdtempSync(join(tmpdir(), 'security-engine-report-'));
|
||||
made.push(d);
|
||||
return d;
|
||||
}
|
||||
|
||||
const MEDIUM = record({ id: 'F3', title: 'Path joined from a query value', file: 'src/run.mjs', line: 30, confidence: 'medium', cwe_id: 'CWE-22', category: 'Path Traversal' });
|
||||
const LOW = record({ id: 'F4', title: 'Token compared with ===', file: 'src/run.mjs', line: 44, confidence: 'low', severity: 'LOW', cwe_id: 'CWE-208' });
|
||||
const OUTSIDE = record({ id: 'F2', title: 'Old helper leaks a handle path', file: 'lib/old.mjs', line: 4 });
|
||||
|
||||
test('[CNV] a verified changes scan of base..head: confidence high becomes an engine finding, the rest go to below_threshold and never into findings', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, records: [record(), OUTSIDE, MEDIUM, LOW] });
|
||||
const r = convertReport({ reportDir: dir, base: BASE, head: HEAD, scopeFiles: ['src/run.mjs', 'src/other.mjs'] });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
const p = r.payload;
|
||||
assert.equal(p.reviewer, 'claude-security');
|
||||
assert.equal(p.findings.length, 1, JSON.stringify(p.findings.map((f) => f.title)));
|
||||
const [f] = p.findings;
|
||||
assert.deepEqual([f.rule_key, f.severity, f.engine, f.file, f.line], ['SECURITY_REVIEW_VULNERABILITY', 'BLOCKER', 'claude-security', 'src/run.mjs', 12]);
|
||||
assert.equal(f.title, record().title, 'summary <- title');
|
||||
assert.ok(f.detail.includes(record().exploit_scenario), `failure_scenario <- exploit_scenario: ${f.detail}`);
|
||||
assert.ok(f.detail.includes('CWE-78'), `category <- cwe_id: ${f.detail}`);
|
||||
assert.equal(f.recommended_action, record().recommendation, 'fix <- recommendation');
|
||||
assert.equal(f.brief_ref, ENGINES['claude-security'].brief_ref);
|
||||
assert.match(f.id, HEX40);
|
||||
assert.ok(!('rule_key' in record()), 'the record carries no narrower key');
|
||||
assert.deepEqual(p.out_of_scope.map((o) => [o.file, o.engine]), [['lib/old.mjs', 'claude-security']], 'a high finding outside the scope is listed, never dropped');
|
||||
assert.deepEqual(p.below_threshold.map((b) => b.title).sort(), [LOW.title, MEDIUM.title].sort(), 'medium and low go to below_threshold');
|
||||
for (const b of p.below_threshold) assert.equal(b.engine, 'claude-security');
|
||||
const inFindings = new Set([...p.findings, ...p.out_of_scope].map((x) => x.title));
|
||||
for (const b of p.below_threshold) assert.ok(!inFindings.has(b.title), `${b.title} is never in findings or out_of_scope`);
|
||||
});
|
||||
|
||||
test('[CNV] the CLI prints the payload and exits 0', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, records: [record(), MEDIUM] });
|
||||
const scope = join(dir, 'scope.txt');
|
||||
writeFileSync(scope, 'src/run.mjs\n');
|
||||
const r = run(['convert', '--report-dir', dir, '--base', BASE, '--head', HEAD, '--scope', scope]);
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
const p = parse(r);
|
||||
assert.equal(p.reviewer, 'claude-security');
|
||||
assert.equal(p.findings.length, 1);
|
||||
assert.equal(p.below_threshold.length, 1);
|
||||
});
|
||||
|
||||
test('[CNV] a -dirty stamp counts', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, dirty: true });
|
||||
const r = convertReport({ reportDir: dir, base: BASE, head: HEAD });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
assert.equal(r.payload.findings.length, 1);
|
||||
});
|
||||
|
||||
test('[CNV] the newest folder whose stamp matches is read; a newer folder of another commit is passed over', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, ts: '20261108-100000', records: [record({ title: 'older scan' })] });
|
||||
writeReport(dir, { head: HEAD, base: BASE, ts: '20261108-110000', records: [record({ title: 'newest matching scan' })] });
|
||||
writeReport(dir, { head: 'c'.repeat(40), base: BASE, ts: '20261108-120000', records: [record({ title: 'another commit' })] });
|
||||
const r = convertReport({ reportDir: dir, base: BASE, head: HEAD });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
assert.deepEqual(r.payload.findings.map((f) => f.title), ['newest matching scan']);
|
||||
});
|
||||
|
||||
test('[CNV] a finding file relative to the scan root is made repo-relative with the stamp\'s scan_prefix', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, records: [record({ file: 'run.mjs' })], stamp: (s) => ({ ...s, scan_prefix: 'src/' }) });
|
||||
const r = convertReport({ reportDir: dir, base: BASE, head: HEAD, scopeFiles: ['src/run.mjs'] });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
assert.deepEqual(r.payload.findings.map((f) => f.file), ['src/run.mjs']);
|
||||
});
|
||||
|
||||
test('[CNV] a scan that found nothing is a report: exit 0, no findings', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE, records: [] });
|
||||
const r = convertReport({ reportDir: dir, base: BASE, head: HEAD });
|
||||
assert.equal(r.exit, 0, r.reason);
|
||||
assert.deepEqual([r.payload.findings, r.payload.out_of_scope, r.payload.below_threshold], [[], [], []]);
|
||||
});
|
||||
|
||||
test('[CNV] not measured: exit 2, the reason on stderr, nothing on stdout', () => {
|
||||
const setV = (key, value) => (s) => ({ ...s, verification: { ...s.verification, [key]: value } });
|
||||
const dropV = (key) => (s) => { const v = { ...s.verification }; delete v[key]; return { ...s, verification: v }; };
|
||||
const setShape = (key, value) => (s) => ({ ...s, run_shape: { ...s.run_shape, [key]: value } });
|
||||
const cases = [
|
||||
['no report folder', null],
|
||||
['a stamp of another commit', { head: 'c'.repeat(40) }],
|
||||
['a stamp of another merge base', { stamp: (s) => ({ ...s, revision: { ...s.revision, merge_base: 'd'.repeat(40) } }) }],
|
||||
['a commit scan, not a changes scan', { stamp: (s) => ({ ...s, mode: 'commit' }) }],
|
||||
['verification unverified', { stamp: setV('status', 'unverified') }],
|
||||
['fewer researchers returned than dispatched', { stamp: setV('researchers_returned', 2) }],
|
||||
['researchers_returned missing', { stamp: dropV('researchers_returned') }],
|
||||
['researchers_dispatched missing', { stamp: dropV('researchers_dispatched') }],
|
||||
['the scan saw an empty diff', { stamp: setShape('empty_diff', true) }],
|
||||
['the scan saw an empty scope', { stamp: setShape('empty_scope', true) }],
|
||||
['no JSONL', { jsonl: null }],
|
||||
['a JSONL line that does not parse', { jsonl: `${JSON.stringify(record())}\n{"title": \n` }],
|
||||
['a record with an unknown confidence', { records: [record({ confidence: 'certain' })] }],
|
||||
['a record with no title', { records: [record({ title: '' })] }],
|
||||
];
|
||||
for (const [name, opts] of cases) {
|
||||
const dir = reportDir();
|
||||
if (opts !== null) writeReport(dir, { head: HEAD, base: BASE, ...opts });
|
||||
const r = run(['convert', '--report-dir', dir, '--base', BASE, '--head', HEAD]);
|
||||
assert.equal(r.status, 2, `${name}: exit 2, got ${r.status}: ${r.stdout}${r.stderr}`);
|
||||
assert.equal(r.stdout, '', `${name}: nothing on stdout`);
|
||||
assert.match(r.stderr, /not measured/, `${name}: the reason says not measured: ${r.stderr}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('[CNV] an empty scope list is exit 1 (the review scope is unknown), as engine-findings.mjs', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE });
|
||||
const scope = join(dir, 'scope.txt');
|
||||
writeFileSync(scope, '\n');
|
||||
const r = run(['convert', '--report-dir', dir, '--base', BASE, '--head', HEAD, '--scope', scope]);
|
||||
assert.equal(r.status, 1, r.stdout + r.stderr);
|
||||
assert.equal(r.stdout, '');
|
||||
});
|
||||
|
||||
test('[CNV] bad usage is exit 2', () => {
|
||||
const dir = reportDir();
|
||||
writeReport(dir, { head: HEAD, base: BASE });
|
||||
for (const [name, args] of [
|
||||
['--head missing', ['convert', '--report-dir', dir, '--base', BASE]],
|
||||
['--base not a full sha', ['convert', '--report-dir', dir, '--base', 'main', '--head', HEAD]],
|
||||
['--report-dir missing', ['convert', '--base', BASE, '--head', HEAD]],
|
||||
['a scope file that does not exist', ['convert', '--report-dir', dir, '--base', BASE, '--head', HEAD, '--scope', join(dir, 'none.txt')]],
|
||||
]) {
|
||||
const r = run(args);
|
||||
assert.equal(r.status, 2, `${name}: exit 2, got ${r.status}: ${r.stdout}${r.stderr}`);
|
||||
assert.equal(r.stdout, '', `${name}: nothing on stdout`);
|
||||
}
|
||||
});
|
||||
|
||||
// ---- ENGINES --------------------------------------------------------------------
|
||||
|
||||
test('[ENG3] ENGINES has claude-security: reviewer claude-security, the engines\' key, its own brief_ref', () => {
|
||||
const s = ENGINES['claude-security'];
|
||||
assert.ok(s, 'ENGINES has claude-security');
|
||||
assert.equal(s.reviewer, 'claude-security');
|
||||
assert.equal(s.rule_key, 'SECURITY_REVIEW_VULNERABILITY');
|
||||
assert.equal(typeof s.brief_ref, 'string');
|
||||
assert.notEqual(s.brief_ref, ENGINES['security-review'].brief_ref, 'its own brief_ref');
|
||||
assert.match(s.brief_ref, /Claude Security/);
|
||||
});
|
||||
|
||||
test('[ENG3] the action written for a finding without a fix names the engine that reported it', () => {
|
||||
const bare = [{ file: 'src/run.mjs', line: 3, summary: 's' }];
|
||||
const action = (engine) => {
|
||||
const r = toReviewerPayload({ engine, findings: bare });
|
||||
assert.equal(r.ok, true, `${engine}: ${r.errors.join('; ')}`);
|
||||
return r.payload.findings[0].recommended_action;
|
||||
};
|
||||
assert.match(action('code-review'), /\/code-review reported/);
|
||||
assert.match(action('security-review'), /\/security-review reported/);
|
||||
assert.match(action('claude-security'), /Claude Security reported/);
|
||||
assert.doesNotMatch(action('claude-security'), /\/code-review/, 'another engine is never called /code-review');
|
||||
});
|
||||
|
||||
// ---- no engine: not measured, never ALLOW -----------------------------------------
|
||||
|
||||
// Copied from tests/lib/verdict-gate.test.mjs:29-74 (review), never imported. Changed:
|
||||
// allow_blocked_by is the one argument.
|
||||
function review(allowBlockedBy) {
|
||||
const json = JSON.stringify({ verdict: 'WARN', counts: { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 }, allow_blocked_by: allowBlockedBy, findings: [] }, null, 2);
|
||||
return `---
|
||||
type: trekreview
|
||||
review_version: "1.1"
|
||||
task: "t"
|
||||
slug: s
|
||||
project_dir: .claude/projects/2026-11-08-s/
|
||||
brief_path: .claude/projects/2026-11-08-s/brief.md
|
||||
scope_sha_end: abc
|
||||
reviewed_files_count: 1
|
||||
verdict: WARN
|
||||
findings: []
|
||||
---
|
||||
|
||||
# Review: t
|
||||
|
||||
## Executive Summary
|
||||
|
||||
x
|
||||
|
||||
## Coverage
|
||||
|
||||
| File | Treatment | Reason |
|
||||
|------|-----------|--------|
|
||||
| a.mjs | deep-review | default |
|
||||
|
||||
## Remediation Summary
|
||||
|
||||
x
|
||||
|
||||
\`\`\`json
|
||||
${json}
|
||||
\`\`\`
|
||||
`;
|
||||
}
|
||||
|
||||
test('[NOE] no engine chosen: the expected reviewer security-engine never reports, so the review is not measured and the verdict gate exits 2, never ALLOW', () => {
|
||||
assert.equal(NO_ENGINE_REVIEWER, 'security-engine');
|
||||
const expected = ['brief-conformance-reviewer', 'code-correctness-reviewer', ENGINES['code-review'].reviewer, NO_ENGINE_REVIEWER];
|
||||
const c = runContract([
|
||||
{ reviewer: 'brief-conformance-reviewer', findings: [] },
|
||||
{ reviewer: 'code-correctness-reviewer', findings: [] },
|
||||
{ reviewer: ENGINES['code-review'].reviewer, findings: [] },
|
||||
], { expectedReviewers: expected });
|
||||
assert.deepEqual(c.missing_reviewers, ['security-engine']);
|
||||
assert.ok(c.allow_blocked_by.includes('missing-reviewer:security-engine'), JSON.stringify(c.allow_blocked_by));
|
||||
assert.notEqual(c.verdict, 'ALLOW');
|
||||
const d = decideVerdict(review(c.allow_blocked_by));
|
||||
assert.equal(d.exit, 2, d.reason);
|
||||
assert.match(d.reason, /missing-reviewer:security-engine/);
|
||||
});
|
||||
|
|
@ -210,6 +210,9 @@ test('backedLine: only a security finding not from an engine can be outside (218
|
|||
const engine = own({ engine: 'security-review' });
|
||||
assert.equal(isOwnSecurityFinding(engine), false);
|
||||
assert.equal(backedLine(engine, tm), null, 'an engine finding blocks as today');
|
||||
const claudeSecurity = own({ engine: 'claude-security' });
|
||||
assert.equal(isOwnSecurityFinding(claudeSecurity), false);
|
||||
assert.equal(backedLine(claudeSecurity, tm), null, 'a Claude Security finding blocks as today (E 3.1)');
|
||||
const enginesKey = own({ rule_key: 'SECURITY_REVIEW_VULNERABILITY' });
|
||||
assert.equal(backedLine(enginesKey, tm), null, "the engines' key is never Voyage's own");
|
||||
const notSecurity = own({ rule_key: 'MISSING_TEST' });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue