voyage/tests/helpers/claude-security-report.mjs
Kjell Tore Guttormsen f6e8529bdd
fix(review): convert refuses a partial or scoped Claude Security report; the stamp check is a trace (decisions 223, 224)
Two fail-closed checks in security-engine.mjs convert (decision 224), both
"not measured" (exit 2, the reason on stderr, nothing on stdout):
- the stamp's findings.total is not a non-negative integer, or the JSONL
  holds another number of records (a JSONL that lost the only BLOCKER line
  converted clean before; part 3 verifier, F3);
- the stamp's scope is not an empty list: a scan limited by --scope, an
  absent scope or a non-list scope (Claude Security 0.12.0 writes scope: []
  for an unscoped scan; F4).
The real Claude Security 0.12.0 probe report still converts: exit 0, one
BLOCKER, two below_threshold.

Decision 223, text only: the module header and SECURITY.md keep the
head/base check and say it is a trace, not a signature: code the reviewed
run executed can write a report folder of its own (convert reads the
matching one whose name sorts last) or delete the real one first (F2, a
known issue). /trekreview's not-measured list names the two new reasons.

The test helper writes the real stamp shape by default (findings.total =
the records in the JSONL written, scope: []). New tests for behaviour that
had none (F6): no fallback from an unverified newest match, a two-stamp
folder passed over, a symlinked stamp, JSONL or folder refused, a malformed
or absent scan_prefix, an unrecorded empty_diff or empty_scope.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 18:24:39 +01:00

101 lines
4.9 KiB
JavaScript

// tests/helpers/claude-security-report.mjs
// A Claude Security report folder as a changes scan writes it (Claude Security
// 0.12.0: scripts/render_report.py, scripts/lib/finding.py, scripts/lib/plugin.py),
// for the tests of lib/review/security-engine.mjs `convert`:
//
// <dir>/CLAUDE-SECURITY-<UTC YYYYMMDD-HHMMSS>/
// .gitignore `*`
// CLAUDE-SECURITY-RESULTS.md
// CLAUDE-SECURITY-RESULTS.jsonl one finding per line
// CLAUDE-SECURITY-RESULTS.sarif
// CLAUDE-SECURITY-REVISION-<first 12 hex of the commit>[-dirty].json the stamp
//
// `writeReport` takes the commit range and lets a test change any part of the
// stamp (`stamp: (s) => ...`) or write the JSONL text itself (`jsonl`).
// By default the stamp has the real shape of an unscoped changes scan:
// `findings.total` is the number of records in the JSONL written (the
// non-empty lines of `jsonl` when a test gives it; render_report.py writes
// `len(findings)`), and `scope` is `[]` (write_scan_meta.py: no `--scope`).
// A test changes either through `stamp`.
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
/** One JSONL record in the field order the products carry, with `over` applied. */
export function record(over = {}) {
return {
id: 'F1',
title: 'Shell command built from the branch name',
impact: 'A pushed branch name runs a command on the machine that checks it out',
file: 'src/run.mjs',
line: 12,
description: 'checkout() interpolates the branch name into a shell string.',
exploit_scenario: 'A branch named `x;touch pwned` runs `touch pwned` when it is checked out',
preconditions: ['the attacker can push a branch'],
category: 'Injection',
severity: 'HIGH',
confidence: 'high',
recommendation: 'Pass the branch name as an argv element through execFile, never through a shell string',
cwe_id: 'CWE-78',
snippet: 'exec(`git checkout ${branch}`)',
symbol: 'checkout',
declared_line: 12,
via_change: null,
other_cwe_ids: [],
claudeSecurityPluginFindingId: 'a3f9c2d1e0b4',
...over,
};
}
/** The stamp of a verified, unscoped changes scan of `base..head` whose JSONL holds `total` records. */
export function stampOf({ head, base, dirty = false, records = [], total = records.length }) {
const count = (sev) => records.filter((r) => r.severity === sev).length;
return {
generated_at: '2026-11-08T12:00:00+00:00',
duration_s: 412,
scan_id: '6f1d2a4e-5b7c-4e1a-9d3f-2c8b0a7e6d51',
mode: 'changes',
scan_prefix: '',
scope: [],
revision: { versioned: true, commit: head, branch: 'feat/x', dirty, base, merge_base: base },
revision_source: 'git',
model: 'claude-opus-5-5',
effort: 'high',
run_shape: {
requested_effort: 'high', collapsed: null, source: 'coverage.json',
diff_files: 2, diff_lines: 30, pre_existing: 0, scope_files: 2,
empty_diff: false, empty_scope: false, researchers_dispatched: 3, verification_runs: 1,
},
findings: { total, critical: count('CRITICAL'), high: count('HIGH'), medium: count('MEDIUM'), low: count('LOW') },
verification: {
status: 'verified', candidates: records.length, candidates_deduped: records.length,
panel_votes: 3 * records.length, panel_reviewed_findings: records.length,
panel_quorum_findings: records.length, unreviewed_candidate_sites: 0,
incomplete_panel_candidates: 0, attested_findings: 0, reason: null, reason_kind: null,
researchers_dispatched: 3, researchers_returned: 3,
},
};
}
/**
* Write one report folder under `dir` and return its path.
* @param {string} dir - the scanned repo's root
* @param {{ head: string, base: string, ts?: string, dirty?: boolean, records?: object[],
* jsonl?: string|null, stamp?: (s: object) => object, stampName?: string }} opts
* `jsonl: null` writes no JSONL file; `stamp` returns the stamp to write.
*/
export function writeReport(dir, { head, base, ts = '20261108-120000', dirty = false, records = [record()], jsonl, stamp = (s) => s, stampName }) {
const folder = join(dir, `CLAUDE-SECURITY-${ts}`);
mkdirSync(folder, { recursive: true });
writeFileSync(join(folder, '.gitignore'), '*\n');
writeFileSync(join(folder, 'CLAUDE-SECURITY-RESULTS.md'), `# Claude Security results\n\n${records.length} finding(s).\n`);
writeFileSync(join(folder, 'CLAUDE-SECURITY-RESULTS.sarif'), JSON.stringify({ version: '2.1.0', runs: [] }));
if (jsonl !== null) {
writeFileSync(join(folder, 'CLAUDE-SECURITY-RESULTS.jsonl'),
jsonl ?? records.map((r) => `${JSON.stringify(r)}\n`).join(''));
}
const total = typeof jsonl === 'string' ? jsonl.split('\n').filter((l) => l.trim() !== '').length : records.length;
const name = stampName ?? `CLAUDE-SECURITY-REVISION-${head.slice(0, 12)}${dirty ? '-dirty' : ''}.json`;
writeFileSync(join(folder, name), JSON.stringify(stamp(stampOf({ head, base, dirty, records, total })), null, 2));
return folder;
}