3cd68dc9fb
Critical-review §2 B3 finding: `riskScore({info: N}) = 0` silently masks
info-volume findings. The behavior was correct (info is scoring-inert by
design) but undocumented. Operators reading a report with N info findings
had no way to know they contribute zero to verdict/band.
Three coordinated edits:
- scanners/lib/severity.mjs JSDoc — explicit "Info severity" subsection
spelling out: scoring-inert, surfaced in owaspCategorize aggregates,
treat as observability telemetry not verdict input. @param updated to
mark info as accepted but ignored.
- CLAUDE.md v7.0.0 risk-score-v2 line — one-sentence anchor pointing to
severity.mjs JSDoc.
- tests/lib/severity.test.mjs — anchor test alongside the existing
4-critical=93 anchor: asserts riskScore({info: 50}) === 0,
riskScore({info: 1000}) === 0, verdict({info: 100}) === 'ALLOW',
riskBand(riskScore({info: 500})) === 'Low'.
Decision: skip the optional `infoScore()` helper from the brief. No
current consumer would use it; doc-only fix keeps API surface minimal.
Revisit if a consumer emerges.
Tests: 1522 → 1523 (+1 anchor block, 4 assertions). All green.
500 lines
17 KiB
JavaScript
500 lines
17 KiB
JavaScript
// severity.test.mjs — Tests for scanners/lib/severity.mjs
|
|
// Zero external dependencies: node:test + node:assert only.
|
|
|
|
import { describe, it } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {
|
|
SEVERITY,
|
|
riskScore,
|
|
verdict,
|
|
riskBand,
|
|
gradeFromPassRate,
|
|
OWASP_MAP,
|
|
OWASP_AGENTIC_MAP,
|
|
OWASP_SKILLS_MAP,
|
|
OWASP_MCP_MAP,
|
|
owaspCategorize,
|
|
} from '../../scanners/lib/severity.mjs';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SEVERITY
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('SEVERITY', () => {
|
|
it('exports all five severity levels', () => {
|
|
assert.ok('CRITICAL' in SEVERITY);
|
|
assert.ok('HIGH' in SEVERITY);
|
|
assert.ok('MEDIUM' in SEVERITY);
|
|
assert.ok('LOW' in SEVERITY);
|
|
assert.ok('INFO' in SEVERITY);
|
|
});
|
|
|
|
it('has lowercase string values', () => {
|
|
assert.equal(SEVERITY.CRITICAL, 'critical');
|
|
assert.equal(SEVERITY.HIGH, 'high');
|
|
assert.equal(SEVERITY.MEDIUM, 'medium');
|
|
assert.equal(SEVERITY.LOW, 'low');
|
|
assert.equal(SEVERITY.INFO, 'info');
|
|
});
|
|
|
|
it('is frozen (immutable)', () => {
|
|
assert.ok(Object.isFrozen(SEVERITY));
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// riskScore
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('riskScore (v2 — severity-dominated log-scaled, v7.0.0+)', () => {
|
|
it('returns 0 when all counts are zero', () => {
|
|
assert.equal(riskScore({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }), 0);
|
|
});
|
|
|
|
it('returns 0 for empty counts object', () => {
|
|
assert.equal(riskScore({}), 0);
|
|
});
|
|
|
|
it('returns 0 for info-only findings (info tier is non-scoring)', () => {
|
|
assert.equal(riskScore({ info: 100 }), 0);
|
|
});
|
|
|
|
// --- Low tier: 1 + log2(n+1)*3, capped at 11 ---
|
|
it('returns 4 for one low finding', () => {
|
|
assert.equal(riskScore({ low: 1 }), 4);
|
|
});
|
|
|
|
it('returns 11 for twenty low findings (tier-capped)', () => {
|
|
assert.equal(riskScore({ low: 20 }), 11);
|
|
});
|
|
|
|
// --- Medium tier: 15 + log2(n+1)*5, capped at 35 ---
|
|
it('returns 20 for one medium finding (tier base + log scale)', () => {
|
|
assert.equal(riskScore({ medium: 1 }), 20);
|
|
});
|
|
|
|
it('returns 28 for five medium findings', () => {
|
|
assert.equal(riskScore({ medium: 5 }), 28);
|
|
});
|
|
|
|
it('returns 29 for six medium findings (still inside Medium band)', () => {
|
|
assert.equal(riskScore({ medium: 6 }), 29);
|
|
});
|
|
|
|
// --- High tier: 40 + log2(n+1)*8, capped at 65 ---
|
|
it('returns 48 for one high finding', () => {
|
|
assert.equal(riskScore({ high: 1 }), 48);
|
|
});
|
|
|
|
it('returns 64 for seven high findings (just below Critical band)', () => {
|
|
assert.equal(riskScore({ high: 7 }), 64);
|
|
});
|
|
|
|
it('returns 65 when high tier saturates — many high + many medium', () => {
|
|
// 17 high + 136 medium (hyperframes-like) → high-tier dominates, cap 65
|
|
assert.equal(riskScore({ high: 17, medium: 136 }), 65);
|
|
});
|
|
|
|
// --- Critical tier: 70 + log2(n+1)*10, capped at 95 ---
|
|
it('returns 80 for one critical finding', () => {
|
|
assert.equal(riskScore({ critical: 1 }), 80);
|
|
});
|
|
|
|
it('returns 86 for two critical findings (enters Extreme band)', () => {
|
|
assert.equal(riskScore({ critical: 2 }), 86);
|
|
});
|
|
|
|
it('returns 93 for four critical findings', () => {
|
|
assert.equal(riskScore({ critical: 4 }), 93);
|
|
});
|
|
|
|
it('returns 95 for ten critical findings (tier-capped)', () => {
|
|
assert.equal(riskScore({ critical: 10 }), 95);
|
|
});
|
|
|
|
it('does not exceed 100 even with huge critical counts', () => {
|
|
assert.ok(riskScore({ critical: 1000 }) <= 100);
|
|
});
|
|
|
|
it('critical dominates high — mixed critical+high scored at critical tier', () => {
|
|
// {critical:1, high:2} → critical tier: 70 + log2(2)*10 = 80
|
|
assert.equal(riskScore({ critical: 1, high: 2, medium: 3, low: 4, info: 5 }), 80);
|
|
});
|
|
|
|
it('high dominates medium — {high:1, medium:100} scored at high tier', () => {
|
|
// 40 + log2(2)*8 = 48
|
|
assert.equal(riskScore({ high: 1, medium: 100 }), 48);
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// verdict
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('verdict (v7.0.0 — co-monotonic with riskBand)', () => {
|
|
it('returns ALLOW for zero findings', () => {
|
|
assert.equal(verdict({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }), 'ALLOW');
|
|
});
|
|
|
|
it('returns ALLOW for empty counts', () => {
|
|
assert.equal(verdict({}), 'ALLOW');
|
|
});
|
|
|
|
it('returns BLOCK when critical >= 1 (score=80)', () => {
|
|
assert.equal(verdict({ critical: 1 }), 'BLOCK');
|
|
});
|
|
|
|
it('returns BLOCK when score >= 65 without critical (17 high + 136 medium = 65)', () => {
|
|
assert.equal(verdict({ high: 17, medium: 136 }), 'BLOCK');
|
|
});
|
|
|
|
it('returns WARNING for 7 high findings (score=64, Critical band boundary not crossed)', () => {
|
|
assert.equal(verdict({ high: 7 }), 'WARNING');
|
|
});
|
|
|
|
it('returns WARNING when high >= 1 (and no critical)', () => {
|
|
assert.equal(verdict({ high: 1 }), 'WARNING');
|
|
});
|
|
|
|
it('returns WARNING for 1 medium (score=20, inside Medium band)', () => {
|
|
assert.equal(verdict({ medium: 1 }), 'WARNING');
|
|
});
|
|
|
|
it('returns WARNING for 6 medium (score=29)', () => {
|
|
assert.equal(verdict({ medium: 6 }), 'WARNING');
|
|
});
|
|
|
|
it('returns ALLOW for 20 low findings (score=11, firmly Low band)', () => {
|
|
assert.equal(verdict({ low: 20 }), 'ALLOW');
|
|
});
|
|
|
|
it('returns ALLOW for 1 low finding (score=4)', () => {
|
|
assert.equal(verdict({ low: 1 }), 'ALLOW');
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// riskBand
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('riskBand (v7.0.0 cutoffs: 14/39/64/84)', () => {
|
|
it('returns Low for score 0', () => {
|
|
assert.equal(riskBand(0), 'Low');
|
|
});
|
|
|
|
it('returns Low for score 14 (upper boundary)', () => {
|
|
assert.equal(riskBand(14), 'Low');
|
|
});
|
|
|
|
it('returns Medium for score 15 (Medium tier start)', () => {
|
|
assert.equal(riskBand(15), 'Medium');
|
|
});
|
|
|
|
it('returns Medium for score 20 (one medium finding)', () => {
|
|
assert.equal(riskBand(20), 'Medium');
|
|
});
|
|
|
|
it('returns Medium for score 39 (upper boundary)', () => {
|
|
assert.equal(riskBand(39), 'Medium');
|
|
});
|
|
|
|
it('returns High for score 40 (High tier start — one high finding is 48)', () => {
|
|
assert.equal(riskBand(40), 'High');
|
|
});
|
|
|
|
it('returns High for score 48 (one high finding)', () => {
|
|
assert.equal(riskBand(48), 'High');
|
|
});
|
|
|
|
it('returns High for score 64 (seven high findings, upper boundary)', () => {
|
|
assert.equal(riskBand(64), 'High');
|
|
});
|
|
|
|
it('returns Critical for score 65 (many high without critical)', () => {
|
|
assert.equal(riskBand(65), 'Critical');
|
|
});
|
|
|
|
it('returns Critical for score 80 (one critical finding)', () => {
|
|
assert.equal(riskBand(80), 'Critical');
|
|
});
|
|
|
|
it('returns Critical for score 84 (upper boundary)', () => {
|
|
assert.equal(riskBand(84), 'Critical');
|
|
});
|
|
|
|
it('returns Extreme for score 85 (two critical findings reach here)', () => {
|
|
assert.equal(riskBand(85), 'Extreme');
|
|
});
|
|
|
|
it('returns Extreme for score 95 (ten critical findings, tier-capped)', () => {
|
|
assert.equal(riskBand(95), 'Extreme');
|
|
});
|
|
|
|
it('returns Extreme for score 100', () => {
|
|
assert.equal(riskBand(100), 'Extreme');
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Verdict / riskBand co-monotonicity sweep (critical-review §5.4)
|
|
//
|
|
// Asserts that for every representative count vector, (verdict, riskBand)
|
|
// agree under the v7.0.0 contract:
|
|
// BLOCK ⇔ band ∈ {Critical, Extreme} OR critical ≥ 1
|
|
// WARNING ⇔ band ∈ {Medium, High} OR (high ≥ 1 AND verdict != BLOCK)
|
|
// ALLOW ⇔ band == Low AND no high/critical
|
|
//
|
|
// Catches regressions where a future change to riskScore tiers, verdict
|
|
// cutoffs, or riskBand cutoffs would re-introduce contradictions like
|
|
// "ALLOW + High band" or "BLOCK + Medium band".
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('verdict/riskBand co-monotonicity (v7.0.0 §5.4)', () => {
|
|
const cases = [
|
|
{ critical: 0, high: 0, medium: 0, low: 0, info: 0 },
|
|
{ low: 1 },
|
|
{ low: 10 },
|
|
{ medium: 1 },
|
|
{ medium: 5 },
|
|
{ medium: 50 },
|
|
{ high: 1 },
|
|
{ high: 5 },
|
|
{ high: 7 },
|
|
{ high: 8 },
|
|
{ high: 17 },
|
|
{ critical: 1 },
|
|
{ critical: 2 },
|
|
{ critical: 4 },
|
|
{ critical: 10 },
|
|
];
|
|
|
|
for (const counts of cases) {
|
|
const label = JSON.stringify(counts);
|
|
it(`(${label}) — verdict and riskBand agree`, () => {
|
|
const score = riskScore(counts);
|
|
const v = verdict(counts);
|
|
const band = riskBand(score);
|
|
const hasCritical = (counts.critical || 0) >= 1;
|
|
const hasHigh = (counts.high || 0) >= 1;
|
|
|
|
if (v === 'BLOCK') {
|
|
assert.ok(
|
|
band === 'Critical' || band === 'Extreme' || hasCritical,
|
|
`BLOCK requires Critical/Extreme band or critical>=1; got band=${band}, score=${score}, counts=${label}`,
|
|
);
|
|
} else if (v === 'WARNING') {
|
|
assert.ok(
|
|
band === 'Medium' || band === 'High' || hasHigh,
|
|
`WARNING requires Medium/High band or high>=1; got band=${band}, score=${score}, counts=${label}`,
|
|
);
|
|
assert.ok(!hasCritical, `WARNING must not have critical>=1; counts=${label}`);
|
|
} else {
|
|
assert.equal(v, 'ALLOW');
|
|
assert.equal(band, 'Low', `ALLOW requires Low band; got band=${band}, score=${score}, counts=${label}`);
|
|
assert.ok(!hasHigh && !hasCritical, `ALLOW must not have high/critical>=1; counts=${label}`);
|
|
}
|
|
});
|
|
}
|
|
|
|
it('JSDoc arithmetic anchor — 4 critical = 93 (not 90)', () => {
|
|
// Pin against doc/code drift documented in critical-review §5 (B4).
|
|
// 70 + min(25, log2(5)*10) = 70 + 23.219... = 93.219 → round → 93.
|
|
assert.equal(riskScore({ critical: 4 }), 93);
|
|
});
|
|
|
|
it('info severity is scoring-inert (B3, v7.2.0)', () => {
|
|
// Documented contract: `info` counts contribute zero to risk_score,
|
|
// do not affect verdict, do not affect riskBand. Pinned here against
|
|
// any future change that would (intentionally or not) start scoring info.
|
|
assert.equal(riskScore({ info: 50 }), 0);
|
|
assert.equal(riskScore({ info: 1000 }), 0);
|
|
assert.equal(verdict({ info: 100 }), 'ALLOW');
|
|
assert.equal(riskBand(riskScore({ info: 500 })), 'Low');
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// gradeFromPassRate
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('gradeFromPassRate', () => {
|
|
it('returns A for perfect pass rate with no critical failures', () => {
|
|
assert.equal(gradeFromPassRate(1.0, 0, 0), 'A');
|
|
});
|
|
|
|
it('returns A for passRate >= 0.89 with no critical category fails and no crits', () => {
|
|
assert.equal(gradeFromPassRate(0.9, 0, 0), 'A');
|
|
});
|
|
|
|
it('does NOT return A if passRate >= 0.89 but has a critical category fail', () => {
|
|
const grade = gradeFromPassRate(0.9, 1, 0);
|
|
assert.notEqual(grade, 'A');
|
|
});
|
|
|
|
it('returns B for passRate >= 0.72 with no critical findings', () => {
|
|
assert.equal(gradeFromPassRate(0.8, 0, 0), 'B');
|
|
});
|
|
|
|
it('returns B for passRate >= 0.72 even with critical category fails (if no critical findings)', () => {
|
|
assert.equal(gradeFromPassRate(0.75, 2, 0), 'B');
|
|
});
|
|
|
|
it('returns C for passRate >= 0.56', () => {
|
|
assert.equal(gradeFromPassRate(0.6, 0, 0), 'C');
|
|
});
|
|
|
|
it('returns C for passRate = 0.56 (lower boundary)', () => {
|
|
assert.equal(gradeFromPassRate(0.56, 0, 0), 'C');
|
|
});
|
|
|
|
it('returns D for passRate >= 0.33 but < 0.56', () => {
|
|
assert.equal(gradeFromPassRate(0.45, 0, 0), 'D');
|
|
});
|
|
|
|
it('returns D for passRate = 0.33 (lower boundary)', () => {
|
|
assert.equal(gradeFromPassRate(0.33, 0, 0), 'D');
|
|
});
|
|
|
|
it('returns F for passRate < 0.33', () => {
|
|
assert.equal(gradeFromPassRate(0.2, 0, 0), 'F');
|
|
});
|
|
|
|
it('returns F for passRate = 0', () => {
|
|
assert.equal(gradeFromPassRate(0, 0, 0), 'F');
|
|
});
|
|
|
|
it('returns F when critCount >= 3 regardless of passRate', () => {
|
|
assert.equal(gradeFromPassRate(1.0, 0, 3), 'F');
|
|
assert.equal(gradeFromPassRate(0.9, 0, 5), 'F');
|
|
});
|
|
|
|
it('uses default values for optional parameters', () => {
|
|
// gradeFromPassRate(passRate) with no optional args — should not throw
|
|
const grade = gradeFromPassRate(0.95);
|
|
assert.ok(['A', 'B', 'C', 'D', 'F'].includes(grade));
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// OWASP Framework Maps
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('OWASP framework maps', () => {
|
|
it('OWASP_MAP includes TFA scanner prefix', () => {
|
|
assert.ok(OWASP_MAP.TFA, 'expected TFA key in OWASP_MAP');
|
|
assert.ok(OWASP_MAP.TFA.includes('LLM01'));
|
|
assert.ok(OWASP_MAP.TFA.includes('LLM02'));
|
|
assert.ok(OWASP_MAP.TFA.includes('LLM06'));
|
|
});
|
|
|
|
it('OWASP_AGENTIC_MAP has all 8 scanner prefixes', () => {
|
|
for (const prefix of ['UNI', 'ENT', 'PRM', 'DEP', 'TNT', 'GIT', 'NET', 'TFA']) {
|
|
assert.ok(OWASP_AGENTIC_MAP[prefix], `expected ${prefix} in OWASP_AGENTIC_MAP`);
|
|
assert.ok(OWASP_AGENTIC_MAP[prefix].length > 0);
|
|
for (const cat of OWASP_AGENTIC_MAP[prefix]) {
|
|
assert.ok(cat.startsWith('ASI'), `expected ASI prefix, got ${cat}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('OWASP_SKILLS_MAP has all 8 scanner prefixes', () => {
|
|
for (const prefix of ['UNI', 'ENT', 'PRM', 'DEP', 'TNT', 'GIT', 'NET', 'TFA']) {
|
|
assert.ok(OWASP_SKILLS_MAP[prefix], `expected ${prefix} in OWASP_SKILLS_MAP`);
|
|
for (const cat of OWASP_SKILLS_MAP[prefix]) {
|
|
assert.ok(cat.startsWith('AST'), `expected AST prefix, got ${cat}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('OWASP_MCP_MAP has all 8 scanner prefixes', () => {
|
|
for (const prefix of ['UNI', 'ENT', 'PRM', 'DEP', 'TNT', 'GIT', 'NET', 'TFA']) {
|
|
assert.ok(OWASP_MCP_MAP[prefix], `expected ${prefix} in OWASP_MCP_MAP`);
|
|
for (const cat of OWASP_MCP_MAP[prefix]) {
|
|
assert.ok(cat.startsWith('MCP'), `expected MCP prefix, got ${cat}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('all framework maps are frozen', () => {
|
|
assert.ok(Object.isFrozen(OWASP_MAP));
|
|
assert.ok(Object.isFrozen(OWASP_AGENTIC_MAP));
|
|
assert.ok(Object.isFrozen(OWASP_SKILLS_MAP));
|
|
assert.ok(Object.isFrozen(OWASP_MCP_MAP));
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// owaspCategorize — multi-framework support
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('owaspCategorize — multi-framework', () => {
|
|
it('categorizes findings with LLM prefix', () => {
|
|
const findings = [
|
|
{ owasp: 'LLM01', severity: 'critical' },
|
|
{ owasp: 'LLM01', severity: 'high' },
|
|
{ owasp: 'LLM06', severity: 'medium' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['LLM01'].count, 2);
|
|
assert.equal(cats['LLM01'].critical, 1);
|
|
assert.equal(cats['LLM01'].high, 1);
|
|
assert.equal(cats['LLM06'].count, 1);
|
|
});
|
|
|
|
it('categorizes findings with ASI prefix', () => {
|
|
const findings = [
|
|
{ owasp: 'ASI01', severity: 'critical' },
|
|
{ owasp: 'ASI02 ASI05', severity: 'high' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['ASI01'].count, 1);
|
|
assert.equal(cats['ASI02'].count, 1);
|
|
assert.equal(cats['ASI05'].count, 1);
|
|
});
|
|
|
|
it('categorizes findings with AST prefix', () => {
|
|
const findings = [
|
|
{ owasp: 'AST03', severity: 'high' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['AST03'].count, 1);
|
|
assert.equal(cats['AST03'].high, 1);
|
|
});
|
|
|
|
it('categorizes findings with MCP prefix', () => {
|
|
const findings = [
|
|
{ owasp: 'MCP1 MCP6', severity: 'critical' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['MCP1'].count, 1);
|
|
assert.equal(cats['MCP6'].count, 1);
|
|
});
|
|
|
|
it('categorizes mixed-framework findings in same owasp field', () => {
|
|
const findings = [
|
|
{ owasp: 'LLM01 ASI01 AST01', severity: 'critical' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['LLM01'].count, 1);
|
|
assert.equal(cats['ASI01'].count, 1);
|
|
assert.equal(cats['AST01'].count, 1);
|
|
});
|
|
|
|
it('falls back to TFA in OWASP_MAP for scanner prefix', () => {
|
|
const findings = [
|
|
{ scanner: 'TFA', severity: 'high' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.ok(cats['LLM01'], 'expected LLM01 from TFA fallback');
|
|
assert.ok(cats['LLM02'], 'expected LLM02 from TFA fallback');
|
|
assert.ok(cats['LLM06'], 'expected LLM06 from TFA fallback');
|
|
});
|
|
|
|
it('returns Unmapped for findings with no owasp and unknown scanner', () => {
|
|
const findings = [
|
|
{ severity: 'low' },
|
|
];
|
|
const cats = owaspCategorize(findings);
|
|
assert.equal(cats['Unmapped'].count, 1);
|
|
});
|
|
});
|