diff --git a/plugins/config-audit/scanners/lib/scoring.mjs b/plugins/config-audit/scanners/lib/scoring.mjs index 4b579ac..79a4653 100644 --- a/plugins/config-audit/scanners/lib/scoring.mjs +++ b/plugins/config-audit/scanners/lib/scoring.mjs @@ -3,7 +3,7 @@ * Zero external dependencies. */ -import { gradeFromPassRate } from './severity.mjs'; +import { gradeFromPassRate, WEIGHTS } from './severity.mjs'; // --- Tier weights for utilization calculation --- const TIER_WEIGHTS = { t1: 3, t2: 2, t3: 1, t4: 1 }; @@ -162,9 +162,23 @@ function slugify(name) { } /** - * Score per config area from scanner results. + * Compute raw severity-weighted penalty from scanner counts. + * Critical/high findings dominate; lows barely move the needle. + * @param {{ critical?: number, high?: number, medium?: number, low?: number, info?: number }} counts + * @returns {number} + */ +function severityPenalty(counts) { + let penalty = 0; + for (const [sev, weight] of Object.entries(WEIGHTS)) { + penalty += (counts[sev] || 0) * weight; + } + return penalty; +} + +/** + * Score per config area from scanner results (v5: severity-weighted). * @param {object[]} scannerResults - Array of scanner result objects from envelope.scanners - * @returns {{ areas: Array<{ id: string, name: string, grade: string, score: number, findingCount: number }>, overallGrade: string }} + * @returns {{ areas: Array<{ id: string, name: string, grade: string, score: number, findingCount: number }>, overallGrade: string, scoringVersion: string }} */ export function scoreByArea(scannerResults) { const areas = []; @@ -175,14 +189,16 @@ export function scoreByArea(scannerResults) { let score; if (result.scanner === 'GAP') { - // Feature coverage: utilization-based const util = calculateUtilization(result.findings); score = util.score; } else { - // Quality-based: fewer findings = higher pass rate - // Use a reasonable max checks per scanner for pass rate - const maxChecks = Math.max(findingCount + 5, 10); - const passRate = ((maxChecks - findingCount) / maxChecks) * 100; + // v5 severity-weighted: penalty proportional to a per-scanner budget. + // maxBudget = max(10, findingCount * 4) — adding more lows doesn't crater the + // grade, but a single high-severity finding consumes a large fraction of budget. + const counts = result.counts || {}; + const penalty = severityPenalty(counts); + const maxBudget = Math.max(10, findingCount * 4); + const passRate = Math.max(0, 100 - (penalty / maxBudget) * 100); score = Math.round(passRate); } @@ -196,7 +212,7 @@ export function scoreByArea(scannerResults) { const avgScore = qualityAreas.length > 0 ? Math.round(totalScore / qualityAreas.length) : 0; const overallGrade = gradeFromPassRate(avgScore); - return { areas, overallGrade }; + return { areas, overallGrade, scoringVersion: 'v5' }; } /** diff --git a/plugins/config-audit/tests/lib/scoring.test.mjs b/plugins/config-audit/tests/lib/scoring.test.mjs index df2127b..3b4a7ff 100644 --- a/plugins/config-audit/tests/lib/scoring.test.mjs +++ b/plugins/config-audit/tests/lib/scoring.test.mjs @@ -59,6 +59,29 @@ function makeScannerResult(scanner, findingCount) { }; } +function makeScannerResultWithSeverities(scanner, severities) { + const counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 }; + const findings = severities.map((sev, i) => { + if (counts[sev] !== undefined) counts[sev]++; + return { + id: `CA-${scanner}-${String(i + 1).padStart(3, '0')}`, + scanner, + severity: sev, + title: `Finding ${i + 1}`, + category: scanner === 'GAP' ? 't2' : null, + recommendation: 'Fix', + }; + }); + return { + scanner, + status: 'ok', + files_scanned: 5, + duration_ms: 10, + findings, + counts, + }; +} + // ======================================== // calculateUtilization // ======================================== @@ -279,8 +302,8 @@ describe('scoreByArea', () => { assert.equal(result.areas[0].score, 100); }); - it('many findings → lower grade', () => { - const scanners = [makeScannerResult('CML', 8)]; + it('many high-severity findings → lower grade (v5 severity-weighted)', () => { + const scanners = [makeScannerResultWithSeverities('CML', ['high', 'high', 'high'])]; const result = scoreByArea(scanners); assert.ok(result.areas[0].score < 50); }); @@ -327,6 +350,52 @@ describe('scoreByArea', () => { assert.ok('score' in area); assert.ok('findingCount' in area); }); + + it('exposes scoringVersion: v5', () => { + const result = scoreByArea([makeScannerResult('CML', 0)]); + assert.equal(result.scoringVersion, 'v5'); + }); +}); + +// ======================================== +// scoreByArea — severity weighting (v5 F3) +// ======================================== +describe('scoreByArea — severity weighting (v5 F3)', () => { + it('clean scanner → 100/A', () => { + const result = scoreByArea([makeScannerResultWithSeverities('CML', [])]); + assert.equal(result.areas[0].score, 100); + assert.equal(result.areas[0].grade, 'A'); + }); + + it('5 lows scores higher than 1 critical', () => { + const fiveLows = scoreByArea([makeScannerResultWithSeverities('CML', ['low', 'low', 'low', 'low', 'low'])]); + const oneCritical = scoreByArea([makeScannerResultWithSeverities('CML', ['critical'])]); + assert.ok(fiveLows.areas[0].score > oneCritical.areas[0].score, + `5 lows (${fiveLows.areas[0].score}) should score higher than 1 critical (${oneCritical.areas[0].score})`); + }); + + it('1 critical → grade is D or F (penalty exceeds budget)', () => { + const result = scoreByArea([makeScannerResultWithSeverities('CML', ['critical'])]); + assert.ok(['D', 'F'].includes(result.areas[0].grade), + `1 critical produced grade ${result.areas[0].grade}, expected D or F`); + }); + + it('a few lows still score A (low impact respected)', () => { + const result = scoreByArea([makeScannerResultWithSeverities('CML', ['low', 'low', 'low'])]); + assert.ok(result.areas[0].score >= 75, + `3 lows scored ${result.areas[0].score}, expected >= 75 (B+ range)`); + }); + + it('info-only findings are not penalized', () => { + const result = scoreByArea([makeScannerResultWithSeverities('CML', ['info', 'info', 'info'])]); + assert.equal(result.areas[0].score, 100); + }); + + it('1 high → grade is C or worse', () => { + const result = scoreByArea([makeScannerResultWithSeverities('CML', ['high'])]); + assert.ok(['C', 'D', 'F'].includes(result.areas[0].grade), + `1 high produced grade ${result.areas[0].grade}, expected C/D/F`); + }); }); // ========================================