config-audit/tests/lib/scoring.test.mjs
Kjell Tore Guttormsen 3ac313fa45 feat(config-audit): severity-weighted scoreByArea (v5 F3)
Replace count-based pass-rate with severity-weighted penalty:
- penalty = sum(count[s] * WEIGHTS[s])
- maxBudget = max(10, findingCount * 4)
- passRate = max(0, 100 - penalty / maxBudget * 100)

A few lows no longer crater an area's grade; a single high or critical
consumes a large fraction of budget. Mirrors the operator intuition that
severity, not count, is the signal.

BREAKING (intentional): scoring semantics differ from v4 for non-clean
configs. Add scoringVersion: 'v5' to the returned struct so consumers
can detect the version. baseline-all-a remains all-A (no critical/high
on that fixture).

Tests: +6 cases for severity weighting; existing "many findings" test
updated to use highs (where v5 still drops the grade as expected).
2026-05-01 06:20:08 +02:00

614 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
calculateUtilization,
determineMaturityLevel,
determineSegment,
scoreByArea,
topActions,
generateScorecard,
generateHealthScorecard,
TITLE_TO_ID,
TIER_WEIGHTS,
TIER_COUNTS,
MAX_WEIGHTED,
MATURITY_LEVELS,
SEGMENTS,
} from '../../scanners/lib/scoring.mjs';
// --- Helpers ---
function makeGapFinding(title, tier) {
return { id: 'CA-GAP-999', scanner: 'GAP', severity: 'info', title, category: tier, recommendation: 'Fix it' };
}
function allGapFindings() {
return Object.entries(TITLE_TO_ID).map(([title, id]) => {
const tier = id.split('_')[0];
return makeGapFinding(title, tier);
});
}
function t1GapFindings() {
return Object.entries(TITLE_TO_ID)
.filter(([, id]) => id.startsWith('t1'))
.map(([title]) => makeGapFinding(title, 't1'));
}
function t4GapFindings() {
return Object.entries(TITLE_TO_ID)
.filter(([, id]) => id.startsWith('t4'))
.map(([title]) => makeGapFinding(title, 't4'));
}
function makeScannerResult(scanner, findingCount) {
const findings = Array.from({ length: findingCount }, (_, i) => ({
id: `CA-${scanner}-${String(i + 1).padStart(3, '0')}`,
scanner,
severity: 'low',
title: `Finding ${i + 1}`,
category: scanner === 'GAP' ? 't2' : null,
recommendation: 'Fix',
}));
return {
scanner,
status: 'ok',
files_scanned: 5,
duration_ms: 10,
findings,
counts: { critical: 0, high: 0, medium: 0, low: findingCount, info: 0 },
};
}
function makeScannerResultWithSeverities(scanner, severities) {
const counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
const findings = severities.map((sev, i) => {
if (counts[sev] !== undefined) counts[sev]++;
return {
id: `CA-${scanner}-${String(i + 1).padStart(3, '0')}`,
scanner,
severity: sev,
title: `Finding ${i + 1}`,
category: scanner === 'GAP' ? 't2' : null,
recommendation: 'Fix',
};
});
return {
scanner,
status: 'ok',
files_scanned: 5,
duration_ms: 10,
findings,
counts,
};
}
// ========================================
// calculateUtilization
// ========================================
describe('calculateUtilization', () => {
it('returns 100% with no gap findings', () => {
const result = calculateUtilization([]);
assert.equal(result.score, 100);
assert.equal(result.overhang, 0);
});
it('returns 0% when all 25 dimensions are gaps', () => {
const result = calculateUtilization(allGapFindings());
assert.equal(result.score, 0);
assert.equal(result.overhang, 100);
});
it('weighs T1 gaps heavier (3x)', () => {
const onlyT1 = t1GapFindings(); // 5 T1 gaps = 15 weight lost
const result = calculateUtilization(onlyT1);
// Lost: 5 × 3 = 15 out of 42. Present: 27/42 = 64%
assert.equal(result.score, 64);
});
it('weighs T4 gaps lighter (1x)', () => {
const onlyT4 = t4GapFindings(); // 5 T4 gaps = 5 weight lost
const result = calculateUtilization(onlyT4);
// Lost: 5 × 1 = 5 out of 42. Present: 37/42 = 88%
assert.equal(result.score, 88);
});
it('T1+T2 present but no T3+T4 scores ~69%', () => {
// T3: 8 dims × 1 = 8, T4: 5 dims × 1 = 5. Lost = 13 out of 42. Present = 29/42 = 69%
const t3t4Gaps = Object.entries(TITLE_TO_ID)
.filter(([, id]) => id.startsWith('t3') || id.startsWith('t4'))
.map(([title, id]) => makeGapFinding(title, id.split('_')[0]));
const result = calculateUtilization(t3t4Gaps);
assert.equal(result.score, 69);
});
it('score + overhang = 100', () => {
const result = calculateUtilization(t1GapFindings());
assert.equal(result.score + result.overhang, 100);
});
it('handles empty array', () => {
const result = calculateUtilization([]);
assert.equal(typeof result.score, 'number');
assert.equal(typeof result.overhang, 'number');
});
it('ignores findings with unknown category', () => {
const weird = [{ category: 'tx' }, { category: undefined }];
const result = calculateUtilization(weird);
assert.equal(result.score, 100); // unknown tiers don't count
});
});
// ========================================
// determineMaturityLevel
// ========================================
describe('determineMaturityLevel', () => {
const discovery = { files: [] };
it('returns Level 0 when CLAUDE.md is missing', () => {
const gaps = [makeGapFinding('No CLAUDE.md file', 't1')];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 0);
assert.equal(result.name, 'Bare');
});
it('returns Level 1 when CLAUDE.md present but no permissions', () => {
const gaps = [
makeGapFinding('No permissions configured', 't1'),
makeGapFinding('No hooks configured', 't1'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 1);
});
it('returns Level 2 when permissions + hooks + modular present but no MCP', () => {
const gaps = [
makeGapFinding('No MCP servers configured', 't1'),
makeGapFinding('Low hook diversity', 't2'),
makeGapFinding('No custom subagents', 't2'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 2);
assert.equal(result.name, 'Structured');
});
it('returns Level 3 when MCP + hook diversity + subagents present but no plugin', () => {
const gaps = [
makeGapFinding('No custom plugin', 't4'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 3);
assert.equal(result.name, 'Automated');
});
it('returns Level 4 when all requirements met', () => {
const result = determineMaturityLevel([], discovery);
assert.equal(result.level, 4);
assert.equal(result.name, 'Governed');
});
it('Level 2 requires modular OR path-rules (modular)', () => {
// Has permissions, hooks, modular — but no path-rules. Should still be level 2.
const gaps = [
makeGapFinding('No path-scoped rules', 't2'),
makeGapFinding('No MCP servers configured', 't1'),
makeGapFinding('Low hook diversity', 't2'),
makeGapFinding('No custom subagents', 't2'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 2);
});
it('Level 2 requires modular OR path-rules (path-rules)', () => {
// Has permissions, hooks, path-rules — but not modular. Should still be level 2.
const gaps = [
makeGapFinding('CLAUDE.md not modular', 't2'),
makeGapFinding('No MCP servers configured', 't1'),
makeGapFinding('Low hook diversity', 't2'),
makeGapFinding('No custom subagents', 't2'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 2);
});
it('stays Level 1 when neither modular nor path-rules', () => {
const gaps = [
makeGapFinding('CLAUDE.md not modular', 't2'),
makeGapFinding('No path-scoped rules', 't2'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 1);
});
it('Level 3 blocked by missing hook diversity', () => {
const gaps = [
makeGapFinding('Low hook diversity', 't2'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 2);
});
it('Level 4 blocked by missing project MCP in git', () => {
const gaps = [
makeGapFinding('No project .mcp.json in git', 't4'),
];
const result = determineMaturityLevel(gaps, discovery);
assert.equal(result.level, 3);
});
it('all MATURITY_LEVELS have required fields', () => {
for (const ml of MATURITY_LEVELS) {
assert.ok(typeof ml.level === 'number');
assert.ok(typeof ml.name === 'string');
assert.ok(typeof ml.description === 'string');
}
});
});
// ========================================
// determineSegment
// ========================================
describe('determineSegment', () => {
it('Top Performer at score > 80', () => {
assert.equal(determineSegment(81).segment, 'Top Performer');
assert.equal(determineSegment(100).segment, 'Top Performer');
});
it('Strong at 65-80', () => {
assert.equal(determineSegment(65).segment, 'Strong');
assert.equal(determineSegment(80).segment, 'Strong');
});
it('Competent at 45-64', () => {
assert.equal(determineSegment(45).segment, 'Competent');
assert.equal(determineSegment(64).segment, 'Competent');
});
it('Developing at 25-44', () => {
assert.equal(determineSegment(25).segment, 'Developing');
assert.equal(determineSegment(44).segment, 'Developing');
});
it('Beginner at < 25', () => {
assert.equal(determineSegment(0).segment, 'Beginner');
assert.equal(determineSegment(24).segment, 'Beginner');
});
it('returns description string', () => {
const result = determineSegment(50);
assert.ok(result.description.length > 0);
});
it('edge case: exactly 80 is Strong', () => {
assert.equal(determineSegment(80).segment, 'Strong');
});
});
// ========================================
// scoreByArea
// ========================================
describe('scoreByArea', () => {
it('returns areas for all 9 scanners', () => {
const scanners = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP', 'TOK']
.map(s => makeScannerResult(s, 0));
const result = scoreByArea(scanners);
assert.equal(result.areas.length, 9);
});
it('zero findings → A grade', () => {
const scanners = [makeScannerResult('CML', 0)];
const result = scoreByArea(scanners);
assert.equal(result.areas[0].grade, 'A');
assert.equal(result.areas[0].score, 100);
});
it('many high-severity findings → lower grade (v5 severity-weighted)', () => {
const scanners = [makeScannerResultWithSeverities('CML', ['high', 'high', 'high'])];
const result = scoreByArea(scanners);
assert.ok(result.areas[0].score < 50);
});
it('GAP scanner uses utilization-based scoring', () => {
const gapResult = makeScannerResult('GAP', 0);
const result = scoreByArea([gapResult]);
assert.equal(result.areas[0].name, 'Feature Coverage');
assert.equal(result.areas[0].score, 100); // 0 gaps = 100%
});
it('overall grade is average of quality areas (excludes GAP)', () => {
const scanners = [
makeScannerResult('CML', 0), // 100
makeScannerResult('SET', 0), // 100
makeScannerResult('GAP', 20), // low utilization — should NOT drag down grade
];
const result = scoreByArea(scanners);
assert.equal(result.overallGrade, 'A'); // only CML+SET averaged
assert.equal(result.areas.length, 3); // GAP still in areas for display
});
it('mixed grades produce mixed overall', () => {
const scanners = [
makeScannerResult('CML', 0), // 100 → A
makeScannerResult('SET', 10), // low → F range
];
const result = scoreByArea(scanners);
assert.ok(['A', 'B', 'C', 'D', 'F'].includes(result.overallGrade));
});
it('empty scanner array → overallGrade F', () => {
const result = scoreByArea([]);
assert.equal(result.areas.length, 0);
assert.equal(result.overallGrade, 'F');
});
it('area objects have required fields', () => {
const scanners = [makeScannerResult('CML', 2)];
const result = scoreByArea(scanners);
const area = result.areas[0];
assert.ok('name' in area);
assert.ok('grade' in area);
assert.ok('score' in area);
assert.ok('findingCount' in area);
});
it('exposes scoringVersion: v5', () => {
const result = scoreByArea([makeScannerResult('CML', 0)]);
assert.equal(result.scoringVersion, 'v5');
});
});
// ========================================
// scoreByArea — severity weighting (v5 F3)
// ========================================
describe('scoreByArea — severity weighting (v5 F3)', () => {
it('clean scanner → 100/A', () => {
const result = scoreByArea([makeScannerResultWithSeverities('CML', [])]);
assert.equal(result.areas[0].score, 100);
assert.equal(result.areas[0].grade, 'A');
});
it('5 lows scores higher than 1 critical', () => {
const fiveLows = scoreByArea([makeScannerResultWithSeverities('CML', ['low', 'low', 'low', 'low', 'low'])]);
const oneCritical = scoreByArea([makeScannerResultWithSeverities('CML', ['critical'])]);
assert.ok(fiveLows.areas[0].score > oneCritical.areas[0].score,
`5 lows (${fiveLows.areas[0].score}) should score higher than 1 critical (${oneCritical.areas[0].score})`);
});
it('1 critical → grade is D or F (penalty exceeds budget)', () => {
const result = scoreByArea([makeScannerResultWithSeverities('CML', ['critical'])]);
assert.ok(['D', 'F'].includes(result.areas[0].grade),
`1 critical produced grade ${result.areas[0].grade}, expected D or F`);
});
it('a few lows still score A (low impact respected)', () => {
const result = scoreByArea([makeScannerResultWithSeverities('CML', ['low', 'low', 'low'])]);
assert.ok(result.areas[0].score >= 75,
`3 lows scored ${result.areas[0].score}, expected >= 75 (B+ range)`);
});
it('info-only findings are not penalized', () => {
const result = scoreByArea([makeScannerResultWithSeverities('CML', ['info', 'info', 'info'])]);
assert.equal(result.areas[0].score, 100);
});
it('1 high → grade is C or worse', () => {
const result = scoreByArea([makeScannerResultWithSeverities('CML', ['high'])]);
assert.ok(['C', 'D', 'F'].includes(result.areas[0].grade),
`1 high produced grade ${result.areas[0].grade}, expected C/D/F`);
});
});
// ========================================
// topActions
// ========================================
describe('topActions', () => {
it('returns max 3 actions', () => {
const gaps = allGapFindings();
const result = topActions(gaps);
assert.equal(result.length, 3);
});
it('prioritizes T1 over T2', () => {
const gaps = [
{ ...makeGapFinding('Low hook diversity', 't2'), recommendation: 'Add hooks' },
{ ...makeGapFinding('No CLAUDE.md file', 't1'), recommendation: 'Create CLAUDE.md' },
];
const result = topActions(gaps);
assert.equal(result[0], 'Create CLAUDE.md');
});
it('returns empty array for no gaps', () => {
assert.deepEqual(topActions([]), []);
});
it('returns all items if fewer than 3', () => {
const gaps = [makeGapFinding('No CLAUDE.md file', 't1')];
assert.equal(topActions(gaps).length, 1);
});
});
// ========================================
// generateScorecard
// ========================================
describe('generateScorecard', () => {
const sampleAreas = {
areas: [
{ name: 'CLAUDE.md', grade: 'A', score: 92 },
{ name: 'Settings', grade: 'B', score: 78 },
{ name: 'Hooks', grade: 'C', score: 55 },
{ name: 'Rules', grade: 'B', score: 71 },
],
overallGrade: 'B',
};
const sampleUtil = { score: 68, overhang: 32 };
const sampleMaturity = { level: 2, name: 'Structured' };
const sampleSegment = { segment: 'Strong' };
const sampleActions = ['Configure MCP', 'Add hooks', 'Create agents'];
it('returns a string', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.equal(typeof result, 'string');
});
it('contains header line', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('Config-Audit Posture Score'));
});
it('contains overall grade', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('Overall: B'));
});
it('contains maturity level', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('Level 2 (Structured)'));
});
it('contains utilization', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('Utilization: 68%'));
});
it('contains segment', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('Segment: Strong'));
});
it('contains all area names', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('CLAUDE.md'));
assert.ok(result.includes('Settings'));
assert.ok(result.includes('Hooks'));
assert.ok(result.includes('Rules'));
});
it('contains top actions', () => {
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
assert.ok(result.includes('1. Configure MCP'));
assert.ok(result.includes('2. Add hooks'));
assert.ok(result.includes('3. Create agents'));
});
it('works with empty areas', () => {
const result = generateScorecard({ areas: [], overallGrade: 'F' }, sampleUtil, sampleMaturity, sampleSegment, []);
assert.equal(typeof result, 'string');
assert.ok(result.includes('Config-Audit Posture Score'));
});
it('works with odd number of areas', () => {
const odd = { areas: [{ name: 'Test', grade: 'A', score: 95 }], overallGrade: 'A' };
const result = generateScorecard(odd, sampleUtil, sampleMaturity, sampleSegment, []);
assert.ok(result.includes('Test'));
});
});
// ========================================
// generateHealthScorecard (v3)
// ========================================
describe('generateHealthScorecard', () => {
const sampleAreas = {
areas: [
{ name: 'CLAUDE.md', grade: 'A', score: 92 },
{ name: 'Settings', grade: 'B', score: 78 },
{ name: 'Hooks', grade: 'C', score: 55 },
{ name: 'Feature Coverage', grade: 'F', score: 20 },
],
overallGrade: 'B',
};
it('returns a string', () => {
const result = generateHealthScorecard(sampleAreas, 12);
assert.equal(typeof result, 'string');
});
it('contains Health header (not Overall)', () => {
const result = generateHealthScorecard(sampleAreas, 5);
assert.ok(result.includes('Config-Audit Health Score'));
assert.ok(result.includes('Health: B'));
assert.ok(!result.includes('Overall:'));
});
it('does NOT contain Maturity, Utilization, or Segment', () => {
const result = generateHealthScorecard(sampleAreas, 5);
assert.ok(!result.includes('Maturity:'));
assert.ok(!result.includes('Utilization:'));
assert.ok(!result.includes('Segment:'));
});
it('excludes Feature Coverage from area display', () => {
const result = generateHealthScorecard(sampleAreas, 5);
assert.ok(!result.includes('Feature Coverage'));
assert.ok(result.includes('CLAUDE.md'));
assert.ok(result.includes('Settings'));
assert.ok(result.includes('Hooks'));
});
it('shows opportunity count', () => {
const result = generateHealthScorecard(sampleAreas, 12);
assert.ok(result.includes('12 opportunities available'));
});
it('uses singular for 1 opportunity', () => {
const result = generateHealthScorecard(sampleAreas, 1);
assert.ok(result.includes('1 opportunity available'));
});
it('hides opportunity line when count is 0', () => {
const result = generateHealthScorecard(sampleAreas, 0);
assert.ok(!result.includes('opportunit'));
});
it('shows areas scanned count', () => {
const result = generateHealthScorecard(sampleAreas, 5);
assert.ok(result.includes('3 areas scanned')); // 3 quality areas (excl Feature Coverage)
});
it('computes avgScore from quality areas only', () => {
// Quality areas: A(92), B(78), C(55) → avg = 75
const result = generateHealthScorecard(sampleAreas, 5);
assert.ok(result.includes('(75/100)'));
});
it('works with empty areas', () => {
const result = generateHealthScorecard({ areas: [], overallGrade: 'F' }, 0);
assert.equal(typeof result, 'string');
assert.ok(result.includes('Config-Audit Health Score'));
});
});
// ========================================
// Constants and exports
// ========================================
describe('scoring constants', () => {
it('TITLE_TO_ID has 25 entries', () => {
assert.equal(Object.keys(TITLE_TO_ID).length, 25);
});
it('TIER_COUNTS sum to 25', () => {
const sum = Object.values(TIER_COUNTS).reduce((a, b) => a + b, 0);
assert.equal(sum, 25);
});
it('MAX_WEIGHTED is 42', () => {
assert.equal(MAX_WEIGHTED, 42);
});
it('TIER_WEIGHTS match spec', () => {
assert.equal(TIER_WEIGHTS.t1, 3);
assert.equal(TIER_WEIGHTS.t2, 2);
assert.equal(TIER_WEIGHTS.t3, 1);
assert.equal(TIER_WEIGHTS.t4, 1);
});
it('SEGMENTS covers full 0-100 range', () => {
assert.equal(SEGMENTS[SEGMENTS.length - 1].min, 0);
assert.ok(SEGMENTS[0].min >= 80);
});
it('MATURITY_LEVELS has 5 levels (0-4)', () => {
assert.equal(MATURITY_LEVELS.length, 5);
assert.equal(MATURITY_LEVELS[0].level, 0);
assert.equal(MATURITY_LEVELS[4].level, 4);
});
});