config-audit/scanners/lib/scoring.mjs
Kjell Tore Guttormsen 3ac313fa45 feat(config-audit): severity-weighted scoreByArea (v5 F3)
Replace count-based pass-rate with severity-weighted penalty:
- penalty = sum(count[s] * WEIGHTS[s])
- maxBudget = max(10, findingCount * 4)
- passRate = max(0, 100 - penalty / maxBudget * 100)

A few lows no longer crater an area's grade; a single high or critical
consumes a large fraction of budget. Mirrors the operator intuition that
severity, not count, is the signal.

BREAKING (intentional): scoring semantics differ from v4 for non-clean
configs. Add scoringVersion: 'v5' to the returned struct so consumers
can detect the version. baseline-all-a remains all-A (no critical/high
on that fixture).

Tests: +6 cases for severity weighting; existing "many findings" test
updated to use highs (where v5 still drops the grade as expected).
2026-05-01 06:20:08 +02:00

335 lines
13 KiB
JavaScript

/**
* Scoring, maturity, and posture assessment for config-audit.
* Zero external dependencies.
*/
import { gradeFromPassRate, WEIGHTS } from './severity.mjs';
// --- Tier weights for utilization calculation ---
const TIER_WEIGHTS = { t1: 3, t2: 2, t3: 1, t4: 1 };
const TIER_COUNTS = { t1: 5, t2: 7, t3: 8, t4: 5 };
const TOTAL_DIMENSIONS = 25;
const MAX_WEIGHTED = Object.entries(TIER_COUNTS).reduce(
(sum, [tier, count]) => sum + count * TIER_WEIGHTS[tier],
0,
); // 5*3 + 7*2 + 8*1 + 5*1 = 42
/**
* Calculate weighted utilization from GAP scanner findings.
* @param {object[]} gapFindings - Array of GAP scanner findings (each has .category = t1|t2|t3|t4)
* @param {number} [totalDimensions=25]
* @returns {{ score: number, overhang: number }}
*/
export function calculateUtilization(gapFindings, totalDimensions = TOTAL_DIMENSIONS) {
// Count gaps per tier
const gapsByTier = { t1: 0, t2: 0, t3: 0, t4: 0 };
for (const f of gapFindings) {
const tier = f.category;
if (tier in gapsByTier) gapsByTier[tier]++;
}
// Present (non-gap) weight
let presentWeight = 0;
for (const [tier, totalCount] of Object.entries(TIER_COUNTS)) {
const presentCount = totalCount - gapsByTier[tier];
presentWeight += presentCount * TIER_WEIGHTS[tier];
}
const score = Math.round((presentWeight / MAX_WEIGHTED) * 100);
return { score, overhang: 100 - score };
}
// --- Maturity levels ---
const MATURITY_LEVELS = [
{ level: 0, name: 'Bare', description: 'No CLAUDE.md, default everything' },
{ level: 1, name: 'Configured', description: 'CLAUDE.md + basic settings' },
{ level: 2, name: 'Structured', description: 'Rules, skills, hooks' },
{ level: 3, name: 'Automated', description: 'MCP, custom agents, diverse hooks' },
{ level: 4, name: 'Governed', description: 'Plugins, managed settings, full monitoring' },
];
/**
* Determine config maturity level (threshold-based: highest level where ALL requirements met).
* @param {object[]} gapFindings - GAP scanner findings
* @param {{ files: Array<{ type: string, absPath?: string, scope?: string }> }} discovery
* @returns {{ level: number, name: string, description: string }}
*/
export function determineMaturityLevel(gapFindings, discovery) {
const gapIds = new Set(gapFindings.map(f => {
// Extract the gap check id from the title — match against known titles
return findGapId(f);
}));
const has = (id) => !gapIds.has(id); // feature is present if NOT in gaps
// Level 1: CLAUDE.md present
if (!has('t1_1')) return MATURITY_LEVELS[0];
// Level 2: Level 1 + permissions + hooks + (modular OR path-rules)
const level2 = has('t1_2') && has('t1_3') && (has('t2_2') || has('t2_3'));
if (!level2) return MATURITY_LEVELS[1];
// Level 3: Level 2 + MCP + hook diversity + custom subagents
const level3 = has('t1_5') && has('t2_5') && has('t2_6');
if (!level3) return MATURITY_LEVELS[2];
// Level 4: Level 3 + project MCP in git + custom plugin
const level4 = has('t4_1') && has('t4_2');
if (!level4) return MATURITY_LEVELS[3];
return MATURITY_LEVELS[4];
}
/**
* Map a GAP finding to its gap check ID based on known title→id mapping.
* @param {object} finding
* @returns {string}
*/
function findGapId(finding) {
return TITLE_TO_ID[finding.title] || 'unknown';
}
/** Title→ID mapping for all 25 gap checks */
const TITLE_TO_ID = {
'No CLAUDE.md file': 't1_1',
'No permissions configured': 't1_2',
'No hooks configured': 't1_3',
'No custom skills or commands': 't1_4',
'No MCP servers configured': 't1_5',
'Settings only at one scope': 't2_1',
'CLAUDE.md not modular': 't2_2',
'No path-scoped rules': 't2_3',
'Auto-memory explicitly disabled': 't2_4',
'Low hook diversity': 't2_5',
'No custom subagents': 't2_6',
'No model configuration': 't2_7',
'No status line configured': 't3_1',
'No custom keybindings': 't3_2',
'Using default output style': 't3_3',
'No worktree workflow': 't3_4',
'No advanced skill frontmatter': 't3_5',
'No subagent isolation': 't3_6',
'No dynamic skill context': 't3_7',
'No autoMode classifier': 't3_8',
'No project .mcp.json in git': 't4_1',
'No custom plugin': 't4_2',
'Agent teams not enabled': 't4_3',
'No managed settings': 't4_4',
'No LSP plugins': 't4_5',
};
// --- Segments ---
const SEGMENTS = [
{ min: 81, segment: 'Top Performer', description: 'Exceptional configuration — leveraging most of Claude Code\'s capabilities' },
{ min: 65, segment: 'Strong', description: 'Well-configured — using advanced features effectively' },
{ min: 45, segment: 'Competent', description: 'Solid foundation — room to leverage more features' },
{ min: 25, segment: 'Developing', description: 'Basic setup — significant features untapped' },
{ min: 0, segment: 'Beginner', description: 'Minimal configuration — most capabilities unused' },
];
/**
* Determine segment from utilization score.
* @param {number} score - 0-100
* @param {number} [_maturityLevel] - unused, kept for API compatibility
* @returns {{ segment: string, description: string }}
*/
export function determineSegment(score, _maturityLevel) {
for (const s of SEGMENTS) {
if (score >= s.min) return { segment: s.segment, description: s.description };
}
return SEGMENTS[SEGMENTS.length - 1];
}
// --- Area scoring ---
const SCANNER_AREA_MAP = {
CML: 'CLAUDE.md',
SET: 'Settings',
HKV: 'Hooks',
RUL: 'Rules',
MCP: 'MCP',
IMP: 'Imports',
CNF: 'Conflicts',
GAP: 'Feature Coverage',
TOK: 'Token Efficiency',
};
/**
* Slugify an area name into a stable id.
* Example: "Token Efficiency" → "token_efficiency", "CLAUDE.md" → "claude_md".
*/
function slugify(name) {
return String(name).toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
}
/**
* Compute raw severity-weighted penalty from scanner counts.
* Critical/high findings dominate; lows barely move the needle.
* @param {{ critical?: number, high?: number, medium?: number, low?: number, info?: number }} counts
* @returns {number}
*/
function severityPenalty(counts) {
let penalty = 0;
for (const [sev, weight] of Object.entries(WEIGHTS)) {
penalty += (counts[sev] || 0) * weight;
}
return penalty;
}
/**
* Score per config area from scanner results (v5: severity-weighted).
* @param {object[]} scannerResults - Array of scanner result objects from envelope.scanners
* @returns {{ areas: Array<{ id: string, name: string, grade: string, score: number, findingCount: number }>, overallGrade: string, scoringVersion: string }}
*/
export function scoreByArea(scannerResults) {
const areas = [];
for (const result of scannerResults) {
const name = SCANNER_AREA_MAP[result.scanner] || result.scanner;
const findingCount = result.findings.length;
let score;
if (result.scanner === 'GAP') {
const util = calculateUtilization(result.findings);
score = util.score;
} else {
// v5 severity-weighted: penalty proportional to a per-scanner budget.
// maxBudget = max(10, findingCount * 4) — adding more lows doesn't crater the
// grade, but a single high-severity finding consumes a large fraction of budget.
const counts = result.counts || {};
const penalty = severityPenalty(counts);
const maxBudget = Math.max(10, findingCount * 4);
const passRate = Math.max(0, 100 - (penalty / maxBudget) * 100);
score = Math.round(passRate);
}
const grade = gradeFromPassRate(score);
areas.push({ id: slugify(name), name, grade, score, findingCount });
}
// Overall grade: quality areas only (exclude GAP — feature coverage is informational, not a quality issue)
const qualityAreas = areas.filter(a => a.name !== 'Feature Coverage');
const totalScore = qualityAreas.reduce((sum, a) => sum + a.score, 0);
const avgScore = qualityAreas.length > 0 ? Math.round(totalScore / qualityAreas.length) : 0;
const overallGrade = gradeFromPassRate(avgScore);
return { areas, overallGrade, scoringVersion: 'v5' };
}
/**
* Derive top 3 actions from GAP findings (T1 first, then T2).
* @param {object[]} gapFindings
* @returns {string[]}
*/
export function topActions(gapFindings) {
const tierOrder = ['t1', 't2', 't3', 't4'];
const sorted = [...gapFindings].sort(
(a, b) => tierOrder.indexOf(a.category) - tierOrder.indexOf(b.category),
);
return sorted.slice(0, 3).map(f => f.recommendation);
}
/**
* Generate a terminal-friendly scorecard string (v2 format — kept for backward compat).
* @param {{ areas: Array<{ name: string, grade: string, score: number }>, overallGrade: string }} areaScores
* @param {{ score: number, overhang: number }} utilization
* @param {{ level: number, name: string }} maturity
* @param {{ segment: string }} segment
* @param {string[]} actions
* @returns {string}
* @deprecated Use generateHealthScorecard for v3+ terminal output
*/
export function generateScorecard(areaScores, utilization, maturity, segment, actions) {
// Bug fix: exclude GAP from displayed avgScore (was inconsistent with overallGrade)
const qualityAreas = areaScores.areas.filter(a => a.name !== 'Feature Coverage');
const avgScore = qualityAreas.length > 0
? Math.round(qualityAreas.reduce((s, a) => s + a.score, 0) / qualityAreas.length)
: 0;
const lines = [];
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push(' Config-Audit Posture Score');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push('');
lines.push(` Overall: ${areaScores.overallGrade} (${avgScore}/100) Maturity: Level ${maturity.level} (${maturity.name})`);
lines.push(` Segment: ${segment.segment} Utilization: ${utilization.score}%`);
lines.push('');
lines.push(' Area Scores');
lines.push(' ───────────');
// Format areas in 2-column layout
const areas = areaScores.areas;
for (let i = 0; i < areas.length; i += 2) {
const left = areas[i];
const right = areas[i + 1];
const leftStr = ` ${left.name} ${'.'.repeat(Math.max(1, 20 - left.name.length))} ${left.grade} (${left.score})`;
if (right) {
const rightStr = `${right.name} ${'.'.repeat(Math.max(1, 20 - right.name.length))} ${right.grade} (${right.score})`;
lines.push(`${leftStr.padEnd(35)}${rightStr}`);
} else {
lines.push(leftStr);
}
}
if (actions.length > 0) {
lines.push('');
lines.push(' Top 3 Actions');
lines.push(' ─────────────');
for (let i = 0; i < actions.length; i++) {
lines.push(` ${i + 1}. ${actions[i]}`);
}
}
lines.push('');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
return lines.join('\n');
}
/**
* Generate a v3 health-focused terminal scorecard.
* Shows only the quality areas (currently 8) — no utilization, maturity, or segment.
* @param {{ areas: Array<{ name: string, grade: string, score: number }>, overallGrade: string }} areaScores
* @param {number} opportunityCount - Number of GAP findings (shown as opportunity count)
* @returns {string}
*/
export function generateHealthScorecard(areaScores, opportunityCount) {
const qualityAreas = areaScores.areas.filter(a => a.name !== 'Feature Coverage');
const avgScore = qualityAreas.length > 0
? Math.round(qualityAreas.reduce((s, a) => s + a.score, 0) / qualityAreas.length)
: 0;
const lines = [];
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push(' Config-Audit Health Score');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push('');
lines.push(` Health: ${areaScores.overallGrade} (${avgScore}/100) ${qualityAreas.length} areas scanned`);
lines.push('');
lines.push(' Area Scores');
lines.push(' ───────────');
// Format areas in 2-column layout (quality areas only)
for (let i = 0; i < qualityAreas.length; i += 2) {
const left = qualityAreas[i];
const right = qualityAreas[i + 1];
const leftStr = ` ${left.name} ${'.'.repeat(Math.max(1, 20 - left.name.length))} ${left.grade} (${left.score})`;
if (right) {
const rightStr = `${right.name} ${'.'.repeat(Math.max(1, 20 - right.name.length))} ${right.grade} (${right.score})`;
lines.push(`${leftStr.padEnd(35)}${rightStr}`);
} else {
lines.push(leftStr);
}
}
if (opportunityCount > 0) {
lines.push('');
lines.push(` ${opportunityCount} ${opportunityCount === 1 ? 'opportunity' : 'opportunities'} available — run /config-audit feature-gap for recommendations`);
}
lines.push('');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
return lines.join('\n');
}
export { TITLE_TO_ID, TIER_WEIGHTS, TIER_COUNTS, MAX_WEIGHTED, MATURITY_LEVELS, SEGMENTS };