config-audit/scanners/lib/scoring.mjs
Kjell Tore Guttormsen fcfb2979ef feat(scanner): add AGT agent-listing always-loaded budget finding (v5.9 B1)
New AGT scanner flags the aggregate always-loaded cost of the agent listing
(every active agent's name+description is injected each turn so the model
knows what it can delegate to). Mirrors the SKL skill-listing pattern but
encodes the key honesty caveat: the agent-listing mechanism is INFERRED
(agents are absent from Claude Code's documented context breakdown), so the
token figure is an UPPER-BOUND estimate and the aggregate budget is a
config-audit heuristic anchored on 200k — not a CC-documented allotment.

- scanners/agent-listing-scanner.mjs + scanners/lib/agent-listing-budget.mjs
- 8 TDD tests (tests/scanners/agent-listing-scanner.test.mjs)
- AGT folds into the Token Efficiency health area (scoring.mjs)
- byte-stability: AGT added to strip-added-scanner (frozen v5.0.0 baselines
  left untouched, same as OST/OPT); SC-5 default-output snapshot refreshed
- suite 1168 -> 1176 green

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-23 15:02:59 +02:00

407 lines
16 KiB
JavaScript

/**
* Scoring, maturity, and posture assessment for config-audit.
* Zero external dependencies.
*/
import { gradeFromPassRate, WEIGHTS } from './severity.mjs';
import { humanizeFinding } from './humanizer.mjs';
/**
* One-line plain-language context per overall grade. Used when a scorecard
* is rendered with `options.humanized: true`.
*/
const GRADE_CONTEXT = {
A: 'Healthy setup, only minor polish needed',
B: 'Good shape — a few items to address',
C: 'Some attention needed',
D: 'Several issues — prioritize the urgent ones',
F: 'Important issues need attention',
};
// --- Tier weights for utilization calculation ---
const TIER_WEIGHTS = { t1: 3, t2: 2, t3: 1, t4: 1 };
const TIER_COUNTS = { t1: 5, t2: 7, t3: 8, t4: 5 };
const TOTAL_DIMENSIONS = 25;
const MAX_WEIGHTED = Object.entries(TIER_COUNTS).reduce(
(sum, [tier, count]) => sum + count * TIER_WEIGHTS[tier],
0,
); // 5*3 + 7*2 + 8*1 + 5*1 = 42
/**
* Calculate weighted utilization from GAP scanner findings.
* @param {object[]} gapFindings - Array of GAP scanner findings (each has .category = t1|t2|t3|t4)
* @param {number} [totalDimensions=25]
* @returns {{ score: number, overhang: number }}
*/
export function calculateUtilization(gapFindings, totalDimensions = TOTAL_DIMENSIONS) {
// Count gaps per tier
const gapsByTier = { t1: 0, t2: 0, t3: 0, t4: 0 };
for (const f of gapFindings) {
const tier = f.category;
if (tier in gapsByTier) gapsByTier[tier]++;
}
// Present (non-gap) weight
let presentWeight = 0;
for (const [tier, totalCount] of Object.entries(TIER_COUNTS)) {
const presentCount = totalCount - gapsByTier[tier];
presentWeight += presentCount * TIER_WEIGHTS[tier];
}
const score = Math.round((presentWeight / MAX_WEIGHTED) * 100);
return { score, overhang: 100 - score };
}
// --- Maturity levels ---
const MATURITY_LEVELS = [
{ level: 0, name: 'Bare', description: 'No CLAUDE.md, default everything' },
{ level: 1, name: 'Configured', description: 'CLAUDE.md + basic settings' },
{ level: 2, name: 'Structured', description: 'Rules, skills, hooks' },
{ level: 3, name: 'Automated', description: 'MCP, custom agents, diverse hooks' },
{ level: 4, name: 'Governed', description: 'Plugins, managed settings, full monitoring' },
];
/**
* Determine config maturity level (threshold-based: highest level where ALL requirements met).
* @param {object[]} gapFindings - GAP scanner findings
* @param {{ files: Array<{ type: string, absPath?: string, scope?: string }> }} discovery
* @returns {{ level: number, name: string, description: string }}
*/
export function determineMaturityLevel(gapFindings, discovery) {
const gapIds = new Set(gapFindings.map(f => {
// Extract the gap check id from the title — match against known titles
return findGapId(f);
}));
const has = (id) => !gapIds.has(id); // feature is present if NOT in gaps
// Level 1: CLAUDE.md present
if (!has('t1_1')) return MATURITY_LEVELS[0];
// Level 2: Level 1 + permissions + hooks + (modular OR path-rules)
const level2 = has('t1_2') && has('t1_3') && (has('t2_2') || has('t2_3'));
if (!level2) return MATURITY_LEVELS[1];
// Level 3: Level 2 + MCP + hook diversity + custom subagents
const level3 = has('t1_5') && has('t2_5') && has('t2_6');
if (!level3) return MATURITY_LEVELS[2];
// Level 4: Level 3 + project MCP in git + custom plugin
const level4 = has('t4_1') && has('t4_2');
if (!level4) return MATURITY_LEVELS[3];
return MATURITY_LEVELS[4];
}
/**
* Map a GAP finding to its gap check ID based on known title→id mapping.
* @param {object} finding
* @returns {string}
*/
function findGapId(finding) {
return TITLE_TO_ID[finding.title] || 'unknown';
}
/** Title→ID mapping for all 25 gap checks */
const TITLE_TO_ID = {
'No CLAUDE.md file': 't1_1',
'No permissions configured': 't1_2',
'No hooks configured': 't1_3',
'No custom skills or commands': 't1_4',
'No MCP servers configured': 't1_5',
'Settings only at one scope': 't2_1',
'CLAUDE.md not modular': 't2_2',
'No path-scoped rules': 't2_3',
'Auto-memory explicitly disabled': 't2_4',
'Low hook diversity': 't2_5',
'No custom subagents': 't2_6',
'No model configuration': 't2_7',
'No status line configured': 't3_1',
'No custom keybindings': 't3_2',
'Using default output style': 't3_3',
'No worktree workflow': 't3_4',
'No advanced skill frontmatter': 't3_5',
'No subagent isolation': 't3_6',
'No dynamic skill context': 't3_7',
'No autoMode classifier': 't3_8',
'No project .mcp.json in git': 't4_1',
'No custom plugin': 't4_2',
'Agent teams not enabled': 't4_3',
'No managed settings': 't4_4',
'No LSP plugins': 't4_5',
};
// --- Segments ---
const SEGMENTS = [
{ min: 81, segment: 'Top Performer', description: 'Exceptional configuration — leveraging most of Claude Code\'s capabilities' },
{ min: 65, segment: 'Strong', description: 'Well-configured — using advanced features effectively' },
{ min: 45, segment: 'Competent', description: 'Solid foundation — room to leverage more features' },
{ min: 25, segment: 'Developing', description: 'Basic setup — significant features untapped' },
{ min: 0, segment: 'Beginner', description: 'Minimal configuration — most capabilities unused' },
];
/**
* Determine segment from utilization score.
* @param {number} score - 0-100
* @param {number} [_maturityLevel] - unused, kept for API compatibility
* @returns {{ segment: string, description: string }}
*/
export function determineSegment(score, _maturityLevel) {
for (const s of SEGMENTS) {
if (score >= s.min) return { segment: s.segment, description: s.description };
}
return SEGMENTS[SEGMENTS.length - 1];
}
// --- Area scoring ---
const SCANNER_AREA_MAP = {
CML: 'CLAUDE.md',
SET: 'Settings',
HKV: 'Hooks',
RUL: 'Rules',
MCP: 'MCP',
IMP: 'Imports',
CNF: 'Conflicts',
GAP: 'Feature Coverage',
TOK: 'Token Efficiency',
CPS: 'Token Efficiency',
SKL: 'Token Efficiency',
AGT: 'Token Efficiency',
DIS: 'Settings',
COL: 'Plugin Hygiene',
OST: 'Settings',
OPT: 'CLAUDE.md',
};
/**
* Slugify an area name into a stable id.
* Example: "Token Efficiency" → "token_efficiency", "CLAUDE.md" → "claude_md".
*/
function slugify(name) {
return String(name).toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
}
/**
* Compute raw severity-weighted penalty from scanner counts.
* Critical/high findings dominate; lows barely move the needle.
* @param {{ critical?: number, high?: number, medium?: number, low?: number, info?: number }} counts
* @returns {number}
*/
function severityPenalty(counts) {
let penalty = 0;
for (const [sev, weight] of Object.entries(WEIGHTS)) {
penalty += (counts[sev] || 0) * weight;
}
return penalty;
}
/**
* Score per config area from scanner results (v5: severity-weighted).
* @param {object[]} scannerResults - Array of scanner result objects from envelope.scanners
* @returns {{ areas: Array<{ id: string, name: string, grade: string, score: number, findingCount: number }>, overallGrade: string, scoringVersion: string }}
*/
export function scoreByArea(scannerResults) {
// Group scanner results by area name so multiple scanners that share an area
// (e.g. TOK + CPS both → "Token Efficiency") produce one combined row.
const grouped = new Map();
for (const result of scannerResults) {
const name = SCANNER_AREA_MAP[result.scanner] || result.scanner;
if (!grouped.has(name)) grouped.set(name, []);
grouped.get(name).push(result);
}
const areas = [];
for (const [name, results] of grouped) {
const findings = results.flatMap(r => r.findings || []);
const findingCount = findings.length;
let score;
if (results.some(r => r.scanner === 'GAP')) {
// GAP scoring uses utilization, not severity penalty
const util = calculateUtilization(findings);
score = util.score;
} else {
// v5 severity-weighted: penalty proportional to a per-area budget.
// Combine counts across all scanners contributing to this area.
const counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
for (const r of results) {
for (const k of Object.keys(counts)) {
counts[k] += (r.counts && r.counts[k]) || 0;
}
}
const penalty = severityPenalty(counts);
const maxBudget = Math.max(10, findingCount * 4);
const passRate = Math.max(0, 100 - (penalty / maxBudget) * 100);
score = Math.round(passRate);
}
const grade = gradeFromPassRate(score);
areas.push({ id: slugify(name), name, grade, score, findingCount });
}
// Overall grade: quality areas only (exclude GAP — feature coverage is informational, not a quality issue)
const qualityAreas = areas.filter(a => a.name !== 'Feature Coverage');
const totalScore = qualityAreas.reduce((sum, a) => sum + a.score, 0);
const avgScore = qualityAreas.length > 0 ? Math.round(totalScore / qualityAreas.length) : 0;
const overallGrade = gradeFromPassRate(avgScore);
return { areas, overallGrade, scoringVersion: 'v5' };
}
/**
* Derive top 3 actions from GAP findings (T1 first, then T2).
* @param {object[]} gapFindings
* @param {object} [options]
* @param {boolean} [options.humanized=false] - When true, return humanized
* recommendations (looked up via humanizer translations).
* @returns {string[]}
*/
export function topActions(gapFindings, options = {}) {
const tierOrder = ['t1', 't2', 't3', 't4'];
const sorted = [...gapFindings].sort(
(a, b) => tierOrder.indexOf(a.category) - tierOrder.indexOf(b.category),
);
const top3 = sorted.slice(0, 3);
if (options.humanized) {
return top3.map(f => humanizeFinding(f).recommendation);
}
return top3.map(f => f.recommendation);
}
/**
* Generate a terminal-friendly scorecard string (v2 format — kept for backward compat).
* @param {{ areas: Array<{ name: string, grade: string, score: number }>, overallGrade: string }} areaScores
* @param {{ score: number, overhang: number }} utilization
* @param {{ level: number, name: string }} maturity
* @param {{ segment: string }} segment
* @param {string[]} actions
* @returns {string}
* @deprecated Use generateHealthScorecard for v3+ terminal output
*/
export function generateScorecard(areaScores, utilization, maturity, segment, actions) {
// Bug fix: exclude GAP from displayed avgScore (was inconsistent with overallGrade)
const qualityAreas = areaScores.areas.filter(a => a.name !== 'Feature Coverage');
const avgScore = qualityAreas.length > 0
? Math.round(qualityAreas.reduce((s, a) => s + a.score, 0) / qualityAreas.length)
: 0;
const lines = [];
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push(' Config-Audit Posture Score');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push('');
lines.push(` Overall: ${areaScores.overallGrade} (${avgScore}/100) Maturity: Level ${maturity.level} (${maturity.name})`);
lines.push(` Segment: ${segment.segment} Utilization: ${utilization.score}%`);
lines.push('');
lines.push(' Area Scores');
lines.push(' ───────────');
// Format areas in 2-column layout
const areas = areaScores.areas;
for (let i = 0; i < areas.length; i += 2) {
const left = areas[i];
const right = areas[i + 1];
const leftStr = ` ${left.name} ${'.'.repeat(Math.max(1, 20 - left.name.length))} ${left.grade} (${left.score})`;
if (right) {
const rightStr = `${right.name} ${'.'.repeat(Math.max(1, 20 - right.name.length))} ${right.grade} (${right.score})`;
lines.push(`${leftStr.padEnd(35)}${rightStr}`);
} else {
lines.push(leftStr);
}
}
if (actions.length > 0) {
lines.push('');
lines.push(' Top 3 Actions');
lines.push(' ─────────────');
for (let i = 0; i < actions.length; i++) {
lines.push(` ${i + 1}. ${actions[i]}`);
}
}
lines.push('');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
return lines.join('\n');
}
/**
* Generate a v3 health-focused terminal scorecard.
* Shows only the quality areas (currently 8) — no utilization, maturity, or segment.
* @param {{ areas: Array<{ name: string, grade: string, score: number }>, overallGrade: string }} areaScores
* @param {number} opportunityCount - Number of GAP findings (shown as opportunity count)
* @param {object} [options]
* @param {boolean} [options.humanized=false] - When true, render with plain-language
* grade context and friendlier opportunity phrasing. When false (default),
* render the v5.0.0 verbatim scorecard (backwards-compatible).
* @returns {string}
*/
export function generateHealthScorecard(areaScores, opportunityCount, options = {}) {
const qualityAreas = areaScores.areas.filter(a => a.name !== 'Feature Coverage');
const avgScore = qualityAreas.length > 0
? Math.round(qualityAreas.reduce((s, a) => s + a.score, 0) / qualityAreas.length)
: 0;
const humanized = options.humanized === true;
const lines = [];
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push(humanized ? ' Configuration health' : ' Config-Audit Health Score');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
lines.push('');
if (humanized) {
const context = GRADE_CONTEXT[areaScores.overallGrade] || '';
const headline = context
? ` Health: ${areaScores.overallGrade} (${avgScore}/100) — ${context}`
: ` Health: ${areaScores.overallGrade} (${avgScore}/100)`;
lines.push(headline);
lines.push(` ${qualityAreas.length} areas reviewed`);
} else {
lines.push(` Health: ${areaScores.overallGrade} (${avgScore}/100) ${qualityAreas.length} areas scanned`);
}
lines.push('');
lines.push(humanized ? ' Area scores' : ' Area Scores');
lines.push(' ───────────');
// Format areas in 2-column layout (quality areas only).
// In humanized mode, area names are wrapped in backticks so SC-3 can treat
// them as code references (technical identifiers like CLAUDE.md, MCP, Hooks
// are tier3 jargon outside backtick spans). Padding compensates for the
// two extra characters so column alignment matches the v5.0.0 layout.
const padBase = humanized ? 22 : 20;
const padCol = humanized ? 37 : 35;
const labelOf = (a) => (humanized ? `\`${a.name}\`` : a.name);
for (let i = 0; i < qualityAreas.length; i += 2) {
const left = qualityAreas[i];
const right = qualityAreas[i + 1];
const leftLabel = labelOf(left);
const leftStr = ` ${leftLabel} ${'.'.repeat(Math.max(1, padBase - leftLabel.length))} ${left.grade} (${left.score})`;
if (right) {
const rightLabel = labelOf(right);
const rightStr = `${rightLabel} ${'.'.repeat(Math.max(1, padBase - rightLabel.length))} ${right.grade} (${right.score})`;
lines.push(`${leftStr.padEnd(padCol)}${rightStr}`);
} else {
lines.push(leftStr);
}
}
if (opportunityCount > 0) {
lines.push('');
if (humanized) {
const noun = opportunityCount === 1 ? 'way' : 'ways';
lines.push(` ${opportunityCount} ${noun} you could get more out of Claude Code — see /config-audit feature-gap`);
} else {
lines.push(` ${opportunityCount} ${opportunityCount === 1 ? 'opportunity' : 'opportunities'} available — run /config-audit feature-gap for recommendations`);
}
}
lines.push('');
lines.push('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
return lines.join('\n');
}
export { TITLE_TO_ID, TIER_WEIGHTS, TIER_COUNTS, MAX_WEIGHTED, MATURITY_LEVELS, SEGMENTS };