caee558e79
Add /ultraresearch-local for structured research combining local codebase analysis with external knowledge via parallel agent swarms. Produces research briefs with triangulation, confidence ratings, and source quality assessment. New command: /ultraresearch-local with modes --quick, --local, --external, --fg. New agents: research-orchestrator (opus), docs-researcher, community-researcher, security-researcher, contrarian-researcher, gemini-bridge (all sonnet). New template: research-brief-template.md. Integration: --research flag in /ultraplan-local accepts pre-built research briefs (up to 3), enriches the interview and exploration phases. Planning orchestrator cross-references brief findings during synthesis. Design principle: Context Engineering — right information to right agent at right time. Research briefs are structured artifacts in the pipeline: ultraresearch → brief → ultraplan --research → plan → ultraexecute. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
545 lines
18 KiB
JavaScript
545 lines
18 KiB
JavaScript
import { describe, it } from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import {
|
||
calculateUtilization,
|
||
determineMaturityLevel,
|
||
determineSegment,
|
||
scoreByArea,
|
||
topActions,
|
||
generateScorecard,
|
||
generateHealthScorecard,
|
||
TITLE_TO_ID,
|
||
TIER_WEIGHTS,
|
||
TIER_COUNTS,
|
||
MAX_WEIGHTED,
|
||
MATURITY_LEVELS,
|
||
SEGMENTS,
|
||
} from '../../scanners/lib/scoring.mjs';
|
||
|
||
// --- Helpers ---
|
||
function makeGapFinding(title, tier) {
|
||
return { id: 'CA-GAP-999', scanner: 'GAP', severity: 'info', title, category: tier, recommendation: 'Fix it' };
|
||
}
|
||
|
||
function allGapFindings() {
|
||
return Object.entries(TITLE_TO_ID).map(([title, id]) => {
|
||
const tier = id.split('_')[0];
|
||
return makeGapFinding(title, tier);
|
||
});
|
||
}
|
||
|
||
function t1GapFindings() {
|
||
return Object.entries(TITLE_TO_ID)
|
||
.filter(([, id]) => id.startsWith('t1'))
|
||
.map(([title]) => makeGapFinding(title, 't1'));
|
||
}
|
||
|
||
function t4GapFindings() {
|
||
return Object.entries(TITLE_TO_ID)
|
||
.filter(([, id]) => id.startsWith('t4'))
|
||
.map(([title]) => makeGapFinding(title, 't4'));
|
||
}
|
||
|
||
function makeScannerResult(scanner, findingCount) {
|
||
const findings = Array.from({ length: findingCount }, (_, i) => ({
|
||
id: `CA-${scanner}-${String(i + 1).padStart(3, '0')}`,
|
||
scanner,
|
||
severity: 'low',
|
||
title: `Finding ${i + 1}`,
|
||
category: scanner === 'GAP' ? 't2' : null,
|
||
recommendation: 'Fix',
|
||
}));
|
||
return {
|
||
scanner,
|
||
status: 'ok',
|
||
files_scanned: 5,
|
||
duration_ms: 10,
|
||
findings,
|
||
counts: { critical: 0, high: 0, medium: 0, low: findingCount, info: 0 },
|
||
};
|
||
}
|
||
|
||
// ========================================
|
||
// calculateUtilization
|
||
// ========================================
|
||
describe('calculateUtilization', () => {
|
||
it('returns 100% with no gap findings', () => {
|
||
const result = calculateUtilization([]);
|
||
assert.equal(result.score, 100);
|
||
assert.equal(result.overhang, 0);
|
||
});
|
||
|
||
it('returns 0% when all 25 dimensions are gaps', () => {
|
||
const result = calculateUtilization(allGapFindings());
|
||
assert.equal(result.score, 0);
|
||
assert.equal(result.overhang, 100);
|
||
});
|
||
|
||
it('weighs T1 gaps heavier (3x)', () => {
|
||
const onlyT1 = t1GapFindings(); // 5 T1 gaps = 15 weight lost
|
||
const result = calculateUtilization(onlyT1);
|
||
// Lost: 5 × 3 = 15 out of 42. Present: 27/42 = 64%
|
||
assert.equal(result.score, 64);
|
||
});
|
||
|
||
it('weighs T4 gaps lighter (1x)', () => {
|
||
const onlyT4 = t4GapFindings(); // 5 T4 gaps = 5 weight lost
|
||
const result = calculateUtilization(onlyT4);
|
||
// Lost: 5 × 1 = 5 out of 42. Present: 37/42 = 88%
|
||
assert.equal(result.score, 88);
|
||
});
|
||
|
||
it('T1+T2 present but no T3+T4 scores ~69%', () => {
|
||
// T3: 8 dims × 1 = 8, T4: 5 dims × 1 = 5. Lost = 13 out of 42. Present = 29/42 = 69%
|
||
const t3t4Gaps = Object.entries(TITLE_TO_ID)
|
||
.filter(([, id]) => id.startsWith('t3') || id.startsWith('t4'))
|
||
.map(([title, id]) => makeGapFinding(title, id.split('_')[0]));
|
||
const result = calculateUtilization(t3t4Gaps);
|
||
assert.equal(result.score, 69);
|
||
});
|
||
|
||
it('score + overhang = 100', () => {
|
||
const result = calculateUtilization(t1GapFindings());
|
||
assert.equal(result.score + result.overhang, 100);
|
||
});
|
||
|
||
it('handles empty array', () => {
|
||
const result = calculateUtilization([]);
|
||
assert.equal(typeof result.score, 'number');
|
||
assert.equal(typeof result.overhang, 'number');
|
||
});
|
||
|
||
it('ignores findings with unknown category', () => {
|
||
const weird = [{ category: 'tx' }, { category: undefined }];
|
||
const result = calculateUtilization(weird);
|
||
assert.equal(result.score, 100); // unknown tiers don't count
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// determineMaturityLevel
|
||
// ========================================
|
||
describe('determineMaturityLevel', () => {
|
||
const discovery = { files: [] };
|
||
|
||
it('returns Level 0 when CLAUDE.md is missing', () => {
|
||
const gaps = [makeGapFinding('No CLAUDE.md file', 't1')];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 0);
|
||
assert.equal(result.name, 'Bare');
|
||
});
|
||
|
||
it('returns Level 1 when CLAUDE.md present but no permissions', () => {
|
||
const gaps = [
|
||
makeGapFinding('No permissions configured', 't1'),
|
||
makeGapFinding('No hooks configured', 't1'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 1);
|
||
});
|
||
|
||
it('returns Level 2 when permissions + hooks + modular present but no MCP', () => {
|
||
const gaps = [
|
||
makeGapFinding('No MCP servers configured', 't1'),
|
||
makeGapFinding('Low hook diversity', 't2'),
|
||
makeGapFinding('No custom subagents', 't2'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 2);
|
||
assert.equal(result.name, 'Structured');
|
||
});
|
||
|
||
it('returns Level 3 when MCP + hook diversity + subagents present but no plugin', () => {
|
||
const gaps = [
|
||
makeGapFinding('No custom plugin', 't4'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 3);
|
||
assert.equal(result.name, 'Automated');
|
||
});
|
||
|
||
it('returns Level 4 when all requirements met', () => {
|
||
const result = determineMaturityLevel([], discovery);
|
||
assert.equal(result.level, 4);
|
||
assert.equal(result.name, 'Governed');
|
||
});
|
||
|
||
it('Level 2 requires modular OR path-rules (modular)', () => {
|
||
// Has permissions, hooks, modular — but no path-rules. Should still be level 2.
|
||
const gaps = [
|
||
makeGapFinding('No path-scoped rules', 't2'),
|
||
makeGapFinding('No MCP servers configured', 't1'),
|
||
makeGapFinding('Low hook diversity', 't2'),
|
||
makeGapFinding('No custom subagents', 't2'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 2);
|
||
});
|
||
|
||
it('Level 2 requires modular OR path-rules (path-rules)', () => {
|
||
// Has permissions, hooks, path-rules — but not modular. Should still be level 2.
|
||
const gaps = [
|
||
makeGapFinding('CLAUDE.md not modular', 't2'),
|
||
makeGapFinding('No MCP servers configured', 't1'),
|
||
makeGapFinding('Low hook diversity', 't2'),
|
||
makeGapFinding('No custom subagents', 't2'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 2);
|
||
});
|
||
|
||
it('stays Level 1 when neither modular nor path-rules', () => {
|
||
const gaps = [
|
||
makeGapFinding('CLAUDE.md not modular', 't2'),
|
||
makeGapFinding('No path-scoped rules', 't2'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 1);
|
||
});
|
||
|
||
it('Level 3 blocked by missing hook diversity', () => {
|
||
const gaps = [
|
||
makeGapFinding('Low hook diversity', 't2'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 2);
|
||
});
|
||
|
||
it('Level 4 blocked by missing project MCP in git', () => {
|
||
const gaps = [
|
||
makeGapFinding('No project .mcp.json in git', 't4'),
|
||
];
|
||
const result = determineMaturityLevel(gaps, discovery);
|
||
assert.equal(result.level, 3);
|
||
});
|
||
|
||
it('all MATURITY_LEVELS have required fields', () => {
|
||
for (const ml of MATURITY_LEVELS) {
|
||
assert.ok(typeof ml.level === 'number');
|
||
assert.ok(typeof ml.name === 'string');
|
||
assert.ok(typeof ml.description === 'string');
|
||
}
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// determineSegment
|
||
// ========================================
|
||
describe('determineSegment', () => {
|
||
it('Top Performer at score > 80', () => {
|
||
assert.equal(determineSegment(81).segment, 'Top Performer');
|
||
assert.equal(determineSegment(100).segment, 'Top Performer');
|
||
});
|
||
|
||
it('Strong at 65-80', () => {
|
||
assert.equal(determineSegment(65).segment, 'Strong');
|
||
assert.equal(determineSegment(80).segment, 'Strong');
|
||
});
|
||
|
||
it('Competent at 45-64', () => {
|
||
assert.equal(determineSegment(45).segment, 'Competent');
|
||
assert.equal(determineSegment(64).segment, 'Competent');
|
||
});
|
||
|
||
it('Developing at 25-44', () => {
|
||
assert.equal(determineSegment(25).segment, 'Developing');
|
||
assert.equal(determineSegment(44).segment, 'Developing');
|
||
});
|
||
|
||
it('Beginner at < 25', () => {
|
||
assert.equal(determineSegment(0).segment, 'Beginner');
|
||
assert.equal(determineSegment(24).segment, 'Beginner');
|
||
});
|
||
|
||
it('returns description string', () => {
|
||
const result = determineSegment(50);
|
||
assert.ok(result.description.length > 0);
|
||
});
|
||
|
||
it('edge case: exactly 80 is Strong', () => {
|
||
assert.equal(determineSegment(80).segment, 'Strong');
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// scoreByArea
|
||
// ========================================
|
||
describe('scoreByArea', () => {
|
||
it('returns areas for all 8 scanners', () => {
|
||
const scanners = ['CML', 'SET', 'HKV', 'RUL', 'MCP', 'IMP', 'CNF', 'GAP']
|
||
.map(s => makeScannerResult(s, 0));
|
||
const result = scoreByArea(scanners);
|
||
assert.equal(result.areas.length, 8);
|
||
});
|
||
|
||
it('zero findings → A grade', () => {
|
||
const scanners = [makeScannerResult('CML', 0)];
|
||
const result = scoreByArea(scanners);
|
||
assert.equal(result.areas[0].grade, 'A');
|
||
assert.equal(result.areas[0].score, 100);
|
||
});
|
||
|
||
it('many findings → lower grade', () => {
|
||
const scanners = [makeScannerResult('CML', 8)];
|
||
const result = scoreByArea(scanners);
|
||
assert.ok(result.areas[0].score < 50);
|
||
});
|
||
|
||
it('GAP scanner uses utilization-based scoring', () => {
|
||
const gapResult = makeScannerResult('GAP', 0);
|
||
const result = scoreByArea([gapResult]);
|
||
assert.equal(result.areas[0].name, 'Feature Coverage');
|
||
assert.equal(result.areas[0].score, 100); // 0 gaps = 100%
|
||
});
|
||
|
||
it('overall grade is average of quality areas (excludes GAP)', () => {
|
||
const scanners = [
|
||
makeScannerResult('CML', 0), // 100
|
||
makeScannerResult('SET', 0), // 100
|
||
makeScannerResult('GAP', 20), // low utilization — should NOT drag down grade
|
||
];
|
||
const result = scoreByArea(scanners);
|
||
assert.equal(result.overallGrade, 'A'); // only CML+SET averaged
|
||
assert.equal(result.areas.length, 3); // GAP still in areas for display
|
||
});
|
||
|
||
it('mixed grades produce mixed overall', () => {
|
||
const scanners = [
|
||
makeScannerResult('CML', 0), // 100 → A
|
||
makeScannerResult('SET', 10), // low → F range
|
||
];
|
||
const result = scoreByArea(scanners);
|
||
assert.ok(['A', 'B', 'C', 'D', 'F'].includes(result.overallGrade));
|
||
});
|
||
|
||
it('empty scanner array → overallGrade F', () => {
|
||
const result = scoreByArea([]);
|
||
assert.equal(result.areas.length, 0);
|
||
assert.equal(result.overallGrade, 'F');
|
||
});
|
||
|
||
it('area objects have required fields', () => {
|
||
const scanners = [makeScannerResult('CML', 2)];
|
||
const result = scoreByArea(scanners);
|
||
const area = result.areas[0];
|
||
assert.ok('name' in area);
|
||
assert.ok('grade' in area);
|
||
assert.ok('score' in area);
|
||
assert.ok('findingCount' in area);
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// topActions
|
||
// ========================================
|
||
describe('topActions', () => {
|
||
it('returns max 3 actions', () => {
|
||
const gaps = allGapFindings();
|
||
const result = topActions(gaps);
|
||
assert.equal(result.length, 3);
|
||
});
|
||
|
||
it('prioritizes T1 over T2', () => {
|
||
const gaps = [
|
||
{ ...makeGapFinding('Low hook diversity', 't2'), recommendation: 'Add hooks' },
|
||
{ ...makeGapFinding('No CLAUDE.md file', 't1'), recommendation: 'Create CLAUDE.md' },
|
||
];
|
||
const result = topActions(gaps);
|
||
assert.equal(result[0], 'Create CLAUDE.md');
|
||
});
|
||
|
||
it('returns empty array for no gaps', () => {
|
||
assert.deepEqual(topActions([]), []);
|
||
});
|
||
|
||
it('returns all items if fewer than 3', () => {
|
||
const gaps = [makeGapFinding('No CLAUDE.md file', 't1')];
|
||
assert.equal(topActions(gaps).length, 1);
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// generateScorecard
|
||
// ========================================
|
||
describe('generateScorecard', () => {
|
||
const sampleAreas = {
|
||
areas: [
|
||
{ name: 'CLAUDE.md', grade: 'A', score: 92 },
|
||
{ name: 'Settings', grade: 'B', score: 78 },
|
||
{ name: 'Hooks', grade: 'C', score: 55 },
|
||
{ name: 'Rules', grade: 'B', score: 71 },
|
||
],
|
||
overallGrade: 'B',
|
||
};
|
||
const sampleUtil = { score: 68, overhang: 32 };
|
||
const sampleMaturity = { level: 2, name: 'Structured' };
|
||
const sampleSegment = { segment: 'Strong' };
|
||
const sampleActions = ['Configure MCP', 'Add hooks', 'Create agents'];
|
||
|
||
it('returns a string', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.equal(typeof result, 'string');
|
||
});
|
||
|
||
it('contains header line', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('Config-Audit Posture Score'));
|
||
});
|
||
|
||
it('contains overall grade', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('Overall: B'));
|
||
});
|
||
|
||
it('contains maturity level', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('Level 2 (Structured)'));
|
||
});
|
||
|
||
it('contains utilization', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('Utilization: 68%'));
|
||
});
|
||
|
||
it('contains segment', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('Segment: Strong'));
|
||
});
|
||
|
||
it('contains all area names', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('CLAUDE.md'));
|
||
assert.ok(result.includes('Settings'));
|
||
assert.ok(result.includes('Hooks'));
|
||
assert.ok(result.includes('Rules'));
|
||
});
|
||
|
||
it('contains top actions', () => {
|
||
const result = generateScorecard(sampleAreas, sampleUtil, sampleMaturity, sampleSegment, sampleActions);
|
||
assert.ok(result.includes('1. Configure MCP'));
|
||
assert.ok(result.includes('2. Add hooks'));
|
||
assert.ok(result.includes('3. Create agents'));
|
||
});
|
||
|
||
it('works with empty areas', () => {
|
||
const result = generateScorecard({ areas: [], overallGrade: 'F' }, sampleUtil, sampleMaturity, sampleSegment, []);
|
||
assert.equal(typeof result, 'string');
|
||
assert.ok(result.includes('Config-Audit Posture Score'));
|
||
});
|
||
|
||
it('works with odd number of areas', () => {
|
||
const odd = { areas: [{ name: 'Test', grade: 'A', score: 95 }], overallGrade: 'A' };
|
||
const result = generateScorecard(odd, sampleUtil, sampleMaturity, sampleSegment, []);
|
||
assert.ok(result.includes('Test'));
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// generateHealthScorecard (v3)
|
||
// ========================================
|
||
describe('generateHealthScorecard', () => {
|
||
const sampleAreas = {
|
||
areas: [
|
||
{ name: 'CLAUDE.md', grade: 'A', score: 92 },
|
||
{ name: 'Settings', grade: 'B', score: 78 },
|
||
{ name: 'Hooks', grade: 'C', score: 55 },
|
||
{ name: 'Feature Coverage', grade: 'F', score: 20 },
|
||
],
|
||
overallGrade: 'B',
|
||
};
|
||
|
||
it('returns a string', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 12);
|
||
assert.equal(typeof result, 'string');
|
||
});
|
||
|
||
it('contains Health header (not Overall)', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 5);
|
||
assert.ok(result.includes('Config-Audit Health Score'));
|
||
assert.ok(result.includes('Health: B'));
|
||
assert.ok(!result.includes('Overall:'));
|
||
});
|
||
|
||
it('does NOT contain Maturity, Utilization, or Segment', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 5);
|
||
assert.ok(!result.includes('Maturity:'));
|
||
assert.ok(!result.includes('Utilization:'));
|
||
assert.ok(!result.includes('Segment:'));
|
||
});
|
||
|
||
it('excludes Feature Coverage from area display', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 5);
|
||
assert.ok(!result.includes('Feature Coverage'));
|
||
assert.ok(result.includes('CLAUDE.md'));
|
||
assert.ok(result.includes('Settings'));
|
||
assert.ok(result.includes('Hooks'));
|
||
});
|
||
|
||
it('shows opportunity count', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 12);
|
||
assert.ok(result.includes('12 opportunities available'));
|
||
});
|
||
|
||
it('uses singular for 1 opportunity', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 1);
|
||
assert.ok(result.includes('1 opportunity available'));
|
||
});
|
||
|
||
it('hides opportunity line when count is 0', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 0);
|
||
assert.ok(!result.includes('opportunit'));
|
||
});
|
||
|
||
it('shows areas scanned count', () => {
|
||
const result = generateHealthScorecard(sampleAreas, 5);
|
||
assert.ok(result.includes('3 areas scanned')); // 3 quality areas (excl Feature Coverage)
|
||
});
|
||
|
||
it('computes avgScore from quality areas only', () => {
|
||
// Quality areas: A(92), B(78), C(55) → avg = 75
|
||
const result = generateHealthScorecard(sampleAreas, 5);
|
||
assert.ok(result.includes('(75/100)'));
|
||
});
|
||
|
||
it('works with empty areas', () => {
|
||
const result = generateHealthScorecard({ areas: [], overallGrade: 'F' }, 0);
|
||
assert.equal(typeof result, 'string');
|
||
assert.ok(result.includes('Config-Audit Health Score'));
|
||
});
|
||
});
|
||
|
||
// ========================================
|
||
// Constants and exports
|
||
// ========================================
|
||
describe('scoring constants', () => {
|
||
it('TITLE_TO_ID has 25 entries', () => {
|
||
assert.equal(Object.keys(TITLE_TO_ID).length, 25);
|
||
});
|
||
|
||
it('TIER_COUNTS sum to 25', () => {
|
||
const sum = Object.values(TIER_COUNTS).reduce((a, b) => a + b, 0);
|
||
assert.equal(sum, 25);
|
||
});
|
||
|
||
it('MAX_WEIGHTED is 42', () => {
|
||
assert.equal(MAX_WEIGHTED, 42);
|
||
});
|
||
|
||
it('TIER_WEIGHTS match spec', () => {
|
||
assert.equal(TIER_WEIGHTS.t1, 3);
|
||
assert.equal(TIER_WEIGHTS.t2, 2);
|
||
assert.equal(TIER_WEIGHTS.t3, 1);
|
||
assert.equal(TIER_WEIGHTS.t4, 1);
|
||
});
|
||
|
||
it('SEGMENTS covers full 0-100 range', () => {
|
||
assert.equal(SEGMENTS[SEGMENTS.length - 1].min, 0);
|
||
assert.ok(SEGMENTS[0].min >= 80);
|
||
});
|
||
|
||
it('MATURITY_LEVELS has 5 levels (0-4)', () => {
|
||
assert.equal(MATURITY_LEVELS.length, 5);
|
||
assert.equal(MATURITY_LEVELS[0].level, 0);
|
||
assert.equal(MATURITY_LEVELS[4].level, 4);
|
||
});
|
||
});
|