/** * C4 — agent model/effort routing lever (CA-GAP-028). * * A LEVER, not a dimension: it fires only when authored agents exist, so it has * no meaningful "present/absent" reading on a config with no agents at all — * exactly the shape the three existing levers already have. The design was * decided by measurement, not taste: as a GAP_CHECKS dimension it would have * counted as vacuously-present on the agent-less marketplace-medium fixture, * moving MAX_WEIGHTED 41→42 and utilization 44→45, which flips `segment` * "Developing"→"Competent" in the frozen v5.0.0 posture baseline (segment is * NOT among the fields strip-retired-gap.mjs drops from comparison). * * The silence has two independent causes and they must not be conflated: * - no authored agents at all (t2_6 "No custom subagents" owns that case), and * - the only agents on disk being plugin-bundled, which isAuthoredConfig * excludes. * P5 below pins the second one specifically, so that arm cannot pass for the * first one's reason. */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { resolve, join } from 'node:path'; import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { scan, agentModelRoutingLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; import { withHermeticHome } from '../helpers/hermetic-home.mjs'; const LEVER_TITLE = 'Subagents pin neither model nor effort'; /** Build a throwaway project; `agents` maps a relative dir to agent frontmatter lines. */ async function makeProject(agents) { const root = await mkdtemp(join(tmpdir(), 'config-audit-c4-')); // A CLAUDE.md keeps the fixture from being a bare directory; irrelevant to the lever. await writeFile(join(root, 'CLAUDE.md'), '# Project\n'); for (const [relDir, files] of Object.entries(agents)) { const dir = join(root, relDir); await mkdir(dir, { recursive: true }); for (const [name, frontmatter] of Object.entries(files)) { await writeFile( join(dir, name), `---\nname: ${name.replace(/\.md$/, '')}\ndescription: does a thing\n${frontmatter}---\nBody.\n`, ); } } return root; } /** Scan a project with hermetic HOME and no global discovery. */ async function scanProject(root) { const discovery = await discoverConfigFiles(resolve(root)); const result = await withHermeticHome(() => scan(resolve(root), discovery)); return { result, discovery }; } const leverFindings = (result) => result.findings.filter(f => f.title === LEVER_TITLE); const hasNoSubagentsGap = (result) => result.findings.some(f => f.title === 'No custom subagents'); // --------------------------------------------------------------------------- // Unit — the pure lever function // --------------------------------------------------------------------------- describe('agentModelRoutingLeverFinding (pure)', () => { it('stays silent when no authored agents exist', () => { assert.equal( agentModelRoutingLeverFinding({ agentCount: 0, modelPinned: 0, effortPinned: 0 }), null, ); }); it('stays silent when at least one agent pins model', () => { assert.equal( agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 1, effortPinned: 0 }), null, ); }); it('stays silent when at least one agent pins effort (the separate axis)', () => { assert.equal( agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 0, effortPinned: 1 }), null, ); }); it('fires when agents exist and neither axis is used anywhere', () => { const f = agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 0, effortPinned: 0 }); assert.ok(f, 'expected a finding'); assert.equal(f.id, 'CA-GAP-028'); assert.equal(f.scanner, 'GAP'); assert.equal(f.severity, 'info'); assert.equal(f.category, 'model-fit'); assert.equal(f.title, LEVER_TITLE); }); it('cites both register entries and their primary sources', () => { const f = agentModelRoutingLeverFinding({ agentCount: 2, modelPinned: 0, effortPinned: 0 }); const text = `${f.description} ${f.recommendation}`; assert.match(text, /BP-MODEL-001/); assert.match(text, /BP-MODEL-002/); assert.match(text, /code\.claude\.com\/docs\/en\/sub-agents/); assert.match(text, /code\.claude\.com\/docs\/en\/model-config/); }); it('carries the measured counts as evidence', () => { const f = agentModelRoutingLeverFinding({ agentCount: 4, modelPinned: 0, effortPinned: 0 }); assert.match(f.evidence, /authored_agents=4/); assert.match(f.evidence, /model_pinned=0/); assert.match(f.evidence, /effort_pinned=0/); }); it('frames the opportunity without asserting the config is wrong', () => { const f = agentModelRoutingLeverFinding({ agentCount: 2, modelPinned: 0, effortPinned: 0 }); // `inherit` is the documented default and the reason the opportunity exists. assert.match(f.description, /inherit/i); }); }); // --------------------------------------------------------------------------- // Integration — fire/silent matrix through scan() // --------------------------------------------------------------------------- describe('GAP scanner — C4 fire/silent matrix', () => { it('P1: fires once when authored agents pin neither axis', async () => { const root = await makeProject({ '.claude/agents': { 'alpha.md': '', 'beta.md': '' }, }); const { result } = await scanProject(root); const hits = leverFindings(result); assert.equal(hits.length, 1, 'expected exactly one lever finding'); assert.equal(hits[0].id, 'CA-GAP-028'); }); // Found by dogfooding this repo's own machine: several installed agents write // `model: inherit` explicitly. `inherit` IS the documented default // (BP-MODEL-001), so naming it routes nothing and must not buy silence — the // opportunity is exactly as open as with the field absent. it('P2b: fires when the only "pin" is the default value spelled out', async () => { const root = await makeProject({ '.claude/agents': { 'alpha.md': 'model: inherit\n', 'beta.md': '' }, }); const { result } = await scanProject(root); const hits = leverFindings(result); assert.equal(hits.length, 1, 'model: inherit must not count as routing'); assert.match(hits[0].evidence, /model_pinned=0/); }); it('P2: silent when one agent pins model', async () => { const root = await makeProject({ '.claude/agents': { 'alpha.md': 'model: haiku\n', 'beta.md': '' }, }); const { result } = await scanProject(root); assert.equal(leverFindings(result).length, 0); }); it('P3: silent when one agent pins effort and none pins model', async () => { const root = await makeProject({ '.claude/agents': { 'alpha.md': 'effort: low\n', 'beta.md': '' }, }); const { result } = await scanProject(root); assert.equal(leverFindings(result).length, 0); }); it('P4: silent when there are no agents at all (t2_6 owns that case)', async () => { const root = await makeProject({}); const { result } = await scanProject(root); assert.equal(leverFindings(result).length, 0); assert.ok(hasNoSubagentsGap(result), 'expected the "No custom subagents" dimension instead'); }); it('P5: silent for the EXCLUSION reason when the only agent is plugin-bundled', async () => { const root = await makeProject({ '.claude/plugins/somePlugin/agents': { 'vendored.md': '' }, }); const { result, discovery } = await scanProject(root); // The discriminator: the file IS on disk and IS discovered as an agent — // so silence here cannot be the P4 "no agent files" reason. assert.ok( discovery.files.some(f => f.type === 'agent-md'), 'fixture invalid: no agent file was discovered at all', ); assert.equal(leverFindings(result).length, 0); assert.ok( hasNoSubagentsGap(result), 'the plugin-bundled agent must not count as an authored subagent either', ); }); });