diff --git a/scanners/agent-listing-scanner.mjs b/scanners/agent-listing-scanner.mjs index b91a872..24dc434 100644 --- a/scanners/agent-listing-scanner.mjs +++ b/scanners/agent-listing-scanner.mjs @@ -8,8 +8,15 @@ * the dominant single always-loaded source). * * Detection: + * CA-AGT-NNN per-agent description over the soft bloat cap (low, advisory) * CA-AGT-NNN aggregate agent-listing estimate exceeds the listing budget (low) * + * Per-agent advisories are emitted FIRST (mirroring SKL 001→002). They are a + * soft bloat heuristic (the 500-char threshold TOK pattern F uses for SKILL.md + * descriptions), NOT a truncation finding — agents have no verified per- + * description cap, so nothing is dropped; the description is simply re-sent in + * full every turn. + * * INTELLECTUAL-HONESTY CONTRACT (the reason this is `low`, not a hard finding): * unlike the skill listing, the agent-listing mechanism is NOT documented (agents * are absent from Claude Code's published context breakdown). So the finding is an @@ -26,6 +33,7 @@ import { SEVERITY } from './lib/severity.mjs'; import { AGGREGATE_BUDGET_TOKENS, BUDGET_CALIBRATION_NOTE, + PER_AGENT_DESC_SOFT_CAP, measureActiveAgentListing, } from './lib/agent-listing-budget.mjs'; @@ -41,7 +49,44 @@ export async function scan(_targetPath, _discovery) { const start = Date.now(); const findings = []; - const { aggregate } = await measureActiveAgentListing(); + const { agents, aggregate } = await measureActiveAgentListing(); + + // Per-agent advisory (emitted FIRST so the common "long agent + aggregate" + // case reads 001=per-agent, 002=aggregate, mirroring SKL). This is a soft + // heuristic, NOT a truncation finding — agents have no verified per-description + // cap, so the framing is "this is large and re-sent every turn", never "Claude + // Code drops the tail". + for (const agent of agents) { + if (agent.descLength <= PER_AGENT_DESC_SOFT_CAP) continue; + + const sourceLabel = agent.source === 'plugin' + ? `plugin:${agent.pluginName}` + : 'user'; + + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Agent description is long (re-sent every turn in the always-loaded listing)', + description: + `Agent "${agent.name}" (${sourceLabel}) has a description of ${agent.descLength} ` + + `characters (>${PER_AGENT_DESC_SOFT_CAP}). Claude Code injects every active agent's ` + + 'name+description into the agent listing on every turn so it knows which subagents it ' + + 'can delegate to, so every character of this description re-enters context each turn ' + + 'whether or not you delegate. Unlike the skill listing there is no verified ' + + 'per-description cap, so nothing is dropped — this is a bloat advisory, not a ' + + 'hard-cap finding.', + file: agent.path, + evidence: + `description_chars=${agent.descLength}; soft_cap=${PER_AGENT_DESC_SOFT_CAP} ` + + `(heuristic, same bloat threshold TOK pattern F uses for SKILL.md descriptions; agents ` + + `have NO verified per-description cap); agent="${agent.name}"; source=${sourceLabel}`, + recommendation: + 'Trim the description toward its trigger phrases / "when to use this agent" cues and move ' + + 'long examples into the agent body, or disable the plugin if you never delegate to this ' + + 'agent (the whole agent block then leaves the always-loaded listing).', + category: 'token-efficiency', + })); + } if (aggregate.overBudget) { findings.push(finding({ diff --git a/scanners/lib/agent-listing-budget.mjs b/scanners/lib/agent-listing-budget.mjs index c3c1eba..c062abf 100644 --- a/scanners/lib/agent-listing-budget.mjs +++ b/scanners/lib/agent-listing-budget.mjs @@ -35,6 +35,15 @@ export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WIND export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000 export { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas }; +// Per-agent soft cap for the description-bloat advisory. This is a HEURISTIC, +// NOT a truncation cap: agents have no verified per-description limit (unlike +// the skill listing's 1,536-char cap, CC 2.1.105). We reuse the 500-char +// bloat threshold TOK pattern F already applies to SKILL.md descriptions, so a +// long agent description is flagged for the same reason — every char re-enters +// context in the always-loaded agent listing on every turn — without ever +// claiming Claude Code drops the tail. +export const PER_AGENT_DESC_SOFT_CAP = 500; + // The honest framing required because (a) the mechanism is inferred and (b) the // budget depends on a context window we cannot observe. Appended to overflow evidence. export const BUDGET_CALIBRATION_NOTE = diff --git a/tests/scanners/agent-listing-scanner.test.mjs b/tests/scanners/agent-listing-scanner.test.mjs index cfb2e12..76efee1 100644 --- a/tests/scanners/agent-listing-scanner.test.mjs +++ b/tests/scanners/agent-listing-scanner.test.mjs @@ -5,6 +5,7 @@ import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/agent-listing-scanner.mjs'; +import { PER_AGENT_DESC_SOFT_CAP } from '../../scanners/lib/agent-listing-budget.mjs'; /** * AGT scanner — agent-listing always-loaded token budget. @@ -62,7 +63,16 @@ async function homeWithNUserAgents(count, descLen, prefix = 'agg') { return home; } +/** Build a fake HOME with a single user agent whose description is `descLen` chars. */ +async function homeWithOneAgent(descLen, name = 'solo') { + const home = uniqueDir(`one-${name}-${descLen}`); + await addUserAgent(home, name, descLen); + return home; +} + const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title)); +// Per-agent advisories are every finding that is NOT the aggregate roll-up. +const findPerAgent = (findings) => findings.filter(f => !/agent listing/i.test(f.title)); describe('AGT scanner — basic structure', () => { it('reports scanner prefix AGT', async () => { @@ -172,3 +182,102 @@ describe('AGT scanner — aggregate always-loaded budget', () => { } }); }); + +/** + * Per-agent description advisory (the B1-rest companion to the aggregate). + * + * A single agent with a long description is flagged on its own: every char of + * that description re-enters context in the always-loaded agent listing on every + * turn. UNLIKE CA-SKL-001, this is NOT a truncation finding — agents have no + * verified per-description cap — so the advisory is a soft heuristic (low, the + * same 500-char bloat threshold TOK pattern F uses for SKILL.md descriptions), + * not a hard "Claude Code drops the tail" claim. + */ +describe('AGT scanner — per-agent description advisory', () => { + it('fires a low-severity advisory when one agent description exceeds the soft cap', async () => { + // One agent, description over the cap; aggregate stays well under budget so + // the per-agent advisory is the only finding (clean isolation). + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + assert.equal(findAggregate(result.findings), undefined, + 'a single small-aggregate agent must not trip the aggregate budget'); + const perAgent = findPerAgent(result.findings); + assert.equal(perAgent.length, 1, + `expected exactly one per-agent advisory; got: ${result.findings.map(f => f.title).join(' | ')}`); + assert.equal(perAgent[0].severity, 'low', `per-agent advisory should be low, got ${perAgent[0].severity}`); + assert.match(perAgent[0].id, /^CA-AGT-\d{3}$/); + assert.match(`${perAgent[0].description} ${perAgent[0].evidence}`, /solo/, + 'the advisory should name the offending agent'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('treats the soft cap as a strict boundary: at-cap no fire, one-over fires', async () => { + const atCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP, 'atcap'); + try { + const result = await runScannerWithHome(atCap); + assert.equal(findPerAgent(result.findings).length, 0, + 'a description exactly at the soft cap must not fire (strictly-greater rule)'); + } finally { + await rm(atCap, { recursive: true, force: true }); + } + const overCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 1, 'overcap'); + try { + const result = await runScannerWithHome(overCap); + assert.equal(findPerAgent(result.findings).length, 1, + 'a description one char over the soft cap must fire'); + } finally { + await rm(overCap, { recursive: true, force: true }); + } + }); + + it('does NOT claim truncation and DISCLOSES the advisory/heuristic framing (intellectual honesty)', async () => { + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + const advisory = findPerAgent(result.findings)[0]; + assert.ok(advisory, 'expected a per-agent advisory'); + const text = `${advisory.description} ${advisory.evidence}`; + assert.doesNotMatch(text, /truncat/i, + 'agents have no verified cap — the advisory must NOT claim Claude Code truncates the description'); + assert.match(text, /every turn|always.?loaded/i, + 'the advisory must explain the description re-enters context every turn'); + assert.match(String(advisory.evidence), new RegExp(String(PER_AGENT_DESC_SOFT_CAP)), + 'evidence must state the soft-cap threshold so it reads as a heuristic'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('recommends trimming the description toward its trigger phrases', async () => { + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + const advisory = findPerAgent(result.findings)[0]; + assert.ok(advisory, 'expected a per-agent advisory to carry remediation'); + assert.match(String(advisory.recommendation), /trim|trigger|shorten/i, + 'recommendation should point at trimming the description'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('emits per-agent advisories BEFORE the aggregate roll-up', async () => { + // 17 large agents → both per-agent advisories AND the aggregate fire. + const home = await homeWithNUserAgents(17, 1000); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg, 'expected the aggregate to fire alongside per-agent advisories'); + const aggIndex = result.findings.indexOf(agg); + assert.equal(aggIndex, result.findings.length - 1, + 'the aggregate roll-up must be emitted last (per-agent advisories precede it, mirroring SKL 001→002)'); + assert.equal(findPerAgent(result.findings).length, 17, + 'each of the 17 over-cap agents should carry its own advisory'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +});