feat(scanner): add AGT per-agent description bloat advisory (v5.9 B1-rest)
Flag any single active agent whose description exceeds a 500-char soft cap (the same bloat threshold TOK pattern F applies to SKILL.md descriptions). Every char of an agent description re-enters context in the always-loaded agent listing on every turn, so a long one is a per-turn cost. Framing is deliberately ADVISORY, severity low: unlike CA-SKL-001 agents have NO verified per-description cap, so nothing is dropped — this is a bloat advisory, never a truncation claim. Per-agent advisories are emitted BEFORE the aggregate roll-up (mirrors SKL 001->002). - PER_AGENT_DESC_SOFT_CAP (=500) added to lib/agent-listing-budget.mjs as the single source of truth, disclosed as a heuristic. - Per-agent loop reuses measureActiveAgentListing()'s agents[] (name, source, pluginName, path, descLength) — no new enumeration. - 5 new tests (fire/strict-boundary/no-truncation-claim/recommendation/order). Byte-stable: AGT already stripped from frozen v5.0.0 baselines; SC-5 unchanged (HOME-scoped, hermetic HOME has no agents). Suite green: 1181 tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
fcfb2979ef
commit
2a3cb537f9
3 changed files with 164 additions and 1 deletions
|
|
@ -8,8 +8,15 @@
|
|||
* the dominant single always-loaded source).
|
||||
*
|
||||
* Detection:
|
||||
* CA-AGT-NNN per-agent description over the soft bloat cap (low, advisory)
|
||||
* CA-AGT-NNN aggregate agent-listing estimate exceeds the listing budget (low)
|
||||
*
|
||||
* Per-agent advisories are emitted FIRST (mirroring SKL 001→002). They are a
|
||||
* soft bloat heuristic (the 500-char threshold TOK pattern F uses for SKILL.md
|
||||
* descriptions), NOT a truncation finding — agents have no verified per-
|
||||
* description cap, so nothing is dropped; the description is simply re-sent in
|
||||
* full every turn.
|
||||
*
|
||||
* INTELLECTUAL-HONESTY CONTRACT (the reason this is `low`, not a hard finding):
|
||||
* unlike the skill listing, the agent-listing mechanism is NOT documented (agents
|
||||
* are absent from Claude Code's published context breakdown). So the finding is an
|
||||
|
|
@ -26,6 +33,7 @@ import { SEVERITY } from './lib/severity.mjs';
|
|||
import {
|
||||
AGGREGATE_BUDGET_TOKENS,
|
||||
BUDGET_CALIBRATION_NOTE,
|
||||
PER_AGENT_DESC_SOFT_CAP,
|
||||
measureActiveAgentListing,
|
||||
} from './lib/agent-listing-budget.mjs';
|
||||
|
||||
|
|
@ -41,7 +49,44 @@ export async function scan(_targetPath, _discovery) {
|
|||
const start = Date.now();
|
||||
const findings = [];
|
||||
|
||||
const { aggregate } = await measureActiveAgentListing();
|
||||
const { agents, aggregate } = await measureActiveAgentListing();
|
||||
|
||||
// Per-agent advisory (emitted FIRST so the common "long agent + aggregate"
|
||||
// case reads 001=per-agent, 002=aggregate, mirroring SKL). This is a soft
|
||||
// heuristic, NOT a truncation finding — agents have no verified per-description
|
||||
// cap, so the framing is "this is large and re-sent every turn", never "Claude
|
||||
// Code drops the tail".
|
||||
for (const agent of agents) {
|
||||
if (agent.descLength <= PER_AGENT_DESC_SOFT_CAP) continue;
|
||||
|
||||
const sourceLabel = agent.source === 'plugin'
|
||||
? `plugin:${agent.pluginName}`
|
||||
: 'user';
|
||||
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.low,
|
||||
title: 'Agent description is long (re-sent every turn in the always-loaded listing)',
|
||||
description:
|
||||
`Agent "${agent.name}" (${sourceLabel}) has a description of ${agent.descLength} ` +
|
||||
`characters (>${PER_AGENT_DESC_SOFT_CAP}). Claude Code injects every active agent's ` +
|
||||
'name+description into the agent listing on every turn so it knows which subagents it ' +
|
||||
'can delegate to, so every character of this description re-enters context each turn ' +
|
||||
'whether or not you delegate. Unlike the skill listing there is no verified ' +
|
||||
'per-description cap, so nothing is dropped — this is a bloat advisory, not a ' +
|
||||
'hard-cap finding.',
|
||||
file: agent.path,
|
||||
evidence:
|
||||
`description_chars=${agent.descLength}; soft_cap=${PER_AGENT_DESC_SOFT_CAP} ` +
|
||||
`(heuristic, same bloat threshold TOK pattern F uses for SKILL.md descriptions; agents ` +
|
||||
`have NO verified per-description cap); agent="${agent.name}"; source=${sourceLabel}`,
|
||||
recommendation:
|
||||
'Trim the description toward its trigger phrases / "when to use this agent" cues and move ' +
|
||||
'long examples into the agent body, or disable the plugin if you never delegate to this ' +
|
||||
'agent (the whole agent block then leaves the always-loaded listing).',
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
}
|
||||
|
||||
if (aggregate.overBudget) {
|
||||
findings.push(finding({
|
||||
|
|
|
|||
|
|
@ -35,6 +35,15 @@ export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WIND
|
|||
export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000
|
||||
export { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas };
|
||||
|
||||
// Per-agent soft cap for the description-bloat advisory. This is a HEURISTIC,
|
||||
// NOT a truncation cap: agents have no verified per-description limit (unlike
|
||||
// the skill listing's 1,536-char cap, CC 2.1.105). We reuse the 500-char
|
||||
// bloat threshold TOK pattern F already applies to SKILL.md descriptions, so a
|
||||
// long agent description is flagged for the same reason — every char re-enters
|
||||
// context in the always-loaded agent listing on every turn — without ever
|
||||
// claiming Claude Code drops the tail.
|
||||
export const PER_AGENT_DESC_SOFT_CAP = 500;
|
||||
|
||||
// The honest framing required because (a) the mechanism is inferred and (b) the
|
||||
// budget depends on a context window we cannot observe. Appended to overflow evidence.
|
||||
export const BUDGET_CALIBRATION_NOTE =
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { mkdir, writeFile, rm } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { scan } from '../../scanners/agent-listing-scanner.mjs';
|
||||
import { PER_AGENT_DESC_SOFT_CAP } from '../../scanners/lib/agent-listing-budget.mjs';
|
||||
|
||||
/**
|
||||
* AGT scanner — agent-listing always-loaded token budget.
|
||||
|
|
@ -62,7 +63,16 @@ async function homeWithNUserAgents(count, descLen, prefix = 'agg') {
|
|||
return home;
|
||||
}
|
||||
|
||||
/** Build a fake HOME with a single user agent whose description is `descLen` chars. */
|
||||
async function homeWithOneAgent(descLen, name = 'solo') {
|
||||
const home = uniqueDir(`one-${name}-${descLen}`);
|
||||
await addUserAgent(home, name, descLen);
|
||||
return home;
|
||||
}
|
||||
|
||||
const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title));
|
||||
// Per-agent advisories are every finding that is NOT the aggregate roll-up.
|
||||
const findPerAgent = (findings) => findings.filter(f => !/agent listing/i.test(f.title));
|
||||
|
||||
describe('AGT scanner — basic structure', () => {
|
||||
it('reports scanner prefix AGT', async () => {
|
||||
|
|
@ -172,3 +182,102 @@ describe('AGT scanner — aggregate always-loaded budget', () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Per-agent description advisory (the B1-rest companion to the aggregate).
|
||||
*
|
||||
* A single agent with a long description is flagged on its own: every char of
|
||||
* that description re-enters context in the always-loaded agent listing on every
|
||||
* turn. UNLIKE CA-SKL-001, this is NOT a truncation finding — agents have no
|
||||
* verified per-description cap — so the advisory is a soft heuristic (low, the
|
||||
* same 500-char bloat threshold TOK pattern F uses for SKILL.md descriptions),
|
||||
* not a hard "Claude Code drops the tail" claim.
|
||||
*/
|
||||
describe('AGT scanner — per-agent description advisory', () => {
|
||||
it('fires a low-severity advisory when one agent description exceeds the soft cap', async () => {
|
||||
// One agent, description over the cap; aggregate stays well under budget so
|
||||
// the per-agent advisory is the only finding (clean isolation).
|
||||
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
assert.equal(findAggregate(result.findings), undefined,
|
||||
'a single small-aggregate agent must not trip the aggregate budget');
|
||||
const perAgent = findPerAgent(result.findings);
|
||||
assert.equal(perAgent.length, 1,
|
||||
`expected exactly one per-agent advisory; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||
assert.equal(perAgent[0].severity, 'low', `per-agent advisory should be low, got ${perAgent[0].severity}`);
|
||||
assert.match(perAgent[0].id, /^CA-AGT-\d{3}$/);
|
||||
assert.match(`${perAgent[0].description} ${perAgent[0].evidence}`, /solo/,
|
||||
'the advisory should name the offending agent');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('treats the soft cap as a strict boundary: at-cap no fire, one-over fires', async () => {
|
||||
const atCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP, 'atcap');
|
||||
try {
|
||||
const result = await runScannerWithHome(atCap);
|
||||
assert.equal(findPerAgent(result.findings).length, 0,
|
||||
'a description exactly at the soft cap must not fire (strictly-greater rule)');
|
||||
} finally {
|
||||
await rm(atCap, { recursive: true, force: true });
|
||||
}
|
||||
const overCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 1, 'overcap');
|
||||
try {
|
||||
const result = await runScannerWithHome(overCap);
|
||||
assert.equal(findPerAgent(result.findings).length, 1,
|
||||
'a description one char over the soft cap must fire');
|
||||
} finally {
|
||||
await rm(overCap, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('does NOT claim truncation and DISCLOSES the advisory/heuristic framing (intellectual honesty)', async () => {
|
||||
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const advisory = findPerAgent(result.findings)[0];
|
||||
assert.ok(advisory, 'expected a per-agent advisory');
|
||||
const text = `${advisory.description} ${advisory.evidence}`;
|
||||
assert.doesNotMatch(text, /truncat/i,
|
||||
'agents have no verified cap — the advisory must NOT claim Claude Code truncates the description');
|
||||
assert.match(text, /every turn|always.?loaded/i,
|
||||
'the advisory must explain the description re-enters context every turn');
|
||||
assert.match(String(advisory.evidence), new RegExp(String(PER_AGENT_DESC_SOFT_CAP)),
|
||||
'evidence must state the soft-cap threshold so it reads as a heuristic');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('recommends trimming the description toward its trigger phrases', async () => {
|
||||
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const advisory = findPerAgent(result.findings)[0];
|
||||
assert.ok(advisory, 'expected a per-agent advisory to carry remediation');
|
||||
assert.match(String(advisory.recommendation), /trim|trigger|shorten/i,
|
||||
'recommendation should point at trimming the description');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('emits per-agent advisories BEFORE the aggregate roll-up', async () => {
|
||||
// 17 large agents → both per-agent advisories AND the aggregate fire.
|
||||
const home = await homeWithNUserAgents(17, 1000);
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg, 'expected the aggregate to fire alongside per-agent advisories');
|
||||
const aggIndex = result.findings.indexOf(agg);
|
||||
assert.equal(aggIndex, result.findings.length - 1,
|
||||
'the aggregate roll-up must be emitted last (per-agent advisories precede it, mirroring SKL 001→002)');
|
||||
assert.equal(findPerAgent(result.findings).length, 17,
|
||||
'each of the 17 over-cap agents should carry its own advisory');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue