feat(scanner): add AGT per-agent description bloat advisory (v5.9 B1-rest)

Flag any single active agent whose description exceeds a 500-char soft cap
(the same bloat threshold TOK pattern F applies to SKILL.md descriptions).
Every char of an agent description re-enters context in the always-loaded
agent listing on every turn, so a long one is a per-turn cost.

Framing is deliberately ADVISORY, severity low: unlike CA-SKL-001 agents
have NO verified per-description cap, so nothing is dropped — this is a bloat
advisory, never a truncation claim. Per-agent advisories are emitted BEFORE
the aggregate roll-up (mirrors SKL 001->002).

- PER_AGENT_DESC_SOFT_CAP (=500) added to lib/agent-listing-budget.mjs as the
  single source of truth, disclosed as a heuristic.
- Per-agent loop reuses measureActiveAgentListing()'s agents[] (name, source,
  pluginName, path, descLength) — no new enumeration.
- 5 new tests (fire/strict-boundary/no-truncation-claim/recommendation/order).

Byte-stable: AGT already stripped from frozen v5.0.0 baselines; SC-5 unchanged
(HOME-scoped, hermetic HOME has no agents). Suite green: 1181 tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 16:27:24 +02:00
commit 2a3cb537f9
3 changed files with 164 additions and 1 deletions

View file

@ -5,6 +5,7 @@ import { mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/agent-listing-scanner.mjs';
import { PER_AGENT_DESC_SOFT_CAP } from '../../scanners/lib/agent-listing-budget.mjs';
/**
* AGT scanner agent-listing always-loaded token budget.
@ -62,7 +63,16 @@ async function homeWithNUserAgents(count, descLen, prefix = 'agg') {
return home;
}
/** Build a fake HOME with a single user agent whose description is `descLen` chars. */
async function homeWithOneAgent(descLen, name = 'solo') {
const home = uniqueDir(`one-${name}-${descLen}`);
await addUserAgent(home, name, descLen);
return home;
}
const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title));
// Per-agent advisories are every finding that is NOT the aggregate roll-up.
const findPerAgent = (findings) => findings.filter(f => !/agent listing/i.test(f.title));
describe('AGT scanner — basic structure', () => {
it('reports scanner prefix AGT', async () => {
@ -172,3 +182,102 @@ describe('AGT scanner — aggregate always-loaded budget', () => {
}
});
});
/**
* Per-agent description advisory (the B1-rest companion to the aggregate).
*
* A single agent with a long description is flagged on its own: every char of
* that description re-enters context in the always-loaded agent listing on every
* turn. UNLIKE CA-SKL-001, this is NOT a truncation finding agents have no
* verified per-description cap so the advisory is a soft heuristic (low, the
* same 500-char bloat threshold TOK pattern F uses for SKILL.md descriptions),
* not a hard "Claude Code drops the tail" claim.
*/
describe('AGT scanner — per-agent description advisory', () => {
it('fires a low-severity advisory when one agent description exceeds the soft cap', async () => {
// One agent, description over the cap; aggregate stays well under budget so
// the per-agent advisory is the only finding (clean isolation).
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
try {
const result = await runScannerWithHome(home);
assert.equal(findAggregate(result.findings), undefined,
'a single small-aggregate agent must not trip the aggregate budget');
const perAgent = findPerAgent(result.findings);
assert.equal(perAgent.length, 1,
`expected exactly one per-agent advisory; got: ${result.findings.map(f => f.title).join(' | ')}`);
assert.equal(perAgent[0].severity, 'low', `per-agent advisory should be low, got ${perAgent[0].severity}`);
assert.match(perAgent[0].id, /^CA-AGT-\d{3}$/);
assert.match(`${perAgent[0].description} ${perAgent[0].evidence}`, /solo/,
'the advisory should name the offending agent');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('treats the soft cap as a strict boundary: at-cap no fire, one-over fires', async () => {
const atCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP, 'atcap');
try {
const result = await runScannerWithHome(atCap);
assert.equal(findPerAgent(result.findings).length, 0,
'a description exactly at the soft cap must not fire (strictly-greater rule)');
} finally {
await rm(atCap, { recursive: true, force: true });
}
const overCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 1, 'overcap');
try {
const result = await runScannerWithHome(overCap);
assert.equal(findPerAgent(result.findings).length, 1,
'a description one char over the soft cap must fire');
} finally {
await rm(overCap, { recursive: true, force: true });
}
});
it('does NOT claim truncation and DISCLOSES the advisory/heuristic framing (intellectual honesty)', async () => {
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
try {
const result = await runScannerWithHome(home);
const advisory = findPerAgent(result.findings)[0];
assert.ok(advisory, 'expected a per-agent advisory');
const text = `${advisory.description} ${advisory.evidence}`;
assert.doesNotMatch(text, /truncat/i,
'agents have no verified cap — the advisory must NOT claim Claude Code truncates the description');
assert.match(text, /every turn|always.?loaded/i,
'the advisory must explain the description re-enters context every turn');
assert.match(String(advisory.evidence), new RegExp(String(PER_AGENT_DESC_SOFT_CAP)),
'evidence must state the soft-cap threshold so it reads as a heuristic');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('recommends trimming the description toward its trigger phrases', async () => {
const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
try {
const result = await runScannerWithHome(home);
const advisory = findPerAgent(result.findings)[0];
assert.ok(advisory, 'expected a per-agent advisory to carry remediation');
assert.match(String(advisory.recommendation), /trim|trigger|shorten/i,
'recommendation should point at trimming the description');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('emits per-agent advisories BEFORE the aggregate roll-up', async () => {
// 17 large agents → both per-agent advisories AND the aggregate fire.
const home = await homeWithNUserAgents(17, 1000);
try {
const result = await runScannerWithHome(home);
const agg = findAggregate(result.findings);
assert.ok(agg, 'expected the aggregate to fire alongside per-agent advisories');
const aggIndex = result.findings.indexOf(agg);
assert.equal(aggIndex, result.findings.length - 1,
'the aggregate roll-up must be emitted last (per-agent advisories precede it, mirroring SKL 001→002)');
assert.equal(findPerAgent(result.findings).length, 17,
'each of the 17 over-cap agents should carry its own advisory');
} finally {
await rm(home, { recursive: true, force: true });
}
});
});