feat(scanner): add AGT agent-listing always-loaded budget finding (v5.9 B1)
New AGT scanner flags the aggregate always-loaded cost of the agent listing (every active agent's name+description is injected each turn so the model knows what it can delegate to). Mirrors the SKL skill-listing pattern but encodes the key honesty caveat: the agent-listing mechanism is INFERRED (agents are absent from Claude Code's documented context breakdown), so the token figure is an UPPER-BOUND estimate and the aggregate budget is a config-audit heuristic anchored on 200k — not a CC-documented allotment. - scanners/agent-listing-scanner.mjs + scanners/lib/agent-listing-budget.mjs - 8 TDD tests (tests/scanners/agent-listing-scanner.test.mjs) - AGT folds into the Token Efficiency health area (scoring.mjs) - byte-stability: AGT added to strip-added-scanner (frozen v5.0.0 baselines left untouched, same as OST/OPT); SC-5 default-output snapshot refreshed - suite 1168 -> 1176 green Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
759daa7201
commit
fcfb2979ef
9 changed files with 399 additions and 13 deletions
71
scanners/agent-listing-scanner.mjs
Normal file
71
scanners/agent-listing-scanner.mjs
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
/**
|
||||||
|
* AGT Scanner — Agent-listing always-loaded token budget
|
||||||
|
*
|
||||||
|
* Claude Code injects a listing of every active agent's name+description into the
|
||||||
|
* system prompt so it knows which subagents it can delegate to. That listing is
|
||||||
|
* re-sent on EVERY turn, whether or not a delegation happens — so with many
|
||||||
|
* installed agents it is a large always-loaded cost (on a heavily-plugged machine
|
||||||
|
* the dominant single always-loaded source).
|
||||||
|
*
|
||||||
|
* Detection:
|
||||||
|
* CA-AGT-NNN aggregate agent-listing estimate exceeds the listing budget (low)
|
||||||
|
*
|
||||||
|
* INTELLECTUAL-HONESTY CONTRACT (the reason this is `low`, not a hard finding):
|
||||||
|
* unlike the skill listing, the agent-listing mechanism is NOT documented (agents
|
||||||
|
* are absent from Claude Code's published context breakdown). So the finding is an
|
||||||
|
* INFERRED, UPPER-BOUND ESTIMATE, the budget is a config-audit heuristic (no
|
||||||
|
* documented agent allotment) anchored on a conservative 200k window, and the
|
||||||
|
* evidence discloses all of that. The cap, budget, and enumerate-and-measure step
|
||||||
|
* live in `lib/agent-listing-budget.mjs` — AGT only constructs the finding.
|
||||||
|
*
|
||||||
|
* Zero external dependencies.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { finding, scannerResult } from './lib/output.mjs';
|
||||||
|
import { SEVERITY } from './lib/severity.mjs';
|
||||||
|
import {
|
||||||
|
AGGREGATE_BUDGET_TOKENS,
|
||||||
|
BUDGET_CALIBRATION_NOTE,
|
||||||
|
measureActiveAgentListing,
|
||||||
|
} from './lib/agent-listing-budget.mjs';
|
||||||
|
|
||||||
|
const SCANNER = 'AGT';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main scanner entry point.
|
||||||
|
*
|
||||||
|
* @param {string} _targetPath unused (agent listing is HOME-scoped)
|
||||||
|
* @param {object} _discovery unused (ignores project discovery)
|
||||||
|
*/
|
||||||
|
export async function scan(_targetPath, _discovery) {
|
||||||
|
const start = Date.now();
|
||||||
|
const findings = [];
|
||||||
|
|
||||||
|
const { aggregate } = await measureActiveAgentListing();
|
||||||
|
|
||||||
|
if (aggregate.overBudget) {
|
||||||
|
findings.push(finding({
|
||||||
|
scanner: SCANNER,
|
||||||
|
severity: SEVERITY.low,
|
||||||
|
title: 'Aggregate agent listing may exceed the always-loaded budget',
|
||||||
|
description:
|
||||||
|
`The ${aggregate.scanned} active agents carry about ${aggregate.aggregateTokens} tokens of ` +
|
||||||
|
'name+description text that Claude Code injects every turn so it knows which subagents it can ' +
|
||||||
|
`delegate to — above the ${AGGREGATE_BUDGET_TOKENS}-token budget this scanner anchors on a 200k ` +
|
||||||
|
'context window. Every one of those tokens is re-sent on every turn whether or not you delegate. ' +
|
||||||
|
'Note: unlike the skill listing, the agent-listing always-loaded mechanism is inferred (not ' +
|
||||||
|
'documented), so this is an upper-bound estimate (see evidence).',
|
||||||
|
evidence:
|
||||||
|
`active_agents_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars}; ` +
|
||||||
|
`description_tokens~${aggregate.aggregateTokens}; budget@200k=${AGGREGATE_BUDGET_TOKENS} tok; ` +
|
||||||
|
`over_by~${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
|
||||||
|
recommendation:
|
||||||
|
'Shrink the always-loaded agent listing: disable plugins whose agents you do not use (the whole ' +
|
||||||
|
'agent block leaves the listing), remove dead user agents from ~/.claude/agents/, and trim long ' +
|
||||||
|
'agent descriptions toward their trigger phrases.',
|
||||||
|
category: 'token-efficiency',
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start);
|
||||||
|
}
|
||||||
123
scanners/lib/agent-listing-budget.mjs
Normal file
123
scanners/lib/agent-listing-budget.mjs
Normal file
|
|
@ -0,0 +1,123 @@
|
||||||
|
/**
|
||||||
|
* Agent-listing budget — single source of truth for the AGT scanner.
|
||||||
|
*
|
||||||
|
* Claude Code injects a listing of every active agent's name+description into the
|
||||||
|
* system prompt so the model knows which subagents it can delegate to. With many
|
||||||
|
* installed agents that listing is a large always-loaded cost: re-sent every turn
|
||||||
|
* whether or not a delegation actually happens.
|
||||||
|
*
|
||||||
|
* CRUCIAL HONESTY CAVEAT — this is why AGT differs from SKL. Unlike the skill
|
||||||
|
* listing (documented ~2% allotment, CC 2.1.32; verified 1,536-char truncation
|
||||||
|
* cap, CC 2.1.105), the agent-listing mechanism is NOT documented — agents are
|
||||||
|
* absent from Claude Code's published context breakdown. Therefore:
|
||||||
|
* - "always-loaded" is INFERRED (reasoned from the skill analogue + agents'
|
||||||
|
* absence from the deferred/on-demand list), not documented.
|
||||||
|
* - the token figure is an UPPER-BOUND ESTIMATE, not measured telemetry.
|
||||||
|
* - there is no documented per-listing allotment, so the aggregate budget is a
|
||||||
|
* config-audit heuristic anchored — by analogy to the skill listing — on a
|
||||||
|
* conservative 200k window, and the evidence says so loudly.
|
||||||
|
* - there is no verified per-description truncation cap, so (unlike CA-SKL-001)
|
||||||
|
* each description contributes its FULL length to the aggregate.
|
||||||
|
*
|
||||||
|
* Zero external dependencies.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { estimateTokens, enumeratePlugins, enumerateAgents } from './active-config-reader.mjs';
|
||||||
|
import { readTextFile } from './file-discovery.mjs';
|
||||||
|
import { parseFrontmatter } from './yaml-parser.mjs';
|
||||||
|
import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas } from './context-window.mjs';
|
||||||
|
|
||||||
|
// Heuristic budget by analogy to the skill listing's documented ~2% allotment.
|
||||||
|
// Agents have NO documented allotment of their own — disclosed loudly in
|
||||||
|
// BUDGET_CALIBRATION_NOTE so the number is never mistaken for a CC guarantee.
|
||||||
|
export const BUDGET_FRACTION = 0.02;
|
||||||
|
export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000
|
||||||
|
export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000
|
||||||
|
export { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas };
|
||||||
|
|
||||||
|
// The honest framing required because (a) the mechanism is inferred and (b) the
|
||||||
|
// budget depends on a context window we cannot observe. Appended to overflow evidence.
|
||||||
|
export const BUDGET_CALIBRATION_NOTE =
|
||||||
|
'the agent-listing always-loaded mechanism is INFERRED, not documented (agents are absent from ' +
|
||||||
|
"Claude Code's published context breakdown), so this token figure is an UPPER-BOUND ESTIMATE, not " +
|
||||||
|
'measured telemetry. unlike the skill listing there is no documented per-listing allotment, so this ' +
|
||||||
|
'budget is a config-audit heuristic anchored on a conservative 200k window; at ' +
|
||||||
|
`${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` +
|
||||||
|
'tok and you are likely within it';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {object} AgentBudgetAssessment
|
||||||
|
* @property {number} scanned - number of agent descriptions assessed
|
||||||
|
* @property {number} aggregateChars - sum of description lengths (no cap; no verified truncation)
|
||||||
|
* @property {number} aggregateTokens - estimateTokens(aggregateChars, 'markdown')
|
||||||
|
* @property {number} budgetTokens - AGGREGATE_BUDGET_TOKENS (the 200k-anchored heuristic)
|
||||||
|
* @property {boolean} overBudget - aggregateTokens strictly greater than budgetTokens
|
||||||
|
* @property {number} overBy - tokens over budget (0 when not over)
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pure aggregate-budget assessment. Each description contributes its full length:
|
||||||
|
* agents have no verified truncation cap, so nothing is dropped from the estimate.
|
||||||
|
*
|
||||||
|
* @param {number[]} descLengths - one entry per active agent (description char count)
|
||||||
|
* @returns {AgentBudgetAssessment}
|
||||||
|
*/
|
||||||
|
export function assessAgentListingBudget(descLengths) {
|
||||||
|
let aggregateChars = 0;
|
||||||
|
for (const len of descLengths) {
|
||||||
|
const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0;
|
||||||
|
aggregateChars += safe;
|
||||||
|
}
|
||||||
|
const aggregateTokens = estimateTokens(aggregateChars, 'markdown');
|
||||||
|
const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS;
|
||||||
|
return {
|
||||||
|
scanned: descLengths.length,
|
||||||
|
aggregateChars,
|
||||||
|
aggregateTokens,
|
||||||
|
budgetTokens: AGGREGATE_BUDGET_TOKENS,
|
||||||
|
overBudget,
|
||||||
|
overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {object} ActiveAgentEntry
|
||||||
|
* @property {string} name
|
||||||
|
* @property {'user'|'plugin'} source
|
||||||
|
* @property {string|null} pluginName
|
||||||
|
* @property {string} path
|
||||||
|
* @property {number} descLength
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Enumerate every active agent (user + plugin) and measure the listing budget.
|
||||||
|
* HOME-scoped (mirrors the skill listing): resolves ~/.claude via process.env.HOME
|
||||||
|
* and excludes project-scoped agents — those load only inside their own repo.
|
||||||
|
* Callers that run under test MUST override HOME (runScannerWithHome pattern).
|
||||||
|
*
|
||||||
|
* @returns {Promise<{ agents: ActiveAgentEntry[], aggregate: AgentBudgetAssessment }>}
|
||||||
|
*/
|
||||||
|
export async function measureActiveAgentListing() {
|
||||||
|
const plugins = await enumeratePlugins();
|
||||||
|
// enumerateAgents yields project + user + plugin; drop project (repo-local).
|
||||||
|
const allAgents = await enumerateAgents('', plugins);
|
||||||
|
|
||||||
|
const agents = [];
|
||||||
|
for (const agent of allAgents) {
|
||||||
|
if (!agent || agent.source === 'project' || typeof agent.path !== 'string') continue;
|
||||||
|
const content = await readTextFile(agent.path);
|
||||||
|
if (!content) continue;
|
||||||
|
const fm = parseFrontmatter(content)?.frontmatter || null;
|
||||||
|
const desc = (fm && typeof fm.description === 'string') ? fm.description : '';
|
||||||
|
agents.push({
|
||||||
|
name: agent.name,
|
||||||
|
source: agent.source,
|
||||||
|
pluginName: agent.pluginName,
|
||||||
|
path: agent.path,
|
||||||
|
descLength: desc.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const aggregate = assessAgentListingBudget(agents.map((a) => a.descLength));
|
||||||
|
return { agents, aggregate };
|
||||||
|
}
|
||||||
|
|
@ -166,6 +166,7 @@ const SCANNER_AREA_MAP = {
|
||||||
TOK: 'Token Efficiency',
|
TOK: 'Token Efficiency',
|
||||||
CPS: 'Token Efficiency',
|
CPS: 'Token Efficiency',
|
||||||
SKL: 'Token Efficiency',
|
SKL: 'Token Efficiency',
|
||||||
|
AGT: 'Token Efficiency',
|
||||||
DIS: 'Settings',
|
DIS: 'Settings',
|
||||||
COL: 'Plugin Hygiene',
|
COL: 'Plugin Hygiene',
|
||||||
OST: 'Settings',
|
OST: 'Settings',
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ import { scan as scanCachePrefix } from './cache-prefix-scanner.mjs';
|
||||||
import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs';
|
import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs';
|
||||||
import { scan as scanCollision } from './collision-scanner.mjs';
|
import { scan as scanCollision } from './collision-scanner.mjs';
|
||||||
import { scan as scanSkillListing } from './skill-listing-scanner.mjs';
|
import { scan as scanSkillListing } from './skill-listing-scanner.mjs';
|
||||||
|
import { scan as scanAgentListing } from './agent-listing-scanner.mjs';
|
||||||
import { scan as scanOutputStyle } from './output-style-scanner.mjs';
|
import { scan as scanOutputStyle } from './output-style-scanner.mjs';
|
||||||
import { scan as scanOptimizationLens } from './optimization-lens-scanner.mjs';
|
import { scan as scanOptimizationLens } from './optimization-lens-scanner.mjs';
|
||||||
|
|
||||||
|
|
@ -66,6 +67,7 @@ const SCANNERS = [
|
||||||
{ name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' },
|
{ name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' },
|
||||||
{ name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' },
|
{ name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' },
|
||||||
{ name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' },
|
{ name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' },
|
||||||
|
{ name: 'AGT', fn: scanAgentListing, label: 'Agent-Listing Budget' },
|
||||||
{ name: 'OST', fn: scanOutputStyle, label: 'Output-Style Validation' },
|
{ name: 'OST', fn: scanOutputStyle, label: 'Output-Style Validation' },
|
||||||
{ name: 'OPT', fn: scanOptimizationLens, label: 'Optimization Lens' },
|
{ name: 'OPT', fn: scanOptimizationLens, label: 'Optimization Lens' },
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,13 @@
|
||||||
/**
|
/**
|
||||||
* v5.6 C added the OST (Output-Style Validation) scanner — a 14th orchestrated
|
* Post-v5.0.0 additive scanners that the frozen v5.0.0 byte-equal baselines
|
||||||
* scanner. The frozen v5.0.0 byte-equal baselines predate it (13 scanners), so
|
* predate: OST (Output-Style Validation, v5.6 C), OPT (Optimization Lens, v5.7),
|
||||||
* this strips the additive OST scanner entry before comparison, the same way
|
* and AGT (Agent-Listing Budget, v5.9 B1). The baselines froze the original
|
||||||
* strip-hotspot-load-pattern.mjs handles B2's additive hotspot fields. Stripping
|
* scanner set, so this strips these additive scanner entries before comparison,
|
||||||
* (rather than re-seeding the frozen snapshots) keeps the strongest invariant:
|
* the same way strip-hotspot-load-pattern.mjs handles B2's additive hotspot
|
||||||
* the ORIGINAL 13 scanners still emit byte-identical output, and the frozen
|
* fields. Stripping (rather than re-seeding the frozen snapshots) keeps the
|
||||||
* snapshots stay free of post-v5.0.0 drift (the hotspot triple, ancestor token
|
* strongest invariant: the ORIGINAL scanners still emit byte-identical output,
|
||||||
* counts) that re-seeding would bake in.
|
* and the frozen snapshots stay free of post-v5.0.0 drift (the hotspot triple,
|
||||||
|
* ancestor token counts) that re-seeding would bake in.
|
||||||
*
|
*
|
||||||
* Removing a zero-finding scanner entry only moves `aggregate.scanners_ok`
|
* Removing a zero-finding scanner entry only moves `aggregate.scanners_ok`
|
||||||
* (OST is always status 'ok' and emits nothing on the deterministic fixture, so
|
* (OST is always status 'ok' and emits nothing on the deterministic fixture, so
|
||||||
|
|
@ -17,7 +18,7 @@
|
||||||
* return stripAddedScanners(stripHotspotLoadPattern(out));
|
* return stripAddedScanners(stripHotspotLoadPattern(out));
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const ADDED_SCANNERS = new Set(['OST', 'OPT']);
|
const ADDED_SCANNERS = new Set(['OST', 'OPT', 'AGT']);
|
||||||
|
|
||||||
function stripFromEnvelope(env) {
|
function stripFromEnvelope(env) {
|
||||||
if (!env || typeof env !== 'object' || !Array.isArray(env.scanners)) return;
|
if (!env || typeof env !== 'object' || !Array.isArray(env.scanners)) return;
|
||||||
|
|
@ -58,5 +59,5 @@ export function stripAddedScanners(payload) {
|
||||||
*/
|
*/
|
||||||
export function stripAddedScannerStderr(text) {
|
export function stripAddedScannerStderr(text) {
|
||||||
if (typeof text !== 'string') return text;
|
if (typeof text !== 'string') return text;
|
||||||
return text.replace(/^[ \t]*\[(OST|OPT)\][^\n]*\n/gm, '');
|
return text.replace(/^[ \t]*\[(OST|OPT|AGT)\][^\n]*\n/gm, '');
|
||||||
}
|
}
|
||||||
|
|
|
||||||
174
tests/scanners/agent-listing-scanner.test.mjs
Normal file
174
tests/scanners/agent-listing-scanner.test.mjs
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
import { describe, it } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { mkdir, writeFile, rm } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||||
|
import { scan } from '../../scanners/agent-listing-scanner.mjs';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AGT scanner — agent-listing always-loaded token budget.
|
||||||
|
*
|
||||||
|
* Claude Code injects a listing of every active agent's name+description into the
|
||||||
|
* system prompt so the model knows which subagents it can delegate to. With ~100
|
||||||
|
* agents that listing is a large always-loaded cost. UNLIKE the skill listing,
|
||||||
|
* the agent-listing mechanism is NOT documented (agents are absent from CC's
|
||||||
|
* deferred / always-loaded context breakdown), so AGT findings are an INFERRED
|
||||||
|
* upper-bound estimate, not a verified truncation like SKL-001. The aggregate
|
||||||
|
* budget is a config-audit heuristic anchored on a conservative 200k window.
|
||||||
|
*
|
||||||
|
* Token heuristic mirrors estimateTokens('markdown'): ceil(chars/4).
|
||||||
|
* 16 agents * 1000 desc-chars = 16000 chars -> 4000 tok == budget (NOT over).
|
||||||
|
* 17 agents * 1000 desc-chars = 17000 chars -> 4250 tok > budget (over).
|
||||||
|
*/
|
||||||
|
const AGGREGATE_BUDGET_TOKENS = 4000; // heuristic: 0.02 * 200_000 (anchored, disclosed)
|
||||||
|
|
||||||
|
function uniqueDir(suffix) {
|
||||||
|
return join(tmpdir(), `config-audit-agt-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The AGT scanner enumerates active agents via process.env.HOME. Tests must
|
||||||
|
* override HOME, run, restore — never rely on the developer's real ~/.claude.
|
||||||
|
*/
|
||||||
|
async function runScannerWithHome(home) {
|
||||||
|
resetCounter();
|
||||||
|
const original = process.env.HOME;
|
||||||
|
process.env.HOME = home;
|
||||||
|
try {
|
||||||
|
return await scan('/unused', { files: [] });
|
||||||
|
} finally {
|
||||||
|
process.env.HOME = original;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Write one user agent (name + description of `descLen` chars) into a HOME. */
|
||||||
|
async function addUserAgent(home, name, descLen) {
|
||||||
|
const dir = join(home, '.claude', 'agents');
|
||||||
|
await mkdir(dir, { recursive: true });
|
||||||
|
const desc = 'a'.repeat(descLen);
|
||||||
|
await writeFile(
|
||||||
|
join(dir, `${name}.md`),
|
||||||
|
`---\nname: ${name}\ndescription: ${desc}\n---\nYou are the ${name} agent. Body of the agent prompt.\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Build a fake HOME with `count` user agents, each description `descLen` chars. */
|
||||||
|
async function homeWithNUserAgents(count, descLen, prefix = 'agg') {
|
||||||
|
const home = uniqueDir(`${prefix}-${count}x${descLen}`);
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
await addUserAgent(home, `${prefix}${i}`, descLen);
|
||||||
|
}
|
||||||
|
return home;
|
||||||
|
}
|
||||||
|
|
||||||
|
const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title));
|
||||||
|
|
||||||
|
describe('AGT scanner — basic structure', () => {
|
||||||
|
it('reports scanner prefix AGT', async () => {
|
||||||
|
const home = uniqueDir('empty');
|
||||||
|
try {
|
||||||
|
await mkdir(join(home, '.claude'), { recursive: true });
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
assert.equal(result.scanner, 'AGT');
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('finding IDs match CA-AGT-NNN pattern', async () => {
|
||||||
|
const home = await homeWithNUserAgents(17, 1000);
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
for (const f of result.findings) assert.match(f.id, /^CA-AGT-\d{3}$/);
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('AGT scanner — aggregate always-loaded budget', () => {
|
||||||
|
it('fires a low-severity finding when the agent listing exceeds the 200k-anchored budget', async () => {
|
||||||
|
const home = await homeWithNUserAgents(17, 1000); // 17000 chars -> ~4250 tok > 4000
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
const agg = findAggregate(result.findings);
|
||||||
|
assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||||
|
assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`);
|
||||||
|
assert.match(agg.id, /^CA-AGT-\d{3}$/);
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does NOT fire when only a few small agents are active', async () => {
|
||||||
|
const home = await homeWithNUserAgents(3, 1000); // 3000 chars -> 750 tok < 4000
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
assert.equal(findAggregate(result.findings), undefined,
|
||||||
|
`expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('treats the budget as a strict boundary: 16x1000 (at budget) no fire, 17x1000 (over) fires', async () => {
|
||||||
|
const atBudget = await homeWithNUserAgents(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(atBudget);
|
||||||
|
assert.equal(findAggregate(result.findings), undefined,
|
||||||
|
'aggregate exactly at the budget must not fire (strictly-greater rule)');
|
||||||
|
} finally {
|
||||||
|
await rm(atBudget, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
const overBudget = await homeWithNUserAgents(17, 1000, 'over'); // 17000 chars -> 4250 tok
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(overBudget);
|
||||||
|
assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire');
|
||||||
|
} finally {
|
||||||
|
await rm(overBudget, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('DISCLOSES the inferred mechanism + upper-bound estimate + anchor (intellectual honesty)', async () => {
|
||||||
|
const home = await homeWithNUserAgents(17, 1000);
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
const agg = findAggregate(result.findings);
|
||||||
|
assert.ok(agg, 'expected an aggregate finding');
|
||||||
|
const ev = String(agg.evidence);
|
||||||
|
assert.match(ev, /infer/i, 'evidence must disclose the always-loaded mechanism is INFERRED, not documented');
|
||||||
|
assert.match(ev, /estimate|upper.?bound/i, 'evidence must flag the figure as an estimate / upper bound');
|
||||||
|
assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence must state the budget');
|
||||||
|
assert.match(ev, /200k/, 'evidence must anchor the budget on a 200k window');
|
||||||
|
assert.match(ev, /1,000,000|1M|20,000/, 'evidence must note the 1M-context scaling');
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('recommends the reduction levers (disable unused plugins / trim descriptions)', async () => {
|
||||||
|
const home = await homeWithNUserAgents(17, 1000);
|
||||||
|
try {
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
const agg = findAggregate(result.findings);
|
||||||
|
assert.ok(agg, 'expected an aggregate finding to carry remediation');
|
||||||
|
const rec = String(agg.recommendation);
|
||||||
|
assert.match(rec, /disable|plugin/i, 'recommendation should mention disabling unused plugins');
|
||||||
|
assert.match(rec, /trim|description/i, 'recommendation should mention trimming descriptions');
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('an empty HOME (no agents) yields zero findings', async () => {
|
||||||
|
const home = uniqueDir('noagents');
|
||||||
|
try {
|
||||||
|
await mkdir(join(home, '.claude'), { recursive: true });
|
||||||
|
const result = await runScannerWithHome(home);
|
||||||
|
assert.equal(result.findings.length, 0);
|
||||||
|
} finally {
|
||||||
|
await rm(home, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -103,7 +103,7 @@ describe('scan-orchestrator humanizer wiring (Step 5)', () => {
|
||||||
assert.ok(actual.meta, 'meta present');
|
assert.ok(actual.meta, 'meta present');
|
||||||
assert.ok(Array.isArray(actual.scanners), 'scanners array present');
|
assert.ok(Array.isArray(actual.scanners), 'scanners array present');
|
||||||
assert.ok(actual.aggregate, 'aggregate present');
|
assert.ok(actual.aggregate, 'aggregate present');
|
||||||
assert.equal(actual.scanners.length, 15, 'all 15 scanners present');
|
assert.equal(actual.scanners.length, 16, 'all 16 scanners present');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('preserves scanner shape (scanner/status/findings/counts)', async () => {
|
it('preserves scanner shape (scanner/status/findings/counts)', async () => {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
{
|
{
|
||||||
"kind": "text",
|
"kind": "text",
|
||||||
"payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n `[OST] Output-Style Validation`: 0 finding(s) (0ms)\n `[OPT] Optimization Lens`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
"payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n `[AGT] Agent-Listing Budget`: 0 finding(s) (0ms)\n `[OST] Output-Style Validation`: 0 finding(s) (0ms)\n `[OPT] Optimization Lens`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -569,6 +569,20 @@
|
||||||
"info": 0
|
"info": 0
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"scanner": "AGT",
|
||||||
|
"status": "ok",
|
||||||
|
"files_scanned": 0,
|
||||||
|
"duration_ms": 0,
|
||||||
|
"findings": [],
|
||||||
|
"counts": {
|
||||||
|
"critical": 0,
|
||||||
|
"high": 0,
|
||||||
|
"medium": 0,
|
||||||
|
"low": 0,
|
||||||
|
"info": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"scanner": "OST",
|
"scanner": "OST",
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
|
|
@ -610,7 +624,7 @@
|
||||||
"risk_score": 11,
|
"risk_score": 11,
|
||||||
"risk_band": "Medium",
|
"risk_band": "Medium",
|
||||||
"verdict": "PASS",
|
"verdict": "PASS",
|
||||||
"scanners_ok": 14,
|
"scanners_ok": 15,
|
||||||
"scanners_error": 0,
|
"scanners_error": 0,
|
||||||
"scanners_skipped": 1
|
"scanners_skipped": 1
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue