config-audit/tests/scanners/agent-listing-scanner.test.mjs

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { join } from 'node:path';
import { mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/agent-listing-scanner.mjs';
import { PER_AGENT_DESC_SOFT_CAP } from '../../scanners/lib/agent-listing-budget.mjs';

/**
 * AGT scanner — agent-listing always-loaded token budget.
 *
 * Claude Code injects a listing of every active agent's name+description into the
 * system prompt so the model knows which subagents it can delegate to. With ~100
 * agents that listing is a large always-loaded cost. UNLIKE the skill listing,
 * the agent-listing mechanism is NOT documented (agents are absent from CC's
 * deferred / always-loaded context breakdown), so AGT findings are an INFERRED
 * upper-bound estimate, not a verified truncation like SKL-001. The aggregate
 * budget is a config-audit heuristic anchored on a conservative 200k window.
 *
 * Token heuristic mirrors estimateTokens('markdown'): ceil(chars/4).
 *   16 agents * 1000 desc-chars = 16000 chars -> 4000 tok == budget (NOT over).
 *   17 agents * 1000 desc-chars = 17000 chars -> 4250 tok  > budget (over).
 */
const AGGREGATE_BUDGET_TOKENS = 4000; // heuristic: 0.02 * 200_000 (anchored, disclosed)

function uniqueDir(suffix) {
  return join(tmpdir(), `config-audit-agt-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
}

/**
 * The AGT scanner enumerates active agents via process.env.HOME. Tests must
 * override HOME, run, restore — never rely on the developer's real ~/.claude.
 */
async function runScannerWithHome(home) {
  resetCounter();
  const original = process.env.HOME;
  process.env.HOME = home;
  try {
    return await scan('/unused', { files: [] });
  } finally {
    process.env.HOME = original;
  }
}

/** Write one user agent (name + description of `descLen` chars) into a HOME. */
async function addUserAgent(home, name, descLen) {
  const dir = join(home, '.claude', 'agents');
  await mkdir(dir, { recursive: true });
  const desc = 'a'.repeat(descLen);
  await writeFile(
    join(dir, `${name}.md`),
    `---\nname: ${name}\ndescription: ${desc}\n---\nYou are the ${name} agent. Body of the agent prompt.\n`,
  );
}

/** Build a fake HOME with `count` user agents, each description `descLen` chars. */
async function homeWithNUserAgents(count, descLen, prefix = 'agg') {
  const home = uniqueDir(`${prefix}-${count}x${descLen}`);
  for (let i = 0; i < count; i++) {
    await addUserAgent(home, `${prefix}${i}`, descLen);
  }
  return home;
}

/** Build a fake HOME with a single user agent whose description is `descLen` chars. */
async function homeWithOneAgent(descLen, name = 'solo') {
  const home = uniqueDir(`one-${name}-${descLen}`);
  await addUserAgent(home, name, descLen);
  return home;
}

const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title));
// Per-agent advisories are every finding that is NOT the aggregate roll-up.
const findPerAgent = (findings) => findings.filter(f => !/agent listing/i.test(f.title));

describe('AGT scanner — basic structure', () => {
  it('reports scanner prefix AGT', async () => {
    const home = uniqueDir('empty');
    try {
      await mkdir(join(home, '.claude'), { recursive: true });
      const result = await runScannerWithHome(home);
      assert.equal(result.scanner, 'AGT');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('finding IDs match CA-AGT-NNN pattern', async () => {
    const home = await homeWithNUserAgents(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      for (const f of result.findings) assert.match(f.id, /^CA-AGT-\d{3}$/);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('AGT scanner — aggregate always-loaded budget', () => {
  it('fires a low-severity finding when the agent listing exceeds the 200k-anchored budget', async () => {
    const home = await homeWithNUserAgents(17, 1000); // 17000 chars -> ~4250 tok > 4000
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
      assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`);
      assert.match(agg.id, /^CA-AGT-\d{3}$/);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT fire when only a few small agents are active', async () => {
    const home = await homeWithNUserAgents(3, 1000); // 3000 chars -> 750 tok < 4000
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findAggregate(result.findings), undefined,
        `expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('treats the budget as a strict boundary: 16x1000 (at budget) no fire, 17x1000 (over) fires', async () => {
    const atBudget = await homeWithNUserAgents(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget
    try {
      const result = await runScannerWithHome(atBudget);
      assert.equal(findAggregate(result.findings), undefined,
        'aggregate exactly at the budget must not fire (strictly-greater rule)');
    } finally {
      await rm(atBudget, { recursive: true, force: true });
    }
    const overBudget = await homeWithNUserAgents(17, 1000, 'over'); // 17000 chars -> 4250 tok
    try {
      const result = await runScannerWithHome(overBudget);
      assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire');
    } finally {
      await rm(overBudget, { recursive: true, force: true });
    }
  });

  it('DISCLOSES the inferred mechanism + upper-bound estimate + anchor (intellectual honesty)', async () => {
    const home = await homeWithNUserAgents(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, 'expected an aggregate finding');
      const ev = String(agg.evidence);
      assert.match(ev, /infer/i, 'evidence must disclose the always-loaded mechanism is INFERRED, not documented');
      assert.match(ev, /estimate|upper.?bound/i, 'evidence must flag the figure as an estimate / upper bound');
      assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence must state the budget');
      assert.match(ev, /200k/, 'evidence must anchor the budget on a 200k window');
      assert.match(ev, /1,000,000|1M|20,000/, 'evidence must note the 1M-context scaling');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('recommends the reduction levers (disable unused plugins / trim descriptions)', async () => {
    const home = await homeWithNUserAgents(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, 'expected an aggregate finding to carry remediation');
      const rec = String(agg.recommendation);
      assert.match(rec, /disable|plugin/i, 'recommendation should mention disabling unused plugins');
      assert.match(rec, /trim|description/i, 'recommendation should mention trimming descriptions');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('an empty HOME (no agents) yields zero findings', async () => {
    const home = uniqueDir('noagents');
    try {
      await mkdir(join(home, '.claude'), { recursive: true });
      const result = await runScannerWithHome(home);
      assert.equal(result.findings.length, 0);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

/**
 * Per-agent description advisory (the B1-rest companion to the aggregate).
 *
 * A single agent with a long description is flagged on its own: every char of
 * that description re-enters context in the always-loaded agent listing on every
 * turn. UNLIKE CA-SKL-001, this is NOT a truncation finding — agents have no
 * verified per-description cap — so the advisory is a soft heuristic (low, the
 * same 500-char bloat threshold TOK pattern F uses for SKILL.md descriptions),
 * not a hard "Claude Code drops the tail" claim.
 */
describe('AGT scanner — per-agent description advisory', () => {
  it('fires a low-severity advisory when one agent description exceeds the soft cap', async () => {
    // One agent, description over the cap; aggregate stays well under budget so
    // the per-agent advisory is the only finding (clean isolation).
    const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findAggregate(result.findings), undefined,
        'a single small-aggregate agent must not trip the aggregate budget');
      const perAgent = findPerAgent(result.findings);
      assert.equal(perAgent.length, 1,
        `expected exactly one per-agent advisory; got: ${result.findings.map(f => f.title).join(' | ')}`);
      assert.equal(perAgent[0].severity, 'low', `per-agent advisory should be low, got ${perAgent[0].severity}`);
      assert.match(perAgent[0].id, /^CA-AGT-\d{3}$/);
      assert.match(`${perAgent[0].description} ${perAgent[0].evidence}`, /solo/,
        'the advisory should name the offending agent');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('treats the soft cap as a strict boundary: at-cap no fire, one-over fires', async () => {
    const atCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP, 'atcap');
    try {
      const result = await runScannerWithHome(atCap);
      assert.equal(findPerAgent(result.findings).length, 0,
        'a description exactly at the soft cap must not fire (strictly-greater rule)');
    } finally {
      await rm(atCap, { recursive: true, force: true });
    }
    const overCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 1, 'overcap');
    try {
      const result = await runScannerWithHome(overCap);
      assert.equal(findPerAgent(result.findings).length, 1,
        'a description one char over the soft cap must fire');
    } finally {
      await rm(overCap, { recursive: true, force: true });
    }
  });

  it('does NOT claim truncation and DISCLOSES the advisory/heuristic framing (intellectual honesty)', async () => {
    const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
    try {
      const result = await runScannerWithHome(home);
      const advisory = findPerAgent(result.findings)[0];
      assert.ok(advisory, 'expected a per-agent advisory');
      const text = `${advisory.description} ${advisory.evidence}`;
      assert.doesNotMatch(text, /truncat/i,
        'agents have no verified cap — the advisory must NOT claim Claude Code truncates the description');
      assert.match(text, /every turn|always.?loaded/i,
        'the advisory must explain the description re-enters context every turn');
      assert.match(String(advisory.evidence), new RegExp(String(PER_AGENT_DESC_SOFT_CAP)),
        'evidence must state the soft-cap threshold so it reads as a heuristic');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('recommends trimming the description toward its trigger phrases', async () => {
    const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100);
    try {
      const result = await runScannerWithHome(home);
      const advisory = findPerAgent(result.findings)[0];
      assert.ok(advisory, 'expected a per-agent advisory to carry remediation');
      assert.match(String(advisory.recommendation), /trim|trigger|shorten/i,
        'recommendation should point at trimming the description');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('emits per-agent advisories BEFORE the aggregate roll-up', async () => {
    // 17 large agents → both per-agent advisories AND the aggregate fire.
    const home = await homeWithNUserAgents(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, 'expected the aggregate to fire alongside per-agent advisories');
      const aggIndex = result.findings.indexOf(agg);
      assert.equal(aggIndex, result.findings.length - 1,
        'the aggregate roll-up must be emitted last (per-agent advisories precede it, mirroring SKL 001→002)');
      assert.equal(findPerAgent(result.findings).length, 17,
        'each of the 17 over-cap agents should carry its own advisory');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});