config-audit/tests/scanners/gap-agent-model-routing.test.mjs

/**
 * C4 — agent model/effort routing lever (CA-GAP-028).
 *
 * A LEVER, not a dimension: it fires only when authored agents exist, so it has
 * no meaningful "present/absent" reading on a config with no agents at all —
 * exactly the shape the three existing levers already have. The design was
 * decided by measurement, not taste: as a GAP_CHECKS dimension it would have
 * counted as vacuously-present on the agent-less marketplace-medium fixture,
 * moving MAX_WEIGHTED 41→42 and utilization 44→45, which flips `segment`
 * "Developing"→"Competent" in the frozen v5.0.0 posture baseline (segment is
 * NOT among the fields strip-retired-gap.mjs drops from comparison).
 *
 * The silence has two independent causes and they must not be conflated:
 *   - no authored agents at all (t2_6 "No custom subagents" owns that case), and
 *   - the only agents on disk being plugin-bundled, which isAuthoredConfig
 *     excludes.
 * P5 below pins the second one specifically, so that arm cannot pass for the
 * first one's reason.
 */
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { mkdtemp, mkdir, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { scan, agentModelRoutingLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
import { withHermeticHome } from '../helpers/hermetic-home.mjs';

const LEVER_TITLE = 'Subagents pin neither model nor effort';

/** Build a throwaway project; `agents` maps a relative dir to agent frontmatter lines. */
async function makeProject(agents) {
  const root = await mkdtemp(join(tmpdir(), 'config-audit-c4-'));
  // A CLAUDE.md keeps the fixture from being a bare directory; irrelevant to the lever.
  await writeFile(join(root, 'CLAUDE.md'), '# Project\n');
  for (const [relDir, files] of Object.entries(agents)) {
    const dir = join(root, relDir);
    await mkdir(dir, { recursive: true });
    for (const [name, frontmatter] of Object.entries(files)) {
      await writeFile(
        join(dir, name),
        `---\nname: ${name.replace(/\.md$/, '')}\ndescription: does a thing\n${frontmatter}---\nBody.\n`,
      );
    }
  }
  return root;
}

/** Scan a project with hermetic HOME and no global discovery. */
async function scanProject(root) {
  const discovery = await discoverConfigFiles(resolve(root));
  const result = await withHermeticHome(() => scan(resolve(root), discovery));
  return { result, discovery };
}

const leverFindings = (result) => result.findings.filter(f => f.title === LEVER_TITLE);
const hasNoSubagentsGap = (result) => result.findings.some(f => f.title === 'No custom subagents');

// ---------------------------------------------------------------------------
// Unit — the pure lever function
// ---------------------------------------------------------------------------

describe('agentModelRoutingLeverFinding (pure)', () => {
  it('stays silent when no authored agents exist', () => {
    assert.equal(
      agentModelRoutingLeverFinding({ agentCount: 0, modelPinned: 0, effortPinned: 0 }),
      null,
    );
  });

  it('stays silent when at least one agent pins model', () => {
    assert.equal(
      agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 1, effortPinned: 0 }),
      null,
    );
  });

  it('stays silent when at least one agent pins effort (the separate axis)', () => {
    assert.equal(
      agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 0, effortPinned: 1 }),
      null,
    );
  });

  it('fires when agents exist and neither axis is used anywhere', () => {
    const f = agentModelRoutingLeverFinding({ agentCount: 3, modelPinned: 0, effortPinned: 0 });
    assert.ok(f, 'expected a finding');
    assert.equal(f.id, 'CA-GAP-028');
    assert.equal(f.scanner, 'GAP');
    assert.equal(f.severity, 'info');
    assert.equal(f.category, 'model-fit');
    assert.equal(f.title, LEVER_TITLE);
  });

  it('cites both register entries and their primary sources', () => {
    const f = agentModelRoutingLeverFinding({ agentCount: 2, modelPinned: 0, effortPinned: 0 });
    const text = `${f.description} ${f.recommendation}`;
    assert.match(text, /BP-MODEL-001/);
    assert.match(text, /BP-MODEL-002/);
    assert.match(text, /code\.claude\.com\/docs\/en\/sub-agents/);
    assert.match(text, /code\.claude\.com\/docs\/en\/model-config/);
  });

  it('carries the measured counts as evidence', () => {
    const f = agentModelRoutingLeverFinding({ agentCount: 4, modelPinned: 0, effortPinned: 0 });
    assert.match(f.evidence, /authored_agents=4/);
    assert.match(f.evidence, /model_pinned=0/);
    assert.match(f.evidence, /effort_pinned=0/);
  });

  it('frames the opportunity without asserting the config is wrong', () => {
    const f = agentModelRoutingLeverFinding({ agentCount: 2, modelPinned: 0, effortPinned: 0 });
    // `inherit` is the documented default and the reason the opportunity exists.
    assert.match(f.description, /inherit/i);
  });
});

// ---------------------------------------------------------------------------
// Integration — fire/silent matrix through scan()
// ---------------------------------------------------------------------------

describe('GAP scanner — C4 fire/silent matrix', () => {
  it('P1: fires once when authored agents pin neither axis', async () => {
    const root = await makeProject({
      '.claude/agents': { 'alpha.md': '', 'beta.md': '' },
    });
    const { result } = await scanProject(root);
    const hits = leverFindings(result);
    assert.equal(hits.length, 1, 'expected exactly one lever finding');
    assert.equal(hits[0].id, 'CA-GAP-028');
  });

  // Found by dogfooding this repo's own machine: several installed agents write
  // `model: inherit` explicitly. `inherit` IS the documented default
  // (BP-MODEL-001), so naming it routes nothing and must not buy silence — the
  // opportunity is exactly as open as with the field absent.
  it('P2b: fires when the only "pin" is the default value spelled out', async () => {
    const root = await makeProject({
      '.claude/agents': { 'alpha.md': 'model: inherit\n', 'beta.md': '' },
    });
    const { result } = await scanProject(root);
    const hits = leverFindings(result);
    assert.equal(hits.length, 1, 'model: inherit must not count as routing');
    assert.match(hits[0].evidence, /model_pinned=0/);
  });

  it('P2: silent when one agent pins model', async () => {
    const root = await makeProject({
      '.claude/agents': { 'alpha.md': 'model: haiku\n', 'beta.md': '' },
    });
    const { result } = await scanProject(root);
    assert.equal(leverFindings(result).length, 0);
  });

  it('P3: silent when one agent pins effort and none pins model', async () => {
    const root = await makeProject({
      '.claude/agents': { 'alpha.md': 'effort: low\n', 'beta.md': '' },
    });
    const { result } = await scanProject(root);
    assert.equal(leverFindings(result).length, 0);
  });

  it('P4: silent when there are no agents at all (t2_6 owns that case)', async () => {
    const root = await makeProject({});
    const { result } = await scanProject(root);
    assert.equal(leverFindings(result).length, 0);
    assert.ok(hasNoSubagentsGap(result), 'expected the "No custom subagents" dimension instead');
  });

  it('P5: silent for the EXCLUSION reason when the only agent is plugin-bundled', async () => {
    const root = await makeProject({
      '.claude/plugins/somePlugin/agents': { 'vendored.md': '' },
    });
    const { result, discovery } = await scanProject(root);
    // The discriminator: the file IS on disk and IS discovered as an agent —
    // so silence here cannot be the P4 "no agent files" reason.
    assert.ok(
      discovery.files.some(f => f.type === 'agent-md'),
      'fixture invalid: no agent file was discovered at all',
    );
    assert.equal(leverFindings(result).length, 0);
    assert.ok(
      hasNoSubagentsGap(result),
      'the plugin-bundled agent must not count as an authored subagent either',
    );
  });
});