config-audit/tests/scanners/skill-listing-scanner.test.mjs

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { join } from 'node:path';
import { mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/skill-listing-scanner.mjs';

const CAP = 1536; // verified per-description listing cap (CC 2.1.105, changelog L1502)

function uniqueDir(suffix) {
  return join(tmpdir(), `config-audit-skl-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
}

/**
 * The SKL scanner enumerates active skills via process.env.HOME
 * (enumeratePlugins/enumerateSkills). Tests must override HOME, run, restore —
 * never rely on the developer's real ~/.claude.
 */
async function runScannerWithHome(home) {
  resetCounter();
  const original = process.env.HOME;
  process.env.HOME = home;
  try {
    return await scan('/unused', { files: [] });
  } finally {
    process.env.HOME = original;
  }
}

/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */
async function runScannerWithCtx(home, contextWindow) {
  resetCounter();
  const original = process.env.HOME;
  process.env.HOME = home;
  try {
    return await scan('/unused', { files: [] }, { contextWindow });
  } finally {
    process.env.HOME = original;
  }
}

/** Build a fake HOME with one user skill whose description has `len` chars. */
async function homeWithUserSkill(name, descLen) {
  const home = uniqueDir(name);
  const dir = join(home, '.claude', 'skills', name);
  await mkdir(dir, { recursive: true });
  const desc = 'a'.repeat(descLen);
  await writeFile(
    join(dir, 'SKILL.md'),
    `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`,
  );
  return home;
}

/** Write one user skill (name + description of `descLen` chars) into an existing HOME. */
async function addUserSkill(home, name, descLen) {
  const dir = join(home, '.claude', 'skills', name);
  await mkdir(dir, { recursive: true });
  const desc = 'a'.repeat(descLen);
  await writeFile(
    join(dir, 'SKILL.md'),
    `---\nname: ${name}\ndescription: ${desc}\n---\nBody of the skill.\n`,
  );
}

/** Build a fake HOME with `count` user skills, each description `descLen` chars. */
async function homeWithNUserSkills(count, descLen, prefix = 'agg') {
  const home = uniqueDir(`${prefix}-${count}x${descLen}`);
  for (let i = 0; i < count; i++) {
    await addUserSkill(home, `${prefix}${i}`, descLen);
  }
  return home;
}

/**
 * Write one user skill whose SKILL.md *body* (below the frontmatter) is about
 * `bodyChars` chars, spread across multiple lines (81 chars incl. newline).
 * The description stays tiny so body-size tests stay isolated from the cap/aggregate checks.
 */
async function addUserSkillWithBody(home, name, bodyChars) {
  const dir = join(home, '.claude', 'skills', name);
  await mkdir(dir, { recursive: true });
  const line = `${'x'.repeat(80)}\n`;
  const body = line.repeat(Math.ceil(bodyChars / line.length));
  await writeFile(
    join(dir, 'SKILL.md'),
    `---\nname: ${name}\ndescription: short.\n---\n${body}`,
  );
}

// Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor.
const AGGREGATE_BUDGET_TOKENS = 4000;   // 0.02 * 200_000
// Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4).
// 16 skills * 1000 chars = 16000 chars -> 4000 tok == budget (NOT over).
// 17 skills * 1000 chars = 17000 chars -> 4250 tok  > budget (over).
const findAggregate = (findings) => findings.find(f => /aggregate/i.test(f.title));
const findCaps = (findings) => findings.filter(f => /listing cap/i.test(f.title));

describe('SKL scanner — basic structure', () => {
  it('reports scanner prefix SKL', async () => {
    const home = uniqueDir('empty');
    try {
      await mkdir(join(home, '.claude'), { recursive: true });
      const result = await runScannerWithHome(home);
      assert.equal(result.scanner, 'SKL');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('finding IDs match CA-SKL-NNN pattern', async () => {
    const home = await homeWithUserSkill('longdesc', CAP + 64);
    try {
      const result = await runScannerWithHome(home);
      for (const f of result.findings) {
        assert.match(f.id, /^CA-SKL-\d{3}$/);
      }
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — per-description 1536-char cap (CA-SKL-001)', () => {
  it('flags a skill whose description exceeds the cap', async () => {
    const home = await homeWithUserSkill('toolong', CAP + 100);
    try {
      const result = await runScannerWithHome(home);
      const f = result.findings.find(x => /toolong/.test(x.evidence || x.description || ''));
      assert.ok(f, `expected a cap finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
      assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`);
      assert.match(f.id, /^CA-SKL-001$/);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('evidence carries the measured char count and the 1536 cap', async () => {
    const home = await homeWithUserSkill('measured', CAP + 200);
    try {
      const result = await runScannerWithHome(home);
      const f = result.findings.find(x => /measured/.test(x.evidence || ''));
      assert.ok(f, 'expected a finding for the oversized skill');
      assert.match(String(f.evidence), new RegExp(String(CAP + 200)));
      assert.match(String(f.evidence), new RegExp(String(CAP)));
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('points file at the offending SKILL.md', async () => {
    const home = await homeWithUserSkill('filepath', CAP + 1);
    try {
      const result = await runScannerWithHome(home);
      const f = result.findings.find(x => /filepath/.test(x.evidence || ''));
      assert.ok(f);
      assert.match(String(f.file), /filepath[\\/]SKILL\.md$/);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — boundary and negative cases', () => {
  it('a description exactly at the cap (1536) yields no finding', async () => {
    const home = await homeWithUserSkill('exact', CAP);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(result.findings.length, 0,
        `expected 0 findings at the cap; got: ${result.findings.map(f => f.title).join(' | ')}`);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('one char over the cap (1537) yields a finding', async () => {
    const home = await homeWithUserSkill('over', CAP + 1);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(result.findings.length, 1,
        `expected 1 finding at cap+1; got: ${result.findings.map(f => f.title).join(' | ')}`);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('a short description yields no finding', async () => {
    const home = await homeWithUserSkill('short', 80);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(result.findings.length, 0,
        `expected 0 findings; got: ${result.findings.map(f => f.title).join(' | ')}`);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('an empty HOME (no skills) yields zero findings', async () => {
    const home = uniqueDir('noskills');
    try {
      await mkdir(join(home, '.claude'), { recursive: true });
      const result = await runScannerWithHome(home);
      assert.equal(result.findings.length, 0);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — remediation levers (Item 2b folded in)', () => {
  it('recommendation lists disableBundledSkills, skillOverrides, and trimming', async () => {
    const home = await homeWithUserSkill('levers', CAP + 300);
    try {
      const result = await runScannerWithHome(home);
      const f = result.findings.find(x => /levers/.test(x.evidence || ''));
      assert.ok(f, 'expected a finding to carry remediation');
      const rec = String(f.recommendation);
      assert.match(rec, /disableBundledSkills/);
      assert.match(rec, /skillOverrides/);
      assert.match(rec, /trim/i);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
  it('fires a low-severity aggregate finding when the description sum exceeds the 200k-anchored budget', async () => {
    const home = await homeWithNUserSkills(17, 1000); // 17000 chars -> ~4250 tok > 4000
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
      assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`);
      assert.match(agg.id, /^CA-SKL-\d{3}$/);
      // 17 skills of 1000 chars each are all under the 1536 cap, so no cap findings.
      assert.equal(findCaps(result.findings).length, 0,
        'no per-description cap findings expected when every description is under 1536');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT fire when the aggregate is under budget', async () => {
    const home = await homeWithNUserSkills(3, 1000); // 3000 chars -> 750 tok < 4000
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findAggregate(result.findings), undefined,
        `expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('treats the budget as a boundary: exactly at budget yields no finding, one skill over yields one', async () => {
    const atBudget = await homeWithNUserSkills(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget
    try {
      const result = await runScannerWithHome(atBudget);
      assert.equal(findAggregate(result.findings), undefined,
        'aggregate exactly at the budget must not fire (strictly-greater rule)');
    } finally {
      await rm(atBudget, { recursive: true, force: true });
    }
    const overBudget = await homeWithNUserSkills(17, 1000, 'over'); // 17000 chars -> 4250 tok
    try {
      const result = await runScannerWithHome(overBudget);
      assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire');
    } finally {
      await rm(overBudget, { recursive: true, force: true });
    }
  });

  it('counts each description capped at 1536 — a single huge skill does NOT inflate the aggregate', async () => {
    // One 5000-char skill: capped contribution = 1536 chars -> 384 tok, far under budget.
    // It must trigger the per-description cap finding but NOT the aggregate.
    const home = await homeWithUserSkill('huge', 5000);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findCaps(result.findings).length, 1, 'the oversized skill should trip the cap finding');
      assert.equal(findAggregate(result.findings), undefined,
        'aggregate must count the capped 1536 contribution, not the full 5000 chars');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('cap and aggregate findings are independent — both can fire in one run', async () => {
    // 1 oversized skill (cap) + 16 normal skills. Aggregate = min(5000,1536)+16000 = 17536 chars -> ~4384 tok.
    const home = await homeWithNUserSkills(16, 1000, 'norm');
    await addUserSkill(home, 'oversized', 5000);
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findCaps(result.findings).length, 1, 'expected exactly one cap finding');
      assert.ok(findAggregate(result.findings), 'expected an aggregate finding alongside the cap finding');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('evidence carries the measured sum, the 200k budget, and an honest context-window calibration note', async () => {
    const home = await homeWithNUserSkills(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, 'expected an aggregate finding');
      const ev = String(agg.evidence);
      assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence should state the 4000-token budget');
      assert.match(ev, /200k/, 'evidence should anchor the budget on a 200k window');
      assert.match(ev, /CC 2\.1\.32/, 'evidence should cite the 2%-of-context changelog fact');
      assert.match(ev, /1,000,000|20,000/, 'evidence should note the 1M-context scaling');
      assert.match(ev, /estimate/i, 'evidence should flag this as an estimate, not measured telemetry');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('aggregate recommendation lists the same levers (disableBundledSkills, skillOverrides, trim)', async () => {
    const home = await homeWithNUserSkills(17, 1000);
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg, 'expected an aggregate finding to carry remediation');
      const rec = String(agg.recommendation);
      assert.match(rec, /disableBundledSkills/);
      assert.match(rec, /skillOverrides/);
      assert.match(rec, /trim/i);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — context-window calibration (B8)', () => {
  it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => {
    // 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k
    // budget, under the 20,000-tok 1M budget — the acceptance case.
    const home = await homeWithNUserSkills(50, 400, 'cw');
    try {
      const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false });
      assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor');

      const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false });
      assert.equal(findAggregate(at1m.findings), undefined,
        'at a 1M context window the listing is within budget and must not fire');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => {
    const home = await homeWithNUserSkills(17, 1000, 'cwadv');
    try {
      const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true });
      const agg = findAggregate(advisory.findings);
      assert.ok(agg, 'still surfaces the measurement (conservative anchor)');
      assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('no opts (default) is unchanged: fires low at the 200k anchor', async () => {
    const home = await homeWithNUserSkills(17, 1000, 'cwdef');
    try {
      const result = await runScannerWithHome(home);
      const agg = findAggregate(result.findings);
      assert.ok(agg);
      assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
  const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));

  it('flags a skill whose body exceeds ~5,000 tokens', async () => {
    const home = uniqueDir('bigbody');
    await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body
    try {
      const result = await runScannerWithHome(home);
      const f = findBody(result.findings);
      assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body');
      assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity');
      assert.match(String(f.file), /heavy[\\/]SKILL\.md$/);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT flag a skill whose body is under the threshold', async () => {
    const home = uniqueDir('smallbody');
    await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findBody(result.findings), undefined, 'a small body must not fire');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => {
    const home = uniqueDir('bodyev');
    await addUserSkillWithBody(home, 'heavy', 20_400);
    try {
      const result = await runScannerWithHome(home);
      const f = findBody(result.findings);
      assert.ok(f);
      const ev = String(f.evidence);
      assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens');
      assert.match(ev, /body_lines=\d+/, 'evidence states the body line count');
      assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost');
      assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing');
      assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('recommends context:fork and moving reference content into supporting files', async () => {
    const home = uniqueDir('bodyrec');
    await addUserSkillWithBody(home, 'heavy', 20_400);
    try {
      const result = await runScannerWithHome(home);
      const f = findBody(result.findings);
      assert.ok(f);
      const rec = String(f.recommendation);
      assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork');
      assert.match(rec, /supporting file/i, 'recommendation should mention supporting files');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => {
    const home = uniqueDir('bodyonly');
    await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description
    try {
      const result = await runScannerWithHome(home);
      assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap');
      assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget');
      assert.ok(findBody(result.findings), 'only the body finding should fire');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('SKL scanner — suppression compatibility', () => {
  it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => {
    const { applySuppressions } = await import('../../scanners/lib/suppression.mjs');
    const home = await homeWithUserSkill('suppress', CAP + 50);
    try {
      const result = await runScannerWithHome(home);
      assert.ok(result.findings.length > 0, 'precondition: at least one SKL finding');
      const { active, suppressed } = applySuppressions(result.findings, [
        { pattern: 'CA-TOK-*', source: 'test', sourceLine: 1 },
      ]);
      assert.equal(active.length, result.findings.length,
        'CA-TOK-* glob should not match CA-SKL-* findings');
      assert.equal(suppressed.length, 0);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });
});