config-audit/tests/scanners/feature-gap-scanner.test.mjs

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding, filterHookLeverFinding } from '../../scanners/feature-gap-scanner.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
import { withHermeticHome } from '../helpers/hermetic-home.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURES = resolve(__dirname, '../fixtures');

// Pre-discover fixture files WITHOUT includeGlobal so tests are environment-independent.
// The GAP scanner uses shared discovery when it has files, avoiding its own includeGlobal scan.
async function fixtureDiscovery(name) {
  return discoverConfigFiles(resolve(FIXTURES, name));
}

describe('GAP scanner — healthy project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('healthy-project');
    // Hermetic HOME: scan() now enumerates active skills via process.env.HOME
    // (the disableBundledSkills lever check). An empty HOME keeps these fixture
    // tests environment-independent — no skills, no budget pressure, no lever finding.
    result = await withHermeticHome(() => scan(resolve(FIXTURES, 'healthy-project'), discovery));
  });

  it('returns status ok', () => {
    assert.equal(result.status, 'ok');
  });

  it('reports scanner prefix GAP', () => {
    assert.equal(result.scanner, 'GAP');
  });

  it('scans multiple files', () => {
    assert.ok(result.files_scanned >= 1);
  });

  it('finding IDs match CA-GAP-NNN pattern', () => {
    for (const f of result.findings) {
      assert.match(f.id, /^CA-GAP-\d{3}$/);
    }
  });

  it('does NOT report missing CLAUDE.md', () => {
    assert.ok(!result.findings.some(f =>
      f.scanner === 'GAP' && f.category === 't1' && /CLAUDE\.md/.test(f.recommendation || '')
    ));
  });

  it('does NOT report missing MCP', () => {
    assert.ok(!result.findings.some(f =>
      f.scanner === 'GAP' && f.category === 't1' && /\.mcp\.json/.test(f.recommendation || '')
    ));
  });

  it('does NOT report missing hooks', () => {
    assert.ok(!result.findings.some(f =>
      f.scanner === 'GAP' && f.category === 't1' && /hook/i.test(f.recommendation || '')
    ));
  });

  it('has counts object with all severity levels', () => {
    assert.ok('critical' in result.counts);
    assert.ok('high' in result.counts);
    assert.ok('medium' in result.counts);
    assert.ok('low' in result.counts);
    assert.ok('info' in result.counts);
  });

  it('has no critical or high findings', () => {
    assert.equal(result.counts.critical, 0);
    assert.equal(result.counts.high, 0);
  });

  it('all findings have recommendations', () => {
    for (const f of result.findings) {
      assert.ok(f.recommendation, `Finding ${f.id} missing recommendation`);
    }
  });

  it('T3/T4 findings are info severity', () => {
    const infoFindings = result.findings.filter(f => f.category === 't3' || f.category === 't4');
    for (const f of infoFindings) {
      assert.equal(f.severity, 'info', `${f.id} (${f.category}) should be info, got ${f.severity}`);
    }
  });
});

describe('GAP scanner — minimal project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('minimal-project');
    result = await withHermeticHome(() => scan(resolve(FIXTURES, 'minimal-project'), discovery));
  });

  it('returns status ok', () => {
    assert.equal(result.status, 'ok');
  });

  it('reports missing hooks', () => {
    // CA-GAP-002 in minimal-project = t1_3 (No hooks configured); see docs/v5.1.0-test-audit.md.
    assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-002'));
  });

  it('reports missing MCP', () => {
    // CA-GAP-004 in minimal-project = t1_5 (No MCP servers configured).
    assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-004'));
  });

  it('T1 gaps are medium severity', () => {
    const t1 = result.findings.filter(f => f.category === 't1');
    for (const f of t1) {
      assert.equal(f.severity, 'medium', `${f.id} should be medium, got ${f.severity}`);
    }
  });

  it('T2 gaps are low severity', () => {
    const t2 = result.findings.filter(f => f.category === 't2');
    for (const f of t2) {
      assert.equal(f.severity, 'low', `${f.id} should be low, got ${f.severity}`);
    }
  });

  it('has more findings than healthy project', async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('healthy-project');
    const healthyResult = await withHermeticHome(() => scan(resolve(FIXTURES, 'healthy-project'), discovery));
    assert.ok(result.findings.length > healthyResult.findings.length);
  });
});

describe('GAP scanner — empty project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('empty-project');
    result = await withHermeticHome(() => scan(resolve(FIXTURES, 'empty-project'), discovery));
  });

  it('returns status ok (never skips)', () => {
    assert.equal(result.status, 'ok');
  });

  it('has multiple medium findings (T1 gaps)', () => {
    const mediums = result.findings.filter(f => f.severity === 'medium');
    assert.ok(mediums.length >= 1);
  });

  it('all findings have category field', () => {
    for (const f of result.findings) {
      assert.ok(f.category, `Finding ${f.id} missing category`);
      assert.match(f.category, /^t[1-4]$/);
    }
  });

  it('reports T1 gaps including missing CLAUDE.md', () => {
    // CA-GAP-001 in empty-project = t1_1 (No CLAUDE.md file).
    assert.ok(result.findings.some(f => f.scanner === 'GAP' && f.id === 'CA-GAP-001'));
  });
});

describe('opportunitySummary', () => {
  it('returns empty arrays for no findings', () => {
    const result = opportunitySummary([]);
    assert.deepEqual(result.highImpact, []);
    assert.deepEqual(result.mediumImpact, []);
    assert.deepEqual(result.explore, []);
  });

  it('routes T1 to highImpact', () => {
    const findings = [{ category: 't1', title: 'No CLAUDE.md' }];
    const result = opportunitySummary(findings);
    assert.equal(result.highImpact.length, 1);
    assert.equal(result.mediumImpact.length, 0);
    assert.equal(result.explore.length, 0);
  });

  it('routes T2 to mediumImpact', () => {
    const findings = [{ category: 't2', title: 'Low hook diversity' }];
    const result = opportunitySummary(findings);
    assert.equal(result.highImpact.length, 0);
    assert.equal(result.mediumImpact.length, 1);
  });

  it('routes T3 and T4 to explore', () => {
    const findings = [
      { category: 't3', title: 'No status line' },
      { category: 't4', title: 'No custom plugin' },
    ];
    const result = opportunitySummary(findings);
    assert.equal(result.explore.length, 2);
  });

  it('handles mixed tiers', () => {
    const findings = [
      { category: 't1', title: 'A' },
      { category: 't2', title: 'B' },
      { category: 't2', title: 'C' },
      { category: 't3', title: 'D' },
      { category: 't4', title: 'E' },
    ];
    const result = opportunitySummary(findings);
    assert.equal(result.highImpact.length, 1);
    assert.equal(result.mediumImpact.length, 2);
    assert.equal(result.explore.length, 2);
  });
});

// ── disableBundledSkills lever (CA-GAP) — remediation companion to SKL CA-SKL-002 ──

describe('bundledSkillsLeverFinding — pure decision', () => {
  const overBudget = { scanned: 17, aggregateChars: 17000, aggregateTokens: 4250, budgetTokens: 4000, overBudget: true, overBy: 250 };
  const underBudget = { scanned: 3, aggregateChars: 3000, aggregateTokens: 750, budgetTokens: 4000, overBudget: false, overBy: 0 };

  it('returns null when the lever is already pulled, even over budget', () => {
    assert.equal(bundledSkillsLeverFinding({ leverPulled: true, aggregate: overBudget }), null);
  });

  it('returns null when the listing is under budget', () => {
    assert.equal(bundledSkillsLeverFinding({ leverPulled: false, aggregate: underBudget }), null);
  });

  it('returns a finding when the lever is un-pulled AND the listing is over budget', () => {
    resetCounter();
    const f = bundledSkillsLeverFinding({ leverPulled: false, aggregate: overBudget });
    assert.ok(f, 'expected a finding');
    assert.match(f.id, /^CA-GAP-\d{3}$/);
    assert.equal(f.scanner, 'GAP');
    assert.equal(f.severity, 'low');
    assert.equal(f.category, 'token-efficiency');
    assert.match(f.recommendation, /disableBundledSkills/);
    assert.match(`${f.description} ${f.recommendation}`, /2\.1\.169/);
  });

  it('handles a null/garbage aggregate without throwing', () => {
    assert.equal(bundledSkillsLeverFinding({ leverPulled: false, aggregate: null }), null);
  });
});

describe('GAP scanner — disableBundledSkills lever wiring (HOME-scoped)', () => {
  async function buildHome(count, descLen, settings) {
    const home = await mkdtemp(join(tmpdir(), 'config-audit-gap-home-'));
    for (let i = 0; i < count; i++) {
      const dir = join(home, '.claude', 'skills', `s${i}`);
      await mkdir(dir, { recursive: true });
      await writeFile(join(dir, 'SKILL.md'), `---\nname: s${i}\ndescription: ${'a'.repeat(descLen)}\n---\nBody.\n`);
    }
    if (settings) {
      await mkdir(join(home, '.claude'), { recursive: true });
      await writeFile(join(home, '.claude', 'settings.json'), JSON.stringify(settings));
    }
    return home;
  }

  /** Run the GAP scanner with HOME pointed at `home`; an empty throwaway project. */
  async function runGapWithHome(home) {
    const project = await mkdtemp(join(tmpdir(), 'config-audit-gap-proj-'));
    const original = process.env.HOME;
    process.env.HOME = home;
    try {
      resetCounter();
      const discovery = await discoverConfigFiles(project, { includeGlobal: true });
      const result = await scan(project, discovery);
      return result;
    } finally {
      process.env.HOME = original;
      await rm(project, { recursive: true, force: true });
    }
  }

  const hasLever = (result) =>
    result.findings.some(f => f.scanner === 'GAP' && /disableBundledSkills/.test(f.recommendation || ''));

  it('fires when the listing is over budget and the lever is un-pulled', async () => {
    const home = await buildHome(17, 1000); // 17000 chars -> 4250 tok > 4000 budget
    try {
      const result = await runGapWithHome(home);
      const f = result.findings.find(x => x.scanner === 'GAP' && /disableBundledSkills/.test(x.recommendation || ''));
      assert.ok(f, `expected a disableBundledSkills finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
      assert.equal(f.severity, 'low');
      assert.equal(f.category, 'token-efficiency');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT fire when the listing is under budget', async () => {
    const home = await buildHome(3, 1000); // 3000 chars -> 750 tok < 4000
    try {
      const result = await runGapWithHome(home);
      assert.equal(hasLever(result), false);
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT fire when disableBundledSkills is already true in settings', async () => {
    const home = await buildHome(17, 1000, { disableBundledSkills: true });
    try {
      const result = await runGapWithHome(home);
      assert.equal(hasLever(result), false,
        'lever already pulled in settings — should not recommend it again');
    } finally {
      await rm(home, { recursive: true, force: true });
    }
  });

  it('does NOT fire when the CLAUDE_CODE_DISABLE_BUNDLED_SKILLS env var is set', async () => {
    const home = await buildHome(17, 1000);
    const originalEnv = process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS;
    process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS = '1';
    try {
      const result = await runGapWithHome(home);
      assert.equal(hasLever(result), false, 'env lever should suppress the recommendation');
    } finally {
      if (originalEnv === undefined) delete process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS;
      else process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS = originalEnv;
      await rm(home, { recursive: true, force: true });
    }
  });

  it('treats CLAUDE_CODE_DISABLE_BUNDLED_SKILLS=0 as un-pulled (still fires)', async () => {
    const home = await buildHome(17, 1000);
    const originalEnv = process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS;
    process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS = '0';
    try {
      const result = await runGapWithHome(home);
      assert.equal(hasLever(result), true, '"0" means off — the lever is NOT pulled, so the finding should fire');
    } finally {
      if (originalEnv === undefined) delete process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS;
      else process.env.CLAUDE_CODE_DISABLE_BUNDLED_SKILLS = originalEnv;
      await rm(home, { recursive: true, force: true });
    }
  });
});

describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => {
  it('returns null when nothing is forced upfront', () => {
    assert.equal(cliOverMcpLeverFinding({ assessment: { forcedUpfront: false } }), null);
    assert.equal(cliOverMcpLeverFinding({ assessment: null }), null);
    assert.equal(cliOverMcpLeverFinding({}), null);
  });

  it('fires a low-severity opportunity when MCP schemas are forced upfront', () => {
    resetCounter();
    const f = cliOverMcpLeverFinding({
      assessment: {
        forcedUpfront: true,
        aggregateTokens: 2500,
        affectedServers: [{ name: 'srv-a' }],
        reason: 'enable-tool-search-false',
      },
    });
    assert.ok(f, 'expected a lever finding');
    assert.equal(f.severity, 'low');
    assert.equal(f.category, 'token-efficiency');
    assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/);
  });
});

describe('filterHookLeverFinding (filter-before-Claude-reads lever, v5.10 B5)', () => {
  it('returns null when no chatty hook was detected', () => {
    assert.equal(filterHookLeverFinding({ flaggedHooks: [] }), null);
    assert.equal(filterHookLeverFinding({}), null);
    assert.equal(filterHookLeverFinding(), null);
  });

  it('fires an info opportunity when ≥1 chatty hook is detected', () => {
    resetCounter();
    const f = filterHookLeverFinding({
      flaggedHooks: [{ event: 'SessionStart', scriptPath: '/x/hooks/scripts/chatty.sh' }],
    });
    assert.ok(f, 'expected a lever finding');
    assert.equal(f.severity, 'info');
    assert.equal(f.category, 'token-efficiency');
    assert.match(f.description || '', /filter-test-output\.sh|filter/i);
    assert.match(f.evidence || '', /chatty_hooks=1/);
  });
});

describe('GAP scanner — filter-before lever wiring (chatty hook fixture)', () => {
  it('emits the filter-before lever when scanning a repo with a chatty hook', async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('hooks-additional-context');
    const result = await withHermeticHome(() =>
      scan(resolve(FIXTURES, 'hooks-additional-context'), discovery),
    );
    const lever = result.findings.find(
      f => f.scanner === 'GAP' && /chatty_hooks=/.test(f.evidence || ''),
    );
    assert.ok(lever, `expected filter-before lever; got: ${result.findings.map(x => x.title).join(' | ')}`);
    assert.equal(lever.severity, 'info');
  });

  it('does NOT emit the lever for a repo with only quiet hooks', async () => {
    resetCounter();
    const discovery = await fixtureDiscovery('hooks-quiet');
    const result = await withHermeticHome(() =>
      scan(resolve(FIXTURES, 'hooks-quiet'), discovery),
    );
    const lever = result.findings.find(
      f => f.scanner === 'GAP' && /chatty_hooks=/.test(f.evidence || ''),
    );
    assert.equal(lever, undefined, `expected no filter-before lever; got id=${lever?.id}`);
  });
});

describe('GAP scanner — test/demo data must not mask real gaps (M-BUG-13)', () => {
  // Build a throwaway project (+ optional hermetic HOME settings) and run GAP
  // exactly as posture --global does: includeGlobal discovery + scan().
  async function runGap({ projectFiles = {}, homeSettings = null } = {}) {
    const project = await mkdtemp(join(tmpdir(), 'config-audit-gap-mask-proj-'));
    const home = await mkdtemp(join(tmpdir(), 'config-audit-gap-mask-home-'));
    for (const [rel, content] of Object.entries(projectFiles)) {
      const abs = join(project, rel);
      await mkdir(dirname(abs), { recursive: true });
      await writeFile(abs, content);
    }
    if (homeSettings) {
      await mkdir(join(home, '.claude'), { recursive: true });
      await writeFile(join(home, '.claude', 'settings.json'), JSON.stringify(homeSettings));
    }
    const original = process.env.HOME;
    process.env.HOME = home;
    try {
      resetCounter();
      const discovery = await discoverConfigFiles(project, { includeGlobal: true });
      const result = await scan(project, discovery);
      return result;
    } finally {
      process.env.HOME = original;
      await rm(project, { recursive: true, force: true });
      await rm(home, { recursive: true, force: true });
    }
  }

  const hasGap = (result, titleRe) =>
    result.findings.some(f => f.scanner === 'GAP' && titleRe.test(f.title || ''));

  it('a nested examples/ settings.json does NOT satisfy the outputStyle check (settings-key)', async () => {
    const result = await runGap({
      projectFiles: {
        '.claude/CLAUDE.md': '# proj',
        'examples/optimal-setup/.claude/settings.json': JSON.stringify({ outputStyle: 'Explanatory' }),
      },
    });
    assert.ok(
      hasGap(result, /output style/i),
      `example settings.json must not mask the outputStyle gap; got: ${result.findings.map(f => f.title).join(' | ')}`,
    );
  });

  it('a nested examples/ keybindings.json does NOT satisfy the keybindings check (file-type)', async () => {
    const result = await runGap({
      projectFiles: {
        '.claude/CLAUDE.md': '# proj',
        'examples/optimal-setup/.claude/keybindings.json': JSON.stringify({}),
      },
    });
    assert.ok(
      hasGap(result, /keybinding/i),
      `example keybindings.json must not mask the keybindings gap; got: ${result.findings.map(f => f.title).join(' | ')}`,
    );
  });

  it('a nested tests/fixtures/ settings.json does NOT satisfy the model check', async () => {
    const result = await runGap({
      projectFiles: {
        '.claude/CLAUDE.md': '# proj',
        'tests/fixtures/demo/.claude/settings.json': JSON.stringify({ model: 'opus' }),
      },
    });
    assert.ok(
      hasGap(result, /model config/i),
      `fixture settings.json must not mask the model gap; got: ${result.findings.map(f => f.title).join(' | ')}`,
    );
  });

  it("the project's OWN real .claude/settings.json IS counted (not over-excluded)", async () => {
    const result = await runGap({
      projectFiles: {
        '.claude/CLAUDE.md': '# proj',
        '.claude/settings.json': JSON.stringify({ outputStyle: 'Explanatory' }),
      },
    });
    assert.ok(
      !hasGap(result, /output style/i),
      `real project settings.json must satisfy outputStyle; got: ${result.findings.map(f => f.title).join(' | ')}`,
    );
  });

  it('the real ~/.claude/settings.json IS seen for settings-key checks (gotcha fix end-to-end)', async () => {
    const result = await runGap({
      homeSettings: { statusLine: { type: 'command', command: 'x' } },
    });
    assert.ok(
      !hasGap(result, /status line/i),
      `~/.claude/settings.json statusLine must be discovered; got: ${result.findings.map(f => f.title).join(' | ')}`,
    );
  });
});