config-audit/tests/scanners/settings-validator.test.mjs

import { describe, it, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/settings-validator.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURES = resolve(__dirname, '../fixtures');

describe('SET scanner — healthy project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await discoverConfigFiles(resolve(FIXTURES, 'healthy-project'));
    result = await scan(resolve(FIXTURES, 'healthy-project'), discovery);
  });

  it('returns status ok', () => {
    assert.strictEqual(result.status, 'ok');
  });

  it('has scanner prefix SET', () => {
    assert.strictEqual(result.scanner, 'SET');
  });

  it('finds no critical issues', () => {
    const critical = result.findings.filter(f => f.severity === 'critical');
    assert.strictEqual(critical.length, 0);
  });

  it('all finding IDs match CA-SET-NNN', () => {
    for (const f of result.findings) {
      assert.match(f.id, /^CA-SET-\d{3}$/);
    }
  });
});

describe('SET scanner — broken project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await discoverConfigFiles(resolve(FIXTURES, 'broken-project'));
    result = await scan(resolve(FIXTURES, 'broken-project'), discovery);
  });

  it('detects unknown settings key', () => {
    // CA-SET-001 in broken-project, evidence='unknownKey123'.
    const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || ''));
    assert.ok(found, 'Should detect unknownKey123');
  });

  it('detects deprecated key (includeCoAuthoredBy)', () => {
    // CA-SET-002 in broken-project, evidence='includeCoAuthoredBy: true'.
    const found = result.findings.some(f => f.scanner === 'SET' && /includeCoAuthoredBy/.test(f.evidence || ''));
    assert.ok(found, 'Should detect includeCoAuthoredBy');
  });

  it('detects type mismatch (alwaysThinkingEnabled as string)', () => {
    // CA-SET-003 in broken-project, evidence='alwaysThinkingEnabled: "yes" (string)'.
    const found = result.findings.some(f => f.scanner === 'SET' && /alwaysThinkingEnabled/.test(f.evidence || ''));
    assert.ok(found, 'Should detect boolean/string mismatch');
  });

  it('detects invalid effortLevel value', () => {
    // CA-SET-004 in broken-project, evidence='effortLevel: "turbo"'.
    const found = result.findings.some(f => f.scanner === 'SET' && /effortLevel:\s*"turbo"/.test(f.evidence || ''));
    assert.ok(found, 'Should detect effortLevel "turbo"');
  });

  it('detects hooks as array', () => {
    // CA-SET-006 in broken-project, evidence='"hooks": [...]'.
    const found = result.findings.some(f => f.scanner === 'SET' && /"hooks":\s*\[/.test(f.evidence || ''));
    assert.ok(found, 'Should detect hooks array format');
  });

  it('marks hooks-as-array as critical', () => {
    const f = result.findings.find(x => x.scanner === 'SET' && /"hooks":\s*\[/.test(x.evidence || ''));
    assert.strictEqual(f?.severity, 'critical');
  });
});

describe('SET scanner — additionalDirectories (v5 M6)', () => {
  it('does NOT flag additionalDirectories as unknown key', async () => {
    resetCounter();
    const path = resolve(FIXTURES, 'additional-dirs-ok');
    const discovery = await discoverConfigFiles(path);
    const result = await scan(path, discovery);
    // SET findings preserve evidence verbatim; an unknown-key finding for additionalDirectories
    // would carry "additionalDirectories" in evidence regardless of humanizer rewriting the title.
    const unknown = result.findings.find(f =>
      f.scanner === 'SET' && /additionalDirectories/.test(f.evidence || ''));
    assert.equal(unknown, undefined,
      'additionalDirectories should be in KNOWN_KEYS');
  });

  it('does NOT flag 2 entries as too many', async () => {
    resetCounter();
    const path = resolve(FIXTURES, 'additional-dirs-ok');
    const discovery = await discoverConfigFiles(path);
    const result = await scan(path, discovery);
    // The additionalDirectories threshold finding writes paths into evidence (e.g., "~/work/repo-a", ...).
    // additional-dirs-ok is below threshold, so no SET finding fires at all.
    const f = result.findings.find(x => x.scanner === 'SET');
    assert.equal(f, undefined,
      `expected no SET findings for 2 entries, got id=${f?.id}`);
  });

  it('flags > 2 entries as low finding', async () => {
    resetCounter();
    const path = resolve(FIXTURES, 'additional-dirs-many');
    const discovery = await discoverConfigFiles(path);
    const result = await scan(path, discovery);
    // CA-SET-001 in additional-dirs-many = the additionalDirectories threshold finding.
    const f = result.findings.find(x => x.scanner === 'SET' && x.id === 'CA-SET-001');
    assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.id).join(' | ')}`);
    assert.equal(f.severity, 'low', `expected low severity, got ${f.severity}`);
  });
});

describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fix)', () => {
  // The pre-write path-guard blocks the agent from committing settings.json
  // fixtures, so this suite materializes a hermetic temp fixture at runtime.
  let tmpRoot;
  let result;

  const NEW_KEYS = [
    'sandbox', 'fallbackModel', 'enforceAvailableModels', 'disableBundledSkills',
    'pluginSuggestionMarketplaces', 'requiredMinimumVersion', 'requiredMaximumVersion',
    'allowAllClaudeAiMcps', 'footerLinksRegexes', 'wheelScrollAccelerationEnabled',
    'parentSettingsBehavior', 'skillOverrides',
  ];

  beforeEach(async () => {
    resetCounter();
    tmpRoot = await mkdtemp(join(tmpdir(), 'ca-set-2181-'));
    await mkdir(join(tmpRoot, '.claude'), { recursive: true });
    const settings = {
      $schema: 'https://json.schemastore.org/claude-code-settings.json',
      effortLevel: 'xhigh',                                      // CC 2.1.154
      sandbox: { allowAppleEvents: false },                      // CC 2.1.181
      fallbackModel: ['claude-opus-4-8', 'claude-sonnet-4-6'],   // CC 2.1.166 (string|array)
      enforceAvailableModels: true,                              // CC 2.1.175
      disableBundledSkills: true,                                // CC 2.1.169
      pluginSuggestionMarketplaces: ['acme'],                    // CC 2.1.152
      requiredMinimumVersion: '2.1.114',                         // CC 2.1.163
      requiredMaximumVersion: '2.2.0',                           // CC 2.1.163
      allowAllClaudeAiMcps: false,                               // CC 2.1.149
      footerLinksRegexes: ['^https://internal\\.'],              // CC 2.1.181
      wheelScrollAccelerationEnabled: true,                      // CC 2.1.174
      parentSettingsBehavior: 'merge',                           // CC 2.1.133
      skillOverrides: { 'example-skill': 'name-only' },          // CC 2.1.129 (per-skill object)
      permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] },
    };
    await writeFile(
      join(tmpRoot, '.claude', 'settings.json'),
      JSON.stringify(settings, null, 2) + '\n',
      'utf8',
    );
    const discovery = await discoverConfigFiles(tmpRoot);
    result = await scan(tmpRoot, discovery);
  });

  afterEach(async () => {
    if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
  });

  it('accepts effortLevel "xhigh" (CC 2.1.154)', () => {
    const bad = result.findings.find(f => /xhigh/.test(f.evidence || ''));
    assert.equal(bad, undefined, 'xhigh must be a valid effortLevel');
  });

  for (const key of NEW_KEYS) {
    it(`does NOT flag "${key}" as an unknown key`, () => {
      const unknown = result.findings.find(f =>
        f.title === 'Unknown settings key' && f.evidence === key);
      assert.equal(unknown, undefined, `${key} should be in KNOWN_KEYS`);
    });
  }

  it('treats fallbackModel array value without a single-type mismatch', () => {
    const f = result.findings.find(x => /fallbackModel/.test(x.evidence || ''));
    assert.equal(f, undefined, 'fallbackModel (string|array) must not be type-checked');
  });

  it('produces zero findings for an all-valid CC 2.1.181 settings file', () => {
    assert.equal(result.findings.length, 0,
      `expected clean scan; got: ${result.findings.map(f => `${f.title}:${f.evidence || ''}`).join(' | ')}`);
  });
});

describe('SET scanner — empty project', () => {
  let result;
  beforeEach(async () => {
    resetCounter();
    const discovery = await discoverConfigFiles(resolve(FIXTURES, 'empty-project'));
    result = await scan(resolve(FIXTURES, 'empty-project'), discovery);
  });

  it('returns skipped when no settings files', () => {
    assert.strictEqual(result.status, 'skipped');
  });

  it('has 0 findings', () => {
    assert.strictEqual(result.findings.length, 0);
  });
});