Kjell Tore Guttormsen
3cf5c714a2
The CC settings schema is passthrough (verified against the 2.1.193 binary): it forwards unrecognized keys unchanged rather than rejecting them, so an arbitrary unknown key is valid/forward-compatible, not an error — the finding's "silently ignored" claim was factually wrong. The only real risk is a TYPO of a real key (the intended setting then silently has no effect). Fix: flag an unknown key only when it closely matches a known key (new levenshtein helper; edit distance <= 2, both keys >= 4 chars); severity medium -> low; honest passthrough framing in the scanner + humanizer. Also refreshed KNOWN_KEYS with 6 binary-verified keys (agentPushNotifEnabled, remoteControlAtStartup, skipAutoPermissionPrompt, skipDangerousModePermissionPrompt, skipWorkflowUsageWarning, tui). Suite 1341/0 (+12). Frozen v5.0.0 snapshots untouched (0 CA-SET findings there), no re-seed. Dogfood ~/.claude/settings.json 6->0 (all 6 keys above were false unknown-key findings; 0 typo flags introduced across 167 walked files).
345 lines
16 KiB
JavaScript
345 lines
16 KiB
JavaScript
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { resetCounter } from '../../scanners/lib/output.mjs';
|
|
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
|
|
import { scan } from '../../scanners/settings-validator.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const FIXTURES = resolve(__dirname, '../fixtures');
|
|
|
|
describe('SET scanner — healthy project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'healthy-project'));
|
|
result = await scan(resolve(FIXTURES, 'healthy-project'), discovery);
|
|
});
|
|
|
|
it('returns status ok', () => {
|
|
assert.strictEqual(result.status, 'ok');
|
|
});
|
|
|
|
it('has scanner prefix SET', () => {
|
|
assert.strictEqual(result.scanner, 'SET');
|
|
});
|
|
|
|
it('finds no critical issues', () => {
|
|
const critical = result.findings.filter(f => f.severity === 'critical');
|
|
assert.strictEqual(critical.length, 0);
|
|
});
|
|
|
|
it('all finding IDs match CA-SET-NNN', () => {
|
|
for (const f of result.findings) {
|
|
assert.match(f.id, /^CA-SET-\d{3}$/);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — broken project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'broken-project'));
|
|
result = await scan(resolve(FIXTURES, 'broken-project'), discovery);
|
|
});
|
|
|
|
it('does NOT flag an arbitrary unknown key (CC schema is passthrough)', () => {
|
|
// M-BUG-10: "unknownKey123" is not close to any known key, so it is a
|
|
// valid forward-compatible / passthrough key — NOT a typo. Claude Code
|
|
// forwards unrecognized keys unchanged, so flagging it is a false positive.
|
|
const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || ''));
|
|
assert.ok(!found, 'unknownKey123 is far from every known key → must not be flagged');
|
|
});
|
|
|
|
it('detects deprecated key (includeCoAuthoredBy)', () => {
|
|
// CA-SET-002 in broken-project, evidence='includeCoAuthoredBy: true'.
|
|
const found = result.findings.some(f => f.scanner === 'SET' && /includeCoAuthoredBy/.test(f.evidence || ''));
|
|
assert.ok(found, 'Should detect includeCoAuthoredBy');
|
|
});
|
|
|
|
it('detects type mismatch (alwaysThinkingEnabled as string)', () => {
|
|
// CA-SET-003 in broken-project, evidence='alwaysThinkingEnabled: "yes" (string)'.
|
|
const found = result.findings.some(f => f.scanner === 'SET' && /alwaysThinkingEnabled/.test(f.evidence || ''));
|
|
assert.ok(found, 'Should detect boolean/string mismatch');
|
|
});
|
|
|
|
it('detects invalid effortLevel value', () => {
|
|
// CA-SET-004 in broken-project, evidence='effortLevel: "turbo"'.
|
|
const found = result.findings.some(f => f.scanner === 'SET' && /effortLevel:\s*"turbo"/.test(f.evidence || ''));
|
|
assert.ok(found, 'Should detect effortLevel "turbo"');
|
|
});
|
|
|
|
it('detects hooks as array', () => {
|
|
// CA-SET-006 in broken-project, evidence='"hooks": [...]'.
|
|
const found = result.findings.some(f => f.scanner === 'SET' && /"hooks":\s*\[/.test(f.evidence || ''));
|
|
assert.ok(found, 'Should detect hooks array format');
|
|
});
|
|
|
|
it('marks hooks-as-array as critical', () => {
|
|
const f = result.findings.find(x => x.scanner === 'SET' && /"hooks":\s*\[/.test(x.evidence || ''));
|
|
assert.strictEqual(f?.severity, 'critical');
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — additionalDirectories (v5 M6)', () => {
|
|
it('does NOT flag additionalDirectories as unknown key', async () => {
|
|
resetCounter();
|
|
const path = resolve(FIXTURES, 'additional-dirs-ok');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
// SET findings preserve evidence verbatim; an unknown-key finding for additionalDirectories
|
|
// would carry "additionalDirectories" in evidence regardless of humanizer rewriting the title.
|
|
const unknown = result.findings.find(f =>
|
|
f.scanner === 'SET' && /additionalDirectories/.test(f.evidence || ''));
|
|
assert.equal(unknown, undefined,
|
|
'additionalDirectories should be in KNOWN_KEYS');
|
|
});
|
|
|
|
it('does NOT flag 2 entries as too many', async () => {
|
|
resetCounter();
|
|
const path = resolve(FIXTURES, 'additional-dirs-ok');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
// The additionalDirectories threshold finding writes paths into evidence (e.g., "~/work/repo-a", ...).
|
|
// additional-dirs-ok is below threshold, so no SET finding fires at all.
|
|
const f = result.findings.find(x => x.scanner === 'SET');
|
|
assert.equal(f, undefined,
|
|
`expected no SET findings for 2 entries, got id=${f?.id}`);
|
|
});
|
|
|
|
it('flags > 2 entries as low finding', async () => {
|
|
resetCounter();
|
|
const path = resolve(FIXTURES, 'additional-dirs-many');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
// CA-SET-001 in additional-dirs-many = the additionalDirectories threshold finding.
|
|
const f = result.findings.find(x => x.scanner === 'SET' && x.id === 'CA-SET-001');
|
|
assert.ok(f, `expected additionalDirectories threshold finding; got: ${result.findings.map(x => x.id).join(' | ')}`);
|
|
assert.equal(f.severity, 'low', `expected low severity, got ${f.severity}`);
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fix)', () => {
|
|
// The pre-write path-guard blocks the agent from committing settings.json
|
|
// fixtures, so this suite materializes a hermetic temp fixture at runtime.
|
|
let tmpRoot;
|
|
let result;
|
|
|
|
const NEW_KEYS = [
|
|
'sandbox', 'fallbackModel', 'enforceAvailableModels', 'disableBundledSkills',
|
|
'pluginSuggestionMarketplaces', 'requiredMinimumVersion', 'requiredMaximumVersion',
|
|
'allowAllClaudeAiMcps', 'footerLinksRegexes', 'wheelScrollAccelerationEnabled',
|
|
'parentSettingsBehavior', 'skillOverrides',
|
|
];
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-set-2181-'));
|
|
await mkdir(join(tmpRoot, '.claude'), { recursive: true });
|
|
const settings = {
|
|
$schema: 'https://json.schemastore.org/claude-code-settings.json',
|
|
effortLevel: 'xhigh', // CC 2.1.154
|
|
sandbox: { allowAppleEvents: false }, // CC 2.1.181
|
|
fallbackModel: ['claude-opus-4-8', 'claude-sonnet-4-6'], // CC 2.1.166 (string|array)
|
|
enforceAvailableModels: true, // CC 2.1.175
|
|
disableBundledSkills: true, // CC 2.1.169
|
|
pluginSuggestionMarketplaces: ['acme'], // CC 2.1.152
|
|
requiredMinimumVersion: '2.1.114', // CC 2.1.163
|
|
requiredMaximumVersion: '2.2.0', // CC 2.1.163
|
|
allowAllClaudeAiMcps: false, // CC 2.1.149
|
|
footerLinksRegexes: ['^https://internal\\.'], // CC 2.1.181
|
|
wheelScrollAccelerationEnabled: true, // CC 2.1.174
|
|
parentSettingsBehavior: 'merge', // CC 2.1.133
|
|
skillOverrides: { 'example-skill': 'name-only' }, // CC 2.1.129 (per-skill object)
|
|
permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] },
|
|
};
|
|
await writeFile(
|
|
join(tmpRoot, '.claude', 'settings.json'),
|
|
JSON.stringify(settings, null, 2) + '\n',
|
|
'utf8',
|
|
);
|
|
const discovery = await discoverConfigFiles(tmpRoot);
|
|
result = await scan(tmpRoot, discovery);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
it('accepts effortLevel "xhigh" (CC 2.1.154)', () => {
|
|
const bad = result.findings.find(f => /xhigh/.test(f.evidence || ''));
|
|
assert.equal(bad, undefined, 'xhigh must be a valid effortLevel');
|
|
});
|
|
|
|
for (const key of NEW_KEYS) {
|
|
it(`does NOT flag "${key}" as an unknown key`, () => {
|
|
const unknown = result.findings.find(f =>
|
|
f.title === 'Unknown settings key' && f.evidence === key);
|
|
assert.equal(unknown, undefined, `${key} should be in KNOWN_KEYS`);
|
|
});
|
|
}
|
|
|
|
it('treats fallbackModel array value without a single-type mismatch', () => {
|
|
const f = result.findings.find(x => /fallbackModel/.test(x.evidence || ''));
|
|
assert.equal(f, undefined, 'fallbackModel (string|array) must not be type-checked');
|
|
});
|
|
|
|
it('produces zero findings for an all-valid CC 2.1.181 settings file', () => {
|
|
assert.equal(result.findings.length, 0,
|
|
`expected clean scan; got: ${result.findings.map(f => `${f.title}:${f.evidence || ''}`).join(' | ')}`);
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — unknown-key typo gate (M-BUG-10)', () => {
|
|
// The CC settings schema is passthrough: it forwards unrecognized keys
|
|
// unchanged rather than rejecting them, so an arbitrary unknown key is
|
|
// valid/forward-compatible and must NOT be flagged. The only real risk is a
|
|
// TYPO of a real key (the intended setting silently has no effect), so the
|
|
// scanner flags ONLY unknown keys that closely match a known key.
|
|
// Path-guard blocks committing settings.json fixtures → materialize a temp one.
|
|
let tmpRoot;
|
|
let result;
|
|
const TYPO_TITLE = 'Possible typo in settings key';
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-set-typo-'));
|
|
await mkdir(join(tmpRoot, '.claude'), { recursive: true });
|
|
const settings = {
|
|
$schema: 'https://json.schemastore.org/claude-code-settings.json',
|
|
permisions: { deny: ['Read(./.env)'] }, // TYPO of "permissions" (edit distance 1)
|
|
someNewKey2027: true, // forward-compat / passthrough — far from every known key
|
|
permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] },
|
|
};
|
|
await writeFile(
|
|
join(tmpRoot, '.claude', 'settings.json'),
|
|
JSON.stringify(settings, null, 2) + '\n',
|
|
'utf8',
|
|
);
|
|
const discovery = await discoverConfigFiles(tmpRoot);
|
|
result = await scan(tmpRoot, discovery);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
it('flags a likely typo ("permisions") and suggests the nearest known key', () => {
|
|
const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions');
|
|
assert.ok(typo, `expected a typo finding for "permisions"; got: ${result.findings.map(f => `${f.title}:${f.evidence || ''}`).join(' | ')}`);
|
|
assert.equal(typo.title, TYPO_TITLE);
|
|
assert.match(typo.recommendation, /permissions/, 'recommendation suggests the nearest known key');
|
|
});
|
|
|
|
it('marks the typo finding as low severity (passthrough → not a hard error)', () => {
|
|
const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions');
|
|
assert.equal(typo.severity, 'low');
|
|
});
|
|
|
|
it('does NOT flag a forward-compatible key far from any known key', () => {
|
|
const fwd = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'someNewKey2027');
|
|
assert.equal(fwd, undefined, 'someNewKey2027 is far from every known key → passthrough, not flagged');
|
|
});
|
|
|
|
it('emits exactly one typo finding for this file', () => {
|
|
const typos = result.findings.filter(f => f.scanner === 'SET' && f.title === TYPO_TITLE);
|
|
assert.equal(typos.length, 1, `expected exactly one typo finding; got: ${typos.map(f => f.evidence).join(', ')}`);
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — empty project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'empty-project'));
|
|
result = await scan(resolve(FIXTURES, 'empty-project'), discovery);
|
|
});
|
|
|
|
it('returns skipped when no settings files', () => {
|
|
assert.strictEqual(result.status, 'skipped');
|
|
});
|
|
|
|
it('has 0 findings', () => {
|
|
assert.strictEqual(result.findings.length, 0);
|
|
});
|
|
});
|
|
|
|
describe('SET scanner — autoMode validation (CA-SET-NNN)', () => {
|
|
// All autoMode findings have a title starting with "autoMode".
|
|
const AM_RE = /^autoMode/;
|
|
const amFindings = (result) => result.findings.filter(f => f.scanner === 'SET' && AM_RE.test(f.title || ''));
|
|
|
|
it('flags structural problems: wrong sub-key type, non-string array entry, unknown sub-key', async () => {
|
|
resetCounter();
|
|
// automode-structure/.claude/settings.local.json (scope=local → no dead-config):
|
|
// environment valid; allow not-array; soft_deny ["ok",7]; bogus unknown sub-key.
|
|
const path = resolve(FIXTURES, 'automode-structure');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
const am = amFindings(result);
|
|
const problems = am.map(f => f.details && f.details.problem).sort();
|
|
assert.deepEqual(
|
|
problems,
|
|
['not-string-array', 'not-string-array', 'unknown-subkey'],
|
|
`Expected 2 not-string-array + 1 unknown-subkey, got ${am.map(f => f.title).join(' | ')}`
|
|
);
|
|
for (const f of am) {
|
|
assert.equal(f.severity, 'medium', `structure problem is medium: ${f.title}`);
|
|
assert.ok(/settings\.local\.json$/.test(f.file || ''), `points at the settings file: ${f.file}`);
|
|
assert.ok(f.details && f.details.key === 'autoMode', 'details.key === autoMode');
|
|
}
|
|
});
|
|
|
|
it('does NOT flag a valid string-array sub-key (incl. "$defaults")', async () => {
|
|
resetCounter();
|
|
const path = resolve(FIXTURES, 'automode-structure');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
// environment: ["$defaults"] is valid → must not appear.
|
|
const envFlagged = amFindings(result).some(f => /environment/.test(f.title || ''));
|
|
assert.equal(envFlagged, false, 'a valid string array (with $defaults) is not flagged');
|
|
});
|
|
|
|
it('flags autoMode that is not an object', async () => {
|
|
resetCounter();
|
|
const path = resolve(FIXTURES, 'automode-nonobject');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
const am = amFindings(result);
|
|
assert.equal(am.length, 1, `Expected one not-an-object finding, got ${am.map(f => f.title).join(' | ')}`);
|
|
assert.equal(am[0].details.problem, 'not-an-object');
|
|
assert.equal(am[0].severity, 'medium');
|
|
});
|
|
|
|
it('flags autoMode in shared project settings as dead config (and not in local scope)', async () => {
|
|
resetCounter();
|
|
// automode-shared/.claude/settings.json → scope=project → dead-config.
|
|
const sharedPath = resolve(FIXTURES, 'automode-shared');
|
|
const sharedResult = await scan(sharedPath, await discoverConfigFiles(sharedPath));
|
|
const dead = amFindings(sharedResult).filter(f => f.details && f.details.problem === 'shared-project-scope');
|
|
assert.equal(dead.length, 1, 'autoMode in shared .claude/settings.json is dead config');
|
|
assert.equal(dead[0].severity, 'low', 'dead-config is low severity');
|
|
// valid structure → no structure findings, only the dead-config one.
|
|
assert.equal(amFindings(sharedResult).length, 1, 'valid structure in shared scope yields only the dead-config finding');
|
|
|
|
// The same-shaped autoMode in LOCAL scope (automode-structure) must NOT be dead config.
|
|
resetCounter();
|
|
const localPath = resolve(FIXTURES, 'automode-structure');
|
|
const localResult = await scan(localPath, await discoverConfigFiles(localPath));
|
|
const localDead = amFindings(localResult).filter(f => f.details && f.details.problem === 'shared-project-scope');
|
|
assert.equal(localDead.length, 0, 'autoMode in settings.local.json is read by CC → not dead config');
|
|
});
|
|
|
|
it('does NOT flag a settings file without autoMode', async () => {
|
|
resetCounter();
|
|
// healthy-project has .claude/settings.json but no autoMode.
|
|
const path = resolve(FIXTURES, 'healthy-project');
|
|
const discovery = await discoverConfigFiles(path);
|
|
const result = await scan(path, discovery);
|
|
assert.equal(amFindings(result).length, 0, 'no autoMode key → no autoMode findings');
|
|
});
|
|
});
|