fix(acr): SET typo-gates unknown-key false positives (M-BUG-10)
The CC settings schema is passthrough (verified against the 2.1.193 binary): it forwards unrecognized keys unchanged rather than rejecting them, so an arbitrary unknown key is valid/forward-compatible, not an error — the finding's "silently ignored" claim was factually wrong. The only real risk is a TYPO of a real key (the intended setting then silently has no effect). Fix: flag an unknown key only when it closely matches a known key (new levenshtein helper; edit distance <= 2, both keys >= 4 chars); severity medium -> low; honest passthrough framing in the scanner + humanizer. Also refreshed KNOWN_KEYS with 6 binary-verified keys (agentPushNotifEnabled, remoteControlAtStartup, skipAutoPermissionPrompt, skipDangerousModePermissionPrompt, skipWorkflowUsageWarning, tui). Suite 1341/0 (+12). Frozen v5.0.0 snapshots untouched (0 CA-SET findings there), no re-seed. Dogfood ~/.claude/settings.json 6->0 (all 6 keys above were false unknown-key findings; 0 typo flags introduced across 167 walked files).
This commit is contained in:
parent
7e94910566
commit
3cf5c714a2
5 changed files with 174 additions and 16 deletions
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { lineCount, truncate, isSimilar, extractKeys, normalizePath } from '../../scanners/lib/string-utils.mjs';
|
||||
import { lineCount, truncate, isSimilar, extractKeys, normalizePath, levenshtein } from '../../scanners/lib/string-utils.mjs';
|
||||
|
||||
describe('lineCount', () => {
|
||||
it('counts lines correctly', () => {
|
||||
|
|
@ -90,6 +90,42 @@ describe('extractKeys', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('levenshtein', () => {
|
||||
it('returns 0 for identical strings', () => {
|
||||
assert.strictEqual(levenshtein('permissions', 'permissions'), 0);
|
||||
});
|
||||
|
||||
it('counts a single substitution as 1', () => {
|
||||
assert.strictEqual(levenshtein('model', 'modet'), 1);
|
||||
});
|
||||
|
||||
it('counts a single deletion as 1 (permisions → permissions)', () => {
|
||||
assert.strictEqual(levenshtein('permisions', 'permissions'), 1);
|
||||
});
|
||||
|
||||
it('counts a single insertion as 1', () => {
|
||||
assert.strictEqual(levenshtein('hooks', 'hooks2'), 1);
|
||||
});
|
||||
|
||||
it('counts a transposition as 2 (standard Levenshtein)', () => {
|
||||
assert.strictEqual(levenshtein('import', 'improt'), 2);
|
||||
});
|
||||
|
||||
it('returns the other length when one string is empty', () => {
|
||||
assert.strictEqual(levenshtein('', 'abc'), 3);
|
||||
assert.strictEqual(levenshtein('abc', ''), 3);
|
||||
assert.strictEqual(levenshtein('', ''), 0);
|
||||
});
|
||||
|
||||
it('is symmetric', () => {
|
||||
assert.strictEqual(levenshtein('effortLevel', 'efortLevel'), levenshtein('efortLevel', 'effortLevel'));
|
||||
});
|
||||
|
||||
it('gives a large distance for unrelated keys', () => {
|
||||
assert.ok(levenshtein('unknownKey123', 'permissions') > 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizePath', () => {
|
||||
it('expands ~ to HOME', () => {
|
||||
const home = process.env.HOME;
|
||||
|
|
|
|||
|
|
@ -47,10 +47,12 @@ describe('SET scanner — broken project', () => {
|
|||
result = await scan(resolve(FIXTURES, 'broken-project'), discovery);
|
||||
});
|
||||
|
||||
it('detects unknown settings key', () => {
|
||||
// CA-SET-001 in broken-project, evidence='unknownKey123'.
|
||||
it('does NOT flag an arbitrary unknown key (CC schema is passthrough)', () => {
|
||||
// M-BUG-10: "unknownKey123" is not close to any known key, so it is a
|
||||
// valid forward-compatible / passthrough key — NOT a typo. Claude Code
|
||||
// forwards unrecognized keys unchanged, so flagging it is a false positive.
|
||||
const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || ''));
|
||||
assert.ok(found, 'Should detect unknownKey123');
|
||||
assert.ok(!found, 'unknownKey123 is far from every known key → must not be flagged');
|
||||
});
|
||||
|
||||
it('detects deprecated key (includeCoAuthoredBy)', () => {
|
||||
|
|
@ -192,6 +194,63 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi
|
|||
});
|
||||
});
|
||||
|
||||
describe('SET scanner — unknown-key typo gate (M-BUG-10)', () => {
|
||||
// The CC settings schema is passthrough: it forwards unrecognized keys
|
||||
// unchanged rather than rejecting them, so an arbitrary unknown key is
|
||||
// valid/forward-compatible and must NOT be flagged. The only real risk is a
|
||||
// TYPO of a real key (the intended setting silently has no effect), so the
|
||||
// scanner flags ONLY unknown keys that closely match a known key.
|
||||
// Path-guard blocks committing settings.json fixtures → materialize a temp one.
|
||||
let tmpRoot;
|
||||
let result;
|
||||
const TYPO_TITLE = 'Possible typo in settings key';
|
||||
|
||||
beforeEach(async () => {
|
||||
resetCounter();
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-set-typo-'));
|
||||
await mkdir(join(tmpRoot, '.claude'), { recursive: true });
|
||||
const settings = {
|
||||
$schema: 'https://json.schemastore.org/claude-code-settings.json',
|
||||
permisions: { deny: ['Read(./.env)'] }, // TYPO of "permissions" (edit distance 1)
|
||||
someNewKey2027: true, // forward-compat / passthrough — far from every known key
|
||||
permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] },
|
||||
};
|
||||
await writeFile(
|
||||
join(tmpRoot, '.claude', 'settings.json'),
|
||||
JSON.stringify(settings, null, 2) + '\n',
|
||||
'utf8',
|
||||
);
|
||||
const discovery = await discoverConfigFiles(tmpRoot);
|
||||
result = await scan(tmpRoot, discovery);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('flags a likely typo ("permisions") and suggests the nearest known key', () => {
|
||||
const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions');
|
||||
assert.ok(typo, `expected a typo finding for "permisions"; got: ${result.findings.map(f => `${f.title}:${f.evidence || ''}`).join(' | ')}`);
|
||||
assert.equal(typo.title, TYPO_TITLE);
|
||||
assert.match(typo.recommendation, /permissions/, 'recommendation suggests the nearest known key');
|
||||
});
|
||||
|
||||
it('marks the typo finding as low severity (passthrough → not a hard error)', () => {
|
||||
const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions');
|
||||
assert.equal(typo.severity, 'low');
|
||||
});
|
||||
|
||||
it('does NOT flag a forward-compatible key far from any known key', () => {
|
||||
const fwd = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'someNewKey2027');
|
||||
assert.equal(fwd, undefined, 'someNewKey2027 is far from every known key → passthrough, not flagged');
|
||||
});
|
||||
|
||||
it('emits exactly one typo finding for this file', () => {
|
||||
const typos = result.findings.filter(f => f.scanner === 'SET' && f.title === TYPO_TITLE);
|
||||
assert.equal(typos.length, 1, `expected exactly one typo finding; got: ${typos.map(f => f.evidence).join(', ')}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SET scanner — empty project', () => {
|
||||
let result;
|
||||
beforeEach(async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue