Kjell Tore Guttormsen
f3aadb5183
Per the official "what survives compaction" model (context-window.md): only the project-root CLAUDE.md (+ unscoped rules) is re-injected after a context compaction. Two additive, structural findings (low severity): - RUL: a large (>50-line) PATH-SCOPED rule — reloads only on a matching file read, and is not re-injected after compaction, so must-hold rules can silently drop mid-session. - CML: a NESTED (subdirectory) CLAUDE.md — not re-injected after compaction (only the project root is). Additive (no new scanner, count stays 13); one humanizer entry each; hermetic temp-fixture tests (positive + negative). Suite 957 -> 961, SC-5 byte-stable, self-audit A/A. Known limitation (pre-existing, broader than A): the lightweight frontmatter parser reads inline `paths:` but not YAML block sequences, so block-sequence-scoped rules are still seen as unscoped. Deferred. Part of v5.5.0 "steering-model I". Foundation (active-config-reader enumeration) deferred to v5.6 with B. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
263 lines
10 KiB
JavaScript
263 lines
10 KiB
JavaScript
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { resetCounter } from '../../scanners/lib/output.mjs';
|
|
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
|
|
import { scan } from '../../scanners/claude-md-linter.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const FIXTURES = resolve(__dirname, '../fixtures');
|
|
|
|
describe('CML scanner — healthy project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'healthy-project'));
|
|
result = await scan(resolve(FIXTURES, 'healthy-project'), discovery);
|
|
});
|
|
|
|
it('returns status ok', () => {
|
|
assert.strictEqual(result.status, 'ok');
|
|
});
|
|
|
|
it('scans at least 1 file', () => {
|
|
assert.ok(result.files_scanned >= 1);
|
|
});
|
|
|
|
it('has scanner prefix CML', () => {
|
|
assert.strictEqual(result.scanner, 'CML');
|
|
});
|
|
|
|
it('has all severity count keys', () => {
|
|
for (const key of ['critical', 'high', 'medium', 'low', 'info']) {
|
|
assert.ok(key in result.counts, `Missing count key: ${key}`);
|
|
}
|
|
});
|
|
|
|
it('finds no critical or high issues in healthy project', () => {
|
|
const serious = result.findings.filter(f => f.severity === 'critical' || f.severity === 'high');
|
|
assert.strictEqual(serious.length, 0, `Found serious issues: ${serious.map(f => f.title).join(', ')}`);
|
|
});
|
|
|
|
it('all finding IDs match CA-CML-NNN pattern', () => {
|
|
for (const f of result.findings) {
|
|
assert.match(f.id, /^CA-CML-\d{3}$/);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — broken project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'broken-project'));
|
|
result = await scan(resolve(FIXTURES, 'broken-project'), discovery);
|
|
});
|
|
|
|
it('detects long CLAUDE.md (>200 lines)', () => {
|
|
const found = result.findings.some(f => f.title.includes('exceeds'));
|
|
assert.ok(found, 'Should detect oversized CLAUDE.md');
|
|
});
|
|
|
|
it('detects missing headings', () => {
|
|
const found = result.findings.some(f => f.title.includes('no markdown headings'));
|
|
assert.ok(found, 'Should detect lack of headings');
|
|
});
|
|
|
|
it('detects TODO markers', () => {
|
|
const found = result.findings.some(f => f.title.includes('TODO'));
|
|
assert.ok(found, 'Should detect TODO markers');
|
|
});
|
|
|
|
it('detects repeated content', () => {
|
|
const found = result.findings.some(f => f.title.includes('Repeated content'));
|
|
assert.ok(found, 'Should detect repeated lines');
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — broken project: 200-tier stays MEDIUM (regression lock)', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'broken-project'));
|
|
result = await scan(resolve(FIXTURES, 'broken-project'), discovery);
|
|
});
|
|
|
|
it('the >200 length finding is MEDIUM', () => {
|
|
const f = result.findings.find(x => /exceeds recommended 200/.test(x.title || ''));
|
|
assert.ok(f, 'expected the 200-line recommendation finding');
|
|
assert.strictEqual(f.severity, 'medium');
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — large cascade (>500 lines): reframed, not absolute-adherence HIGH', () => {
|
|
// large-cascade/CLAUDE.md is 1024 lines. CC 2.1.169 scales the "too long"
|
|
// threshold by context window, and the plugin's own
|
|
// configuration-best-practices.md:97 footnote says raw line count is a
|
|
// Sonnet-era heuristic superseded by cache-prefix stability. So the absolute
|
|
// HIGH@500 + "significantly reduce adherence" claim is now-wrong.
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'large-cascade'));
|
|
result = await scan(resolve(FIXTURES, 'large-cascade'), discovery);
|
|
});
|
|
|
|
it('flags the >500 finding as MEDIUM, not HIGH', () => {
|
|
const f = result.findings.find(x => /exceeds 500/.test(x.title || ''));
|
|
assert.ok(f, 'expected the >500 length finding');
|
|
assert.strictEqual(f.severity, 'medium');
|
|
});
|
|
|
|
it('drops the absolute "significantly reduce adherence" claim', () => {
|
|
const f = result.findings.find(x => /exceeds 500/.test(x.title || ''));
|
|
assert.doesNotMatch(String(f?.description || ''), /significantly reduce/i);
|
|
});
|
|
|
|
it('reframes toward token cost / context window / cache-prefix', () => {
|
|
const f = result.findings.find(x => /exceeds 500/.test(x.title || ''));
|
|
assert.match(
|
|
`${f?.description || ''} ${f?.recommendation || ''}`,
|
|
/every turn|context window|cache/i,
|
|
);
|
|
});
|
|
|
|
it('produces no HIGH-severity length finding', () => {
|
|
const highLen = result.findings.filter(f => f.severity === 'high' && /exceeds/.test(f.title || ''));
|
|
assert.strictEqual(highLen.length, 0, `unexpected HIGH length finding: ${highLen.map(f => f.title).join(', ')}`);
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)', () => {
|
|
// large-claude-chars/CLAUDE.md is 48,531 chars across 100 lines: it crosses
|
|
// Claude Code's ~40.0k-char startup-warning threshold while staying under the
|
|
// 200-line count, so it isolates the char-budget check from the line check.
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'large-claude-chars'));
|
|
result = await scan(resolve(FIXTURES, 'large-claude-chars'), discovery);
|
|
});
|
|
|
|
const charFinding = (r) =>
|
|
r.findings.find((f) => /performance-warning threshold/i.test(f.title || ''));
|
|
|
|
it('flags a CLAUDE.md over ~40k chars', () => {
|
|
assert.ok(charFinding(result), 'expected a char-budget finding for a >40k-char CLAUDE.md');
|
|
});
|
|
|
|
it('the char-budget finding is MEDIUM (token cost, not an adherence cliff)', () => {
|
|
assert.strictEqual(charFinding(result)?.severity, 'medium');
|
|
});
|
|
|
|
it('anchors on CC\'s 40.0k figure and discloses context-window scaling', () => {
|
|
const f = charFinding(result);
|
|
const text = `${f?.description || ''} ${f?.evidence || ''}`;
|
|
assert.match(text, /40\.0k/, 'should mirror CC\'s 40.0k startup-warning figure');
|
|
assert.match(text, /context window|scales|1,000,000/i, 'should disclose context-window scaling');
|
|
// Lock the scaling arithmetic: 40.0k anchor x (1M / 200k) = 200,000 chars @ 1M.
|
|
assert.match(text, /200,000/, 'should disclose the ~200,000-char relaxed threshold at 1M context');
|
|
});
|
|
|
|
it('does NOT fire the line-count findings (chars high, lines under 200)', () => {
|
|
const lineFinding = result.findings.find((f) => /exceeds (recommended 200|500)/.test(f.title || ''));
|
|
assert.ok(!lineFinding, `char fixture should not trip a line finding: ${lineFinding?.title || ''}`);
|
|
});
|
|
|
|
it('the char-budget finding ID matches CA-CML-NNN', () => {
|
|
assert.match(charFinding(result)?.id || '', /^CA-CML-\d{3}$/);
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => {
|
|
// large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines):
|
|
// under CC's 40.0k char threshold, so the char-budget finding must NOT fire —
|
|
// proving the check keys on chars, not raw size or line count.
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'large-cascade'));
|
|
result = await scan(resolve(FIXTURES, 'large-cascade'), discovery);
|
|
});
|
|
|
|
it('does not emit a char-budget finding under 40k chars', () => {
|
|
const f = result.findings.find((x) => /performance-warning threshold/i.test(x.title || ''));
|
|
assert.ok(!f, 'a 37k-char file (under 40.0k) must not trip the char-budget finding');
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — empty project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'empty-project'));
|
|
result = await scan(resolve(FIXTURES, 'empty-project'), discovery);
|
|
});
|
|
|
|
it('detects missing CLAUDE.md', () => {
|
|
const found = result.findings.some(f => f.title.includes('No CLAUDE.md'));
|
|
assert.ok(found, 'Should report missing CLAUDE.md');
|
|
});
|
|
|
|
it('returns high severity for missing CLAUDE.md', () => {
|
|
const f = result.findings.find(f => f.title.includes('No CLAUDE.md'));
|
|
assert.strictEqual(f?.severity, 'high');
|
|
});
|
|
});
|
|
|
|
describe('CML scanner — minimal project', () => {
|
|
let result;
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
const discovery = await discoverConfigFiles(resolve(FIXTURES, 'minimal-project'));
|
|
result = await scan(resolve(FIXTURES, 'minimal-project'), discovery);
|
|
});
|
|
|
|
it('detects nearly empty CLAUDE.md', () => {
|
|
const found = result.findings.some(f => f.title.includes('nearly empty'));
|
|
assert.ok(found, 'Should detect nearly empty CLAUDE.md');
|
|
});
|
|
});
|
|
|
|
describe('CML — nested CLAUDE.md not re-injected after compaction (A)', () => {
|
|
// Only the project-root CLAUDE.md is re-injected after a compaction; a nested
|
|
// (subdirectory) CLAUDE.md is lost until a file in that dir is read again
|
|
// (V3, context-window.md). Hermetic temp fixture.
|
|
let tmpRoot;
|
|
let result;
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-cml-nested-'));
|
|
await mkdir(join(tmpRoot, 'src'), { recursive: true });
|
|
await writeFile(join(tmpRoot, 'CLAUDE.md'), '# Root\n\nProject overview goes here.\n', 'utf8');
|
|
await writeFile(
|
|
join(tmpRoot, 'src', 'CLAUDE.md'),
|
|
'# Src rules\n\n' + Array.from({ length: 10 }, (_, i) => `- nested rule ${i + 1}`).join('\n') + '\n',
|
|
'utf8',
|
|
);
|
|
const discovery = await discoverConfigFiles(tmpRoot);
|
|
result = await scan(tmpRoot, discovery);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
it('flags the nested CLAUDE.md as low (compaction durability)', () => {
|
|
const f = result.findings.find(x =>
|
|
x.scanner === 'CML' && /compaction/i.test(x.title || '') &&
|
|
/src/.test(`${x.file || ''}${x.evidence || ''}${x.description || ''}`));
|
|
assert.ok(f, `expected nested durability finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
|
assert.equal(f.severity, 'low');
|
|
});
|
|
|
|
it('does NOT flag the project-root CLAUDE.md', () => {
|
|
const rootFinding = result.findings.find(x =>
|
|
x.scanner === 'CML' && /compaction/i.test(x.title || '') && !/src/.test(x.file || ''));
|
|
assert.equal(rootFinding, undefined, 'project-root CLAUDE.md must not get the compaction finding');
|
|
});
|
|
});
|