open/config-audit
1
0
Fork 0
Code Issues Releases 1 Activity

fix(acr): token estimator discounts block-level HTML comments (M-BUG-6)

Browse source 
CLAUDE.md token estimates counted block-level <!-- --> HTML comments toward
always-loaded tokens, but CC strips them before injection (preserved only inside
code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments
+ effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade)
and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte
figures stay honest. Block-level only — inline comments retained (conservative,
verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture
has <!--), no re-seed. Dogfood ~/.claude CLAUDE.md ~3386->3301 tok (~85 tok discount,
matches worklist prediction).
This commit is contained in:
Kjell Tore Guttormsen 2026-06-26 14:29:24 +02:00
commit 7e94910566
4 changed files with 234 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

37
tests/scanners/token-hotspots.test.mjs
View file

@ -1,7 +1,9 @@
import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdir, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/token-hotspots.mjs';
import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
@ -405,3 +407,36 @@ describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => {
assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || '')));
});
});
describe('TOK scanner — CLAUDE.md HTML-comment token discount (M-BUG-6)', () => {
it('estimates a comment-padded CLAUDE.md below its raw byte heuristic', async () => {
const dir = join(
tmpdir(),
`config-audit-tok-mbug6-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
);
await mkdir(dir, { recursive: true });
try {
// Big block-level comment CC strips before injection — the hotspot's
// estimated_tokens must reflect the stripped size, not the raw on-disk byte
// count. (Hotspot output exposes `path` + `estimated_tokens`, not `size`.)
const comment = `<!-- ${'maintainer note '.repeat(80)} -->`;
const content = `# Root\n\n${comment}\n\nReal instruction body.\n`;
const rawBytes = Buffer.byteLength(content, 'utf8');
await writeFile(join(dir, 'CLAUDE.md'), content);
resetCounter();
const discovery = await discoverConfigFiles(dir);
const result = await withHermeticHome(() => scan(dir, discovery));
const hs = result.hotspots.find(
h => typeof h.path === 'string' && h.path.endsWith('CLAUDE.md'),
);
assert.ok(hs, 'expected a CLAUDE.md hotspot for the fixture');
assert.ok(
hs.estimated_tokens < Math.ceil(rawBytes / 4),
`expected discounted tokens (${hs.estimated_tokens}) below raw heuristic ` +
`(${Math.ceil(rawBytes / 4)}) for ${rawBytes} raw bytes`,
);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
});