From 7e949105665958f508e144db4834506207c395f6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:29:24 +0200 Subject: [PATCH] fix(acr): token estimator discounts block-level HTML comments (M-BUG-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md token estimates counted block-level HTML comments toward always-loaded tokens, but CC strips them before injection (preserved only inside code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments + effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade) and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte figures stay honest. Block-level only — inline comments retained (conservative, verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture has `) that lie OUTSIDE fenced code + * blocks. Claude Code strips these before injecting a CLAUDE.md / memory file + * into context (code.claude.com/docs/en/memory: "block-level HTML comments are + * stripped before the content is injected"), preserving them only inside fenced + * code blocks (``` / ~~~). A byte-accurate token estimate must therefore discount + * them. (M-BUG-6) + * + * Conservative scope — only *block-level* comments are removed (a comment that + * occupies its own line(s)); inline comments sharing a line with other text are + * retained, since the verified CC behavior covers block-level stripping only. + * + * @param {string} content + * @returns {string} content with out-of-fence block comments removed + */ +export function stripInjectedHtmlComments(content) { + if (typeof content !== 'string' || content === '') return ''; + const lines = content.split('\n'); + const out = []; + let inFence = false; + let inComment = false; + for (const line of lines) { + if (inComment) { + // Inside a multi-line block comment: drop lines until the closing `-->`, + // keeping any real content that trails the close on the same line. + const end = line.indexOf('-->'); + if (end !== -1) { + inComment = false; + const rest = line.slice(end + 3); + if (rest.trim() !== '') out.push(rest); + } + continue; + } + // Fence delimiters (``` / ~~~) toggle a preserve-verbatim region. + if (/^\s*(```|~~~)/.test(line)) { + inFence = !inFence; + out.push(line); + continue; + } + if (inFence) { + out.push(line); + continue; + } + // Whole line is a single self-contained block comment → CC strips it. + if (/^\s*\s*$/.test(line)) continue; + // Block comment opening with nothing but whitespace before it and no close + // on this line → runs onto following lines. + const openIdx = line.indexOf('', openIdx) === -1 && line.slice(0, openIdx).trim() === '') { + inComment = true; + continue; + } + out.push(line); + } + return out.join('\n'); +} + +/** + * Effective injected byte length of a CLAUDE.md / memory source: raw UTF-8 bytes + * minus the block-level HTML comments CC strips before injection. Used wherever a + * CLAUDE.md token estimate must reflect what actually enters context. (M-BUG-6) + * + * @param {string} content + * @returns {number} + */ +export function effectiveMemoryBytes(content) { + if (typeof content !== 'string') return 0; + return Buffer.byteLength(stripInjectedHtmlComments(content), 'utf8'); +} + // ───────────────────────────────────────────────────────────────────────── // Load-pattern model (v5.6 Foundation) // ───────────────────────────────────────────────────────────────────────── @@ -202,7 +272,11 @@ export async function walkClaudeMdCascade(repoPath) { const totalBytes = files.reduce((sum, f) => sum + f.bytes, 0); const totalLines = files.reduce((sum, f) => sum + f.lines, 0); - const estimatedTokens = estimateTokens(totalBytes, 'markdown'); + // Token estimate is computed from the *effective* (injected) byte count — CC + // strips block-level HTML comments before injection — while totalBytes stays + // the honest on-disk figure. (M-BUG-6) + const effectiveBytes = files.reduce((sum, f) => sum + (f.effectiveBytes ?? f.bytes), 0); + const estimatedTokens = estimateTokens(effectiveBytes, 'markdown'); return { files, totalBytes, totalLines, estimatedTokens }; } @@ -217,6 +291,7 @@ async function tryAddClaudeMd(absPath, scope, parent, files, seen) { path: absPath, scope, bytes: s.size, + effectiveBytes: effectiveMemoryBytes(content), lines: lineCount(content), parent, }; diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index 87efcbe..f0f3e6c 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -22,12 +22,12 @@ */ import { resolve, dirname, isAbsolute } from 'node:path'; -import { stat } from 'node:fs/promises'; +import { stat, readFile } from 'node:fs/promises'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; -import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; +import { estimateTokens, effectiveMemoryBytes, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; import { assessMcpDeferralForRepo, severityForForcedSchemas, @@ -261,6 +261,26 @@ function detectRedundantPermissions(settings) { return issues; } +/** + * Byte count to feed the token estimator for a discovered file. CLAUDE.md / + * memory files are sized from their *effective* (injected) content — CC strips + * block-level HTML comments before injection — so a raw byte read over-counts + * them. Every other source uses the raw on-disk size. (M-BUG-6) + * + * @param {{type:string, absPath?:string, size:number}} f + * @returns {Promise} + */ +async function tokenBytesFor(f) { + if (f.type === 'claude-md' && f.absPath) { + try { + return effectiveMemoryBytes(await readFile(f.absPath, 'utf-8')); + } catch { + return f.size; + } + } + return f.size; +} + /** * Build the ranked hotspots array. * @@ -272,7 +292,7 @@ async function buildHotspots(discovery, targetPath, activeConfig) { const ranked = []; for (const f of discovery.files) { const kind = tokenKind(f.type); - const tokens = estimateTokens(f.size, kind); + const tokens = estimateTokens(await tokenBytesFor(f), kind); if (tokens <= 0) continue; ranked.push({ absPath: f.absPath, @@ -651,7 +671,7 @@ export async function scan(targetPath, discovery) { // ── Total estimated tokens (sum of every discovered source + activeConfig MCP) ── let totalTokens = 0; for (const f of discovery.files) { - totalTokens += estimateTokens(f.size, tokenKind(f.type)); + totalTokens += estimateTokens(await tokenBytesFor(f), tokenKind(f.type)); } if (activeConfig && Array.isArray(activeConfig.mcpServers)) { for (const m of activeConfig.mcpServers) { diff --git a/tests/lib/active-config-reader.test.mjs b/tests/lib/active-config-reader.test.mjs index 10c1874..7753022 100644 --- a/tests/lib/active-config-reader.test.mjs +++ b/tests/lib/active-config-reader.test.mjs @@ -5,6 +5,8 @@ import { mkdir, writeFile, rm, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { estimateTokens, + stripInjectedHtmlComments, + effectiveMemoryBytes, detectGitRoot, walkClaudeMdCascade, readClaudeJsonProjectSlice, @@ -196,6 +198,85 @@ describe('estimateTokens', () => { }); }); +// ───────────────────────────────────────────────────────────────────────── +// stripInjectedHtmlComments / effectiveMemoryBytes (M-BUG-6) +// Claude Code strips block-level HTML comments from a CLAUDE.md/memory file +// before injecting it into context (code.claude.com/docs/en/memory), preserving +// them only inside fenced code blocks. A byte-accurate token estimate must +// discount them. Inline comments (text on the same line) are conservatively +// retained — only block-level stripping is verified behavior. +// ───────────────────────────────────────────────────────────────────────── + +describe('stripInjectedHtmlComments (M-BUG-6)', () => { + it('strips a single-line block-level comment outside code fences', () => { + const src = '# Title\n\n\n\nBody text.\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(!out.includes('maintainer note'), 'comment text should be removed'); + assert.ok(out.includes('# Title') && out.includes('Body text.'), 'surrounding content preserved'); + }); + + it('strips a multi-line block comment outside fences', () => { + const src = 'A\n\nB\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(!out.includes('line one') && !out.includes('line two'), 'all comment lines removed'); + assert.ok(out.includes('A') && out.includes('B'), 'surrounding content preserved'); + }); + + it('preserves an HTML comment inside a ``` fenced code block', () => { + const src = '# Title\n\n```html\n\n```\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(out.includes('kept: this is example code'), 'fenced comment must be preserved (CC keeps it)'); + }); + + it('preserves an HTML comment inside a ~~~ fenced code block', () => { + const src = '~~~\n\n~~~\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(out.includes('kept tilde'), 'tilde-fenced comment must be preserved'); + }); + + it('keeps inline comments (only block-level stripping is verified)', () => { + // Text on the same line as the comment → conservatively retained; the + // verified CC behavior covers block-level comments only (Verifiseringsplikt). + const src = 'Visible tail\n'; + assert.equal(stripInjectedHtmlComments(src), src); + }); + + it('returns content unchanged when there are no comments', () => { + const src = '# Plain\n\nNo comments here.\n'; + assert.equal(stripInjectedHtmlComments(src), src); + }); + + it('handles empty and non-string input', () => { + assert.equal(stripInjectedHtmlComments(''), ''); + assert.equal(stripInjectedHtmlComments(undefined), ''); + assert.equal(stripInjectedHtmlComments(null), ''); + }); +}); + +describe('effectiveMemoryBytes (M-BUG-6)', () => { + it('discounts out-of-fence block comments from the byte count', () => { + const src = '# Title\n\n\n\nBody.\n'; + const raw = Buffer.byteLength(src, 'utf8'); + const eff = effectiveMemoryBytes(src); + assert.ok(eff < raw, `effective (${eff}) should be below raw (${raw})`); + }); + + it('counts comments inside fences (CC keeps them)', () => { + const src = '```\n\n```\n'; + assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8')); + }); + + it('equals raw bytes when no comments are present', () => { + const src = '# Plain markdown\n\nbody\n'; + assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8')); + }); + + it('returns 0 for non-string input', () => { + assert.equal(effectiveMemoryBytes(undefined), 0); + assert.equal(effectiveMemoryBytes(null), 0); + }); +}); + // ───────────────────────────────────────────────────────────────────────── // detectGitRoot // ───────────────────────────────────────────────────────────────────────── @@ -291,6 +372,23 @@ describe('walkClaudeMdCascade', () => { assert.equal(result.estimatedTokens, Math.ceil(result.totalBytes / 4)); }); + it('discounts block-level HTML comments from estimatedTokens (M-BUG-6)', async () => { + // CC strips block-level HTML comments before injection, so a CLAUDE.md + // padded with a maintainer-note comment must estimate FEWER tokens than its + // raw byte size would imply — totalBytes stays the honest on-disk figure. + const comment = ``; + await writeFile( + join(fixture.root, 'CLAUDE.md'), + `# Project Instructions\n\n${comment}\n\nBuild with care.\n`, + ); + const result = await walkClaudeMdCascade(fixture.root); + assert.ok( + result.estimatedTokens < Math.ceil(result.totalBytes / 4), + `expected discounted tokens (${result.estimatedTokens}) below raw heuristic ` + + `(${Math.ceil(result.totalBytes / 4)})`, + ); + }); + it('handles missing user CLAUDE.md gracefully', async () => { // Remove user CLAUDE.md await rm(join(fixture.fakeHome, '.claude', 'CLAUDE.md')); diff --git a/tests/scanners/token-hotspots.test.mjs b/tests/scanners/token-hotspots.test.mjs index bb42ed9..7ade2c6 100644 --- a/tests/scanners/token-hotspots.test.mjs +++ b/tests/scanners/token-hotspots.test.mjs @@ -1,7 +1,9 @@ import { describe, it, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve } from 'node:path'; +import { resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/token-hotspots.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; @@ -405,3 +407,36 @@ describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => { assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || ''))); }); }); + +describe('TOK scanner — CLAUDE.md HTML-comment token discount (M-BUG-6)', () => { + it('estimates a comment-padded CLAUDE.md below its raw byte heuristic', async () => { + const dir = join( + tmpdir(), + `config-audit-tok-mbug6-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ); + await mkdir(dir, { recursive: true }); + try { + // Big block-level comment CC strips before injection — the hotspot's + // estimated_tokens must reflect the stripped size, not the raw on-disk byte + // count. (Hotspot output exposes `path` + `estimated_tokens`, not `size`.) + const comment = ``; + const content = `# Root\n\n${comment}\n\nReal instruction body.\n`; + const rawBytes = Buffer.byteLength(content, 'utf8'); + await writeFile(join(dir, 'CLAUDE.md'), content); + resetCounter(); + const discovery = await discoverConfigFiles(dir); + const result = await withHermeticHome(() => scan(dir, discovery)); + const hs = result.hotspots.find( + h => typeof h.path === 'string' && h.path.endsWith('CLAUDE.md'), + ); + assert.ok(hs, 'expected a CLAUDE.md hotspot for the fixture'); + assert.ok( + hs.estimated_tokens < Math.ceil(rawBytes / 4), + `expected discounted tokens (${hs.estimated_tokens}) below raw heuristic ` + + `(${Math.ceil(rawBytes / 4)}) for ${rawBytes} raw bytes`, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +});