fix(acr): token estimator discounts block-level HTML comments (M-BUG-6)

CLAUDE.md token estimates counted block-level <!-- --> HTML comments toward
always-loaded tokens, but CC strips them before injection (preserved only inside
code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments
+ effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade)
and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte
figures stay honest. Block-level only — inline comments retained (conservative,
verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture
has <!--), no re-seed. Dogfood ~/.claude CLAUDE.md ~3386->3301 tok (~85 tok discount,
matches worklist prediction).

scanners/lib/active-config-reader.mjs

@@ -53,6 +53,76 @@ export function estimateTokens(bytes, kind = 'markdown', opts = {}) {
   return Math.ceil(bytes / 4);
 }
 
+/**
+ * Strip block-level HTML comments (`<!-- ... -->`) that lie OUTSIDE fenced code
+ * blocks. Claude Code strips these before injecting a CLAUDE.md / memory file
+ * into context (code.claude.com/docs/en/memory: "block-level HTML comments are
+ * stripped before the content is injected"), preserving them only inside fenced
+ * code blocks (``` / ~~~). A byte-accurate token estimate must therefore discount
+ * them. (M-BUG-6)
+ *
+ * Conservative scope — only *block-level* comments are removed (a comment that
+ * occupies its own line(s)); inline comments sharing a line with other text are
+ * retained, since the verified CC behavior covers block-level stripping only.
+ *
+ * @param {string} content
+ * @returns {string} content with out-of-fence block comments removed
+ */
+export function stripInjectedHtmlComments(content) {
+  if (typeof content !== 'string' || content === '') return '';
+  const lines = content.split('\n');
+  const out = [];
+  let inFence = false;
+  let inComment = false;
+  for (const line of lines) {
+    if (inComment) {
+      // Inside a multi-line block comment: drop lines until the closing `-->`,
+      // keeping any real content that trails the close on the same line.
+      const end = line.indexOf('-->');
+      if (end !== -1) {
+        inComment = false;
+        const rest = line.slice(end + 3);
+        if (rest.trim() !== '') out.push(rest);
+      }
+      continue;
+    }
+    // Fence delimiters (``` / ~~~) toggle a preserve-verbatim region.
+    if (/^\s*(```|~~~)/.test(line)) {
+      inFence = !inFence;
+      out.push(line);
+      continue;
+    }
+    if (inFence) {
+      out.push(line);
+      continue;
+    }
+    // Whole line is a single self-contained block comment → CC strips it.
+    if (/^\s*<!--[\s\S]*?-->\s*$/.test(line)) continue;
+    // Block comment opening with nothing but whitespace before it and no close
+    // on this line → runs onto following lines.
+    const openIdx = line.indexOf('<!--');
+    if (openIdx !== -1 && line.indexOf('-->', openIdx) === -1 && line.slice(0, openIdx).trim() === '') {
+      inComment = true;
+      continue;
+    }
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+/**
+ * Effective injected byte length of a CLAUDE.md / memory source: raw UTF-8 bytes
+ * minus the block-level HTML comments CC strips before injection. Used wherever a
+ * CLAUDE.md token estimate must reflect what actually enters context. (M-BUG-6)
+ *
+ * @param {string} content
+ * @returns {number}
+ */
+export function effectiveMemoryBytes(content) {
+  if (typeof content !== 'string') return 0;
+  return Buffer.byteLength(stripInjectedHtmlComments(content), 'utf8');
+}
+
 // ─────────────────────────────────────────────────────────────────────────
 // Load-pattern model (v5.6 Foundation)
 // ─────────────────────────────────────────────────────────────────────────
@@ -202,7 +272,11 @@ export async function walkClaudeMdCascade(repoPath) {
 
   const totalBytes = files.reduce((sum, f) => sum + f.bytes, 0);
   const totalLines = files.reduce((sum, f) => sum + f.lines, 0);
-  const estimatedTokens = estimateTokens(totalBytes, 'markdown');
+  // Token estimate is computed from the *effective* (injected) byte count — CC
+  // strips block-level HTML comments before injection — while totalBytes stays
+  // the honest on-disk figure. (M-BUG-6)
+  const effectiveBytes = files.reduce((sum, f) => sum + (f.effectiveBytes ?? f.bytes), 0);
+  const estimatedTokens = estimateTokens(effectiveBytes, 'markdown');
 
   return { files, totalBytes, totalLines, estimatedTokens };
 }
@@ -217,6 +291,7 @@ async function tryAddClaudeMd(absPath, scope, parent, files, seen) {
       path: absPath,
       scope,
       bytes: s.size,
+      effectiveBytes: effectiveMemoryBytes(content),
       lines: lineCount(content),
       parent,
     };

scanners/token-hotspots.mjs

@@ -22,12 +22,12 @@
  */
 
 import { resolve, dirname, isAbsolute } from 'node:path';
-import { stat } from 'node:fs/promises';
+import { stat, readFile } from 'node:fs/promises';
 import { readTextFile } from './lib/file-discovery.mjs';
 import { finding, scannerResult } from './lib/output.mjs';
 import { SEVERITY } from './lib/severity.mjs';
 import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs';
-import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs';
+import { estimateTokens, effectiveMemoryBytes, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs';
 import {
   assessMcpDeferralForRepo,
   severityForForcedSchemas,
@@ -261,6 +261,26 @@ function detectRedundantPermissions(settings) {
   return issues;
 }
 
+/**
+ * Byte count to feed the token estimator for a discovered file. CLAUDE.md /
+ * memory files are sized from their *effective* (injected) content — CC strips
+ * block-level HTML comments before injection — so a raw byte read over-counts
+ * them. Every other source uses the raw on-disk size. (M-BUG-6)
+ *
+ * @param {{type:string, absPath?:string, size:number}} f
+ * @returns {Promise<number>}
+ */
+async function tokenBytesFor(f) {
+  if (f.type === 'claude-md' && f.absPath) {
+    try {
+      return effectiveMemoryBytes(await readFile(f.absPath, 'utf-8'));
+    } catch {
+      return f.size;
+    }
+  }
+  return f.size;
+}
+
 /**
  * Build the ranked hotspots array.
  *
@@ -272,7 +292,7 @@ async function buildHotspots(discovery, targetPath, activeConfig) {
   const ranked = [];
   for (const f of discovery.files) {
     const kind = tokenKind(f.type);
-    const tokens = estimateTokens(f.size, kind);
+    const tokens = estimateTokens(await tokenBytesFor(f), kind);
     if (tokens <= 0) continue;
     ranked.push({
       absPath: f.absPath,
@@ -651,7 +671,7 @@ export async function scan(targetPath, discovery) {
   // ── Total estimated tokens (sum of every discovered source + activeConfig MCP) ──
   let totalTokens = 0;
   for (const f of discovery.files) {
-    totalTokens += estimateTokens(f.size, tokenKind(f.type));
+    totalTokens += estimateTokens(await tokenBytesFor(f), tokenKind(f.type));
   }
   if (activeConfig && Array.isArray(activeConfig.mcpServers)) {
     for (const m of activeConfig.mcpServers) {

tests/lib/active-config-reader.test.mjs

@@ -5,6 +5,8 @@ import { mkdir, writeFile, rm, readFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import {
   estimateTokens,
+  stripInjectedHtmlComments,
+  effectiveMemoryBytes,
   detectGitRoot,
   walkClaudeMdCascade,
   readClaudeJsonProjectSlice,
@@ -196,6 +198,85 @@ describe('estimateTokens', () => {
   });
 });
 
+// ─────────────────────────────────────────────────────────────────────────
+// stripInjectedHtmlComments / effectiveMemoryBytes (M-BUG-6)
+// Claude Code strips block-level HTML comments from a CLAUDE.md/memory file
+// before injecting it into context (code.claude.com/docs/en/memory), preserving
+// them only inside fenced code blocks. A byte-accurate token estimate must
+// discount them. Inline comments (text on the same line) are conservatively
+// retained — only block-level stripping is verified behavior.
+// ─────────────────────────────────────────────────────────────────────────
+
+describe('stripInjectedHtmlComments (M-BUG-6)', () => {
+  it('strips a single-line block-level comment outside code fences', () => {
+    const src = '# Title\n\n<!-- maintainer note: regenerate weekly -->\n\nBody text.\n';
+    const out = stripInjectedHtmlComments(src);
+    assert.ok(!out.includes('maintainer note'), 'comment text should be removed');
+    assert.ok(out.includes('# Title') && out.includes('Body text.'), 'surrounding content preserved');
+  });
+
+  it('strips a multi-line block comment outside fences', () => {
+    const src = 'A\n<!--\nline one\nline two\n-->\nB\n';
+    const out = stripInjectedHtmlComments(src);
+    assert.ok(!out.includes('line one') && !out.includes('line two'), 'all comment lines removed');
+    assert.ok(out.includes('A') && out.includes('B'), 'surrounding content preserved');
+  });
+
+  it('preserves an HTML comment inside a ``` fenced code block', () => {
+    const src = '# Title\n\n```html\n<!-- kept: this is example code -->\n```\n';
+    const out = stripInjectedHtmlComments(src);
+    assert.ok(out.includes('kept: this is example code'), 'fenced comment must be preserved (CC keeps it)');
+  });
+
+  it('preserves an HTML comment inside a ~~~ fenced code block', () => {
+    const src = '~~~\n<!-- kept tilde -->\n~~~\n';
+    const out = stripInjectedHtmlComments(src);
+    assert.ok(out.includes('kept tilde'), 'tilde-fenced comment must be preserved');
+  });
+
+  it('keeps inline comments (only block-level stripping is verified)', () => {
+    // Text on the same line as the comment → conservatively retained; the
+    // verified CC behavior covers block-level comments only (Verifiseringsplikt).
+    const src = 'Visible <!-- hidden --> tail\n';
+    assert.equal(stripInjectedHtmlComments(src), src);
+  });
+
+  it('returns content unchanged when there are no comments', () => {
+    const src = '# Plain\n\nNo comments here.\n';
+    assert.equal(stripInjectedHtmlComments(src), src);
+  });
+
+  it('handles empty and non-string input', () => {
+    assert.equal(stripInjectedHtmlComments(''), '');
+    assert.equal(stripInjectedHtmlComments(undefined), '');
+    assert.equal(stripInjectedHtmlComments(null), '');
+  });
+});
+
+describe('effectiveMemoryBytes (M-BUG-6)', () => {
+  it('discounts out-of-fence block comments from the byte count', () => {
+    const src = '# Title\n\n<!-- a fairly long maintainer note that costs real bytes -->\n\nBody.\n';
+    const raw = Buffer.byteLength(src, 'utf8');
+    const eff = effectiveMemoryBytes(src);
+    assert.ok(eff < raw, `effective (${eff}) should be below raw (${raw})`);
+  });
+
+  it('counts comments inside fences (CC keeps them)', () => {
+    const src = '```\n<!-- kept -->\n```\n';
+    assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8'));
+  });
+
+  it('equals raw bytes when no comments are present', () => {
+    const src = '# Plain markdown\n\nbody\n';
+    assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8'));
+  });
+
+  it('returns 0 for non-string input', () => {
+    assert.equal(effectiveMemoryBytes(undefined), 0);
+    assert.equal(effectiveMemoryBytes(null), 0);
+  });
+});
+
 // ─────────────────────────────────────────────────────────────────────────
 // detectGitRoot
 // ─────────────────────────────────────────────────────────────────────────
@@ -291,6 +372,23 @@ describe('walkClaudeMdCascade', () => {
     assert.equal(result.estimatedTokens, Math.ceil(result.totalBytes / 4));
   });
 
+  it('discounts block-level HTML comments from estimatedTokens (M-BUG-6)', async () => {
+    // CC strips block-level HTML comments before injection, so a CLAUDE.md
+    // padded with a maintainer-note comment must estimate FEWER tokens than its
+    // raw byte size would imply — totalBytes stays the honest on-disk figure.
+    const comment = `<!-- ${'maintainer note '.repeat(40)} -->`;
+    await writeFile(
+      join(fixture.root, 'CLAUDE.md'),
+      `# Project Instructions\n\n${comment}\n\nBuild with care.\n`,
+    );
+    const result = await walkClaudeMdCascade(fixture.root);
+    assert.ok(
+      result.estimatedTokens < Math.ceil(result.totalBytes / 4),
+      `expected discounted tokens (${result.estimatedTokens}) below raw heuristic ` +
+        `(${Math.ceil(result.totalBytes / 4)})`,
+    );
+  });
+
   it('handles missing user CLAUDE.md gracefully', async () => {
     // Remove user CLAUDE.md
     await rm(join(fixture.fakeHome, '.claude', 'CLAUDE.md'));

tests/scanners/token-hotspots.test.mjs

@@ -1,7 +1,9 @@
 import { describe, it, beforeEach } from 'node:test';
 import assert from 'node:assert/strict';
-import { resolve } from 'node:path';
+import { resolve, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { mkdir, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
 import { resetCounter } from '../../scanners/lib/output.mjs';
 import { scan } from '../../scanners/token-hotspots.mjs';
 import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs';
@@ -405,3 +407,36 @@ describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => {
     assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || '')));
   });
 });
+
+describe('TOK scanner — CLAUDE.md HTML-comment token discount (M-BUG-6)', () => {
+  it('estimates a comment-padded CLAUDE.md below its raw byte heuristic', async () => {
+    const dir = join(
+      tmpdir(),
+      `config-audit-tok-mbug6-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+    );
+    await mkdir(dir, { recursive: true });
+    try {
+      // Big block-level comment CC strips before injection — the hotspot's
+      // estimated_tokens must reflect the stripped size, not the raw on-disk byte
+      // count. (Hotspot output exposes `path` + `estimated_tokens`, not `size`.)
+      const comment = `<!-- ${'maintainer note '.repeat(80)} -->`;
+      const content = `# Root\n\n${comment}\n\nReal instruction body.\n`;
+      const rawBytes = Buffer.byteLength(content, 'utf8');
+      await writeFile(join(dir, 'CLAUDE.md'), content);
+      resetCounter();
+      const discovery = await discoverConfigFiles(dir);
+      const result = await withHermeticHome(() => scan(dir, discovery));
+      const hs = result.hotspots.find(
+        h => typeof h.path === 'string' && h.path.endsWith('CLAUDE.md'),
+      );
+      assert.ok(hs, 'expected a CLAUDE.md hotspot for the fixture');
+      assert.ok(
+        hs.estimated_tokens < Math.ceil(rawBytes / 4),
+        `expected discounted tokens (${hs.estimated_tokens}) below raw heuristic ` +
+          `(${Math.ceil(rawBytes / 4)}) for ${rawBytes} raw bytes`,
+      );
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});