fix(acr): token estimator discounts block-level HTML comments (M-BUG-6)

CLAUDE.md token estimates counted block-level <!-- --> HTML comments toward
always-loaded tokens, but CC strips them before injection (preserved only inside
code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments
+ effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade)
and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte
figures stay honest. Block-level only — inline comments retained (conservative,
verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture
has <!--), no re-seed. Dogfood ~/.claude CLAUDE.md ~3386->3301 tok (~85 tok discount,
matches worklist prediction).

scanners/lib/active-config-reader.mjs

@@ -53,6 +53,76 @@ export function estimateTokens(bytes, kind = 'markdown', opts = {}) {
   return Math.ceil(bytes / 4);
 }
 
+/**
+ * Strip block-level HTML comments (`<!-- ... -->`) that lie OUTSIDE fenced code
+ * blocks. Claude Code strips these before injecting a CLAUDE.md / memory file
+ * into context (code.claude.com/docs/en/memory: "block-level HTML comments are
+ * stripped before the content is injected"), preserving them only inside fenced
+ * code blocks (``` / ~~~). A byte-accurate token estimate must therefore discount
+ * them. (M-BUG-6)
+ *
+ * Conservative scope — only *block-level* comments are removed (a comment that
+ * occupies its own line(s)); inline comments sharing a line with other text are
+ * retained, since the verified CC behavior covers block-level stripping only.
+ *
+ * @param {string} content
+ * @returns {string} content with out-of-fence block comments removed
+ */
+export function stripInjectedHtmlComments(content) {
+  if (typeof content !== 'string' || content === '') return '';
+  const lines = content.split('\n');
+  const out = [];
+  let inFence = false;
+  let inComment = false;
+  for (const line of lines) {
+    if (inComment) {
+      // Inside a multi-line block comment: drop lines until the closing `-->`,
+      // keeping any real content that trails the close on the same line.
+      const end = line.indexOf('-->');
+      if (end !== -1) {
+        inComment = false;
+        const rest = line.slice(end + 3);
+        if (rest.trim() !== '') out.push(rest);
+      }
+      continue;
+    }
+    // Fence delimiters (``` / ~~~) toggle a preserve-verbatim region.
+    if (/^\s*(```|~~~)/.test(line)) {
+      inFence = !inFence;
+      out.push(line);
+      continue;
+    }
+    if (inFence) {
+      out.push(line);
+      continue;
+    }
+    // Whole line is a single self-contained block comment → CC strips it.
+    if (/^\s*<!--[\s\S]*?-->\s*$/.test(line)) continue;
+    // Block comment opening with nothing but whitespace before it and no close
+    // on this line → runs onto following lines.
+    const openIdx = line.indexOf('<!--');
+    if (openIdx !== -1 && line.indexOf('-->', openIdx) === -1 && line.slice(0, openIdx).trim() === '') {
+      inComment = true;
+      continue;
+    }
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+/**
+ * Effective injected byte length of a CLAUDE.md / memory source: raw UTF-8 bytes
+ * minus the block-level HTML comments CC strips before injection. Used wherever a
+ * CLAUDE.md token estimate must reflect what actually enters context. (M-BUG-6)
+ *
+ * @param {string} content
+ * @returns {number}
+ */
+export function effectiveMemoryBytes(content) {
+  if (typeof content !== 'string') return 0;
+  return Buffer.byteLength(stripInjectedHtmlComments(content), 'utf8');
+}
+
 // ─────────────────────────────────────────────────────────────────────────
 // Load-pattern model (v5.6 Foundation)
 // ─────────────────────────────────────────────────────────────────────────
@@ -202,7 +272,11 @@ export async function walkClaudeMdCascade(repoPath) {
 
   const totalBytes = files.reduce((sum, f) => sum + f.bytes, 0);
   const totalLines = files.reduce((sum, f) => sum + f.lines, 0);
-  const estimatedTokens = estimateTokens(totalBytes, 'markdown');
+  // Token estimate is computed from the *effective* (injected) byte count — CC
+  // strips block-level HTML comments before injection — while totalBytes stays
+  // the honest on-disk figure. (M-BUG-6)
+  const effectiveBytes = files.reduce((sum, f) => sum + (f.effectiveBytes ?? f.bytes), 0);
+  const estimatedTokens = estimateTokens(effectiveBytes, 'markdown');
 
   return { files, totalBytes, totalLines, estimatedTokens };
 }
@@ -217,6 +291,7 @@ async function tryAddClaudeMd(absPath, scope, parent, files, seen) {
       path: absPath,
       scope,
       bytes: s.size,
+      effectiveBytes: effectiveMemoryBytes(content),
       lines: lineCount(content),
       parent,
     };

scanners/token-hotspots.mjs

@@ -22,12 +22,12 @@
  */
 
 import { resolve, dirname, isAbsolute } from 'node:path';
-import { stat } from 'node:fs/promises';
+import { stat, readFile } from 'node:fs/promises';
 import { readTextFile } from './lib/file-discovery.mjs';
 import { finding, scannerResult } from './lib/output.mjs';
 import { SEVERITY } from './lib/severity.mjs';
 import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs';
-import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs';
+import { estimateTokens, effectiveMemoryBytes, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs';
 import {
   assessMcpDeferralForRepo,
   severityForForcedSchemas,
@@ -261,6 +261,26 @@ function detectRedundantPermissions(settings) {
   return issues;
 }
 
+/**
+ * Byte count to feed the token estimator for a discovered file. CLAUDE.md /
+ * memory files are sized from their *effective* (injected) content — CC strips
+ * block-level HTML comments before injection — so a raw byte read over-counts
+ * them. Every other source uses the raw on-disk size. (M-BUG-6)
+ *
+ * @param {{type:string, absPath?:string, size:number}} f
+ * @returns {Promise<number>}
+ */
+async function tokenBytesFor(f) {
+  if (f.type === 'claude-md' && f.absPath) {
+    try {
+      return effectiveMemoryBytes(await readFile(f.absPath, 'utf-8'));
+    } catch {
+      return f.size;
+    }
+  }
+  return f.size;
+}
+
 /**
  * Build the ranked hotspots array.
  *
@@ -272,7 +292,7 @@ async function buildHotspots(discovery, targetPath, activeConfig) {
   const ranked = [];
   for (const f of discovery.files) {
     const kind = tokenKind(f.type);
-    const tokens = estimateTokens(f.size, kind);
+    const tokens = estimateTokens(await tokenBytesFor(f), kind);
     if (tokens <= 0) continue;
     ranked.push({
       absPath: f.absPath,
@@ -651,7 +671,7 @@ export async function scan(targetPath, discovery) {
   // ── Total estimated tokens (sum of every discovered source + activeConfig MCP) ──
   let totalTokens = 0;
   for (const f of discovery.files) {
-    totalTokens += estimateTokens(f.size, tokenKind(f.type));
+    totalTokens += estimateTokens(await tokenBytesFor(f), tokenKind(f.type));
   }
   if (activeConfig && Array.isArray(activeConfig.mcpServers)) {
     for (const m of activeConfig.mcpServers) {