fix(acr): token estimator discounts block-level HTML comments (M-BUG-6)
CLAUDE.md token estimates counted block-level <!-- --> HTML comments toward always-loaded tokens, but CC strips them before injection (preserved only inside code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments + effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade) and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte figures stay honest. Block-level only — inline comments retained (conservative, verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture has <!--), no re-seed. Dogfood ~/.claude CLAUDE.md ~3386->3301 tok (~85 tok discount, matches worklist prediction).
This commit is contained in:
parent
dd9db60fc9
commit
7e94910566
4 changed files with 234 additions and 6 deletions
|
|
@ -53,6 +53,76 @@ export function estimateTokens(bytes, kind = 'markdown', opts = {}) {
|
|||
return Math.ceil(bytes / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip block-level HTML comments (`<!-- ... -->`) that lie OUTSIDE fenced code
|
||||
* blocks. Claude Code strips these before injecting a CLAUDE.md / memory file
|
||||
* into context (code.claude.com/docs/en/memory: "block-level HTML comments are
|
||||
* stripped before the content is injected"), preserving them only inside fenced
|
||||
* code blocks (``` / ~~~). A byte-accurate token estimate must therefore discount
|
||||
* them. (M-BUG-6)
|
||||
*
|
||||
* Conservative scope — only *block-level* comments are removed (a comment that
|
||||
* occupies its own line(s)); inline comments sharing a line with other text are
|
||||
* retained, since the verified CC behavior covers block-level stripping only.
|
||||
*
|
||||
* @param {string} content
|
||||
* @returns {string} content with out-of-fence block comments removed
|
||||
*/
|
||||
export function stripInjectedHtmlComments(content) {
|
||||
if (typeof content !== 'string' || content === '') return '';
|
||||
const lines = content.split('\n');
|
||||
const out = [];
|
||||
let inFence = false;
|
||||
let inComment = false;
|
||||
for (const line of lines) {
|
||||
if (inComment) {
|
||||
// Inside a multi-line block comment: drop lines until the closing `-->`,
|
||||
// keeping any real content that trails the close on the same line.
|
||||
const end = line.indexOf('-->');
|
||||
if (end !== -1) {
|
||||
inComment = false;
|
||||
const rest = line.slice(end + 3);
|
||||
if (rest.trim() !== '') out.push(rest);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Fence delimiters (``` / ~~~) toggle a preserve-verbatim region.
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
inFence = !inFence;
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
if (inFence) {
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
// Whole line is a single self-contained block comment → CC strips it.
|
||||
if (/^\s*<!--[\s\S]*?-->\s*$/.test(line)) continue;
|
||||
// Block comment opening with nothing but whitespace before it and no close
|
||||
// on this line → runs onto following lines.
|
||||
const openIdx = line.indexOf('<!--');
|
||||
if (openIdx !== -1 && line.indexOf('-->', openIdx) === -1 && line.slice(0, openIdx).trim() === '') {
|
||||
inComment = true;
|
||||
continue;
|
||||
}
|
||||
out.push(line);
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Effective injected byte length of a CLAUDE.md / memory source: raw UTF-8 bytes
|
||||
* minus the block-level HTML comments CC strips before injection. Used wherever a
|
||||
* CLAUDE.md token estimate must reflect what actually enters context. (M-BUG-6)
|
||||
*
|
||||
* @param {string} content
|
||||
* @returns {number}
|
||||
*/
|
||||
export function effectiveMemoryBytes(content) {
|
||||
if (typeof content !== 'string') return 0;
|
||||
return Buffer.byteLength(stripInjectedHtmlComments(content), 'utf8');
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Load-pattern model (v5.6 Foundation)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
|
@ -202,7 +272,11 @@ export async function walkClaudeMdCascade(repoPath) {
|
|||
|
||||
const totalBytes = files.reduce((sum, f) => sum + f.bytes, 0);
|
||||
const totalLines = files.reduce((sum, f) => sum + f.lines, 0);
|
||||
const estimatedTokens = estimateTokens(totalBytes, 'markdown');
|
||||
// Token estimate is computed from the *effective* (injected) byte count — CC
|
||||
// strips block-level HTML comments before injection — while totalBytes stays
|
||||
// the honest on-disk figure. (M-BUG-6)
|
||||
const effectiveBytes = files.reduce((sum, f) => sum + (f.effectiveBytes ?? f.bytes), 0);
|
||||
const estimatedTokens = estimateTokens(effectiveBytes, 'markdown');
|
||||
|
||||
return { files, totalBytes, totalLines, estimatedTokens };
|
||||
}
|
||||
|
|
@ -217,6 +291,7 @@ async function tryAddClaudeMd(absPath, scope, parent, files, seen) {
|
|||
path: absPath,
|
||||
scope,
|
||||
bytes: s.size,
|
||||
effectiveBytes: effectiveMemoryBytes(content),
|
||||
lines: lineCount(content),
|
||||
parent,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue