/** * TOK Scanner — Token Hotspots / Opus 4.7 patterns * * Detects three structural Opus 4.7-era token-efficiency patterns: * CA-TOK-001 cache-breaking volatile top in CLAUDE.md (medium) * CA-TOK-002 redundant tool/permission declarations (low) * CA-TOK-003 deep @import chain (>2 hops) (medium) * * Note: the v4 sonnet-era signature pattern was removed in v5 F5 — too noisy * and not actionable; live token costs are better surfaced by the hotspots * ranking and per-pattern findings. * * Also ranks every discovered config source by estimated tokens and exposes * a `hotspots` array (≤10 entries, possibly fewer for tiny projects) on the * scanner result. * * Pattern catalogue: knowledge/opus-4.7-patterns.md * Token heuristic: estimateTokens() in scanners/lib/active-config-reader.mjs * * Zero external dependencies. */ import { resolve, dirname, isAbsolute } from 'node:path'; import { stat } from 'node:fs/promises'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson } from './lib/yaml-parser.mjs'; import { estimateTokens, readActiveConfig } from './lib/active-config-reader.mjs'; const SCANNER = 'TOK'; const VOLATILE_TOP_LINES = 30; const VOLATILE_PATTERNS = [ /\{timestamp\}/i, /\{uuid\}/i, /\{date\}/i, /\{session(?:_id)?\}/i, /\bactivity log\b/i, /^\s*\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/m, // ISO timestamps at line start /^\s*\[\d{4}-\d{2}-\d{2}/m, // [YYYY-MM-DD ...] log lines ]; const MAX_IMPORT_DEPTH = 2; const HOTSPOTS_MAX = 10; /** * Classify a discovered config file into a token-estimation kind. */ function tokenKind(type) { if (type === 'claude-md' || type === 'agent-md' || type === 'command-md' || type === 'skill-md' || type === 'rule') { return 'markdown'; } if (type === 'settings-json' || type === 'mcp-json' || type === 'hooks-json' || type === 'plugin-json' || type === 'claude-json' || type === 'keybindings-json') { return 'json'; } return 'markdown'; } async function fileExists(absPath) { try { await stat(absPath); return true; } catch { return false; } } function resolveImportPath(importPath, fromFile) { let p = importPath.trim(); if (!p) return null; if (p.startsWith('~/')) { const home = process.env.HOME || process.env.USERPROFILE || ''; p = resolve(home, p.slice(2)); } else if (p.startsWith('~')) { const home = process.env.HOME || process.env.USERPROFILE || ''; p = resolve(home, p.slice(1)); } else if (!isAbsolute(p)) { p = resolve(dirname(fromFile), p); } return p; } /** * Compute the deepest @import chain reachable from `startFile`. * Returns max depth observed (0 = no imports, 1 = direct import, etc.). */ async function maxImportDepth(startFile, contentCache) { let maxDepth = 0; async function walk(file, depth, visited) { if (depth > 20 || visited.has(file)) return; visited.add(file); if (depth > maxDepth) maxDepth = depth; let content = contentCache.get(file); if (content === undefined) { content = await readTextFile(file); contentCache.set(file, content); } if (!content) return; const imports = findImports(content); for (const imp of imports) { const target = resolveImportPath(imp.path, file); if (!target) continue; if (!(await fileExists(target))) continue; await walk(target, depth + 1, new Set(visited)); } } await walk(startFile, 0, new Set()); return maxDepth; } /** * Detect cache-breaking volatile content in the first VOLATILE_TOP_LINES * lines of a CLAUDE.md file. */ function detectVolatileTop(content) { if (!content) return false; const top = content.split('\n').slice(0, VOLATILE_TOP_LINES).join('\n'); return VOLATILE_PATTERNS.some(rx => rx.test(top)); } /** * Detect redundant or overlapping permission entries in a settings JSON object. * Returns array of `{list, entry, reason}` for reporting. */ function detectRedundantPermissions(settings) { const issues = []; if (!settings || typeof settings !== 'object') return issues; const perms = settings.permissions; if (!perms || typeof perms !== 'object') return issues; for (const list of ['allow', 'deny', 'ask']) { const arr = perms[list]; if (!Array.isArray(arr)) continue; const seen = new Set(); for (const entry of arr) { if (typeof entry !== 'string') continue; // Exact duplicate if (seen.has(entry)) { issues.push({ list, entry, reason: 'duplicate entry' }); continue; } seen.add(entry); } // Subset detection: an entry like `Read(src/**)` is redundant if `Read(**)` // or bare `Read` is also present in the same list. for (const entry of arr) { if (typeof entry !== 'string') continue; const tool = entry.replace(/\(.*\)$/, '').trim(); const hasBare = arr.includes(tool); const hasWildcard = arr.includes(`${tool}(**)`) || arr.includes(`${tool}(*)`); const isBare = entry === tool; const isWildcard = entry === `${tool}(**)` || entry === `${tool}(*)`; if (!isBare && !isWildcard && (hasBare || hasWildcard)) { issues.push({ list, entry, reason: `overlapped by ${hasBare ? tool : `${tool}(**)`}` }); } } } return issues; } /** * Build the ranked hotspots array. * * v5 F1: when activeConfig is available, expand each MCP server into its own * hotspot entry (richer signal than the parent .mcp.json file). Discovery * files remain the primary source for CLAUDE.md / settings / skills. */ async function buildHotspots(discovery, targetPath, activeConfig) { const ranked = []; for (const f of discovery.files) { const kind = tokenKind(f.type); const tokens = estimateTokens(f.size, kind); if (tokens <= 0) continue; ranked.push({ absPath: f.absPath, relPath: f.relPath || f.absPath.replace(targetPath + '/', ''), type: f.type, scope: f.scope, size: f.size, estimated_tokens: tokens, }); } // Per-MCP-server entries from activeConfig (each ~500+ tokens at runtime, // not represented by the parent .mcp.json file size alone). if (activeConfig && Array.isArray(activeConfig.mcpServers)) { for (const m of activeConfig.mcpServers) { if (!m || !m.enabled) continue; ranked.push({ absPath: m.source || `mcp:${m.name}`, relPath: `mcp:${m.name} (${m.source})`, type: 'mcp-server', scope: m.source, size: 0, estimated_tokens: m.estimatedTokens || 0, }); } } ranked.sort((a, b) => b.estimated_tokens - a.estimated_tokens); const top = ranked.slice(0, HOTSPOTS_MAX); const out = []; for (let i = 0; i < top.length; i++) { const h = top[i]; out.push({ source: h.relPath || h.absPath, estimated_tokens: h.estimated_tokens, rank: i + 1, recommendations: hotspotRecommendations(h), }); } return out; } function hotspotRecommendations(h) { const recs = []; if (h.type === 'claude-md') { recs.push('Move volatile top-of-file content to the bottom or extract to an @import-ed file.'); recs.push('Split overlong CLAUDE.md into focused @imports (≤200 lines each).'); } else if (h.type === 'settings-json' || h.type === 'mcp-json' || h.type === 'hooks-json') { recs.push('Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.'); recs.push('Move rarely-used permissions to a project-local override.'); } else if (h.type === 'skill-md' || h.type === 'agent-md' || h.type === 'command-md') { recs.push('Tighten the description field — it loads on every turn even when the body does not.'); } else { recs.push('Review whether this source needs to load on every turn.'); } // Always cap to 1–3 recommendations return recs.slice(0, 3); } /** * Main scanner entry point. * @param {string} targetPath * @param {{files: Array<{absPath:string, relPath:string, type:string, scope:string, size:number}>, skipped?:number}} discovery */ export async function scan(targetPath, discovery) { const start = Date.now(); const findings = []; let filesScanned = 0; const contentCache = new Map(); // v5 F1: pull active-config snapshot once. Failures are non-fatal — the // scanner falls back to the discovery-only path used in v4. let activeConfig = null; try { activeConfig = await readActiveConfig(targetPath, {}); } catch { activeConfig = null; } // ── Pattern A: cache-breaking volatile top in CLAUDE.md ── for (const f of discovery.files) { if (f.type !== 'claude-md') continue; filesScanned++; const content = await readTextFile(f.absPath); contentCache.set(f.absPath, content); if (detectVolatileTop(content)) { findings.push(finding({ scanner: SCANNER, severity: SEVERITY.medium, title: 'Cache-breaking volatile content at top of CLAUDE.md', description: `The first ${VOLATILE_TOP_LINES} lines of ${f.relPath || f.absPath} contain volatile ` + 'tokens (timestamps, session ids, or activity logs). Volatile content above stable ' + 'content defeats Opus 4.7 prompt-cache reuse on every turn.', file: f.absPath, recommendation: 'Move volatile sections to the bottom of the file, or extract them to an @import-ed ' + 'file outside the cached prefix. Keep the first 30 lines stable across turns.', category: 'token-efficiency', })); } } // ── Pattern B: redundant tool/permission declarations ── for (const f of discovery.files) { if (f.type !== 'settings-json') continue; filesScanned++; const content = await readTextFile(f.absPath); if (!content) continue; const parsed = parseJson(content); if (!parsed) continue; const issues = detectRedundantPermissions(parsed); if (issues.length === 0) continue; findings.push(finding({ scanner: SCANNER, severity: SEVERITY.low, title: 'Redundant permission declarations', description: `${f.relPath || f.absPath} contains ${issues.length} redundant or overlapping ` + `permission entr${issues.length === 1 ? 'y' : 'ies'}. Each duplicate inflates the ` + 'tool-schema payload sent on every turn.', file: f.absPath, evidence: issues.slice(0, 5).map(i => `${i.list}: "${i.entry}" (${i.reason})`).join('; '), recommendation: 'Deduplicate the permissions.allow / permissions.deny arrays. Prefer the most ' + 'specific entry that still grants the intended access.', category: 'token-efficiency', })); } // ── Pattern C: deep @import chain (>2 hops) ── for (const f of discovery.files) { if (f.type !== 'claude-md') continue; const depth = await maxImportDepth(f.absPath, contentCache); if (depth > MAX_IMPORT_DEPTH) { findings.push(finding({ scanner: SCANNER, severity: SEVERITY.medium, title: 'Deep @import chain defeats prompt-cache reuse', description: `${f.relPath || f.absPath} reaches @import depth ${depth} (>${MAX_IMPORT_DEPTH} hops). ` + 'Each @import boundary fragments the prompt-cache prefix; deeply chained imports ' + 'defeat caching for the deepest content even when it never changes.', file: f.absPath, evidence: `Max chain depth: ${depth}`, recommendation: 'Flatten the @import chain to ≤2 hops. Inline the deepest layer back into its parent.', category: 'token-efficiency', })); } } // ── Hotspots ranking ── const hotspots = await buildHotspots(discovery, targetPath, activeConfig); // ── Total estimated tokens (sum of every discovered source + activeConfig MCP) ── let totalTokens = 0; for (const f of discovery.files) { totalTokens += estimateTokens(f.size, tokenKind(f.type)); } if (activeConfig && Array.isArray(activeConfig.mcpServers)) { for (const m of activeConfig.mcpServers) { if (m && m.enabled) totalTokens += m.estimatedTokens || 0; } } const result = scannerResult(SCANNER, 'ok', findings, filesScanned, Date.now() - start); result.hotspots = hotspots; result.total_estimated_tokens = totalTokens; if (activeConfig) { result.activeConfig = { claudeMdEstimatedTokens: activeConfig.claudeMd?.estimatedTokens ?? 0, mcpServerCount: activeConfig.mcpServers?.length ?? 0, pluginCount: activeConfig.plugins?.length ?? 0, skillCount: activeConfig.skills?.length ?? 0, }; } return result; }