feat(config-audit): --accurate-tokens API calibration (v5 N5) [skip-docs]

plugins/config-audit/scanners/lib/tokenizer-api.mjs

@@ -0,0 +1,126 @@
+/**
+ * tokenizer-api.mjs — wrapper around Anthropic's count_tokens API for
+ * --accurate-tokens calibration.
+ *
+ * Surface:
+ *   callCountTokensApi(text, apiKey, options)
+ *     → Promise<{ input_tokens: number }>
+ *
+ * Security:
+ *   - API key is masked to first 8 chars + "..." in ALL error messages and
+ *     ALL thrown errors.
+ *   - Response body is NEVER included in thrown errors (may echo the key).
+ *   - Logs go to stderr only on caller request — this module throws, doesn't log.
+ *
+ * Reliability:
+ *   - 5-second AbortController timeout per request.
+ *   - Exponential backoff on HTTP 429 (max 3 retries: 1s, 2s, 4s by default).
+ *   - Non-429 HTTP errors throw immediately with status code only.
+ *
+ * Zero external dependencies. Requires globalThis.fetch (Node 18+).
+ */
+
+const ENDPOINT = 'https://api.anthropic.com/v1/messages/count_tokens';
+const ANTHROPIC_VERSION = '2023-06-01';
+const TIMEOUT_MS = 5000;
+const DEFAULT_MAX_RETRIES = 3;
+const DEFAULT_BACKOFF_BASE_MS = 1000;
+
+/**
+ * Mask an API key to its first 8 characters plus "..." for safe logging.
+ * Always pass user-provided strings through this before including them in
+ * error messages.
+ */
+export function maskKey(apiKey) {
+  if (typeof apiKey !== 'string' || apiKey.length === 0) {
+    return '<missing>';
+  }
+  return `${apiKey.slice(0, 8)}...`;
+}
+
+function sleep(ms) {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+/**
+ * Call Anthropic's count_tokens API for a single text payload.
+ * Uses claude-haiku-4-5 as the model — count_tokens requires a model param
+ * but token counts are tokenizer-driven, not model-driven for input counting.
+ *
+ * @param {string} text — the content to count
+ * @param {string} apiKey — Anthropic API key
+ * @param {object} [options]
+ * @param {number} [options.maxRetries=3]
+ * @param {number} [options.backoffBaseMs=1000] — base for exponential backoff
+ * @param {string} [options.model='claude-haiku-4-5']
+ * @returns {Promise<{input_tokens: number}>}
+ */
+export async function callCountTokensApi(text, apiKey, options = {}) {
+  const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
+  const backoffBaseMs = options.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
+  const model = options.model ?? 'claude-haiku-4-5';
+
+  if (typeof globalThis.fetch !== 'function') {
+    throw new Error('fetch is not available — Node.js >= 18 required for --accurate-tokens');
+  }
+
+  const masked = maskKey(apiKey);
+  const body = JSON.stringify({
+    model,
+    messages: [{ role: 'user', content: text }],
+  });
+
+  let attempt = 0;
+  while (true) {
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), TIMEOUT_MS);
+
+    let response;
+    try {
+      response = await globalThis.fetch(ENDPOINT, {
+        method: 'POST',
+        headers: {
+          'x-api-key': apiKey,
+          'anthropic-version': ANTHROPIC_VERSION,
+          'content-type': 'application/json',
+        },
+        body,
+        signal: controller.signal,
+      });
+    } catch (err) {
+      clearTimeout(timeoutHandle);
+      // Network or abort error. Mask key in re-thrown error. Do NOT propagate
+      // the original error object — its `cause`/properties may include the
+      // request init we passed.
+      const reason = err && err.name === 'AbortError'
+        ? 'request aborted (timeout 5s)'
+        : (err && err.message ? `network error: ${err.message}` : 'network error');
+      throw new Error(`count_tokens API failed (key ${masked}): ${reason}`);
+    }
+    clearTimeout(timeoutHandle);
+
+    if (response.ok) {
+      let data;
+      try {
+        data = await response.json();
+      } catch {
+        throw new Error(`count_tokens API failed (key ${masked}): malformed JSON response`);
+      }
+      if (typeof data?.input_tokens !== 'number') {
+        throw new Error(`count_tokens API failed (key ${masked}): missing input_tokens in response`);
+      }
+      return { input_tokens: data.input_tokens };
+    }
+
+    if (response.status === 429 && attempt < maxRetries) {
+      const wait = backoffBaseMs * Math.pow(2, attempt);
+      attempt++;
+      await sleep(wait);
+      continue;
+    }
+
+    // Non-retryable HTTP error. Body deliberately NOT included — it may echo
+    // the API key on auth failures.
+    throw new Error(`count_tokens API failed (key ${masked}): HTTP ${response.status}`);
+  }
+}

plugins/config-audit/scanners/token-hotspots-cli.mjs

@@ -6,22 +6,46 @@
  *
  * Usage:
  *   node token-hotspots-cli.mjs [path] [--json] [--output-file <path>] [--global]
- *                               [--with-telemetry-recipe]
+ *                               [--with-telemetry-recipe] [--accurate-tokens]
  *
  * Exit codes: 0=ok, 3=unrecoverable error.
  * Zero external dependencies.
  */
 
-import { resolve, dirname, join } from 'node:path';
+import { resolve, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { writeFile, stat } from 'node:fs/promises';
+import { writeFile, readFile, stat } from 'node:fs/promises';
 import { discoverConfigFiles } from './lib/file-discovery.mjs';
 import { resetCounter } from './lib/output.mjs';
 import { scan } from './token-hotspots.mjs';
+import * as tokenizerApi from './lib/tokenizer-api.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const TELEMETRY_RECIPE_PATH = resolve(__dirname, '..', 'knowledge', 'cache-telemetry-recipe.md');
 
+const ACCURATE_TOKENS_SAMPLE_SIZE = 3;
+
+async function calibrateAgainstApi(hotspots, apiKey) {
+  const sampled = hotspots.slice(0, ACCURATE_TOKENS_SAMPLE_SIZE);
+  let actualTokens = 0;
+  for (const hotspot of sampled) {
+    if (!hotspot?.path) continue;
+    let content;
+    try {
+      content = await readFile(hotspot.path, 'utf-8');
+    } catch {
+      continue;
+    }
+    const result = await tokenizerApi.callCountTokensApi(content, apiKey);
+    actualTokens += result.input_tokens;
+  }
+  return {
+    actual_tokens: actualTokens,
+    source: 'count_tokens_api',
+    sampled_hotspots: sampled.length,
+  };
+}
+
 async function main() {
   const args = process.argv.slice(2);
   let targetPath = '.';
@@ -29,11 +53,13 @@ async function main() {
   let jsonMode = false;
   let includeGlobal = false;
   let withTelemetryRecipe = false;
+  let accurateTokens = false;
 
   for (let i = 0; i < args.length; i++) {
     if (args[i] === '--json') jsonMode = true;
     else if (args[i] === '--global') includeGlobal = true;
     else if (args[i] === '--with-telemetry-recipe') withTelemetryRecipe = true;
+    else if (args[i] === '--accurate-tokens') accurateTokens = true;
     else if (args[i] === '--output-file' && args[i + 1]) outputFile = args[++i];
     else if (!args[i].startsWith('-')) targetPath = args[i];
   }
@@ -69,6 +95,22 @@ async function main() {
     payload.telemetry_recipe_path = TELEMETRY_RECIPE_PATH;
   }
 
+  if (accurateTokens) {
+    const apiKey = process.env.ANTHROPIC_API_KEY;
+    if (!apiKey || apiKey.length === 0) {
+      process.stderr.write('ANTHROPIC_API_KEY not set — skipping API calibration\n');
+      payload.calibration = { skipped: 'no-api-key' };
+    } else {
+      try {
+        payload.calibration = await calibrateAgainstApi(result.hotspots || [], apiKey);
+      } catch (err) {
+        // Error message is already key-masked by tokenizer-api.mjs.
+        process.stderr.write(`Calibration error: ${err.message}\n`);
+        payload.calibration = { skipped: 'api-error', error: err.message };
+      }
+    }
+  }
+
   const json = JSON.stringify(payload, null, 2);
 
   if (outputFile) {