/** * tokenizer-api.mjs — wrapper around Anthropic's count_tokens API for * --accurate-tokens calibration. * * Surface: * callCountTokensApi(text, apiKey, options) * → Promise<{ input_tokens: number }> * * Security: * - API key is masked to first 8 chars + "..." in ALL error messages and * ALL thrown errors. * - Response body is NEVER included in thrown errors (may echo the key). * - Logs go to stderr only on caller request — this module throws, doesn't log. * * Reliability: * - 5-second AbortController timeout per request. * - Exponential backoff on HTTP 429 (max 3 retries: 1s, 2s, 4s by default). * - Non-429 HTTP errors throw immediately with status code only. * * Zero external dependencies. Requires globalThis.fetch (Node 18+). */ const ENDPOINT = 'https://api.anthropic.com/v1/messages/count_tokens'; const ANTHROPIC_VERSION = '2023-06-01'; const TIMEOUT_MS = 5000; const DEFAULT_MAX_RETRIES = 3; const DEFAULT_BACKOFF_BASE_MS = 1000; /** * Mask an API key to its first 8 characters plus "..." for safe logging. * Always pass user-provided strings through this before including them in * error messages. */ export function maskKey(apiKey) { if (typeof apiKey !== 'string' || apiKey.length === 0) { return ''; } return `${apiKey.slice(0, 8)}...`; } function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } /** * Call Anthropic's count_tokens API for a single text payload. * Uses claude-haiku-4-5 as the model — count_tokens requires a model param * but token counts are tokenizer-driven, not model-driven for input counting. * * @param {string} text — the content to count * @param {string} apiKey — Anthropic API key * @param {object} [options] * @param {number} [options.maxRetries=3] * @param {number} [options.backoffBaseMs=1000] — base for exponential backoff * @param {string} [options.model='claude-haiku-4-5'] * @returns {Promise<{input_tokens: number}>} */ export async function callCountTokensApi(text, apiKey, options = {}) { const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES; const backoffBaseMs = options.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS; const model = options.model ?? 'claude-haiku-4-5'; if (typeof globalThis.fetch !== 'function') { throw new Error('fetch is not available — Node.js >= 18 required for --accurate-tokens'); } const masked = maskKey(apiKey); const body = JSON.stringify({ model, messages: [{ role: 'user', content: text }], }); let attempt = 0; while (true) { const controller = new AbortController(); const timeoutHandle = setTimeout(() => controller.abort(), TIMEOUT_MS); let response; try { response = await globalThis.fetch(ENDPOINT, { method: 'POST', headers: { 'x-api-key': apiKey, 'anthropic-version': ANTHROPIC_VERSION, 'content-type': 'application/json', }, body, signal: controller.signal, }); } catch (err) { clearTimeout(timeoutHandle); // Network or abort error. Mask key in re-thrown error. Do NOT propagate // the original error object — its `cause`/properties may include the // request init we passed. const reason = err && err.name === 'AbortError' ? 'request aborted (timeout 5s)' : (err && err.message ? `network error: ${err.message}` : 'network error'); throw new Error(`count_tokens API failed (key ${masked}): ${reason}`); } clearTimeout(timeoutHandle); if (response.ok) { let data; try { data = await response.json(); } catch { throw new Error(`count_tokens API failed (key ${masked}): malformed JSON response`); } if (typeof data?.input_tokens !== 'number') { throw new Error(`count_tokens API failed (key ${masked}): missing input_tokens in response`); } return { input_tokens: data.input_tokens }; } if (response.status === 429 && attempt < maxRetries) { const wait = backoffBaseMs * Math.pow(2, attempt); attempt++; await sleep(wait); continue; } // Non-retryable HTTP error. Body deliberately NOT included — it may echo // the API key on auth failures. throw new Error(`count_tokens API failed (key ${masked}): HTTP ${response.status}`); } }