126 lines
4.3 KiB
JavaScript
126 lines
4.3 KiB
JavaScript
/**
|
|
* tokenizer-api.mjs — wrapper around Anthropic's count_tokens API for
|
|
* --accurate-tokens calibration.
|
|
*
|
|
* Surface:
|
|
* callCountTokensApi(text, apiKey, options)
|
|
* → Promise<{ input_tokens: number }>
|
|
*
|
|
* Security:
|
|
* - API key is masked to first 8 chars + "..." in ALL error messages and
|
|
* ALL thrown errors.
|
|
* - Response body is NEVER included in thrown errors (may echo the key).
|
|
* - Logs go to stderr only on caller request — this module throws, doesn't log.
|
|
*
|
|
* Reliability:
|
|
* - 5-second AbortController timeout per request.
|
|
* - Exponential backoff on HTTP 429 (max 3 retries: 1s, 2s, 4s by default).
|
|
* - Non-429 HTTP errors throw immediately with status code only.
|
|
*
|
|
* Zero external dependencies. Requires globalThis.fetch (Node 18+).
|
|
*/
|
|
|
|
const ENDPOINT = 'https://api.anthropic.com/v1/messages/count_tokens';
|
|
const ANTHROPIC_VERSION = '2023-06-01';
|
|
const TIMEOUT_MS = 5000;
|
|
const DEFAULT_MAX_RETRIES = 3;
|
|
const DEFAULT_BACKOFF_BASE_MS = 1000;
|
|
|
|
/**
|
|
* Mask an API key to its first 8 characters plus "..." for safe logging.
|
|
* Always pass user-provided strings through this before including them in
|
|
* error messages.
|
|
*/
|
|
export function maskKey(apiKey) {
|
|
if (typeof apiKey !== 'string' || apiKey.length === 0) {
|
|
return '<missing>';
|
|
}
|
|
return `${apiKey.slice(0, 8)}...`;
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise(r => setTimeout(r, ms));
|
|
}
|
|
|
|
/**
|
|
* Call Anthropic's count_tokens API for a single text payload.
|
|
* Uses claude-haiku-4-5 as the model — count_tokens requires a model param
|
|
* but token counts are tokenizer-driven, not model-driven for input counting.
|
|
*
|
|
* @param {string} text — the content to count
|
|
* @param {string} apiKey — Anthropic API key
|
|
* @param {object} [options]
|
|
* @param {number} [options.maxRetries=3]
|
|
* @param {number} [options.backoffBaseMs=1000] — base for exponential backoff
|
|
* @param {string} [options.model='claude-haiku-4-5']
|
|
* @returns {Promise<{input_tokens: number}>}
|
|
*/
|
|
export async function callCountTokensApi(text, apiKey, options = {}) {
|
|
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
const backoffBaseMs = options.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
const model = options.model ?? 'claude-haiku-4-5';
|
|
|
|
if (typeof globalThis.fetch !== 'function') {
|
|
throw new Error('fetch is not available — Node.js >= 18 required for --accurate-tokens');
|
|
}
|
|
|
|
const masked = maskKey(apiKey);
|
|
const body = JSON.stringify({
|
|
model,
|
|
messages: [{ role: 'user', content: text }],
|
|
});
|
|
|
|
let attempt = 0;
|
|
while (true) {
|
|
const controller = new AbortController();
|
|
const timeoutHandle = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
|
|
let response;
|
|
try {
|
|
response = await globalThis.fetch(ENDPOINT, {
|
|
method: 'POST',
|
|
headers: {
|
|
'x-api-key': apiKey,
|
|
'anthropic-version': ANTHROPIC_VERSION,
|
|
'content-type': 'application/json',
|
|
},
|
|
body,
|
|
signal: controller.signal,
|
|
});
|
|
} catch (err) {
|
|
clearTimeout(timeoutHandle);
|
|
// Network or abort error. Mask key in re-thrown error. Do NOT propagate
|
|
// the original error object — its `cause`/properties may include the
|
|
// request init we passed.
|
|
const reason = err && err.name === 'AbortError'
|
|
? 'request aborted (timeout 5s)'
|
|
: (err && err.message ? `network error: ${err.message}` : 'network error');
|
|
throw new Error(`count_tokens API failed (key ${masked}): ${reason}`);
|
|
}
|
|
clearTimeout(timeoutHandle);
|
|
|
|
if (response.ok) {
|
|
let data;
|
|
try {
|
|
data = await response.json();
|
|
} catch {
|
|
throw new Error(`count_tokens API failed (key ${masked}): malformed JSON response`);
|
|
}
|
|
if (typeof data?.input_tokens !== 'number') {
|
|
throw new Error(`count_tokens API failed (key ${masked}): missing input_tokens in response`);
|
|
}
|
|
return { input_tokens: data.input_tokens };
|
|
}
|
|
|
|
if (response.status === 429 && attempt < maxRetries) {
|
|
const wait = backoffBaseMs * Math.pow(2, attempt);
|
|
attempt++;
|
|
await sleep(wait);
|
|
continue;
|
|
}
|
|
|
|
// Non-retryable HTTP error. Body deliberately NOT included — it may echo
|
|
// the API key on auth failures.
|
|
throw new Error(`count_tokens API failed (key ${masked}): HTTP ${response.status}`);
|
|
}
|
|
}
|