feat(config-audit): --accurate-tokens API calibration (v5 N5) [skip-docs]

plugins/config-audit/tests/scanners/accurate-tokens.test.mjs

@@ -0,0 +1,215 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const exec = promisify(execFile);
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+const REPO = resolve(__dirname, '../..');
+const CLI = resolve(REPO, 'scanners/token-hotspots-cli.mjs');
+const TOKENIZER_MODULE = resolve(REPO, 'scanners/lib/tokenizer-api.mjs');
+const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-large');
+
+describe('--accurate-tokens (no API key)', () => {
+  it('skips API calibration and reports calibration.skipped === "no-api-key"', async () => {
+    const env = { ...process.env };
+    delete env.ANTHROPIC_API_KEY;
+    const { stdout, stderr } = await exec(
+      'node',
+      [CLI, FIXTURE, '--json', '--accurate-tokens'],
+      { timeout: 30000, cwd: REPO, env },
+    );
+    const json = JSON.parse(stdout);
+    assert.equal(json.calibration?.skipped, 'no-api-key');
+    assert.match(stderr, /ANTHROPIC_API_KEY not set/i);
+  });
+
+  it('does not include calibration field when --accurate-tokens absent', async () => {
+    const { stdout } = await exec('node', [CLI, FIXTURE, '--json'], {
+      timeout: 30000,
+      cwd: REPO,
+    });
+    const json = JSON.parse(stdout);
+    assert.equal(json.calibration, undefined);
+  });
+});
+
+describe('tokenizer-api.mjs — key masking', () => {
+  it('masks API key in error messages to first 8 chars + "..."', async () => {
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-FAKEKEY-1234567890';
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => {
+      const err = new Error('network failure');
+      throw err;
+    };
+
+    let threw = null;
+    try {
+      await tokenizerApi.callCountTokensApi('hello', fakeKey, { maxRetries: 0 });
+    } catch (e) {
+      threw = e;
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+
+    assert.ok(threw, 'expected an error to be thrown');
+    assert.ok(
+      !threw.message.includes('FAKEKEY-1234567890'),
+      `key must NOT appear unmasked in error message; got: ${threw.message}`,
+    );
+    assert.ok(
+      threw.message.includes('sk-ant-F'),
+      `error must mention masked key prefix sk-ant-F...; got: ${threw.message}`,
+    );
+  });
+
+  it('does NOT include response body in thrown errors on non-429 HTTP failure', async () => {
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-LEAKYBODY-9999';
+    const echoBody = `{"error": "invalid api key sk-ant-LEAKYBODY-9999"}`;
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => ({
+      ok: false,
+      status: 401,
+      statusText: 'Unauthorized',
+      text: async () => echoBody,
+      json: async () => JSON.parse(echoBody),
+    });
+
+    let threw = null;
+    try {
+      await tokenizerApi.callCountTokensApi('hi', fakeKey, { maxRetries: 0 });
+    } catch (e) {
+      threw = e;
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+
+    assert.ok(threw);
+    assert.ok(
+      !threw.message.includes('LEAKYBODY-9999'),
+      `body must NOT echo back into thrown message; got: ${threw.message}`,
+    );
+    assert.match(threw.message, /401/);
+  });
+
+  it('uses AbortController with a 5-second timeout', async () => {
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-TIMEOUTKEY-0000';
+    let capturedSignal = null;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async (_url, init) => {
+      capturedSignal = init?.signal;
+      return {
+        ok: true,
+        status: 200,
+        statusText: 'OK',
+        json: async () => ({ input_tokens: 42 }),
+      };
+    };
+
+    try {
+      const result = await tokenizerApi.callCountTokensApi('hi', fakeKey, { maxRetries: 0 });
+      assert.equal(result.input_tokens, 42);
+      assert.ok(capturedSignal, 'fetch must be called with an AbortController signal');
+      assert.ok(typeof capturedSignal.aborted === 'boolean');
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  it('retries on 429 with exponential backoff (max 3 retries)', async () => {
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-RETRYKEY-0000';
+    let calls = 0;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => {
+      calls++;
+      if (calls <= 2) {
+        return {
+          ok: false,
+          status: 429,
+          statusText: 'Too Many Requests',
+          text: async () => '',
+          json: async () => ({}),
+        };
+      }
+      return {
+        ok: true,
+        status: 200,
+        statusText: 'OK',
+        json: async () => ({ input_tokens: 100 }),
+      };
+    };
+
+    try {
+      const result = await tokenizerApi.callCountTokensApi('hello', fakeKey, {
+        maxRetries: 3,
+        backoffBaseMs: 1,
+      });
+      assert.equal(result.input_tokens, 100);
+      assert.equal(calls, 3, 'expected 2 retries before success on third call');
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  it('sends required headers: x-api-key, anthropic-version, content-type', async () => {
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-HEADERTEST-0000';
+    let capturedInit = null;
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async (_url, init) => {
+      capturedInit = init;
+      return {
+        ok: true,
+        status: 200,
+        statusText: 'OK',
+        json: async () => ({ input_tokens: 10 }),
+      };
+    };
+
+    try {
+      await tokenizerApi.callCountTokensApi('hi', fakeKey, { maxRetries: 0 });
+      const headers = capturedInit?.headers || {};
+      assert.equal(headers['x-api-key'], fakeKey);
+      assert.equal(headers['anthropic-version'], '2023-06-01');
+      assert.equal(headers['content-type'], 'application/json');
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+describe('--accurate-tokens (mocked fetch — happy path)', () => {
+  it('returns input_tokens from mocked fetch response', async () => {
+    // Note: the v5 plan specified `mock.method(tokenizerApi, ...)` but ESM
+    // read-only bindings make that pattern unusable. We mock at the
+    // globalThis.fetch boundary instead, which is the actual external
+    // dependency and gives equivalent coverage. Subprocess CLI integration
+    // can't carry the mock across processes, so unit-level fetch mock + the
+    // no-key subprocess test are the two coverage points.
+    const tokenizerApi = await import(TOKENIZER_MODULE);
+    const fakeKey = 'sk-ant-MOCKED-0000';
+
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => ({
+      ok: true,
+      status: 200,
+      statusText: 'OK',
+      json: async () => ({ input_tokens: 4200 }),
+    });
+
+    try {
+      const result = await tokenizerApi.callCountTokensApi('hello world', fakeKey, { maxRetries: 0 });
+      assert.equal(result.input_tokens, 4200);
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});