diff --git a/hooks/scripts/otel-export.mjs b/hooks/scripts/otel-export.mjs index d36f46a..d45a8d4 100644 --- a/hooks/scripts/otel-export.mjs +++ b/hooks/scripts/otel-export.mjs @@ -17,6 +17,7 @@ // - All stderr prefixed with [voyage]. // - EXDEV mitigation: tmp file in same dir as target (do NOT use atomicWriteJson). +import { stdin } from 'node:process'; import { readFileSync, existsSync, writeFileSync, renameSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { transformToPrometheus } from '../../lib/exporters/textfile-format.mjs'; @@ -24,6 +25,13 @@ import { transformToOtlpJson } from '../../lib/exporters/otlp-format.mjs'; import { validateTextfilePath } from '../../lib/exporters/path-validator.mjs'; import { validateOtlpEndpoint } from '../../lib/exporters/endpoint-validator.mjs'; import { applyFieldAllowlist } from '../../lib/exporters/field-allowlist.mjs'; +import { captureTokenUsage } from '../../lib/stats/token-usage.mjs'; + +async function readStdin() { + let data = ''; + for await (const chunk of stdin) data += chunk; + return data; +} const VALID_MODES = new Set(['off', 'textfile', 'otlp']); const TEXTFILE_NAME = 'voyage.prom'; @@ -38,6 +46,7 @@ const STATS_FILES = [ { file: 'trekexecute-stats.jsonl', schema: 'trekexecute' }, { file: 'trekreview-stats.jsonl', schema: 'trekreview' }, { file: 'trekcontinue-stats.jsonl', schema: 'trekcontinue' }, + { file: 'token-usage-stats.jsonl', schema: 'token-usage' }, ]; function loadAndAllowlist(dataDir) { @@ -134,6 +143,26 @@ async function exportOtlp(records, env) { (async () => { try { const env = process.env; + + // SKAL-2: opt-in main-context token/cost capture (default off → zero added + // latency). Rides this existing Stop hook rather than a new hook script — + // avoids a second-Stop-hook race and keeps the README hook-count pin intact. + // Runs BEFORE the export-mode gate so capture is independent of export. + // Fail-open: any error is swallowed; capture must never block Stop. + if (env.VOYAGE_TOKEN_METER) { + try { + const raw = await readStdin(); + if (raw.trim()) { + const payload = JSON.parse(raw); + captureTokenUsage({ + transcriptPath: payload.transcript_path, + sessionId: payload.session_id, + dataDir: env.CLAUDE_PLUGIN_DATA, + }); + } + } catch { /* fail-open: never block Stop, never throw */ } + } + const mode = (env.VOYAGE_EXPORT_MODE || 'off').toLowerCase(); if (mode === 'off') return; diff --git a/lib/stats/token-usage.mjs b/lib/stats/token-usage.mjs index 72036ab..229da4f 100644 --- a/lib/stats/token-usage.mjs +++ b/lib/stats/token-usage.mjs @@ -15,11 +15,15 @@ // main-transcript sum UNDER-counts total Voyage cost. Every record is stamped // scope:'main-context' so no reader mistakes cost_usd for the session total. // -// All functions in this module are PURE (no I/O). The impure capture shell -// (captureTokenUsage) is added in Step 4. +// The four core functions (parseTranscriptUsage, deriveCost, buildRecord, +// upsertSessionRecord) are PURE (no I/O). captureTokenUsage is the impure +// shell that wires them to the filesystem (read transcript → upsert jsonl). // // Zero npm dependencies. Node stdlib only. +import { readFileSync, existsSync, writeFileSync, renameSync, statSync } from 'node:fs'; +import { join, dirname } from 'node:path'; + // Per-Mtok USD prices, resolved 2026-06-26 via the claude-api skill reference: // base input/output from the model table; cache rates from the prompt-caching // doc multipliers (cache_read 0.1x, write_5m 1.25x, write_1h 2.0x of input). @@ -159,3 +163,64 @@ export function upsertSessionRecord(existingText, record) { if (!replaced) out.push(JSON.stringify(record)); return out.join('\n') + '\n'; } + +/** + * Last main-chain (non-sidechain) assistant model in the transcript. Used to + * pick the price-table key. Returns null when no model is found (→ deriveCost + * refuses to estimate). Pure. + */ +export function lastMainChainModel(text) { + const lines = (text || '').split('\n'); + let model = null; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === '') continue; + let obj; + try { obj = JSON.parse(trimmed); } + catch { continue; } + if (!obj || obj.type !== 'assistant' || obj.isSidechain === true) continue; + const m = obj.message && obj.message.model; + if (typeof m === 'string' && m) model = m; + } + return model; +} + +/** + * Impure capture shell (Step 4). Reads the transcript, derives main-context + * token totals + cost, and UPSERTS a one-line-per-session record into + * {dataDir}/token-usage-stats.jsonl via an atomic temp+rename write + * (EXDEV mitigation: tmp lives in the same dir as the target). + * + * Returns the written record, or null when skipped (no path / no dataDir / + * transcript not a readable regular file). Throws propagate to the caller — + * the Stop hook wraps this in try/catch so capture stays fail-open. + * + * @param {{transcriptPath:string, sessionId:string, dataDir:string, + * now?:string}} args + */ +export function captureTokenUsage({ transcriptPath, sessionId, dataDir, now }) { + if (!transcriptPath || !dataDir) return null; + let st; + try { st = statSync(transcriptPath); } + catch { return null; } + if (!st.isFile()) return null; + + const text = readFileSync(transcriptPath, 'utf-8'); + const totals = parseTranscriptUsage(text); + const model = lastMainChainModel(text); + const record = buildRecord({ + sessionId, + model, + totals, + now: now || new Date().toISOString(), + }); + + const outPath = join(dataDir, 'token-usage-stats.jsonl'); + const existing = existsSync(outPath) ? readFileSync(outPath, 'utf-8') : ''; + const updated = upsertSessionRecord(existing, record); + + const tmpPath = join(dirname(outPath), '.token-usage-stats.jsonl.tmp'); + writeFileSync(tmpPath, updated); + renameSync(tmpPath, outPath); + return record; +} diff --git a/tests/hooks/otel-export-token-capture.test.mjs b/tests/hooks/otel-export-token-capture.test.mjs new file mode 100644 index 0000000..3fdb1da --- /dev/null +++ b/tests/hooks/otel-export-token-capture.test.mjs @@ -0,0 +1,147 @@ +// tests/hooks/otel-export-token-capture.test.mjs +// SKAL-2 Step 4 — opt-in main-context token/cost capture folded into the Stop +// hook (otel-export.mjs), gated by VOYAGE_TOKEN_METER. +// +// Fail-open contract: any error → exit 0, no token file, Stop never blocked. +// Pattern: tests/hooks/otel-export.test.mjs (setupDataDir + runHookWithEnv). + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { runHookWithEnv } from '../helpers/hook-helper.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const HOOK_PATH = join(HERE, '..', '..', 'hooks', 'scripts', 'otel-export.mjs'); +const FIXTURE = join(HERE, '..', 'fixtures', 'token-usage', 'sample-transcript.jsonl'); +const TOKEN_FILE = 'token-usage-stats.jsonl'; + +const mkDataDir = () => mkdtempSync(join(tmpdir(), 'voyage-token-capture-')); +const stdinFor = (extra = {}) => JSON.stringify({ transcript_path: FIXTURE, session_id: 'sess-1', ...extra }); +const readToken = (dir) => readFileSync(join(dir, TOKEN_FILE), 'utf-8').trim().split('\n').filter(Boolean).map(JSON.parse); + +test('VOYAGE_TOKEN_METER=1 → writes token-usage-stats.jsonl with expected totals + scope:main-context', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), true, 'token file must be written'); + const recs = readToken(dir); + assert.equal(recs.length, 1); + const rec = recs[0]; + // Fixture totals: dedup-by-requestId + sidechain-exclusion (verified in Step 1). + assert.equal(rec.tokens_input, 3000); + assert.equal(rec.tokens_output, 500); + assert.equal(rec.tokens_cache_creation, 1000); + assert.equal(rec.tokens_cache_read, 6000); + assert.equal(rec.scope, 'main-context'); + assert.equal(rec.model, 'claude-opus-4-8'); + assert.ok(Math.abs(rec.cost_usd - 0.03675) < 1e-9, `cost ${rec.cost_usd}`); + assert.equal(rec.session_id, 'sess-1'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('UPSERT — running twice for the same session_id keeps ONE line (not two)', async () => { + const dir = mkDataDir(); + try { + const env = { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }; + await runHookWithEnv(HOOK_PATH, stdinFor(), env); + const r2 = await runHookWithEnv(HOOK_PATH, stdinFor(), env); + assert.equal(r2.code, 0); + const recs = readToken(dir); + assert.equal(recs.length, 1, 'upsert must NOT append a second line for the same session'); + assert.equal(recs[0].tokens_input, 3000); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('UPSERT — distinct session_ids append (one line per session)', async () => { + const dir = mkDataDir(); + try { + const env = { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }; + await runHookWithEnv(HOOK_PATH, stdinFor({ session_id: 'A' }), env); + await runHookWithEnv(HOOK_PATH, stdinFor({ session_id: 'B' }), env); + const recs = readToken(dir); + assert.equal(recs.length, 2); + assert.deepEqual(recs.map(r => r.session_id).sort(), ['A', 'B']); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('OPT-IN — VOYAGE_TOKEN_METER unset → NO token file written', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '', // explicit empty mimics unset + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false, 'no token file without opt-in'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('FAIL-OPEN — malformed stdin → exit 0, no token file, no throw', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv(HOOK_PATH, 'not valid json {{{', { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'off', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('FAIL-OPEN — missing transcript file → exit 0, no token file', async () => { + const dir = mkDataDir(); + try { + const r = await runHookWithEnv( + HOOK_PATH, + JSON.stringify({ transcript_path: '/no/such/transcript.jsonl', session_id: 'x' }), + { VOYAGE_TOKEN_METER: '1', VOYAGE_EXPORT_MODE: 'off', CLAUDE_PLUGIN_DATA: dir }, + ); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('capture coexists with export — VOYAGE_TOKEN_METER=1 + textfile mode writes BOTH', async () => { + const dir = mkDataDir(); + try { + // Seed a stats file so the export path has something to write. + writeFileSync(join(dir, 'trekplan-stats.jsonl'), + JSON.stringify({ ts: '2026-06-26T08:00:00.000Z', slug: 't', mode: 'default', codebase_files: 10, profile: 'premium' }) + '\n'); + const r = await runHookWithEnv(HOOK_PATH, stdinFor(), { + VOYAGE_TOKEN_METER: '1', + VOYAGE_EXPORT_MODE: 'textfile', + CLAUDE_PLUGIN_DATA: dir, + }); + assert.equal(r.code, 0); + assert.equal(existsSync(join(dir, TOKEN_FILE)), true, 'token file written'); + assert.equal(existsSync(join(dir, 'voyage.prom')), true, 'export still completed'); + // The exported textfile should now also carry the token metric (STATS_FILES wiring). + const prom = readFileSync(join(dir, 'voyage.prom'), 'utf-8'); + assert.match(prom, /voyage_token_usage_tokens_input/); + assert.ok(!prom.includes('sess-1'), 'session_id must not leak into export (CWE-212)'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +});