feat(hooks): enforce research loop cap at PreToolUse or document the gap
This commit is contained in:
parent
f3874946ad
commit
e9ff8ab023
6 changed files with 441 additions and 2 deletions
233
tests/hooks/agent-cap.test.mjs
Normal file
233
tests/hooks/agent-cap.test.mjs
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
// tests/hooks/agent-cap.test.mjs
|
||||
// Step 10 — pins hooks/scripts/pre-agent-cap.mjs, the PreToolUse enforcement
|
||||
// of the /trekresearch Phase 5 loop bound.
|
||||
//
|
||||
// The spike (docs/spike-pretooluse-subagent-reach.md, RESULT: FIRES) proved a
|
||||
// plugin PreToolUse hook observes sub-agent tool calls, so the cap can be
|
||||
// enforced rather than merely documented. This file pins the two properties
|
||||
// that matter in opposite directions:
|
||||
//
|
||||
// (a) it DENIES (exit 2) once the ledger shows the budget spent, and
|
||||
// (b) it does NOT over-block — an unrelated session, unparsable stdin, a
|
||||
// stale marker, or the kill switch all exit 0.
|
||||
//
|
||||
// Pattern: tests/hooks/bash-guard.test.mjs (child process via runHook).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { runHookWithEnv } from '../helpers/hook-helper.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, '..', '..');
|
||||
const CAP_HOOK = join(ROOT, 'hooks', 'scripts', 'pre-agent-cap.mjs');
|
||||
const HOOKS_JSON = join(ROOT, 'hooks', 'hooks.json');
|
||||
|
||||
const SESSION = 'sess-abc123';
|
||||
const RUN_ID = 'run-xyz789';
|
||||
|
||||
/**
|
||||
* Build a throwaway CLAUDE_PLUGIN_DATA dir holding a scope marker for
|
||||
* `sessionId` and `turns` ledger entries for RUN_ID.
|
||||
*/
|
||||
function fixture({ turns = 0, sessionId = SESSION, startedAt = new Date() } = {}) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-cap-'));
|
||||
mkdirSync(join(dir, 'trekresearch-loop-scope'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(dir, 'trekresearch-loop-scope', `${sessionId}.json`),
|
||||
JSON.stringify({ runId: RUN_ID, startedAt: startedAt.toISOString() }),
|
||||
);
|
||||
const ledger = Array.from({ length: turns }, (_, i) =>
|
||||
JSON.stringify({ ts: new Date().toISOString(), runId: RUN_ID, dimension: `d${i}`, effort: 'high' }),
|
||||
).join('\n');
|
||||
writeFileSync(join(dir, 'trekresearch-loop-ledger.jsonl'), ledger ? ledger + '\n' : '');
|
||||
return dir;
|
||||
}
|
||||
|
||||
// TREKRESEARCH_MAX_CONV_TURNS=1 => budget = 1 * MAX_TOTAL_DIMENSIONS (8).
|
||||
const CAPPED_ENV = { VOYAGE_STORM_ENABLED: '1', TREKRESEARCH_MAX_CONV_TURNS: '1' };
|
||||
const BUDGET = 8;
|
||||
|
||||
function searchInput(sessionId = SESSION) {
|
||||
return {
|
||||
session_id: sessionId,
|
||||
hook_event_name: 'PreToolUse',
|
||||
tool_name: 'WebSearch',
|
||||
tool_input: { query: 'claude code hooks reference' },
|
||||
agent_id: 'aa6d19525a4680fe0',
|
||||
agent_type: 'general-purpose',
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// DENY — budget spent
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap DENIES a loop turn once the ledger shows the budget spent', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const { code, stderr } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 2);
|
||||
assert.match(stderr, /loop cap/i, 'stderr must name the cap it enforced');
|
||||
assert.match(stderr, new RegExp(`${BUDGET}`), 'stderr must state the budget');
|
||||
});
|
||||
|
||||
test('pre-agent-cap DENIES above the budget too (used > budget)', async () => {
|
||||
const dir = fixture({ turns: BUDGET + 5 });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 2);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// ALLOW — under the cap
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap ALLOWS a loop turn under the budget', async () => {
|
||||
const dir = fixture({ turns: BUDGET - 1 });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// DOES NOT OVER-BLOCK — the property that keeps this hook safe to wire
|
||||
// globally. A broken PreToolUse hook would brick every session on the box.
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap ALLOWS an unrelated session even when a loop is exhausted', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput('some-other-session'), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0, 'no scope marker for this session_id => out of scope');
|
||||
});
|
||||
|
||||
test('pre-agent-cap ALLOWS when no scope marker directory exists at all', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'voyage-cap-empty-'));
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
test('pre-agent-cap ALLOWS on unparsable stdin', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, 'not json at all', {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
test('pre-agent-cap ALLOWS when the input carries no session_id', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const input = searchInput();
|
||||
delete input.session_id;
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, input, {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// TTL / auto-reset — a marker left behind by a crashed run must not deny
|
||||
// tool calls forever.
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap ALLOWS when the scope marker is older than the TTL', async () => {
|
||||
const dir = fixture({ turns: BUDGET, startedAt: new Date(Date.now() - 48 * 3600 * 1000) });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
VOYAGE_CAP_SCOPE_TTL_MS: '1000',
|
||||
});
|
||||
assert.strictEqual(code, 0, 'a stale marker must auto-reset, not deny forever');
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Kill switch + default-off
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap kill switch VOYAGE_DISABLE_CAP_HOOK=1 allows an exhausted loop', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
VOYAGE_DISABLE_CAP_HOOK: '1',
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
test('pre-agent-cap is inert when VOYAGE_STORM_ENABLED is not 1 (default-off)', async () => {
|
||||
const dir = fixture({ turns: BUDGET });
|
||||
const { code } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
VOYAGE_STORM_ENABLED: '0',
|
||||
CLAUDE_PLUGIN_DATA: dir,
|
||||
});
|
||||
assert.strictEqual(code, 0);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Fail-closed inside scope — in scope but the ledger is unreachable means
|
||||
// the hook cannot count, and a cap that cannot count must not grant.
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap DENIES in scope when CLAUDE_PLUGIN_DATA is absent', async () => {
|
||||
const dir = fixture({ turns: 0 });
|
||||
const { code, stderr } = await runHookWithEnv(CAP_HOOK, searchInput(), {
|
||||
...CAPPED_ENV,
|
||||
CLAUDE_PLUGIN_DATA: '',
|
||||
VOYAGE_CAP_SCOPE_DIR: dir,
|
||||
});
|
||||
assert.strictEqual(code, 2);
|
||||
assert.match(stderr, /CLAUDE_PLUGIN_DATA/, 'stderr must name the missing variable');
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Append-only counting — the hook reads the ledger, it never writes it.
|
||||
// Writing per tool call would make the cap count its own enforcement.
|
||||
// -----------------------------------------------------------------------
|
||||
test('pre-agent-cap never writes to the ledger', async () => {
|
||||
const dir = fixture({ turns: 2 });
|
||||
const ledgerPath = join(dir, 'trekresearch-loop-ledger.jsonl');
|
||||
const before = readFileSync(ledgerPath, 'utf-8');
|
||||
await runHookWithEnv(CAP_HOOK, searchInput(), { ...CAPPED_ENV, CLAUDE_PLUGIN_DATA: dir });
|
||||
assert.strictEqual(readFileSync(ledgerPath, 'utf-8'), before);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Wiring — pattern from tests/hooks/hooks-json-stop-wired.test.mjs
|
||||
// -----------------------------------------------------------------------
|
||||
function invocationOf(h) {
|
||||
return [h.command || '', ...(h.args || [])].join(' ').trim();
|
||||
}
|
||||
|
||||
test('hooks.json wires pre-agent-cap.mjs on PreToolUse with ${CLAUDE_PLUGIN_ROOT}', () => {
|
||||
const cfg = JSON.parse(readFileSync(HOOKS_JSON, 'utf8'));
|
||||
const invocations = (cfg.hooks.PreToolUse || []).flatMap((entry) =>
|
||||
(entry.hooks || []).map(invocationOf),
|
||||
);
|
||||
const capInvocation = invocations.find((cmd) => cmd.includes('pre-agent-cap.mjs'));
|
||||
assert.ok(capInvocation, `no PreToolUse hook references pre-agent-cap.mjs. Found: ${JSON.stringify(invocations)}`);
|
||||
assert.match(capInvocation, /\$\{CLAUDE_PLUGIN_ROOT\}/, 'relative paths fail in headless sessions');
|
||||
assert.match(capInvocation, /^node\s+/);
|
||||
});
|
||||
|
||||
test('hooks.json matcher for pre-agent-cap covers the loop’s outbound surface', () => {
|
||||
const cfg = JSON.parse(readFileSync(HOOKS_JSON, 'utf8'));
|
||||
const entry = (cfg.hooks.PreToolUse || []).find((e) =>
|
||||
(e.hooks || []).some((h) => invocationOf(h).includes('pre-agent-cap.mjs')),
|
||||
);
|
||||
assert.ok(entry, 'pre-agent-cap entry missing from PreToolUse');
|
||||
for (const tool of ['WebSearch', 'WebFetch', 'Task']) {
|
||||
assert.match(entry.matcher, new RegExp(tool), `matcher must cover ${tool}`);
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue