fix(hooks): differentiate burst and edit-ratio heuristics on tool type
The burst counter incremented on any tool call <30s apart regardless of tool type, so a bulk read of many files produced the same "Rapid-fire" alert as a rapid-fire editing sequence. The edit-ratio check reported "possible stuck/spiral" for any session under 10% edits past 30 minutes, which a read-heavy analysis session satisfies structurally whether or not the work is productive. Both mechanisms are confirmed in docs/BRIEF-vurdering-v2.md, verification points 1a and 1b. tool-tracker.mjs now tracks whether the current burst run consists only of read tools and suppresses the rapid-fire alert for such runs. A read-dominant session (>=70% reads) reports its low edit ratio as context rather than as a stuck/spiral claim, and never as the sole reason for a warning. Bursts involving Edit/Write/Bash and non-read-dominant sessions behave exactly as before. No new data is recorded: the differentiation uses tool_name, which events.jsonl already logs. The privacy design is untouched. Tests first (Iron Law): 5 cases in tests/tool-tracker.test.mjs, 3 of which failed against the previous implementation, 2 of which are regression guards that passed before and still pass. Verified: node --test tests/*.test.mjs -> 263 pass, 5 fail. All 5 failures are the perf wall-clock assertions, which fail identically on HEAD without this change (measured by stashing and re-running) — they are a pre-existing property of this machine, not a regression. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011wMhJuPFUiWqKLSQjrzCXP
This commit is contained in:
parent
626140bb6a
commit
2c9e2de00a
5 changed files with 141 additions and 13 deletions
18
CHANGELOG.md
18
CHANGELOG.md
|
|
@ -2,6 +2,24 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file.
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **The burst and edit-ratio heuristics could not tell reading from editing**
|
||||||
|
(`docs/BRIEF-vurdering-v2.md` tiltak 1, verification points 1a/1b). A bulk
|
||||||
|
read of many files produced the same "Rapid-fire: N consecutive fast
|
||||||
|
interactions" alert as a rapid-fire editing sequence, and a read-heavy
|
||||||
|
analysis session was reported as "possible stuck/spiral" purely because its
|
||||||
|
edit ratio is structurally low. `tool-tracker.mjs` now tracks whether a
|
||||||
|
burst run consists only of read tools (`Read`/`Grep`/`Glob`/`NotebookRead`)
|
||||||
|
and suppresses the rapid-fire alert for such runs; a read-dominant session
|
||||||
|
(≥70% reads) reports its low edit ratio as context rather than as a
|
||||||
|
stuck/spiral claim, and never as the sole reason for a warning. Bursts
|
||||||
|
involving `Edit`/`Write`/`Bash` and non-read-dominant sessions are
|
||||||
|
unchanged. No new data is recorded — the differentiation uses `tool_name`,
|
||||||
|
which `events.jsonl` already logs.
|
||||||
|
|
||||||
## [1.2.2] — 2026-08-02
|
## [1.2.2] — 2026-08-02
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ layer4: false # default off
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
Automated test suite using `node:test` (258 cases, zero npm dependencies):
|
Automated test suite using `node:test` (263 cases, zero npm dependencies):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
node --test tests/*.test.mjs
|
node --test tests/*.test.mjs
|
||||||
|
|
@ -75,7 +75,7 @@ node --test tests/*.test.mjs
|
||||||
|------|-------|----------|
|
|------|-------|----------|
|
||||||
| `tests/session-start.test.mjs` | 11 | State init, JSONL, tier-2 cross-session alert |
|
| `tests/session-start.test.mjs` | 11 | State init, JSONL, tier-2 cross-session alert |
|
||||||
| `tests/prompt-analyzer.test.mjs` | 100 | All v1.x patterns × 2 + thresholds + valence + v1.2 pushback contract |
|
| `tests/prompt-analyzer.test.mjs` | 100 | All v1.x patterns × 2 + thresholds + valence + v1.2 pushback contract |
|
||||||
| `tests/tool-tracker.test.mjs` | 8 | Counting, burst, reminders |
|
| `tests/tool-tracker.test.mjs` | 13 | Counting, burst, reminders + read-tool calibration |
|
||||||
| `tests/session-end.test.mjs` | 7 | Finalize, duration, flags, v1.1.0 string + v1.2 array shapes |
|
| `tests/session-end.test.mjs` | 7 | Finalize, duration, flags, v1.1.0 string + v1.2 array shapes |
|
||||||
| `tests/privacy.test.mjs` | 7 | Canary + matched-phrase × original + 5 v1.2 detector variants |
|
| `tests/privacy.test.mjs` | 7 | Canary + matched-phrase × original + 5 v1.2 detector variants |
|
||||||
| `tests/skill-md.test.mjs` | 3 | Constitution citation + Score 5 + 11 guidance criteria |
|
| `tests/skill-md.test.mjs` | 3 | Constitution citation + Score 5 + 11 guidance criteria |
|
||||||
|
|
|
||||||
|
|
@ -133,6 +133,16 @@ export const THRESHOLD_SOFT_DEP_FLAGS = 2;
|
||||||
export const THRESHOLD_HARD_DEP_FLAGS = 5;
|
export const THRESHOLD_HARD_DEP_FLAGS = 5;
|
||||||
export const COOLDOWN_SOFT = 1800;
|
export const COOLDOWN_SOFT = 1800;
|
||||||
export const COOLDOWN_HARD = 3600;
|
export const COOLDOWN_HARD = 3600;
|
||||||
|
|
||||||
|
// Read-only tools. A sequence of these is bulk reading (research, audit,
|
||||||
|
// bulk file review), not rapid-fire editing — the burst and edit-ratio
|
||||||
|
// heuristics must not treat the two as the same signal.
|
||||||
|
export const READ_TOOLS = Object.freeze(['Read', 'Grep', 'Glob', 'NotebookRead']);
|
||||||
|
export function isReadTool(tool) {
|
||||||
|
return READ_TOOLS.includes(tool);
|
||||||
|
}
|
||||||
|
// Share of read tools above which a session counts as read-dominant.
|
||||||
|
export const THRESHOLD_READ_DOMINANT_RATIO = 70;
|
||||||
// v1.1.0 — counting threshold; tier-reduction logic is v1.2 scope
|
// v1.1.0 — counting threshold; tier-reduction logic is v1.2 scope
|
||||||
export const THRESHOLD_PUSHBACK_FLAGS = 2;
|
export const THRESHOLD_PUSHBACK_FLAGS = 2;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import {
|
||||||
THRESHOLD_SOFT_SESSIONS, THRESHOLD_HARD_SESSIONS,
|
THRESHOLD_SOFT_SESSIONS, THRESHOLD_HARD_SESSIONS,
|
||||||
THRESHOLD_SOFT_BURST, THRESHOLD_HARD_BURST, THRESHOLD_BURST_INTERVAL,
|
THRESHOLD_SOFT_BURST, THRESHOLD_HARD_BURST, THRESHOLD_BURST_INTERVAL,
|
||||||
THRESHOLD_LOW_EDIT_RATIO, THRESHOLD_LOW_EDIT_MIN_DURATION,
|
THRESHOLD_LOW_EDIT_RATIO, THRESHOLD_LOW_EDIT_MIN_DURATION,
|
||||||
|
THRESHOLD_READ_DOMINANT_RATIO, isReadTool,
|
||||||
COOLDOWN_SOFT, COOLDOWN_HARD,
|
COOLDOWN_SOFT, COOLDOWN_HARD,
|
||||||
readState, sessionStateFile, writeState, appendJsonl, sessionsToday,
|
readState, sessionStateFile, writeState, appendJsonl, sessionsToday,
|
||||||
outputContinue, outputWithContext
|
outputContinue, outputWithContext
|
||||||
|
|
@ -38,24 +39,37 @@ appendJsonl(EVENTS_LOG, { ts: nowIsoStr, session_id: sid, tool_name: tool });
|
||||||
let state = readState();
|
let state = readState();
|
||||||
let toolCount = (Number(state.tool_count) || 0) + 1;
|
let toolCount = (Number(state.tool_count) || 0) + 1;
|
||||||
let editCount = Number(state.edit_count) || 0;
|
let editCount = Number(state.edit_count) || 0;
|
||||||
|
let readCount = Number(state.read_count) || 0;
|
||||||
const lastEvent = Number(state.last_event_epoch) || 0;
|
const lastEvent = Number(state.last_event_epoch) || 0;
|
||||||
let burstCount = Number(state.burst_count) || 0;
|
let burstCount = Number(state.burst_count) || 0;
|
||||||
|
// Absent in pre-calibration state files — an unseen run starts read-only.
|
||||||
|
let burstReadOnly = state.burst_read_only !== false;
|
||||||
const startEpoch = Number(state.start_epoch) || 0;
|
const startEpoch = Number(state.start_epoch) || 0;
|
||||||
const lastWarning = Number(state.last_warning_epoch) || 0;
|
const lastWarning = Number(state.last_warning_epoch) || 0;
|
||||||
|
|
||||||
|
const toolIsRead = isReadTool(tool);
|
||||||
if (tool === 'Edit') editCount++;
|
if (tool === 'Edit') editCount++;
|
||||||
|
if (toolIsRead) readCount++;
|
||||||
|
|
||||||
// Burst detection: rapid-fire if <30s since last event
|
// Burst detection: rapid-fire if <30s since last event
|
||||||
if (lastEvent > 0) {
|
if (lastEvent > 0) {
|
||||||
const interval = nowTs - lastEvent;
|
const interval = nowTs - lastEvent;
|
||||||
burstCount = interval < THRESHOLD_BURST_INTERVAL ? burstCount + 1 : 0;
|
if (interval < THRESHOLD_BURST_INTERVAL) {
|
||||||
|
burstCount++;
|
||||||
|
burstReadOnly = burstReadOnly && toolIsRead;
|
||||||
|
} else {
|
||||||
|
burstCount = 0;
|
||||||
|
burstReadOnly = toolIsRead;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write updated state
|
// Write updated state
|
||||||
state.tool_count = toolCount;
|
state.tool_count = toolCount;
|
||||||
state.edit_count = editCount;
|
state.edit_count = editCount;
|
||||||
|
state.read_count = readCount;
|
||||||
state.last_event_epoch = nowTs;
|
state.last_event_epoch = nowTs;
|
||||||
state.burst_count = burstCount;
|
state.burst_count = burstCount;
|
||||||
|
state.burst_read_only = burstReadOnly;
|
||||||
writeState(state);
|
writeState(state);
|
||||||
|
|
||||||
// Check thresholds every 25 calls or when burst threshold hit
|
// Check thresholds every 25 calls or when burst threshold hit
|
||||||
|
|
@ -106,19 +120,33 @@ if (dayCount >= THRESHOLD_HARD_SESSIONS) {
|
||||||
messages.push(`${dayCount} sessions today.`);
|
messages.push(`${dayCount} sessions today.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Burst
|
// Burst — a run of read-only tools is bulk reading, not rapid-fire work
|
||||||
if (burstCount >= THRESHOLD_HARD_BURST) {
|
if (!burstReadOnly) {
|
||||||
level = 'hard';
|
if (burstCount >= THRESHOLD_HARD_BURST) {
|
||||||
messages.push(`Rapid-fire: ${burstCount} consecutive fast interactions.`);
|
level = 'hard';
|
||||||
} else if (burstCount >= THRESHOLD_SOFT_BURST) {
|
messages.push(`Rapid-fire: ${burstCount} consecutive fast interactions.`);
|
||||||
if (!level) level = 'soft';
|
} else if (burstCount >= THRESHOLD_SOFT_BURST) {
|
||||||
messages.push(`Rapid-fire: ${burstCount} consecutive fast interactions.`);
|
if (!level) level = 'soft';
|
||||||
|
messages.push(`Rapid-fire: ${burstCount} consecutive fast interactions.`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Low edit ratio (only after minimum duration)
|
// Low edit ratio (only after minimum duration). A read-dominant session has
|
||||||
|
// a structurally low edit ratio whether or not the work is productive, so it
|
||||||
|
// is reported as context — never as a stuck/spiral claim, and never as the
|
||||||
|
// sole reason for a warning.
|
||||||
|
const readRatio = toolCount > 0 ? Math.floor(readCount * 100 / toolCount) : 0;
|
||||||
|
const readDominant = readRatio >= THRESHOLD_READ_DOMINANT_RATIO;
|
||||||
|
|
||||||
if (durationMin >= THRESHOLD_LOW_EDIT_MIN_DURATION && editRatio < THRESHOLD_LOW_EDIT_RATIO) {
|
if (durationMin >= THRESHOLD_LOW_EDIT_MIN_DURATION && editRatio < THRESHOLD_LOW_EDIT_RATIO) {
|
||||||
if (!level) level = 'soft';
|
if (readDominant) {
|
||||||
messages.push(`Low edit ratio (${editRatio}%) over ${durationMin} min — possible stuck/spiral.`);
|
if (level) {
|
||||||
|
messages.push(`Low edit ratio (${editRatio}%) over ${durationMin} min, but ${readRatio}% of tool calls are reads — read-intensive work (research/audit) rather than a stall.`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (!level) level = 'soft';
|
||||||
|
messages.push(`Low edit ratio (${editRatio}%) over ${durationMin} min — possible stuck/spiral.`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Late night check
|
// Late night check
|
||||||
|
|
|
||||||
|
|
@ -93,3 +93,75 @@ describe('tool-tracker', () => {
|
||||||
assert.equal(out.continue, true);
|
assert.equal(out.continue, true);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Tiltak 1 (BRIEF-vurdering-v2.md): burst and edit-ratio heuristics must
|
||||||
|
// differentiate on tool type. A bulk-read sequence is structurally
|
||||||
|
// indistinguishable from a rapid-fire editing sequence today.
|
||||||
|
describe('tool-tracker — task-type calibration', () => {
|
||||||
|
it('does not raise a rapid-fire alert for a pure read burst', () => {
|
||||||
|
dir = setupTestDir();
|
||||||
|
// burst_count 9 + this call = THRESHOLD_HARD_BURST (10), all reads
|
||||||
|
createStateFile(dir, 'b1', freshState({
|
||||||
|
last_event_epoch: Math.floor(Date.now() / 1000) - 5,
|
||||||
|
burst_count: 9,
|
||||||
|
tool_count: 30,
|
||||||
|
read_count: 30,
|
||||||
|
}));
|
||||||
|
const out = runHook('tool-tracker.mjs', { session_id: 'b1', tool_name: 'Read' }, dir);
|
||||||
|
const ctx = out.hookSpecificOutput?.additionalContext || '';
|
||||||
|
assert.ok(!ctx.includes('Rapid-fire'), `expected no rapid-fire alert, got: ${ctx}`);
|
||||||
|
assert.equal(out.continue, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('still raises a rapid-fire alert when the burst includes an edit', () => {
|
||||||
|
dir = setupTestDir();
|
||||||
|
createStateFile(dir, 'b2', freshState({
|
||||||
|
last_event_epoch: Math.floor(Date.now() / 1000) - 5,
|
||||||
|
burst_count: 9,
|
||||||
|
tool_count: 30,
|
||||||
|
read_count: 29,
|
||||||
|
}));
|
||||||
|
const out = runHook('tool-tracker.mjs', { session_id: 'b2', tool_name: 'Edit' }, dir);
|
||||||
|
const ctx = out.hookSpecificOutput?.additionalContext || '';
|
||||||
|
assert.ok(ctx.includes('Rapid-fire'), `expected rapid-fire alert, got: ${ctx}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not call a read-dominant session stuck/spiral', () => {
|
||||||
|
dir = setupTestDir();
|
||||||
|
// 40 min, 50 tool calls, no edits, all reads — a research/audit session
|
||||||
|
createStateFile(dir, 'e1', freshState({
|
||||||
|
start_epoch: Math.floor(Date.now() / 1000) - 40 * 60,
|
||||||
|
tool_count: 49,
|
||||||
|
edit_count: 0,
|
||||||
|
read_count: 49,
|
||||||
|
}));
|
||||||
|
const out = runHook('tool-tracker.mjs', { session_id: 'e1', tool_name: 'Read' }, dir);
|
||||||
|
const ctx = out.hookSpecificOutput?.additionalContext || '';
|
||||||
|
assert.ok(!ctx.includes('stuck/spiral'), `expected no stuck/spiral claim, got: ${ctx}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('still reports low edit ratio when the session is not read-dominant', () => {
|
||||||
|
dir = setupTestDir();
|
||||||
|
// Same duration and volume, but reads are a minority (5/50)
|
||||||
|
createStateFile(dir, 'e2', freshState({
|
||||||
|
start_epoch: Math.floor(Date.now() / 1000) - 40 * 60,
|
||||||
|
tool_count: 49,
|
||||||
|
edit_count: 2,
|
||||||
|
read_count: 5,
|
||||||
|
}));
|
||||||
|
const out = runHook('tool-tracker.mjs', { session_id: 'e2', tool_name: 'Bash' }, dir);
|
||||||
|
const ctx = out.hookSpecificOutput?.additionalContext || '';
|
||||||
|
assert.ok(ctx.includes('stuck/spiral'), `expected stuck/spiral claim, got: ${ctx}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('counts read tools in read_count and leaves it alone for others', () => {
|
||||||
|
dir = setupTestDir();
|
||||||
|
createStateFile(dir, 'r1', freshState());
|
||||||
|
runHook('tool-tracker.mjs', { session_id: 'r1', tool_name: 'Grep' }, dir);
|
||||||
|
assert.equal(readState(dir, 'r1').read_count, 1);
|
||||||
|
runHook('tool-tracker.mjs', { session_id: 'r1', tool_name: 'Glob' }, dir);
|
||||||
|
assert.equal(readState(dir, 'r1').read_count, 2);
|
||||||
|
runHook('tool-tracker.mjs', { session_id: 'r1', tool_name: 'Write' }, dir);
|
||||||
|
assert.equal(readState(dir, 'r1').read_count, 2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue