feat(config-audit): recalibrate TOK severities for tokens/turn (v5 F7) [skip-docs]

- Pattern A (cache-breaking volatile top): medium → high
- Pattern B (redundant permissions): low → medium
- Pattern C (deep @import chain): medium → low
- Add calibration_note evidence on every TOK finding
- Table-driven severity tests (identify by title, IDs are sequential)

563 → 569 tests, all green. Doc sweep deferred to Session 5 (Step 28).
This commit is contained in:
Kjell Tore Guttormsen 2026-05-01 06:47:32 +02:00
commit 58d6b5b9ea
2 changed files with 50 additions and 11 deletions

View file

@ -1,10 +1,11 @@
/**
* TOK Scanner Token Hotspots / Opus 4.7 patterns
*
* Detects three structural Opus 4.7-era token-efficiency patterns:
* CA-TOK-001 cache-breaking volatile top in CLAUDE.md (medium)
* CA-TOK-002 redundant tool/permission declarations (low)
* CA-TOK-003 deep @import chain (>2 hops) (medium)
* Detects three structural Opus 4.7-era token-efficiency patterns
* (severities recalibrated for tokens/turn impact in v5 F7):
* CA-TOK-001 cache-breaking volatile top in CLAUDE.md (high)
* CA-TOK-002 redundant tool/permission declarations (medium)
* CA-TOK-003 deep @import chain (>2 hops) (low)
*
* Note: the v4 sonnet-era signature pattern was removed in v5 F5 too noisy
* and not actionable; live token costs are better surfaced by the hotspots
@ -45,6 +46,13 @@ const MAX_IMPORT_DEPTH = 2;
const HOTSPOTS_MAX = 10;
// v5 F7: shared evidence note appended to every TOK pattern finding.
// Communicates that severity reflects a structural heuristic, not measured
// runtime telemetry — tells reviewers how to interpret the rating.
const CALIBRATION_NOTE =
'severity reflects estimated tokens/turn based on structural heuristic; ' +
'not measured against runtime telemetry';
/**
* Classify a discovered config file into a token-estimation kind.
*/
@ -255,13 +263,14 @@ export async function scan(targetPath, discovery) {
if (detectVolatileTop(content)) {
findings.push(finding({
scanner: SCANNER,
severity: SEVERITY.medium,
severity: SEVERITY.high,
title: 'Cache-breaking volatile content at top of CLAUDE.md',
description:
`The first ${VOLATILE_TOP_LINES} lines of ${f.relPath || f.absPath} contain volatile ` +
'tokens (timestamps, session ids, or activity logs). Volatile content above stable ' +
'content defeats Opus 4.7 prompt-cache reuse on every turn.',
file: f.absPath,
evidence: CALIBRATION_NOTE,
recommendation:
'Move volatile sections to the bottom of the file, or extract them to an @import-ed ' +
'file outside the cached prefix. Keep the first 30 lines stable across turns.',
@ -282,14 +291,16 @@ export async function scan(targetPath, discovery) {
if (issues.length === 0) continue;
findings.push(finding({
scanner: SCANNER,
severity: SEVERITY.low,
severity: SEVERITY.medium,
title: 'Redundant permission declarations',
description:
`${f.relPath || f.absPath} contains ${issues.length} redundant or overlapping ` +
`permission entr${issues.length === 1 ? 'y' : 'ies'}. Each duplicate inflates the ` +
'tool-schema payload sent on every turn.',
file: f.absPath,
evidence: issues.slice(0, 5).map(i => `${i.list}: "${i.entry}" (${i.reason})`).join('; '),
evidence:
issues.slice(0, 5).map(i => `${i.list}: "${i.entry}" (${i.reason})`).join('; ') +
`${CALIBRATION_NOTE}`,
recommendation:
'Deduplicate the permissions.allow / permissions.deny arrays. Prefer the most ' +
'specific entry that still grants the intended access.',
@ -304,14 +315,14 @@ export async function scan(targetPath, discovery) {
if (depth > MAX_IMPORT_DEPTH) {
findings.push(finding({
scanner: SCANNER,
severity: SEVERITY.medium,
severity: SEVERITY.low,
title: 'Deep @import chain defeats prompt-cache reuse',
description:
`${f.relPath || f.absPath} reaches @import depth ${depth} (>${MAX_IMPORT_DEPTH} hops). ` +
'Each @import boundary fragments the prompt-cache prefix; deeply chained imports ' +
'defeat caching for the deepest content even when it never changes.',
file: f.absPath,
evidence: `Max chain depth: ${depth}`,
evidence: `Max chain depth: ${depth}${CALIBRATION_NOTE}`,
recommendation:
'Flatten the @import chain to ≤2 hops. Inline the deepest layer back into its parent.',
category: 'token-efficiency',