From 58d6b5b9ea12ba9b9e1c6903ca213071332e8402 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 06:47:32 +0200 Subject: [PATCH] feat(config-audit): recalibrate TOK severities for tokens/turn (v5 F7) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pattern A (cache-breaking volatile top): medium → high - Pattern B (redundant permissions): low → medium - Pattern C (deep @import chain): medium → low - Add calibration_note evidence on every TOK finding - Table-driven severity tests (identify by title, IDs are sequential) 563 → 569 tests, all green. Doc sweep deferred to Session 5 (Step 28). --- .../config-audit/scanners/token-hotspots.mjs | 29 +++++++++++------ .../tests/scanners/token-hotspots.test.mjs | 32 +++++++++++++++++-- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/plugins/config-audit/scanners/token-hotspots.mjs b/plugins/config-audit/scanners/token-hotspots.mjs index cee0815..b24b221 100644 --- a/plugins/config-audit/scanners/token-hotspots.mjs +++ b/plugins/config-audit/scanners/token-hotspots.mjs @@ -1,10 +1,11 @@ /** * TOK Scanner — Token Hotspots / Opus 4.7 patterns * - * Detects three structural Opus 4.7-era token-efficiency patterns: - * CA-TOK-001 cache-breaking volatile top in CLAUDE.md (medium) - * CA-TOK-002 redundant tool/permission declarations (low) - * CA-TOK-003 deep @import chain (>2 hops) (medium) + * Detects three structural Opus 4.7-era token-efficiency patterns + * (severities recalibrated for tokens/turn impact in v5 F7): + * CA-TOK-001 cache-breaking volatile top in CLAUDE.md (high) + * CA-TOK-002 redundant tool/permission declarations (medium) + * CA-TOK-003 deep @import chain (>2 hops) (low) * * Note: the v4 sonnet-era signature pattern was removed in v5 F5 — too noisy * and not actionable; live token costs are better surfaced by the hotspots @@ -45,6 +46,13 @@ const MAX_IMPORT_DEPTH = 2; const HOTSPOTS_MAX = 10; +// v5 F7: shared evidence note appended to every TOK pattern finding. +// Communicates that severity reflects a structural heuristic, not measured +// runtime telemetry — tells reviewers how to interpret the rating. +const CALIBRATION_NOTE = + 'severity reflects estimated tokens/turn based on structural heuristic; ' + + 'not measured against runtime telemetry'; + /** * Classify a discovered config file into a token-estimation kind. */ @@ -255,13 +263,14 @@ export async function scan(targetPath, discovery) { if (detectVolatileTop(content)) { findings.push(finding({ scanner: SCANNER, - severity: SEVERITY.medium, + severity: SEVERITY.high, title: 'Cache-breaking volatile content at top of CLAUDE.md', description: `The first ${VOLATILE_TOP_LINES} lines of ${f.relPath || f.absPath} contain volatile ` + 'tokens (timestamps, session ids, or activity logs). Volatile content above stable ' + 'content defeats Opus 4.7 prompt-cache reuse on every turn.', file: f.absPath, + evidence: CALIBRATION_NOTE, recommendation: 'Move volatile sections to the bottom of the file, or extract them to an @import-ed ' + 'file outside the cached prefix. Keep the first 30 lines stable across turns.', @@ -282,14 +291,16 @@ export async function scan(targetPath, discovery) { if (issues.length === 0) continue; findings.push(finding({ scanner: SCANNER, - severity: SEVERITY.low, + severity: SEVERITY.medium, title: 'Redundant permission declarations', description: `${f.relPath || f.absPath} contains ${issues.length} redundant or overlapping ` + `permission entr${issues.length === 1 ? 'y' : 'ies'}. Each duplicate inflates the ` + 'tool-schema payload sent on every turn.', file: f.absPath, - evidence: issues.slice(0, 5).map(i => `${i.list}: "${i.entry}" (${i.reason})`).join('; '), + evidence: + issues.slice(0, 5).map(i => `${i.list}: "${i.entry}" (${i.reason})`).join('; ') + + ` — ${CALIBRATION_NOTE}`, recommendation: 'Deduplicate the permissions.allow / permissions.deny arrays. Prefer the most ' + 'specific entry that still grants the intended access.', @@ -304,14 +315,14 @@ export async function scan(targetPath, discovery) { if (depth > MAX_IMPORT_DEPTH) { findings.push(finding({ scanner: SCANNER, - severity: SEVERITY.medium, + severity: SEVERITY.low, title: 'Deep @import chain defeats prompt-cache reuse', description: `${f.relPath || f.absPath} reaches @import depth ${depth} (>${MAX_IMPORT_DEPTH} hops). ` + 'Each @import boundary fragments the prompt-cache prefix; deeply chained imports ' + 'defeat caching for the deepest content even when it never changes.', file: f.absPath, - evidence: `Max chain depth: ${depth}`, + evidence: `Max chain depth: ${depth} — ${CALIBRATION_NOTE}`, recommendation: 'Flatten the @import chain to ≤2 hops. Inline the deepest layer back into its parent.', category: 'token-efficiency', diff --git a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs index a94072d..12e7e3f 100644 --- a/plugins/config-audit/tests/scanners/token-hotspots.test.mjs +++ b/plugins/config-audit/tests/scanners/token-hotspots.test.mjs @@ -57,9 +57,9 @@ describe('TOK scanner — opus-47/cache-breaking', () => { assert.ok(f, 'expected a CA-TOK-001 finding for cache-breaking fixture'); }); - it('CA-TOK-001 severity is medium or low', () => { + it('CA-TOK-001 severity is high (v5 F7 recalibration)', () => { const f = result.findings.find(x => x.id === 'CA-TOK-001'); - assert.ok(['medium', 'low'].includes(f.severity), `unexpected severity ${f.severity}`); + assert.equal(f.severity, 'high', `expected high after F7, got ${f.severity}`); }); }); @@ -188,3 +188,31 @@ describe('TOK scanner — hotspots contract', () => { `expected ≤10 hotspots, got ${result.hotspots.length}`); }); }); + +describe('TOK scanner — F7 severity recalibration (v5)', () => { + // Findings identified by title pattern, not finding ID — TOK IDs are + // sequential per scan run, not semantic per pattern (output.mjs:31). + const SEVERITY_TABLE = [ + { fixture: 'opus-47/cache-breaking', pattern: 'A', titleMatch: /cache-breaking volatile/i, expected: 'high' }, + { fixture: 'opus-47/redundant-tools', pattern: 'B', titleMatch: /redundant permission/i, expected: 'medium' }, + { fixture: 'opus-47/deep-imports', pattern: 'C', titleMatch: /deep @import chain/i, expected: 'low' }, + ]; + + for (const { fixture, pattern, titleMatch, expected } of SEVERITY_TABLE) { + it(`Pattern ${pattern} (${fixture}) has severity ${expected}`, async () => { + const result = await runScanner(fixture); + const f = result.findings.find(x => titleMatch.test(x.title || '')); + assert.ok(f, `expected a finding matching ${titleMatch} in ${fixture}; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, expected, `expected ${expected}, got ${f.severity}`); + }); + + it(`Pattern ${pattern} (${fixture}) carries calibration_note evidence`, async () => { + const result = await runScanner(fixture); + const f = result.findings.find(x => titleMatch.test(x.title || '')); + assert.ok(f, `expected a finding matching ${titleMatch} in ${fixture}`); + const evidence = String(f.evidence || ''); + assert.ok(/severity reflects estimated tokens\/turn/i.test(evidence), + `expected calibration_note phrase in evidence, got: ${evidence}`); + }); + } +});