open/config-audit
1
0
Fork 0
Code Issues Releases 1 Activity

feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]

Browse source 
SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:

- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
  conservative 200k anchor but marks advisory (model→window probing deferred
  to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
  for byte-stability and a calibrated branch; advisory downgrades the budget
  finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
  and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
  150-line volatility heuristic), so there is nothing to calibrate.

15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 21:44:52 +02:00
commit 2082b7d112
10 changed files with 364 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

92
scanners/skill-listing-scanner.mjs
View file

@ -44,6 +44,15 @@ import {
BODY_CALIBRATION_NOTE,
measureActiveSkillListing,
} from './lib/skill-listing-budget.mjs';
import { CONTEXT_WINDOW_ANCHOR, scaleForWindow, withCommas } from './lib/context-window.mjs';
// Shared remediation for the aggregate-budget finding (byte-identical across the
// default and the B8 window-calibrated branches).
const AGGREGATE_RECOMMENDATION =
'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
'`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
'trigger phrases.';
const SCANNER = 'SKL';
@ -53,11 +62,21 @@ const SCANNER = 'SKL';
* @param {string} _targetPath unused (skill listing is HOME-scoped)
* @param {object} _discovery unused (ignores project discovery)
*/
export async function scan(_targetPath, _discovery) {
export async function scan(_targetPath, _discovery, opts = {}) {
const start = Date.now();
const findings = [];
const { skills, aggregate } = await measureActiveSkillListing();
// B8 — calibrate the aggregate budget to the resolved context window. The
// default (no opts) is the conservative 200k anchor at full severity, which is
// byte-identical to the pre-B8 behavior. An unknown (advisory) window keeps the
// anchor but downgrades the finding to info instead of firing it as a breach.
const cw = opts.contextWindow;
const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR;
const advisory = !!(cw && cw.advisory);
const isDefault = window === CONTEXT_WINDOW_ANCHOR && !advisory;
const budgetTokens = scaleForWindow(AGGREGATE_BUDGET_TOKENS, window);
const { skills, aggregate } = await measureActiveSkillListing(budgetTokens);
for (const skill of skills) {
if (skill.descLength <= DESCRIPTION_CAP) continue;
@ -93,29 +112,52 @@ export async function scan(_targetPath, _discovery) {
// CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common
// "one oversized skill + aggregate" case reads 001=cap, 002=aggregate.
if (aggregate.overBudget) {
findings.push(finding({
scanner: SCANNER,
severity: SEVERITY.low,
title: 'Aggregate skill descriptions may exceed the listing budget',
description:
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
`${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
'is an estimate — the budget scales with your actual context window (see evidence).',
evidence:
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` +
`${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
`${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
recommendation:
'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
'`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
'trigger phrases.',
category: 'token-efficiency',
}));
if (isDefault) {
// Conservative 200k anchor — byte-identical to the pre-B8 finding.
findings.push(finding({
scanner: SCANNER,
severity: SEVERITY.low,
title: 'Aggregate skill descriptions may exceed the listing budget',
description:
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
`${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
'is an estimate — the budget scales with your actual context window (see evidence).',
evidence:
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` +
`${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
`${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
recommendation: AGGREGATE_RECOMMENDATION,
category: 'token-efficiency',
}));
} else {
// B8 — window-calibrated. Advisory (unknown window) downgrades to info.
const winLabel = withCommas(window);
findings.push(finding({
scanner: SCANNER,
severity: advisory ? SEVERITY.info : SEVERITY.low,
title: 'Aggregate skill descriptions may exceed the listing budget',
description:
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
`${budgetTokens}-token budget Claude Code allots the skill listing at a ${winLabel}-token ` +
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
'Claude Code drops descriptions, so the model may stop seeing some skills entirely.' +
(advisory
? ' Your context window is unknown, so this is advisory: it anchors on the conservative 200k window.'
: ''),
evidence:
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@${winLabel}=` +
`${budgetTokens} tok (skill listing ~2% of context, CC 2.1.32); over_by~${aggregate.overBy} tok` +
(advisory ? ` - ${BUDGET_CALIBRATION_NOTE}` : ' - this is an estimate, not measured telemetry'),
recommendation: AGGREGATE_RECOMMENDATION,
category: 'token-efficiency',
}));
}
}
// CA-SKL-003 (oversized body). Emitted last so the common single-issue cases