feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]
SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:
- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
conservative 200k anchor but marks advisory (model→window probing deferred
to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
for byte-stability and a calibrated branch; advisory downgrades the budget
finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
150-line volatility heuristic), so there is nothing to calibrate.
15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
27988801be
commit
2082b7d112
10 changed files with 364 additions and 45 deletions
|
|
@ -9,13 +9,18 @@ import { finding, scannerResult, resetCounter } from './lib/output.mjs';
|
|||
import { SEVERITY } from './lib/severity.mjs';
|
||||
import { parseFrontmatter, extractSections, findImports } from './lib/yaml-parser.mjs';
|
||||
import { lineCount, truncate } from './lib/string-utils.mjs';
|
||||
import { LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, withCommas } from './lib/context-window.mjs';
|
||||
import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, scaleForWindow, withCommas } from './lib/context-window.mjs';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
const SCANNER = 'CML';
|
||||
const MAX_RECOMMENDED_LINES = 200;
|
||||
const MAX_ABSOLUTE_LINES = 500;
|
||||
|
||||
// Shared remediation for the char-budget finding (byte-identical across the
|
||||
// default and the B8 window-calibrated branches).
|
||||
const CHAR_BUDGET_RECOMMENDATION =
|
||||
'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.';
|
||||
|
||||
// Claude Code's own startup warning ("Large CLAUDE.md will impact performance
|
||||
// (X chars > 40.0k)") fires once a CLAUDE.md passes ~40.0k chars on a
|
||||
// 200k-context model. CC 2.1.169 made that threshold scale with the model's
|
||||
|
|
@ -39,10 +44,20 @@ const RECOMMENDED_SECTIONS = [
|
|||
* @param {{ files: import('./lib/file-discovery.mjs').ConfigFile[] }} discovery
|
||||
* @returns {Promise<object>}
|
||||
*/
|
||||
export async function scan(targetPath, discovery) {
|
||||
export async function scan(targetPath, discovery, opts = {}) {
|
||||
const start = Date.now();
|
||||
const claudeFiles = discovery.files.filter(f => f.type === 'claude-md');
|
||||
|
||||
// B8 — calibrate the char-budget threshold to the resolved context window. The
|
||||
// default (no opts) is the conservative 200k anchor (40k chars) at full
|
||||
// severity — byte-identical to the pre-B8 finding. An unknown (advisory) window
|
||||
// keeps the anchor but downgrades the finding to info instead of a breach.
|
||||
const cw = opts.contextWindow;
|
||||
const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR;
|
||||
const advisory = !!(cw && cw.advisory);
|
||||
const isDefaultWindow = window === CONTEXT_WINDOW_ANCHOR && !advisory;
|
||||
const charThreshold = scaleForWindow(CLAUDE_MD_CHAR_WARN_ANCHOR, window);
|
||||
|
||||
if (claudeFiles.length === 0) {
|
||||
return scannerResult(SCANNER, 'ok', [
|
||||
finding({
|
||||
|
|
@ -122,17 +137,35 @@ export async function scan(targetPath, discovery) {
|
|||
// this budget (short lines), or short by lines yet over it (long lines), so
|
||||
// this is complementary to the line-count checks above.
|
||||
const chars = content.length;
|
||||
if (chars > CLAUDE_MD_CHAR_WARN_ANCHOR) {
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.medium,
|
||||
title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold',
|
||||
description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`,
|
||||
file: file.absPath,
|
||||
evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`,
|
||||
recommendation: 'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.',
|
||||
autoFixable: false,
|
||||
}));
|
||||
if (chars > charThreshold) {
|
||||
if (isDefaultWindow) {
|
||||
// Conservative 200k anchor — byte-identical to the pre-B8 finding.
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.medium,
|
||||
title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold',
|
||||
description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`,
|
||||
file: file.absPath,
|
||||
evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`,
|
||||
recommendation: CHAR_BUDGET_RECOMMENDATION,
|
||||
autoFixable: false,
|
||||
}));
|
||||
} else {
|
||||
// B8 — window-calibrated. Advisory (unknown window) downgrades to info.
|
||||
const winLabel = withCommas(window);
|
||||
const threshLabel = withCommas(charThreshold);
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: advisory ? SEVERITY.info : SEVERITY.medium,
|
||||
title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold',
|
||||
description: `${file.relPath} is ${withCommas(chars)} chars, over the ~${threshLabel}-char performance-warning threshold Claude Code applies at a ${winLabel}-token context window (it scales the ~40.0k-char @ 200k warning by the context window, CC 2.1.169) — it loads in full on every turn.` +
|
||||
(advisory ? ' Your context window is unknown, so this anchors on the conservative 200k window — advisory.' : ''),
|
||||
file: file.absPath,
|
||||
evidence: `${withCommas(chars)} chars > ${threshLabel} (calibrated to a ${winLabel}-token context window). This is an estimate, not measured telemetry.`,
|
||||
recommendation: CHAR_BUDGET_RECOMMENDATION,
|
||||
autoFixable: false,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// --- Empty file ---
|
||||
|
|
|
|||
|
|
@ -26,3 +26,52 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR;
|
|||
|
||||
// Dependency-free thousands separator (repo invariant: zero external deps).
|
||||
export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
|
||||
|
||||
/**
|
||||
* @typedef {object} ResolvedContextWindow
|
||||
* @property {number} window - the context window budgets calibrate against
|
||||
* @property {boolean} advisory - true when the window is unknown: keep the anchor
|
||||
* but downgrade budget findings to info instead of
|
||||
* firing them as a breach
|
||||
* @property {'default'|'explicit'|'auto-unresolved'} source
|
||||
*/
|
||||
|
||||
/**
|
||||
* Resolve the raw `--context-window` CLI value into a window + advisory flag.
|
||||
*
|
||||
* Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior —
|
||||
* the conservative 200k anchor at full severity. Only an explicit value changes
|
||||
* calibration. `auto` asks the tool to figure out the window; until model→window
|
||||
* probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the
|
||||
* result advisory so SKL/CML downgrade their budget findings to info rather than
|
||||
* "crying wolf" with a breach on a window we cannot confirm.
|
||||
*
|
||||
* @param {string|number|null|undefined} arg
|
||||
* @returns {ResolvedContextWindow}
|
||||
*/
|
||||
export function resolveContextWindow(arg) {
|
||||
if (arg == null) {
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' };
|
||||
}
|
||||
if (String(arg).trim().toLowerCase() === 'auto') {
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' };
|
||||
}
|
||||
const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10);
|
||||
if (Number.isFinite(n) && n > 0) {
|
||||
return { window: n, advisory: false, source: 'explicit' };
|
||||
}
|
||||
// Unparseable / non-positive: fall back to the conservative default (no advisory).
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale a 200k-anchored budget to a given context window. Linear in the window,
|
||||
* so it is the identity at the anchor (keeps the default byte-stable).
|
||||
*
|
||||
* @param {number} anchorValue - the budget/threshold defined at the 200k anchor
|
||||
* @param {number} window - the target context window
|
||||
* @returns {number}
|
||||
*/
|
||||
export function scaleForWindow(anchorValue, window) {
|
||||
return Math.round(anchorValue * (window / CONTEXT_WINDOW_ANCHOR));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,23 +77,26 @@ export const BODY_CALIBRATION_NOTE =
|
|||
* flags it — so the aggregate does not double-count it).
|
||||
*
|
||||
* @param {number[]} descLengths - one entry per active skill (description char count)
|
||||
* @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - the listing budget to
|
||||
* measure against. Defaults to the 200k-anchored 4,000 tok; B8 passes a
|
||||
* window-calibrated budget. Defaulting keeps existing callers byte-stable.
|
||||
* @returns {BudgetAssessment}
|
||||
*/
|
||||
export function assessSkillListingBudget(descLengths) {
|
||||
export function assessSkillListingBudget(descLengths, budgetTokens = AGGREGATE_BUDGET_TOKENS) {
|
||||
let aggregateChars = 0;
|
||||
for (const len of descLengths) {
|
||||
const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0;
|
||||
aggregateChars += Math.min(safe, DESCRIPTION_CAP);
|
||||
}
|
||||
const aggregateTokens = estimateTokens(aggregateChars, 'markdown');
|
||||
const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS;
|
||||
const overBudget = aggregateTokens > budgetTokens;
|
||||
return {
|
||||
scanned: descLengths.length,
|
||||
aggregateChars,
|
||||
aggregateTokens,
|
||||
budgetTokens: AGGREGATE_BUDGET_TOKENS,
|
||||
budgetTokens,
|
||||
overBudget,
|
||||
overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0,
|
||||
overBy: overBudget ? aggregateTokens - budgetTokens : 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -115,9 +118,11 @@ export function assessSkillListingBudget(descLengths) {
|
|||
* enumerateSkills). Callers that run under test MUST override HOME (see the
|
||||
* hermetic-home helper / runScannerWithHome pattern).
|
||||
*
|
||||
* @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - listing budget for the
|
||||
* aggregate assessment (B8 window-calibration); defaults keep callers byte-stable.
|
||||
* @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>}
|
||||
*/
|
||||
export async function measureActiveSkillListing() {
|
||||
export async function measureActiveSkillListing(budgetTokens = AGGREGATE_BUDGET_TOKENS) {
|
||||
const plugins = await enumeratePlugins();
|
||||
const allSkills = await enumerateSkills(plugins);
|
||||
|
||||
|
|
@ -143,7 +148,7 @@ export async function measureActiveSkillListing() {
|
|||
});
|
||||
}
|
||||
|
||||
const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength));
|
||||
const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength), budgetTokens);
|
||||
return { skills, aggregate };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -63,10 +63,13 @@ async function main() {
|
|||
let rawMode = false;
|
||||
let includeGlobal = false;
|
||||
let fullMachine = false;
|
||||
let contextWindow = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--output-file' && args[i + 1]) {
|
||||
outputFile = args[++i];
|
||||
} else if (args[i] === '--context-window' && args[i + 1]) {
|
||||
contextWindow = args[++i];
|
||||
} else if (args[i] === '--json') {
|
||||
jsonMode = true;
|
||||
} else if (args[i] === '--raw') {
|
||||
|
|
@ -89,6 +92,7 @@ async function main() {
|
|||
fullMachine,
|
||||
filterFixtures,
|
||||
humanizedProgress,
|
||||
contextWindow,
|
||||
});
|
||||
|
||||
// stdout JSON path: --json and --raw both write the v5.0.0-shape result
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import { envelope } from './lib/output.mjs';
|
|||
import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths } from './lib/file-discovery.mjs';
|
||||
import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs';
|
||||
import { humanizeEnvelope } from './lib/humanizer.mjs';
|
||||
import { resolveContextWindow } from './lib/context-window.mjs';
|
||||
|
||||
// Scanner registry — import order determines execution order
|
||||
import { scan as scanClaudeMd } from './claude-md-linter.mjs';
|
||||
|
|
@ -94,6 +95,11 @@ export async function runAllScanners(targetPath, opts = {}) {
|
|||
// and CNF duplicate-hook findings with config that loads on zero turns. (B3)
|
||||
const excludeCache = opts.excludeCache !== false;
|
||||
|
||||
// B8 — resolve the context window once and thread it to budget-aware scanners
|
||||
// (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is
|
||||
// byte-identical to the pre-B8 default; other scanners ignore the third arg.
|
||||
const contextWindow = resolveContextWindow(opts.contextWindow);
|
||||
|
||||
// Shared file discovery — scanners reuse this
|
||||
let discovery;
|
||||
if (opts.fullMachine) {
|
||||
|
|
@ -112,7 +118,7 @@ export async function runAllScanners(targetPath, opts = {}) {
|
|||
resetCounter();
|
||||
const scanStart = Date.now();
|
||||
try {
|
||||
const result = await scanner.fn(resolvedPath, discovery);
|
||||
const result = await scanner.fn(resolvedPath, discovery, { contextWindow });
|
||||
results.push(result);
|
||||
const count = result.findings.length;
|
||||
const label = opts.humanizedProgress
|
||||
|
|
@ -206,10 +212,13 @@ async function main() {
|
|||
let outputFile = null;
|
||||
let saveBaseline = false;
|
||||
let baselinePath = null;
|
||||
let contextWindow = null;
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--output-file' && args[i + 1]) {
|
||||
outputFile = args[++i];
|
||||
} else if (args[i] === '--context-window' && args[i + 1]) {
|
||||
contextWindow = args[++i];
|
||||
} else if (args[i] === '--save-baseline') {
|
||||
saveBaseline = true;
|
||||
} else if (args[i] === '--baseline' && args[i + 1]) {
|
||||
|
|
@ -254,6 +263,7 @@ async function main() {
|
|||
filterFixtures,
|
||||
excludeCache,
|
||||
humanizedProgress,
|
||||
contextWindow,
|
||||
});
|
||||
|
||||
// Default mode runs the humanizer; --json and --raw bypass for v5.0.0 byte-equal output.
|
||||
|
|
|
|||
|
|
@ -44,6 +44,15 @@ import {
|
|||
BODY_CALIBRATION_NOTE,
|
||||
measureActiveSkillListing,
|
||||
} from './lib/skill-listing-budget.mjs';
|
||||
import { CONTEXT_WINDOW_ANCHOR, scaleForWindow, withCommas } from './lib/context-window.mjs';
|
||||
|
||||
// Shared remediation for the aggregate-budget finding (byte-identical across the
|
||||
// default and the B8 window-calibrated branches).
|
||||
const AGGREGATE_RECOMMENDATION =
|
||||
'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
|
||||
'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
|
||||
'`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
|
||||
'trigger phrases.';
|
||||
|
||||
const SCANNER = 'SKL';
|
||||
|
||||
|
|
@ -53,11 +62,21 @@ const SCANNER = 'SKL';
|
|||
* @param {string} _targetPath unused (skill listing is HOME-scoped)
|
||||
* @param {object} _discovery unused (ignores project discovery)
|
||||
*/
|
||||
export async function scan(_targetPath, _discovery) {
|
||||
export async function scan(_targetPath, _discovery, opts = {}) {
|
||||
const start = Date.now();
|
||||
const findings = [];
|
||||
|
||||
const { skills, aggregate } = await measureActiveSkillListing();
|
||||
// B8 — calibrate the aggregate budget to the resolved context window. The
|
||||
// default (no opts) is the conservative 200k anchor at full severity, which is
|
||||
// byte-identical to the pre-B8 behavior. An unknown (advisory) window keeps the
|
||||
// anchor but downgrades the finding to info instead of firing it as a breach.
|
||||
const cw = opts.contextWindow;
|
||||
const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR;
|
||||
const advisory = !!(cw && cw.advisory);
|
||||
const isDefault = window === CONTEXT_WINDOW_ANCHOR && !advisory;
|
||||
const budgetTokens = scaleForWindow(AGGREGATE_BUDGET_TOKENS, window);
|
||||
|
||||
const { skills, aggregate } = await measureActiveSkillListing(budgetTokens);
|
||||
|
||||
for (const skill of skills) {
|
||||
if (skill.descLength <= DESCRIPTION_CAP) continue;
|
||||
|
|
@ -93,29 +112,52 @@ export async function scan(_targetPath, _discovery) {
|
|||
// CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common
|
||||
// "one oversized skill + aggregate" case reads 001=cap, 002=aggregate.
|
||||
if (aggregate.overBudget) {
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.low,
|
||||
title: 'Aggregate skill descriptions may exceed the listing budget',
|
||||
description:
|
||||
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
|
||||
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
|
||||
`${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
|
||||
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
|
||||
'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
|
||||
'is an estimate — the budget scales with your actual context window (see evidence).',
|
||||
evidence:
|
||||
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
|
||||
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` +
|
||||
`${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
|
||||
`${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
|
||||
recommendation:
|
||||
'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' +
|
||||
'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' +
|
||||
'`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' +
|
||||
'trigger phrases.',
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
if (isDefault) {
|
||||
// Conservative 200k anchor — byte-identical to the pre-B8 finding.
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: SEVERITY.low,
|
||||
title: 'Aggregate skill descriptions may exceed the listing budget',
|
||||
description:
|
||||
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
|
||||
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
|
||||
`${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` +
|
||||
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
|
||||
'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' +
|
||||
'is an estimate — the budget scales with your actual context window (see evidence).',
|
||||
evidence:
|
||||
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
|
||||
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` +
|
||||
`${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` +
|
||||
`${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`,
|
||||
recommendation: AGGREGATE_RECOMMENDATION,
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
} else {
|
||||
// B8 — window-calibrated. Advisory (unknown window) downgrades to info.
|
||||
const winLabel = withCommas(window);
|
||||
findings.push(finding({
|
||||
scanner: SCANNER,
|
||||
severity: advisory ? SEVERITY.info : SEVERITY.low,
|
||||
title: 'Aggregate skill descriptions may exceed the listing budget',
|
||||
description:
|
||||
`The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` +
|
||||
`(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` +
|
||||
`${budgetTokens}-token budget Claude Code allots the skill listing at a ${winLabel}-token ` +
|
||||
'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' +
|
||||
'Claude Code drops descriptions, so the model may stop seeing some skills entirely.' +
|
||||
(advisory
|
||||
? ' Your context window is unknown, so this is advisory: it anchors on the conservative 200k window.'
|
||||
: ''),
|
||||
evidence:
|
||||
`active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` +
|
||||
`${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@${winLabel}=` +
|
||||
`${budgetTokens} tok (skill listing ~2% of context, CC 2.1.32); over_by~${aggregate.overBy} tok` +
|
||||
(advisory ? ` - ${BUDGET_CALIBRATION_NOTE}` : ' - this is an estimate, not measured telemetry'),
|
||||
recommendation: AGGREGATE_RECOMMENDATION,
|
||||
category: 'token-efficiency',
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// CA-SKL-003 (oversized body). Emitted last so the common single-issue cases
|
||||
|
|
|
|||
76
tests/lib/context-window.test.mjs
Normal file
76
tests/lib/context-window.test.mjs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
CONTEXT_WINDOW_ANCHOR,
|
||||
LARGE_CONTEXT_WINDOW,
|
||||
resolveContextWindow,
|
||||
scaleForWindow,
|
||||
} from '../../scanners/lib/context-window.mjs';
|
||||
|
||||
// B8 — context-window calibration. resolveContextWindow turns the raw
|
||||
// --context-window CLI value into { window, advisory } that SKL/CML calibrate
|
||||
// their budgets with. The DEFAULT (no flag) must be byte-identical to the
|
||||
// pre-B8 behavior: the conservative 200k anchor at full severity (advisory=false).
|
||||
|
||||
describe('resolveContextWindow — default (no flag) is the conservative anchor', () => {
|
||||
it('undefined resolves to the 200k anchor, not advisory (byte-stable default)', () => {
|
||||
const r = resolveContextWindow(undefined);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
|
||||
assert.equal(r.advisory, false);
|
||||
assert.equal(r.source, 'default');
|
||||
});
|
||||
|
||||
it('null resolves to the conservative default', () => {
|
||||
const r = resolveContextWindow(null);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
|
||||
assert.equal(r.advisory, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveContextWindow — explicit window', () => {
|
||||
it('a numeric string calibrates to that window, not advisory', () => {
|
||||
const r = resolveContextWindow('1000000');
|
||||
assert.equal(r.window, 1_000_000);
|
||||
assert.equal(r.advisory, false);
|
||||
assert.equal(r.source, 'explicit');
|
||||
});
|
||||
|
||||
it('a plain number is accepted', () => {
|
||||
const r = resolveContextWindow(1_000_000);
|
||||
assert.equal(r.window, 1_000_000);
|
||||
assert.equal(r.advisory, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveContextWindow — auto / unknown downgrades to advisory', () => {
|
||||
it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => {
|
||||
const r = resolveContextWindow('auto');
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor');
|
||||
assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach');
|
||||
assert.equal(r.source, 'auto-unresolved');
|
||||
});
|
||||
|
||||
it('"AUTO" is case-insensitive', () => {
|
||||
assert.equal(resolveContextWindow('AUTO').advisory, true);
|
||||
});
|
||||
|
||||
it('an invalid value falls back to the conservative default (not advisory)', () => {
|
||||
for (const bad of ['banana', '0', '-5', '']) {
|
||||
const r = resolveContextWindow(bad);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, `"${bad}" -> anchor`);
|
||||
assert.equal(r.advisory, false, `"${bad}" -> not advisory`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('scaleForWindow — linear scaling off the 200k anchor', () => {
|
||||
it('is the identity at the 200k anchor (byte-stable default)', () => {
|
||||
assert.equal(scaleForWindow(4000, CONTEXT_WINDOW_ANCHOR), 4000);
|
||||
assert.equal(scaleForWindow(40_000, CONTEXT_WINDOW_ANCHOR), 40_000);
|
||||
});
|
||||
|
||||
it('scales 5x at the 1M window', () => {
|
||||
assert.equal(scaleForWindow(4000, LARGE_CONTEXT_WINDOW), 20_000);
|
||||
assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000);
|
||||
});
|
||||
});
|
||||
|
|
@ -102,6 +102,20 @@ describe('assessSkillListingBudget — aggregate math', () => {
|
|||
assert.equal(r.aggregateChars, 100);
|
||||
assert.equal(r.scanned, 3);
|
||||
});
|
||||
|
||||
it('accepts a calibrated budget (B8): 17×1000 chars is within a 20,000-tok 1M budget', () => {
|
||||
const r = assessSkillListingBudget(Array(17).fill(1000), 20_000);
|
||||
assert.equal(r.aggregateTokens, 4250);
|
||||
assert.equal(r.budgetTokens, 20_000, 'reports the calibrated budget, not the 200k default');
|
||||
assert.equal(r.overBudget, false, 'within the relaxed 1M budget');
|
||||
assert.equal(r.overBy, 0);
|
||||
});
|
||||
|
||||
it('the budget argument defaults to the 200k anchor (byte-stable for existing callers)', () => {
|
||||
const r = assessSkillListingBudget(Array(17).fill(1000));
|
||||
assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS);
|
||||
assert.equal(r.overBudget, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('envFlag', () => {
|
||||
|
|
|
|||
|
|
@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)',
|
|||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — context-window calibration (B8)', () => {
|
||||
const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars
|
||||
const charFinding = (r) =>
|
||||
r.findings.find((f) => /performance-warning threshold/i.test(f.title || ''));
|
||||
|
||||
async function scanWithCtx(contextWindow) {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
return scan(FIXTURE, discovery, { contextWindow });
|
||||
}
|
||||
|
||||
it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => {
|
||||
const at1m = await scanWithCtx({ window: 1_000_000, advisory: false });
|
||||
assert.equal(charFinding(at1m), undefined,
|
||||
'48,531 chars is under the ~200,000-char threshold at a 1M window');
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => {
|
||||
const advisory = await scanWithCtx({ window: 200_000, advisory: true });
|
||||
const f = charFinding(advisory);
|
||||
assert.ok(f, 'still surfaces the measurement at the conservative anchor');
|
||||
assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info');
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
const result = await scan(FIXTURE, discovery);
|
||||
assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => {
|
||||
// large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines):
|
||||
// under CC's 40.0k char threshold, so the char-budget finding must NOT fire —
|
||||
|
|
|
|||
|
|
@ -28,6 +28,18 @@ async function runScannerWithHome(home) {
|
|||
}
|
||||
}
|
||||
|
||||
/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */
|
||||
async function runScannerWithCtx(home, contextWindow) {
|
||||
resetCounter();
|
||||
const original = process.env.HOME;
|
||||
process.env.HOME = home;
|
||||
try {
|
||||
return await scan('/unused', { files: [] }, { contextWindow });
|
||||
} finally {
|
||||
process.env.HOME = original;
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a fake HOME with one user skill whose description has `len` chars. */
|
||||
async function homeWithUserSkill(name, descLen) {
|
||||
const home = uniqueDir(name);
|
||||
|
|
@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — context-window calibration (B8)', () => {
|
||||
it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => {
|
||||
// 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k
|
||||
// budget, under the 20,000-tok 1M budget — the acceptance case.
|
||||
const home = await homeWithNUserSkills(50, 400, 'cw');
|
||||
try {
|
||||
const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false });
|
||||
assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor');
|
||||
|
||||
const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false });
|
||||
assert.equal(findAggregate(at1m.findings), undefined,
|
||||
'at a 1M context window the listing is within budget and must not fire');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwadv');
|
||||
try {
|
||||
const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true });
|
||||
const agg = findAggregate(advisory.findings);
|
||||
assert.ok(agg, 'still surfaces the measurement (conservative anchor)');
|
||||
assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires low at the 200k anchor', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwdef');
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg);
|
||||
assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
|
||||
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue