From 2082b7d112c0f75db41f37db819282094cfefca7 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 21:44:52 +0200 Subject: [PATCH] feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKL-002 (skill-listing budget) and CML char-budget now calibrate to a resolved context window instead of always anchoring at 200k: - resolveContextWindow(): --context-window calibrates; 'auto' keeps the conservative 200k anchor but marks advisory (model→window probing deferred to B8b); no flag → 200k anchor, byte-identical to pre-B8 default. - scaleForWindow(): linear off the 200k anchor (identity at the anchor). - SKL + CML each keep an untouched default branch (window===200k && !advisory) for byte-stability and a calibrated branch; advisory downgrades the budget finding from a breach (low/medium) to info. - Flag wired through scan-orchestrator + posture; runAllScanners resolves once and threads { contextWindow } to scanners (others ignore the 3rd arg). - CPS intentionally excluded: it has no window-anchored budget (fixed 150-line volatility heuristic), so there is nothing to calibrate. 15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default unchanged); full suite 1279 green; snapshots byte-stable. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/claude-md-linter.mjs | 59 +++++++++--- scanners/lib/context-window.mjs | 49 ++++++++++ scanners/lib/skill-listing-budget.mjs | 17 ++-- scanners/posture.mjs | 4 + scanners/scan-orchestrator.mjs | 12 ++- scanners/skill-listing-scanner.mjs | 92 ++++++++++++++----- tests/lib/context-window.test.mjs | 76 +++++++++++++++ tests/lib/skill-listing-budget.test.mjs | 14 +++ tests/scanners/claude-md-linter.test.mjs | 32 +++++++ tests/scanners/skill-listing-scanner.test.mjs | 54 +++++++++++ 10 files changed, 364 insertions(+), 45 deletions(-) create mode 100644 tests/lib/context-window.test.mjs diff --git a/scanners/claude-md-linter.mjs b/scanners/claude-md-linter.mjs index 316f702..8878416 100644 --- a/scanners/claude-md-linter.mjs +++ b/scanners/claude-md-linter.mjs @@ -9,13 +9,18 @@ import { finding, scannerResult, resetCounter } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { parseFrontmatter, extractSections, findImports } from './lib/yaml-parser.mjs'; import { lineCount, truncate } from './lib/string-utils.mjs'; -import { LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, withCommas } from './lib/context-window.mjs'; +import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, scaleForWindow, withCommas } from './lib/context-window.mjs'; import { dirname } from 'node:path'; const SCANNER = 'CML'; const MAX_RECOMMENDED_LINES = 200; const MAX_ABSOLUTE_LINES = 500; +// Shared remediation for the char-budget finding (byte-identical across the +// default and the B8 window-calibrated branches). +const CHAR_BUDGET_RECOMMENDATION = + 'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.'; + // Claude Code's own startup warning ("Large CLAUDE.md will impact performance // (X chars > 40.0k)") fires once a CLAUDE.md passes ~40.0k chars on a // 200k-context model. CC 2.1.169 made that threshold scale with the model's @@ -39,10 +44,20 @@ const RECOMMENDED_SECTIONS = [ * @param {{ files: import('./lib/file-discovery.mjs').ConfigFile[] }} discovery * @returns {Promise} */ -export async function scan(targetPath, discovery) { +export async function scan(targetPath, discovery, opts = {}) { const start = Date.now(); const claudeFiles = discovery.files.filter(f => f.type === 'claude-md'); + // B8 — calibrate the char-budget threshold to the resolved context window. The + // default (no opts) is the conservative 200k anchor (40k chars) at full + // severity — byte-identical to the pre-B8 finding. An unknown (advisory) window + // keeps the anchor but downgrades the finding to info instead of a breach. + const cw = opts.contextWindow; + const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR; + const advisory = !!(cw && cw.advisory); + const isDefaultWindow = window === CONTEXT_WINDOW_ANCHOR && !advisory; + const charThreshold = scaleForWindow(CLAUDE_MD_CHAR_WARN_ANCHOR, window); + if (claudeFiles.length === 0) { return scannerResult(SCANNER, 'ok', [ finding({ @@ -122,17 +137,35 @@ export async function scan(targetPath, discovery) { // this budget (short lines), or short by lines yet over it (long lines), so // this is complementary to the line-count checks above. const chars = content.length; - if (chars > CLAUDE_MD_CHAR_WARN_ANCHOR) { - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.medium, - title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', - description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`, - file: file.absPath, - evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`, - recommendation: 'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.', - autoFixable: false, - })); + if (chars > charThreshold) { + if (isDefaultWindow) { + // Conservative 200k anchor — byte-identical to the pre-B8 finding. + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.medium, + title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', + description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`, + file: file.absPath, + evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`, + recommendation: CHAR_BUDGET_RECOMMENDATION, + autoFixable: false, + })); + } else { + // B8 — window-calibrated. Advisory (unknown window) downgrades to info. + const winLabel = withCommas(window); + const threshLabel = withCommas(charThreshold); + findings.push(finding({ + scanner: SCANNER, + severity: advisory ? SEVERITY.info : SEVERITY.medium, + title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', + description: `${file.relPath} is ${withCommas(chars)} chars, over the ~${threshLabel}-char performance-warning threshold Claude Code applies at a ${winLabel}-token context window (it scales the ~40.0k-char @ 200k warning by the context window, CC 2.1.169) — it loads in full on every turn.` + + (advisory ? ' Your context window is unknown, so this anchors on the conservative 200k window — advisory.' : ''), + file: file.absPath, + evidence: `${withCommas(chars)} chars > ${threshLabel} (calibrated to a ${winLabel}-token context window). This is an estimate, not measured telemetry.`, + recommendation: CHAR_BUDGET_RECOMMENDATION, + autoFixable: false, + })); + } } // --- Empty file --- diff --git a/scanners/lib/context-window.mjs b/scanners/lib/context-window.mjs index de33374..15c914a 100644 --- a/scanners/lib/context-window.mjs +++ b/scanners/lib/context-window.mjs @@ -26,3 +26,52 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR; // Dependency-free thousands separator (repo invariant: zero external deps). export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); + +/** + * @typedef {object} ResolvedContextWindow + * @property {number} window - the context window budgets calibrate against + * @property {boolean} advisory - true when the window is unknown: keep the anchor + * but downgrade budget findings to info instead of + * firing them as a breach + * @property {'default'|'explicit'|'auto-unresolved'} source + */ + +/** + * Resolve the raw `--context-window` CLI value into a window + advisory flag. + * + * Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior — + * the conservative 200k anchor at full severity. Only an explicit value changes + * calibration. `auto` asks the tool to figure out the window; until model→window + * probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the + * result advisory so SKL/CML downgrade their budget findings to info rather than + * "crying wolf" with a breach on a window we cannot confirm. + * + * @param {string|number|null|undefined} arg + * @returns {ResolvedContextWindow} + */ +export function resolveContextWindow(arg) { + if (arg == null) { + return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; + } + if (String(arg).trim().toLowerCase() === 'auto') { + return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' }; + } + const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10); + if (Number.isFinite(n) && n > 0) { + return { window: n, advisory: false, source: 'explicit' }; + } + // Unparseable / non-positive: fall back to the conservative default (no advisory). + return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; +} + +/** + * Scale a 200k-anchored budget to a given context window. Linear in the window, + * so it is the identity at the anchor (keeps the default byte-stable). + * + * @param {number} anchorValue - the budget/threshold defined at the 200k anchor + * @param {number} window - the target context window + * @returns {number} + */ +export function scaleForWindow(anchorValue, window) { + return Math.round(anchorValue * (window / CONTEXT_WINDOW_ANCHOR)); +} diff --git a/scanners/lib/skill-listing-budget.mjs b/scanners/lib/skill-listing-budget.mjs index d68682e..ff8886b 100644 --- a/scanners/lib/skill-listing-budget.mjs +++ b/scanners/lib/skill-listing-budget.mjs @@ -77,23 +77,26 @@ export const BODY_CALIBRATION_NOTE = * flags it — so the aggregate does not double-count it). * * @param {number[]} descLengths - one entry per active skill (description char count) + * @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - the listing budget to + * measure against. Defaults to the 200k-anchored 4,000 tok; B8 passes a + * window-calibrated budget. Defaulting keeps existing callers byte-stable. * @returns {BudgetAssessment} */ -export function assessSkillListingBudget(descLengths) { +export function assessSkillListingBudget(descLengths, budgetTokens = AGGREGATE_BUDGET_TOKENS) { let aggregateChars = 0; for (const len of descLengths) { const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0; aggregateChars += Math.min(safe, DESCRIPTION_CAP); } const aggregateTokens = estimateTokens(aggregateChars, 'markdown'); - const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS; + const overBudget = aggregateTokens > budgetTokens; return { scanned: descLengths.length, aggregateChars, aggregateTokens, - budgetTokens: AGGREGATE_BUDGET_TOKENS, + budgetTokens, overBudget, - overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0, + overBy: overBudget ? aggregateTokens - budgetTokens : 0, }; } @@ -115,9 +118,11 @@ export function assessSkillListingBudget(descLengths) { * enumerateSkills). Callers that run under test MUST override HOME (see the * hermetic-home helper / runScannerWithHome pattern). * + * @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - listing budget for the + * aggregate assessment (B8 window-calibration); defaults keep callers byte-stable. * @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>} */ -export async function measureActiveSkillListing() { +export async function measureActiveSkillListing(budgetTokens = AGGREGATE_BUDGET_TOKENS) { const plugins = await enumeratePlugins(); const allSkills = await enumerateSkills(plugins); @@ -143,7 +148,7 @@ export async function measureActiveSkillListing() { }); } - const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength)); + const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength), budgetTokens); return { skills, aggregate }; } diff --git a/scanners/posture.mjs b/scanners/posture.mjs index 4a480e5..db12564 100644 --- a/scanners/posture.mjs +++ b/scanners/posture.mjs @@ -63,10 +63,13 @@ async function main() { let rawMode = false; let includeGlobal = false; let fullMachine = false; + let contextWindow = null; for (let i = 0; i < args.length; i++) { if (args[i] === '--output-file' && args[i + 1]) { outputFile = args[++i]; + } else if (args[i] === '--context-window' && args[i + 1]) { + contextWindow = args[++i]; } else if (args[i] === '--json') { jsonMode = true; } else if (args[i] === '--raw') { @@ -89,6 +92,7 @@ async function main() { fullMachine, filterFixtures, humanizedProgress, + contextWindow, }); // stdout JSON path: --json and --raw both write the v5.0.0-shape result diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 096eb12..71864ea 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -14,6 +14,7 @@ import { envelope } from './lib/output.mjs'; import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths } from './lib/file-discovery.mjs'; import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs'; import { humanizeEnvelope } from './lib/humanizer.mjs'; +import { resolveContextWindow } from './lib/context-window.mjs'; // Scanner registry — import order determines execution order import { scan as scanClaudeMd } from './claude-md-linter.mjs'; @@ -94,6 +95,11 @@ export async function runAllScanners(targetPath, opts = {}) { // and CNF duplicate-hook findings with config that loads on zero turns. (B3) const excludeCache = opts.excludeCache !== false; + // B8 — resolve the context window once and thread it to budget-aware scanners + // (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is + // byte-identical to the pre-B8 default; other scanners ignore the third arg. + const contextWindow = resolveContextWindow(opts.contextWindow); + // Shared file discovery — scanners reuse this let discovery; if (opts.fullMachine) { @@ -112,7 +118,7 @@ export async function runAllScanners(targetPath, opts = {}) { resetCounter(); const scanStart = Date.now(); try { - const result = await scanner.fn(resolvedPath, discovery); + const result = await scanner.fn(resolvedPath, discovery, { contextWindow }); results.push(result); const count = result.findings.length; const label = opts.humanizedProgress @@ -206,10 +212,13 @@ async function main() { let outputFile = null; let saveBaseline = false; let baselinePath = null; + let contextWindow = null; for (let i = 0; i < args.length; i++) { if (args[i] === '--output-file' && args[i + 1]) { outputFile = args[++i]; + } else if (args[i] === '--context-window' && args[i + 1]) { + contextWindow = args[++i]; } else if (args[i] === '--save-baseline') { saveBaseline = true; } else if (args[i] === '--baseline' && args[i + 1]) { @@ -254,6 +263,7 @@ async function main() { filterFixtures, excludeCache, humanizedProgress, + contextWindow, }); // Default mode runs the humanizer; --json and --raw bypass for v5.0.0 byte-equal output. diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs index cbbf16e..30ceb08 100644 --- a/scanners/skill-listing-scanner.mjs +++ b/scanners/skill-listing-scanner.mjs @@ -44,6 +44,15 @@ import { BODY_CALIBRATION_NOTE, measureActiveSkillListing, } from './lib/skill-listing-budget.mjs'; +import { CONTEXT_WINDOW_ANCHOR, scaleForWindow, withCommas } from './lib/context-window.mjs'; + +// Shared remediation for the aggregate-budget finding (byte-identical across the +// default and the B8 window-calibrated branches). +const AGGREGATE_RECOMMENDATION = + 'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' + + 'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' + + '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' + + 'trigger phrases.'; const SCANNER = 'SKL'; @@ -53,11 +62,21 @@ const SCANNER = 'SKL'; * @param {string} _targetPath unused (skill listing is HOME-scoped) * @param {object} _discovery unused (ignores project discovery) */ -export async function scan(_targetPath, _discovery) { +export async function scan(_targetPath, _discovery, opts = {}) { const start = Date.now(); const findings = []; - const { skills, aggregate } = await measureActiveSkillListing(); + // B8 — calibrate the aggregate budget to the resolved context window. The + // default (no opts) is the conservative 200k anchor at full severity, which is + // byte-identical to the pre-B8 behavior. An unknown (advisory) window keeps the + // anchor but downgrades the finding to info instead of firing it as a breach. + const cw = opts.contextWindow; + const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR; + const advisory = !!(cw && cw.advisory); + const isDefault = window === CONTEXT_WINDOW_ANCHOR && !advisory; + const budgetTokens = scaleForWindow(AGGREGATE_BUDGET_TOKENS, window); + + const { skills, aggregate } = await measureActiveSkillListing(budgetTokens); for (const skill of skills) { if (skill.descLength <= DESCRIPTION_CAP) continue; @@ -93,29 +112,52 @@ export async function scan(_targetPath, _discovery) { // CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common // "one oversized skill + aggregate" case reads 001=cap, 002=aggregate. if (aggregate.overBudget) { - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.low, - title: 'Aggregate skill descriptions may exceed the listing budget', - description: - `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + - `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + - `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` + - 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + - 'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' + - 'is an estimate — the budget scales with your actual context window (see evidence).', - evidence: - `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + - `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` + - `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` + - `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, - recommendation: - 'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' + - 'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' + - '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' + - 'trigger phrases.', - category: 'token-efficiency', - })); + if (isDefault) { + // Conservative 200k anchor — byte-identical to the pre-B8 finding. + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Aggregate skill descriptions may exceed the listing budget', + description: + `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + + `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + + `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` + + 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + + 'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' + + 'is an estimate — the budget scales with your actual context window (see evidence).', + evidence: + `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + + `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` + + `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` + + `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, + recommendation: AGGREGATE_RECOMMENDATION, + category: 'token-efficiency', + })); + } else { + // B8 — window-calibrated. Advisory (unknown window) downgrades to info. + const winLabel = withCommas(window); + findings.push(finding({ + scanner: SCANNER, + severity: advisory ? SEVERITY.info : SEVERITY.low, + title: 'Aggregate skill descriptions may exceed the listing budget', + description: + `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + + `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + + `${budgetTokens}-token budget Claude Code allots the skill listing at a ${winLabel}-token ` + + 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + + 'Claude Code drops descriptions, so the model may stop seeing some skills entirely.' + + (advisory + ? ' Your context window is unknown, so this is advisory: it anchors on the conservative 200k window.' + : ''), + evidence: + `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + + `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@${winLabel}=` + + `${budgetTokens} tok (skill listing ~2% of context, CC 2.1.32); over_by~${aggregate.overBy} tok` + + (advisory ? ` - ${BUDGET_CALIBRATION_NOTE}` : ' - this is an estimate, not measured telemetry'), + recommendation: AGGREGATE_RECOMMENDATION, + category: 'token-efficiency', + })); + } } // CA-SKL-003 (oversized body). Emitted last so the common single-issue cases diff --git a/tests/lib/context-window.test.mjs b/tests/lib/context-window.test.mjs new file mode 100644 index 0000000..4557d80 --- /dev/null +++ b/tests/lib/context-window.test.mjs @@ -0,0 +1,76 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + CONTEXT_WINDOW_ANCHOR, + LARGE_CONTEXT_WINDOW, + resolveContextWindow, + scaleForWindow, +} from '../../scanners/lib/context-window.mjs'; + +// B8 — context-window calibration. resolveContextWindow turns the raw +// --context-window CLI value into { window, advisory } that SKL/CML calibrate +// their budgets with. The DEFAULT (no flag) must be byte-identical to the +// pre-B8 behavior: the conservative 200k anchor at full severity (advisory=false). + +describe('resolveContextWindow — default (no flag) is the conservative anchor', () => { + it('undefined resolves to the 200k anchor, not advisory (byte-stable default)', () => { + const r = resolveContextWindow(undefined); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, false); + assert.equal(r.source, 'default'); + }); + + it('null resolves to the conservative default', () => { + const r = resolveContextWindow(null); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, false); + }); +}); + +describe('resolveContextWindow — explicit window', () => { + it('a numeric string calibrates to that window, not advisory', () => { + const r = resolveContextWindow('1000000'); + assert.equal(r.window, 1_000_000); + assert.equal(r.advisory, false); + assert.equal(r.source, 'explicit'); + }); + + it('a plain number is accepted', () => { + const r = resolveContextWindow(1_000_000); + assert.equal(r.window, 1_000_000); + assert.equal(r.advisory, false); + }); +}); + +describe('resolveContextWindow — auto / unknown downgrades to advisory', () => { + it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => { + const r = resolveContextWindow('auto'); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor'); + assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach'); + assert.equal(r.source, 'auto-unresolved'); + }); + + it('"AUTO" is case-insensitive', () => { + assert.equal(resolveContextWindow('AUTO').advisory, true); + }); + + it('an invalid value falls back to the conservative default (not advisory)', () => { + for (const bad of ['banana', '0', '-5', '']) { + const r = resolveContextWindow(bad); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, `"${bad}" -> anchor`); + assert.equal(r.advisory, false, `"${bad}" -> not advisory`); + } + }); +}); + +describe('scaleForWindow — linear scaling off the 200k anchor', () => { + it('is the identity at the 200k anchor (byte-stable default)', () => { + assert.equal(scaleForWindow(4000, CONTEXT_WINDOW_ANCHOR), 4000); + assert.equal(scaleForWindow(40_000, CONTEXT_WINDOW_ANCHOR), 40_000); + }); + + it('scales 5x at the 1M window', () => { + assert.equal(scaleForWindow(4000, LARGE_CONTEXT_WINDOW), 20_000); + assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000); + }); +}); diff --git a/tests/lib/skill-listing-budget.test.mjs b/tests/lib/skill-listing-budget.test.mjs index 8c4c826..a20b43a 100644 --- a/tests/lib/skill-listing-budget.test.mjs +++ b/tests/lib/skill-listing-budget.test.mjs @@ -102,6 +102,20 @@ describe('assessSkillListingBudget — aggregate math', () => { assert.equal(r.aggregateChars, 100); assert.equal(r.scanned, 3); }); + + it('accepts a calibrated budget (B8): 17×1000 chars is within a 20,000-tok 1M budget', () => { + const r = assessSkillListingBudget(Array(17).fill(1000), 20_000); + assert.equal(r.aggregateTokens, 4250); + assert.equal(r.budgetTokens, 20_000, 'reports the calibrated budget, not the 200k default'); + assert.equal(r.overBudget, false, 'within the relaxed 1M budget'); + assert.equal(r.overBy, 0); + }); + + it('the budget argument defaults to the 200k anchor (byte-stable for existing callers)', () => { + const r = assessSkillListingBudget(Array(17).fill(1000)); + assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS); + assert.equal(r.overBudget, true); + }); }); describe('envFlag', () => { diff --git a/tests/scanners/claude-md-linter.test.mjs b/tests/scanners/claude-md-linter.test.mjs index f720dbc..5e4dd23 100644 --- a/tests/scanners/claude-md-linter.test.mjs +++ b/tests/scanners/claude-md-linter.test.mjs @@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)', }); }); +describe('CML scanner — context-window calibration (B8)', () => { + const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars + const charFinding = (r) => + r.findings.find((f) => /performance-warning threshold/i.test(f.title || '')); + + async function scanWithCtx(contextWindow) { + resetCounter(); + const discovery = await discoverConfigFiles(FIXTURE); + return scan(FIXTURE, discovery, { contextWindow }); + } + + it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => { + const at1m = await scanWithCtx({ window: 1_000_000, advisory: false }); + assert.equal(charFinding(at1m), undefined, + '48,531 chars is under the ~200,000-char threshold at a 1M window'); + }); + + it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => { + const advisory = await scanWithCtx({ window: 200_000, advisory: true }); + const f = charFinding(advisory); + assert.ok(f, 'still surfaces the measurement at the conservative anchor'); + assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info'); + }); + + it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => { + resetCounter(); + const discovery = await discoverConfigFiles(FIXTURE); + const result = await scan(FIXTURE, discovery); + assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium'); + }); +}); + describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => { // large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines): // under CC's 40.0k char threshold, so the char-budget finding must NOT fire — diff --git a/tests/scanners/skill-listing-scanner.test.mjs b/tests/scanners/skill-listing-scanner.test.mjs index 92d7b97..e50ff59 100644 --- a/tests/scanners/skill-listing-scanner.test.mjs +++ b/tests/scanners/skill-listing-scanner.test.mjs @@ -28,6 +28,18 @@ async function runScannerWithHome(home) { } } +/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */ +async function runScannerWithCtx(home, contextWindow) { + resetCounter(); + const original = process.env.HOME; + process.env.HOME = home; + try { + return await scan('/unused', { files: [] }, { contextWindow }); + } finally { + process.env.HOME = original; + } +} + /** Build a fake HOME with one user skill whose description has `len` chars. */ async function homeWithUserSkill(name, descLen) { const home = uniqueDir(name); @@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { }); }); +describe('SKL scanner — context-window calibration (B8)', () => { + it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => { + // 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k + // budget, under the 20,000-tok 1M budget — the acceptance case. + const home = await homeWithNUserSkills(50, 400, 'cw'); + try { + const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false }); + assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor'); + + const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false }); + assert.equal(findAggregate(at1m.findings), undefined, + 'at a 1M context window the listing is within budget and must not fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => { + const home = await homeWithNUserSkills(17, 1000, 'cwadv'); + try { + const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true }); + const agg = findAggregate(advisory.findings); + assert.ok(agg, 'still surfaces the measurement (conservative anchor)'); + assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('no opts (default) is unchanged: fires low at the 200k anchor', async () => { + const home = await homeWithNUserSkills(17, 1000, 'cwdef'); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg); + assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => { const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));