feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]

SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:

- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
  conservative 200k anchor but marks advisory (model→window probing deferred
  to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
  for byte-stability and a calibrated branch; advisory downgrades the budget
  finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
  and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
  150-line volatility heuristic), so there is nothing to calibrate.

15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-06-23 21:44:52 +02:00
commit 2082b7d112
10 changed files with 364 additions and 45 deletions

View file

@ -0,0 +1,76 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
CONTEXT_WINDOW_ANCHOR,
LARGE_CONTEXT_WINDOW,
resolveContextWindow,
scaleForWindow,
} from '../../scanners/lib/context-window.mjs';
// B8 — context-window calibration. resolveContextWindow turns the raw
// --context-window CLI value into { window, advisory } that SKL/CML calibrate
// their budgets with. The DEFAULT (no flag) must be byte-identical to the
// pre-B8 behavior: the conservative 200k anchor at full severity (advisory=false).
describe('resolveContextWindow — default (no flag) is the conservative anchor', () => {
it('undefined resolves to the 200k anchor, not advisory (byte-stable default)', () => {
const r = resolveContextWindow(undefined);
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
assert.equal(r.advisory, false);
assert.equal(r.source, 'default');
});
it('null resolves to the conservative default', () => {
const r = resolveContextWindow(null);
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
assert.equal(r.advisory, false);
});
});
describe('resolveContextWindow — explicit window', () => {
it('a numeric string calibrates to that window, not advisory', () => {
const r = resolveContextWindow('1000000');
assert.equal(r.window, 1_000_000);
assert.equal(r.advisory, false);
assert.equal(r.source, 'explicit');
});
it('a plain number is accepted', () => {
const r = resolveContextWindow(1_000_000);
assert.equal(r.window, 1_000_000);
assert.equal(r.advisory, false);
});
});
describe('resolveContextWindow — auto / unknown downgrades to advisory', () => {
it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => {
const r = resolveContextWindow('auto');
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor');
assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach');
assert.equal(r.source, 'auto-unresolved');
});
it('"AUTO" is case-insensitive', () => {
assert.equal(resolveContextWindow('AUTO').advisory, true);
});
it('an invalid value falls back to the conservative default (not advisory)', () => {
for (const bad of ['banana', '0', '-5', '']) {
const r = resolveContextWindow(bad);
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, `"${bad}" -> anchor`);
assert.equal(r.advisory, false, `"${bad}" -> not advisory`);
}
});
});
describe('scaleForWindow — linear scaling off the 200k anchor', () => {
it('is the identity at the 200k anchor (byte-stable default)', () => {
assert.equal(scaleForWindow(4000, CONTEXT_WINDOW_ANCHOR), 4000);
assert.equal(scaleForWindow(40_000, CONTEXT_WINDOW_ANCHOR), 40_000);
});
it('scales 5x at the 1M window', () => {
assert.equal(scaleForWindow(4000, LARGE_CONTEXT_WINDOW), 20_000);
assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000);
});
});

View file

@ -102,6 +102,20 @@ describe('assessSkillListingBudget — aggregate math', () => {
assert.equal(r.aggregateChars, 100);
assert.equal(r.scanned, 3);
});
it('accepts a calibrated budget (B8): 17×1000 chars is within a 20,000-tok 1M budget', () => {
const r = assessSkillListingBudget(Array(17).fill(1000), 20_000);
assert.equal(r.aggregateTokens, 4250);
assert.equal(r.budgetTokens, 20_000, 'reports the calibrated budget, not the 200k default');
assert.equal(r.overBudget, false, 'within the relaxed 1M budget');
assert.equal(r.overBy, 0);
});
it('the budget argument defaults to the 200k anchor (byte-stable for existing callers)', () => {
const r = assessSkillListingBudget(Array(17).fill(1000));
assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS);
assert.equal(r.overBudget, true);
});
});
describe('envFlag', () => {

View file

@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)',
});
});
describe('CML scanner — context-window calibration (B8)', () => {
const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars
const charFinding = (r) =>
r.findings.find((f) => /performance-warning threshold/i.test(f.title || ''));
async function scanWithCtx(contextWindow) {
resetCounter();
const discovery = await discoverConfigFiles(FIXTURE);
return scan(FIXTURE, discovery, { contextWindow });
}
it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => {
const at1m = await scanWithCtx({ window: 1_000_000, advisory: false });
assert.equal(charFinding(at1m), undefined,
'48,531 chars is under the ~200,000-char threshold at a 1M window');
});
it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => {
const advisory = await scanWithCtx({ window: 200_000, advisory: true });
const f = charFinding(advisory);
assert.ok(f, 'still surfaces the measurement at the conservative anchor');
assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info');
});
it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => {
resetCounter();
const discovery = await discoverConfigFiles(FIXTURE);
const result = await scan(FIXTURE, discovery);
assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium');
});
});
describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => {
// large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines):
// under CC's 40.0k char threshold, so the char-budget finding must NOT fire —

View file

@ -28,6 +28,18 @@ async function runScannerWithHome(home) {
}
}
/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */
async function runScannerWithCtx(home, contextWindow) {
resetCounter();
const original = process.env.HOME;
process.env.HOME = home;
try {
return await scan('/unused', { files: [] }, { contextWindow });
} finally {
process.env.HOME = original;
}
}
/** Build a fake HOME with one user skill whose description has `len` chars. */
async function homeWithUserSkill(name, descLen) {
const home = uniqueDir(name);
@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
});
});
describe('SKL scanner — context-window calibration (B8)', () => {
it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => {
// 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k
// budget, under the 20,000-tok 1M budget — the acceptance case.
const home = await homeWithNUserSkills(50, 400, 'cw');
try {
const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false });
assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor');
const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false });
assert.equal(findAggregate(at1m.findings), undefined,
'at a 1M context window the listing is within budget and must not fire');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => {
const home = await homeWithNUserSkills(17, 1000, 'cwadv');
try {
const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true });
const agg = findAggregate(advisory.findings);
assert.ok(agg, 'still surfaces the measurement (conservative anchor)');
assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info');
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('no opts (default) is unchanged: fires low at the 200k anchor', async () => {
const home = await homeWithNUserSkills(17, 1000, 'cwdef');
try {
const result = await runScannerWithHome(home);
const agg = findAggregate(result.findings);
assert.ok(agg);
assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity');
} finally {
await rm(home, { recursive: true, force: true });
}
});
});
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));