feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]
SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:
- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
conservative 200k anchor but marks advisory (model→window probing deferred
to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
for byte-stability and a calibrated branch; advisory downgrades the budget
finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
150-line volatility heuristic), so there is nothing to calibrate.
15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
27988801be
commit
2082b7d112
10 changed files with 364 additions and 45 deletions
76
tests/lib/context-window.test.mjs
Normal file
76
tests/lib/context-window.test.mjs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
CONTEXT_WINDOW_ANCHOR,
|
||||
LARGE_CONTEXT_WINDOW,
|
||||
resolveContextWindow,
|
||||
scaleForWindow,
|
||||
} from '../../scanners/lib/context-window.mjs';
|
||||
|
||||
// B8 — context-window calibration. resolveContextWindow turns the raw
|
||||
// --context-window CLI value into { window, advisory } that SKL/CML calibrate
|
||||
// their budgets with. The DEFAULT (no flag) must be byte-identical to the
|
||||
// pre-B8 behavior: the conservative 200k anchor at full severity (advisory=false).
|
||||
|
||||
describe('resolveContextWindow — default (no flag) is the conservative anchor', () => {
|
||||
it('undefined resolves to the 200k anchor, not advisory (byte-stable default)', () => {
|
||||
const r = resolveContextWindow(undefined);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
|
||||
assert.equal(r.advisory, false);
|
||||
assert.equal(r.source, 'default');
|
||||
});
|
||||
|
||||
it('null resolves to the conservative default', () => {
|
||||
const r = resolveContextWindow(null);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR);
|
||||
assert.equal(r.advisory, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveContextWindow — explicit window', () => {
|
||||
it('a numeric string calibrates to that window, not advisory', () => {
|
||||
const r = resolveContextWindow('1000000');
|
||||
assert.equal(r.window, 1_000_000);
|
||||
assert.equal(r.advisory, false);
|
||||
assert.equal(r.source, 'explicit');
|
||||
});
|
||||
|
||||
it('a plain number is accepted', () => {
|
||||
const r = resolveContextWindow(1_000_000);
|
||||
assert.equal(r.window, 1_000_000);
|
||||
assert.equal(r.advisory, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveContextWindow — auto / unknown downgrades to advisory', () => {
|
||||
it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => {
|
||||
const r = resolveContextWindow('auto');
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor');
|
||||
assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach');
|
||||
assert.equal(r.source, 'auto-unresolved');
|
||||
});
|
||||
|
||||
it('"AUTO" is case-insensitive', () => {
|
||||
assert.equal(resolveContextWindow('AUTO').advisory, true);
|
||||
});
|
||||
|
||||
it('an invalid value falls back to the conservative default (not advisory)', () => {
|
||||
for (const bad of ['banana', '0', '-5', '']) {
|
||||
const r = resolveContextWindow(bad);
|
||||
assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, `"${bad}" -> anchor`);
|
||||
assert.equal(r.advisory, false, `"${bad}" -> not advisory`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('scaleForWindow — linear scaling off the 200k anchor', () => {
|
||||
it('is the identity at the 200k anchor (byte-stable default)', () => {
|
||||
assert.equal(scaleForWindow(4000, CONTEXT_WINDOW_ANCHOR), 4000);
|
||||
assert.equal(scaleForWindow(40_000, CONTEXT_WINDOW_ANCHOR), 40_000);
|
||||
});
|
||||
|
||||
it('scales 5x at the 1M window', () => {
|
||||
assert.equal(scaleForWindow(4000, LARGE_CONTEXT_WINDOW), 20_000);
|
||||
assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000);
|
||||
});
|
||||
});
|
||||
|
|
@ -102,6 +102,20 @@ describe('assessSkillListingBudget — aggregate math', () => {
|
|||
assert.equal(r.aggregateChars, 100);
|
||||
assert.equal(r.scanned, 3);
|
||||
});
|
||||
|
||||
it('accepts a calibrated budget (B8): 17×1000 chars is within a 20,000-tok 1M budget', () => {
|
||||
const r = assessSkillListingBudget(Array(17).fill(1000), 20_000);
|
||||
assert.equal(r.aggregateTokens, 4250);
|
||||
assert.equal(r.budgetTokens, 20_000, 'reports the calibrated budget, not the 200k default');
|
||||
assert.equal(r.overBudget, false, 'within the relaxed 1M budget');
|
||||
assert.equal(r.overBy, 0);
|
||||
});
|
||||
|
||||
it('the budget argument defaults to the 200k anchor (byte-stable for existing callers)', () => {
|
||||
const r = assessSkillListingBudget(Array(17).fill(1000));
|
||||
assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS);
|
||||
assert.equal(r.overBudget, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('envFlag', () => {
|
||||
|
|
|
|||
|
|
@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)',
|
|||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — context-window calibration (B8)', () => {
|
||||
const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars
|
||||
const charFinding = (r) =>
|
||||
r.findings.find((f) => /performance-warning threshold/i.test(f.title || ''));
|
||||
|
||||
async function scanWithCtx(contextWindow) {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
return scan(FIXTURE, discovery, { contextWindow });
|
||||
}
|
||||
|
||||
it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => {
|
||||
const at1m = await scanWithCtx({ window: 1_000_000, advisory: false });
|
||||
assert.equal(charFinding(at1m), undefined,
|
||||
'48,531 chars is under the ~200,000-char threshold at a 1M window');
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => {
|
||||
const advisory = await scanWithCtx({ window: 200_000, advisory: true });
|
||||
const f = charFinding(advisory);
|
||||
assert.ok(f, 'still surfaces the measurement at the conservative anchor');
|
||||
assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info');
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
const result = await scan(FIXTURE, discovery);
|
||||
assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => {
|
||||
// large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines):
|
||||
// under CC's 40.0k char threshold, so the char-budget finding must NOT fire —
|
||||
|
|
|
|||
|
|
@ -28,6 +28,18 @@ async function runScannerWithHome(home) {
|
|||
}
|
||||
}
|
||||
|
||||
/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */
|
||||
async function runScannerWithCtx(home, contextWindow) {
|
||||
resetCounter();
|
||||
const original = process.env.HOME;
|
||||
process.env.HOME = home;
|
||||
try {
|
||||
return await scan('/unused', { files: [] }, { contextWindow });
|
||||
} finally {
|
||||
process.env.HOME = original;
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a fake HOME with one user skill whose description has `len` chars. */
|
||||
async function homeWithUserSkill(name, descLen) {
|
||||
const home = uniqueDir(name);
|
||||
|
|
@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — context-window calibration (B8)', () => {
|
||||
it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => {
|
||||
// 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k
|
||||
// budget, under the 20,000-tok 1M budget — the acceptance case.
|
||||
const home = await homeWithNUserSkills(50, 400, 'cw');
|
||||
try {
|
||||
const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false });
|
||||
assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor');
|
||||
|
||||
const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false });
|
||||
assert.equal(findAggregate(at1m.findings), undefined,
|
||||
'at a 1M context window the listing is within budget and must not fire');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwadv');
|
||||
try {
|
||||
const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true });
|
||||
const agg = findAggregate(advisory.findings);
|
||||
assert.ok(agg, 'still surfaces the measurement (conservative anchor)');
|
||||
assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires low at the 200k anchor', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwdef');
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg);
|
||||
assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
|
||||
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue