feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]
SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:
- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
conservative 200k anchor but marks advisory (model→window probing deferred
to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
for byte-stability and a calibrated branch; advisory downgrades the budget
finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
150-line volatility heuristic), so there is nothing to calibrate.
15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
27988801be
commit
2082b7d112
10 changed files with 364 additions and 45 deletions
|
|
@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)',
|
|||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — context-window calibration (B8)', () => {
|
||||
const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars
|
||||
const charFinding = (r) =>
|
||||
r.findings.find((f) => /performance-warning threshold/i.test(f.title || ''));
|
||||
|
||||
async function scanWithCtx(contextWindow) {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
return scan(FIXTURE, discovery, { contextWindow });
|
||||
}
|
||||
|
||||
it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => {
|
||||
const at1m = await scanWithCtx({ window: 1_000_000, advisory: false });
|
||||
assert.equal(charFinding(at1m), undefined,
|
||||
'48,531 chars is under the ~200,000-char threshold at a 1M window');
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => {
|
||||
const advisory = await scanWithCtx({ window: 200_000, advisory: true });
|
||||
const f = charFinding(advisory);
|
||||
assert.ok(f, 'still surfaces the measurement at the conservative anchor');
|
||||
assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info');
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => {
|
||||
resetCounter();
|
||||
const discovery = await discoverConfigFiles(FIXTURE);
|
||||
const result = await scan(FIXTURE, discovery);
|
||||
assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium');
|
||||
});
|
||||
});
|
||||
|
||||
describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => {
|
||||
// large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines):
|
||||
// under CC's 40.0k char threshold, so the char-budget finding must NOT fire —
|
||||
|
|
|
|||
|
|
@ -28,6 +28,18 @@ async function runScannerWithHome(home) {
|
|||
}
|
||||
}
|
||||
|
||||
/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */
|
||||
async function runScannerWithCtx(home, contextWindow) {
|
||||
resetCounter();
|
||||
const original = process.env.HOME;
|
||||
process.env.HOME = home;
|
||||
try {
|
||||
return await scan('/unused', { files: [] }, { contextWindow });
|
||||
} finally {
|
||||
process.env.HOME = original;
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a fake HOME with one user skill whose description has `len` chars. */
|
||||
async function homeWithUserSkill(name, descLen) {
|
||||
const home = uniqueDir(name);
|
||||
|
|
@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — context-window calibration (B8)', () => {
|
||||
it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => {
|
||||
// 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k
|
||||
// budget, under the 20,000-tok 1M budget — the acceptance case.
|
||||
const home = await homeWithNUserSkills(50, 400, 'cw');
|
||||
try {
|
||||
const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false });
|
||||
assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor');
|
||||
|
||||
const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false });
|
||||
assert.equal(findAggregate(at1m.findings), undefined,
|
||||
'at a 1M context window the listing is within budget and must not fire');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwadv');
|
||||
try {
|
||||
const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true });
|
||||
const agg = findAggregate(advisory.findings);
|
||||
assert.ok(agg, 'still surfaces the measurement (conservative anchor)');
|
||||
assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('no opts (default) is unchanged: fires low at the 200k anchor', async () => {
|
||||
const home = await homeWithNUserSkills(17, 1000, 'cwdef');
|
||||
try {
|
||||
const result = await runScannerWithHome(home);
|
||||
const agg = findAggregate(result.findings);
|
||||
assert.ok(agg);
|
||||
assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity');
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => {
|
||||
const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || ''));
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue