feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs]
SKL-002 (skill-listing budget) and CML char-budget now calibrate to a
resolved context window instead of always anchoring at 200k:
- resolveContextWindow(): --context-window <n> calibrates; 'auto' keeps the
conservative 200k anchor but marks advisory (model→window probing deferred
to B8b); no flag → 200k anchor, byte-identical to pre-B8 default.
- scaleForWindow(): linear off the 200k anchor (identity at the anchor).
- SKL + CML each keep an untouched default branch (window===200k && !advisory)
for byte-stability and a calibrated branch; advisory downgrades the budget
finding from a breach (low/medium) to info.
- Flag wired through scan-orchestrator + posture; runAllScanners resolves once
and threads { contextWindow } to scanners (others ignore the 3rd arg).
- CPS intentionally excluded: it has no window-anchored budget (fixed
150-line volatility heuristic), so there is nothing to calibrate.
15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default
unchanged); full suite 1279 green; snapshots byte-stable.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
27988801be
commit
2082b7d112
10 changed files with 364 additions and 45 deletions
|
|
@ -26,3 +26,52 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR;
|
|||
|
||||
// Dependency-free thousands separator (repo invariant: zero external deps).
|
||||
export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',');
|
||||
|
||||
/**
|
||||
* @typedef {object} ResolvedContextWindow
|
||||
* @property {number} window - the context window budgets calibrate against
|
||||
* @property {boolean} advisory - true when the window is unknown: keep the anchor
|
||||
* but downgrade budget findings to info instead of
|
||||
* firing them as a breach
|
||||
* @property {'default'|'explicit'|'auto-unresolved'} source
|
||||
*/
|
||||
|
||||
/**
|
||||
* Resolve the raw `--context-window` CLI value into a window + advisory flag.
|
||||
*
|
||||
* Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior —
|
||||
* the conservative 200k anchor at full severity. Only an explicit value changes
|
||||
* calibration. `auto` asks the tool to figure out the window; until model→window
|
||||
* probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the
|
||||
* result advisory so SKL/CML downgrade their budget findings to info rather than
|
||||
* "crying wolf" with a breach on a window we cannot confirm.
|
||||
*
|
||||
* @param {string|number|null|undefined} arg
|
||||
* @returns {ResolvedContextWindow}
|
||||
*/
|
||||
export function resolveContextWindow(arg) {
|
||||
if (arg == null) {
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' };
|
||||
}
|
||||
if (String(arg).trim().toLowerCase() === 'auto') {
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' };
|
||||
}
|
||||
const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10);
|
||||
if (Number.isFinite(n) && n > 0) {
|
||||
return { window: n, advisory: false, source: 'explicit' };
|
||||
}
|
||||
// Unparseable / non-positive: fall back to the conservative default (no advisory).
|
||||
return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale a 200k-anchored budget to a given context window. Linear in the window,
|
||||
* so it is the identity at the anchor (keeps the default byte-stable).
|
||||
*
|
||||
* @param {number} anchorValue - the budget/threshold defined at the 200k anchor
|
||||
* @param {number} window - the target context window
|
||||
* @returns {number}
|
||||
*/
|
||||
export function scaleForWindow(anchorValue, window) {
|
||||
return Math.round(anchorValue * (window / CONTEXT_WINDOW_ANCHOR));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,23 +77,26 @@ export const BODY_CALIBRATION_NOTE =
|
|||
* flags it — so the aggregate does not double-count it).
|
||||
*
|
||||
* @param {number[]} descLengths - one entry per active skill (description char count)
|
||||
* @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - the listing budget to
|
||||
* measure against. Defaults to the 200k-anchored 4,000 tok; B8 passes a
|
||||
* window-calibrated budget. Defaulting keeps existing callers byte-stable.
|
||||
* @returns {BudgetAssessment}
|
||||
*/
|
||||
export function assessSkillListingBudget(descLengths) {
|
||||
export function assessSkillListingBudget(descLengths, budgetTokens = AGGREGATE_BUDGET_TOKENS) {
|
||||
let aggregateChars = 0;
|
||||
for (const len of descLengths) {
|
||||
const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0;
|
||||
aggregateChars += Math.min(safe, DESCRIPTION_CAP);
|
||||
}
|
||||
const aggregateTokens = estimateTokens(aggregateChars, 'markdown');
|
||||
const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS;
|
||||
const overBudget = aggregateTokens > budgetTokens;
|
||||
return {
|
||||
scanned: descLengths.length,
|
||||
aggregateChars,
|
||||
aggregateTokens,
|
||||
budgetTokens: AGGREGATE_BUDGET_TOKENS,
|
||||
budgetTokens,
|
||||
overBudget,
|
||||
overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0,
|
||||
overBy: overBudget ? aggregateTokens - budgetTokens : 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -115,9 +118,11 @@ export function assessSkillListingBudget(descLengths) {
|
|||
* enumerateSkills). Callers that run under test MUST override HOME (see the
|
||||
* hermetic-home helper / runScannerWithHome pattern).
|
||||
*
|
||||
* @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - listing budget for the
|
||||
* aggregate assessment (B8 window-calibration); defaults keep callers byte-stable.
|
||||
* @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>}
|
||||
*/
|
||||
export async function measureActiveSkillListing() {
|
||||
export async function measureActiveSkillListing(budgetTokens = AGGREGATE_BUDGET_TOKENS) {
|
||||
const plugins = await enumeratePlugins();
|
||||
const allSkills = await enumerateSkills(plugins);
|
||||
|
||||
|
|
@ -143,7 +148,7 @@ export async function measureActiveSkillListing() {
|
|||
});
|
||||
}
|
||||
|
||||
const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength));
|
||||
const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength), budgetTokens);
|
||||
return { skills, aggregate };
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue