From cf75249b5ecf0cca6cf049c03016744f219c0465 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 22:29:12 +0200 Subject: [PATCH] =?UTF-8?q?feat(skl,cml):=20--context-window=20auto=20mode?= =?UTF-8?q?l=E2=86=92window=20probe=20(v5.12=20B8b)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the deferred B8 half. `--context-window auto` now probes the configured model and calibrates SKL/CML budgets to its real window instead of always falling back to the conservative advisory anchor. - lib/context-window.mjs: pure modelToContextWindow() maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit [1m] tier tag, dated/provider-prefixed IDs, and the opus/sonnet/fable aliases) to the 1M window; unknown/unconfirmed -> null (caller keeps the conservative anchor). resolveContextWindow() auto branch now probes opts.model: recognized -> auto-probed (not advisory); unknown or unpinned -> auto-unresolved (advisory, pre-B8b behavior). - lib/active-model.mjs (new): resolveActiveModel() reads the model the way Claude Code resolves it — shell ANTHROPIC_MODEL override, then settings cascade local > project > user. Injectable env, hermetic under test HOME. - scan-orchestrator: resolves the active model only when the flag is `auto` and threads it into resolveContextWindow; posture inherits via runAllScanners. Default (no flag) and explicit --context-window paths ignore the model and stay byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. TDD: 17 new tests (context-window mapping/probe + active-model cascade). Suite 1296 green. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/lib/active-model.mjs | 51 ++++++++++++++++ scanners/lib/context-window.mjs | 67 +++++++++++++++++++-- scanners/scan-orchestrator.mjs | 9 ++- tests/lib/active-model.test.mjs | 96 +++++++++++++++++++++++++++++++ tests/lib/context-window.test.mjs | 93 +++++++++++++++++++++++++++++- 5 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 scanners/lib/active-model.mjs create mode 100644 tests/lib/active-model.test.mjs diff --git a/scanners/lib/active-model.mjs b/scanners/lib/active-model.mjs new file mode 100644 index 0000000..0ccbc76 --- /dev/null +++ b/scanners/lib/active-model.mjs @@ -0,0 +1,51 @@ +/** + * Active-model resolution for the `--context-window auto` probe (B8b). + * + * Reads the configured model the way Claude Code itself resolves it, so the + * window probe (context-window.mjs `modelToContextWindow`) sees the real model: + * 1. the shell `ANTHROPIC_MODEL` override (applies to the launched session); + * 2. otherwise the settings cascade `model` field — user `~/.claude`, then + * project `.claude`, then project-local `.claude` (local > project > user). + * + * Reads the cascade files directly (like isBundledSkillsDisabled) rather than via + * config-discovery classification, and takes an injectable `env` so it is + * deterministic and hermetic under the test HOME. Returns null when no model is + * pinned anywhere — the honest signal that `auto` must fall back to advisory. + * + * Zero external dependencies (repo invariant). + */ + +import { join } from 'node:path'; +import { readTextFile } from './file-discovery.mjs'; +import { parseJson } from './yaml-parser.mjs'; + +/** + * @param {string|null|undefined} projectPath - project root, to also read project + local settings + * @param {{ env?: Record }} [opts] + * @returns {Promise} the resolved model id/alias, or null if unset + */ +export async function resolveActiveModel(projectPath, { env = process.env } = {}) { + // 1. Shell ANTHROPIC_MODEL overrides settings (CC: applies to the session). + const envModel = typeof env?.ANTHROPIC_MODEL === 'string' ? env.ANTHROPIC_MODEL.trim() : ''; + if (envModel) return envModel; + + // 2. Settings cascade: user -> project -> project-local, later wins. + const home = (env && (env.HOME || env.USERPROFILE)) || ''; + const candidates = []; + if (home) candidates.push(join(home, '.claude', 'settings.json')); + if (projectPath) { + candidates.push(join(projectPath, '.claude', 'settings.json')); + candidates.push(join(projectPath, '.claude', 'settings.local.json')); + } + + let model = null; + for (const p of candidates) { + const content = await readTextFile(p); + if (!content) continue; + const parsed = parseJson(content); + if (parsed && typeof parsed.model === 'string' && parsed.model.trim()) { + model = parsed.model.trim(); + } + } + return model; +} diff --git a/scanners/lib/context-window.mjs b/scanners/lib/context-window.mjs index 15c914a..596d846 100644 --- a/scanners/lib/context-window.mjs +++ b/scanners/lib/context-window.mjs @@ -27,13 +27,58 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR; // Dependency-free thousands separator (repo invariant: zero external deps). export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); +// Model families whose context window is the large (1M) tier. Verified June 2026 +// (platform.claude.com models overview): Fable 5, Opus 4.8/4.7/4.6 and Sonnet 4.6 +// all run a 1M context window. Matched by substring so dated IDs +// (claude-opus-4-8-20260528) and provider-prefixed IDs +// (us.anthropic.claude-opus-4-8) resolve too. Models we cannot confirm (e.g. +// Haiku, older 200k-era IDs) are deliberately left out: the caller then keeps the +// conservative anchor rather than guess a relaxed budget. +export const LARGE_CONTEXT_MODEL_IDS = [ + 'claude-fable-5', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-sonnet-4-6', +]; + +// Short aliases Claude Code accepts in the `model` setting / ANTHROPIC_MODEL that +// currently resolve to a 1M-tier model (`opusplan` plans on an Opus-tier model). +export const LARGE_CONTEXT_MODEL_ALIASES = new Set(['opus', 'sonnet', 'fable', 'opusplan']); + +/** + * Map a configured model id/alias to its context window, or null when we cannot + * confirm it. Pure: no IO. Used by the `--context-window auto` probe (B8b) so + * known 1M-tier models calibrate budgets instead of falling back to the + * conservative advisory anchor. + * + * @param {string} modelId - e.g. "claude-opus-4-8[1m]", "claude-sonnet-4-6", "opus" + * @returns {number|null} the context window, or null if unrecognized + */ +export function modelToContextWindow(modelId) { + if (typeof modelId !== 'string') return null; + const id = modelId.trim().toLowerCase(); + if (!id) return null; + // Explicit tier tag wins — the running session model surfaces as e.g. + // "claude-opus-4-8[1m]". This is the strongest, most future-proof signal. + if (id.includes('[1m]')) return LARGE_CONTEXT_WINDOW; + // Known 1M-tier families (substring → tolerant of date/provider-prefix variants). + for (const fam of LARGE_CONTEXT_MODEL_IDS) { + if (id.includes(fam)) return LARGE_CONTEXT_WINDOW; + } + // Short aliases. + if (LARGE_CONTEXT_MODEL_ALIASES.has(id)) return LARGE_CONTEXT_WINDOW; + // Unknown: cannot confirm the window — keep the conservative anchor (null). + return null; +} + /** * @typedef {object} ResolvedContextWindow * @property {number} window - the context window budgets calibrate against * @property {boolean} advisory - true when the window is unknown: keep the anchor * but downgrade budget findings to info instead of * firing them as a breach - * @property {'default'|'explicit'|'auto-unresolved'} source + * @property {'default'|'explicit'|'auto-probed'|'auto-unresolved'} source */ /** @@ -41,19 +86,29 @@ export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',') * * Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior — * the conservative 200k anchor at full severity. Only an explicit value changes - * calibration. `auto` asks the tool to figure out the window; until model→window - * probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the - * result advisory so SKL/CML downgrade their budget findings to info rather than - * "crying wolf" with a breach on a window we cannot confirm. + * calibration. `auto` asks the tool to figure out the window. + * + * B8b: `auto` now probes the configured model (`opts.model`, resolved from the + * settings cascade / ANTHROPIC_MODEL by the orchestrator). A recognized 1M-tier + * model calibrates to its window (source `auto-probed`, not advisory). When the + * model is unknown or unpinned, it keeps the conservative anchor but marks the + * result advisory (source `auto-unresolved`) so SKL/CML downgrade their budget + * findings to info rather than "crying wolf" on a window we cannot confirm. * * @param {string|number|null|undefined} arg + * @param {{ model?: string|null }} [opts] - probe input for `auto` (ignored on the + * default/explicit paths, which stay byte-stable). * @returns {ResolvedContextWindow} */ -export function resolveContextWindow(arg) { +export function resolveContextWindow(arg, opts = {}) { if (arg == null) { return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; } if (String(arg).trim().toLowerCase() === 'auto') { + const probed = modelToContextWindow(opts.model); + if (probed) { + return { window: probed, advisory: false, source: 'auto-probed' }; + } return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' }; } const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10); diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 71864ea..6beed16 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -15,6 +15,7 @@ import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs'; import { humanizeEnvelope } from './lib/humanizer.mjs'; import { resolveContextWindow } from './lib/context-window.mjs'; +import { resolveActiveModel } from './lib/active-model.mjs'; // Scanner registry — import order determines execution order import { scan as scanClaudeMd } from './claude-md-linter.mjs'; @@ -98,7 +99,13 @@ export async function runAllScanners(targetPath, opts = {}) { // B8 — resolve the context window once and thread it to budget-aware scanners // (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is // byte-identical to the pre-B8 default; other scanners ignore the third arg. - const contextWindow = resolveContextWindow(opts.contextWindow); + // B8b — `--context-window auto` probes the configured model (settings cascade / + // ANTHROPIC_MODEL) so a 1M-tier host self-calibrates; unknown/unpinned → advisory. + let probedModel = null; + if (String(opts.contextWindow ?? '').trim().toLowerCase() === 'auto') { + probedModel = await resolveActiveModel(resolvedPath, { env: process.env }); + } + const contextWindow = resolveContextWindow(opts.contextWindow, { model: probedModel }); // Shared file discovery — scanners reuse this let discovery; diff --git a/tests/lib/active-model.test.mjs b/tests/lib/active-model.test.mjs new file mode 100644 index 0000000..1ae1dd3 --- /dev/null +++ b/tests/lib/active-model.test.mjs @@ -0,0 +1,96 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { resolveActiveModel } from '../../scanners/lib/active-model.mjs'; + +// B8b — resolveActiveModel reads the configured model so `--context-window auto` +// can probe it. Source precedence mirrors Claude Code's own resolution: the shell +// ANTHROPIC_MODEL override wins, otherwise the settings cascade (local > project > +// user). Fully file/env-injectable so it is hermetic under the test HOME. + +async function withTempHome(fn) { + const home = await mkdtemp(join(tmpdir(), 'config-audit-model-home-')); + try { + return await fn(home); + } finally { + await rm(home, { recursive: true, force: true }); + } +} + +async function withTempProject(fn) { + const project = await mkdtemp(join(tmpdir(), 'config-audit-model-proj-')); + try { + return await fn(project); + } finally { + await rm(project, { recursive: true, force: true }); + } +} + +async function writeSettings(dir, file, obj) { + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, file), JSON.stringify(obj)); +} + +describe('resolveActiveModel — env override (ANTHROPIC_MODEL)', () => { + it('returns the shell ANTHROPIC_MODEL when set, ignoring settings', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + const env = { HOME: home, ANTHROPIC_MODEL: 'claude-opus-4-8[1m]' }; + assert.equal(await resolveActiveModel(null, { env }), 'claude-opus-4-8[1m]'); + }); + }); + + it('an empty ANTHROPIC_MODEL is treated as unset (falls through to the cascade)', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + const env = { HOME: home, ANTHROPIC_MODEL: ' ' }; + assert.equal(await resolveActiveModel(null, { env }), 'claude-sonnet-4-6'); + }); + }); +}); + +describe('resolveActiveModel — settings cascade (local > project > user)', () => { + it('reads the model from user ~/.claude/settings.json', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-opus-4-8' }); + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), 'claude-opus-4-8'); + }); + }); + + it('project settings.json overrides user settings', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-opus-4-8' }); + await withTempProject(async (project) => { + await writeSettings(join(project, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + assert.equal(await resolveActiveModel(project, { env: { HOME: home } }), 'claude-sonnet-4-6'); + }); + }); + }); + + it('settings.local.json overrides project settings (local wins)', async () => { + await withTempHome(async (home) => { + await withTempProject(async (project) => { + await writeSettings(join(project, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + await writeSettings(join(project, '.claude'), 'settings.local.json', { model: 'claude-opus-4-8[1m]' }); + assert.equal(await resolveActiveModel(project, { env: { HOME: home } }), 'claude-opus-4-8[1m]'); + }); + }); + }); +}); + +describe('resolveActiveModel — nothing configured', () => { + it('returns null on a clean HOME with no model anywhere (honest advisory fallback)', async () => { + await withTempHome(async (home) => { + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), null); + }); + }); + + it('ignores a non-string model field', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 123 }); + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), null); + }); + }); +}); diff --git a/tests/lib/context-window.test.mjs b/tests/lib/context-window.test.mjs index 4557d80..0c003e4 100644 --- a/tests/lib/context-window.test.mjs +++ b/tests/lib/context-window.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, + modelToContextWindow, resolveContextWindow, scaleForWindow, } from '../../scanners/lib/context-window.mjs'; @@ -43,13 +44,20 @@ describe('resolveContextWindow — explicit window', () => { }); describe('resolveContextWindow — auto / unknown downgrades to advisory', () => { - it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => { + it('"auto" with no model keeps the conservative anchor but flags advisory', () => { const r = resolveContextWindow('auto'); assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor'); assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach'); assert.equal(r.source, 'auto-unresolved'); }); + it('"auto" with an unrecognized model also stays advisory', () => { + const r = resolveContextWindow('auto', { model: 'totally-made-up-model' }); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, true); + assert.equal(r.source, 'auto-unresolved'); + }); + it('"AUTO" is case-insensitive', () => { assert.equal(resolveContextWindow('AUTO').advisory, true); }); @@ -74,3 +82,86 @@ describe('scaleForWindow — linear scaling off the 200k anchor', () => { assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000); }); }); + +// B8b — model -> context-window mapping. `--context-window auto` probes the +// configured model and maps known 1M-tier model IDs to the large window so SKL/CML +// self-calibrate instead of falling back to the conservative advisory anchor. +// Verified June 2026 (platform.claude.com models overview): Fable 5, Opus +// 4.8/4.7/4.6 and Sonnet 4.6 run a 1M context window. + +describe('modelToContextWindow — known 1M-tier models map to the large window', () => { + it('the explicit [1m] tier tag wins (the running session model surfaces this way)', () => { + assert.equal(modelToContextWindow('claude-opus-4-8[1m]'), LARGE_CONTEXT_WINDOW); + // Suffix wins even for a base ID we do not otherwise enumerate. + assert.equal(modelToContextWindow('claude-future-9[1m]'), LARGE_CONTEXT_WINDOW); + }); + + it('known 1M base IDs map to 1M', () => { + for (const id of [ + 'claude-fable-5', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-sonnet-4-6', + ]) { + assert.equal(modelToContextWindow(id), LARGE_CONTEXT_WINDOW, id); + } + }); + + it('tolerates dated suffixes and provider prefixes (substring match)', () => { + assert.equal(modelToContextWindow('claude-opus-4-8-20260528'), LARGE_CONTEXT_WINDOW); + assert.equal(modelToContextWindow('us.anthropic.claude-sonnet-4-6'), LARGE_CONTEXT_WINDOW); + assert.equal(modelToContextWindow('CLAUDE-OPUS-4-8'), LARGE_CONTEXT_WINDOW); // case-insensitive + }); + + it('short aliases that resolve to a 1M-tier model map to 1M', () => { + for (const alias of ['opus', 'sonnet', 'fable', 'opusplan']) { + assert.equal(modelToContextWindow(alias), LARGE_CONTEXT_WINDOW, alias); + } + }); +}); + +describe('modelToContextWindow — unknown / unconfirmed models return null (conservative)', () => { + it('returns null for models whose window we do not confirm', () => { + // Haiku is intentionally NOT enumerated: keep the conservative anchor rather + // than guess. null -> caller keeps the advisory 200k anchor. + assert.equal(modelToContextWindow('claude-haiku-4-5-20251001'), null); + assert.equal(modelToContextWindow('haiku'), null); + assert.equal(modelToContextWindow('some-unknown-model'), null); + assert.equal(modelToContextWindow('default'), null); + }); + + it('returns null for non-string / empty input', () => { + assert.equal(modelToContextWindow(undefined), null); + assert.equal(modelToContextWindow(null), null); + assert.equal(modelToContextWindow(''), null); + assert.equal(modelToContextWindow(' '), null); + assert.equal(modelToContextWindow(42), null); + }); +}); + +describe('resolveContextWindow — auto probes the model (B8b)', () => { + it('"auto" + a recognized 1M model calibrates to 1M, not advisory', () => { + const r = resolveContextWindow('auto', { model: 'claude-opus-4-8[1m]' }); + assert.equal(r.window, LARGE_CONTEXT_WINDOW); + assert.equal(r.advisory, false, 'a confirmed window is not advisory'); + assert.equal(r.source, 'auto-probed'); + }); + + it('"auto" + a recognized base ID (no tag) also calibrates', () => { + const r = resolveContextWindow('auto', { model: 'claude-sonnet-4-6' }); + assert.equal(r.window, LARGE_CONTEXT_WINDOW); + assert.equal(r.source, 'auto-probed'); + }); + + it('the model is ignored on the explicit and default paths (byte-stable)', () => { + const explicit = resolveContextWindow('1000000', { model: 'claude-opus-4-8' }); + assert.equal(explicit.source, 'explicit'); + assert.equal(explicit.window, 1_000_000); + + const def = resolveContextWindow(undefined, { model: 'claude-opus-4-8[1m]' }); + assert.equal(def.source, 'default'); + assert.equal(def.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(def.advisory, false); + }); +});