From f93830ce74cf60461cab81762103dcdad5beb7ad Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 22 Jun 2026 13:30:44 +0200 Subject: [PATCH 01/53] feat(campaign): durable machine-wide campaign-ledger core (v5.7 Fase 2 Block 3a THIN) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the durable ledger that sits ABOVE individual config-audit sessions for a machine-wide audit campaign: repo list + per-repo lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by status and severity. scanners/lib/campaign-ledger.mjs — same hybrid split as knowledge-refresh: - PURE transforms (createLedger / addRepo / setRepoStatus / rollUp) with `now` injected (YYYY-MM-DD, never the clock) → deterministic + unit-testable. - soft validateLedger (returns {valid,errors}, never throws) for loaded data; transforms throw on programmer error (invalid status, unknown path). - thin IO shell (defaultLedgerPath / loadLedger→null-on-ENOENT / saveLedger). - persists to ~/.claude/config-audit/campaign-ledger.json — OUTSIDE the plugin dir (next to sessions/) so it survives uninstall/reinstall/upgrade. - schemaVersion stamped from the start → cheap Block 4 migration. THIN scope (Block 3a, operator-approved): ledger + roll-up + persistence only — no CLI/command/execution (Blocks 3b/3c/4). Internal plumbing, byte-stable until consumed: no `export async function scan` + lives in lib/ → scanner count stays 15, no orchestrator wiring, SC-5 unchanged. tests/lib/campaign-ledger.test.mjs — 28 tests (TDD, red→green). Full hermetic suite 1091→1119 green. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 16 ++ scanners/lib/campaign-ledger.mjs | 222 ++++++++++++++++++++++++++++ tests/lib/campaign-ledger.test.mjs | 230 +++++++++++++++++++++++++++++ 3 files changed, 468 insertions(+) create mode 100644 scanners/lib/campaign-ledger.mjs create mode 100644 tests/lib/campaign-ledger.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 37eb44e..8b270d0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -419,6 +419,22 @@ Same hybrid split as Chunk 2b — a deterministic, byte-stable, unit-tested core like `/config-audit optimize`. **No new agent** (web poll runs in the command's own context), **no new orchestrated scanner**. suite 1068→1091. +### campaign-ledger — durable machine-wide campaign core (v5.7 Fase 2, Block 3a THIN) + +`scanners/lib/campaign-ledger.mjs`: the durable ledger that sits ABOVE individual sessions for a +machine-wide audit campaign — repo list + per-repo lifecycle (`STATUSES` = pending→audited→planned +→implemented) + a machine-wide `rollUp` (counts by status + severity aggregated across repos). It +persists to a single JSON file **outside** the plugin dir (`~/.claude/config-audit/campaign-ledger +.json`, next to `sessions/`) so it survives uninstall/reinstall/upgrade. Same hybrid split as +knowledge-refresh: PURE transforms (`createLedger`/`addRepo`/`setRepoStatus`/`rollUp`) with `now` +**injected** (YYYY-MM-DD, never the clock) + soft `validateLedger` (returns `{valid,errors}`, never +throws) + a thin IO shell (`defaultLedgerPath`/`loadLedger`→null-on-ENOENT/`saveLedger`). Transforms +throw on programmer error (invalid status, unknown path); `schemaVersion` stamped from the start so a +Block 4 migration is cheap. **THIN**: ledger + roll-up + persistence only — NO execution, CLI, or +command surface (Blocks 3b/3c/4). **Internal plumbing, byte-stable until consumed**: no `export async +function scan` + lives in `lib/` → scanner count stays 15, no orchestrator wiring, SC-5 unchanged. +28 tests, suite 1091→1119. + ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage diff --git a/scanners/lib/campaign-ledger.mjs b/scanners/lib/campaign-ledger.mjs new file mode 100644 index 0000000..e8a47c9 --- /dev/null +++ b/scanners/lib/campaign-ledger.mjs @@ -0,0 +1,222 @@ +/** + * campaign-ledger — durable, machine-wide campaign ledger (v5.7 Fase 2, Block 3a THIN). + * + * The ledger sits ABOVE individual config-audit sessions: it tracks which repos are part + * of a machine-wide audit campaign, each repo's lifecycle status (pending → audited → + * planned → implemented), and a machine-wide roll-up by status + severity. It is resumable + * across sessions because it persists to a single JSON file OUTSIDE the plugin dir + * (`~/.claude/config-audit/campaign-ledger.json`, next to `sessions/` and `mcp-cache/`), + * so it survives plugin uninstall/reinstall/upgrade. + * + * Design mirrors the knowledge-refresh precedent: the transformations are PURE and + * deterministic (every "now" is injected as a YYYY-MM-DD string, never read from the clock + * here), so they are fully unit-testable; a thin IO shell (load/save, explicit path) does + * the only filesystem work. `validateLedger` is soft (returns a result, never throws) for + * externally-loaded data; the transforms throw on programmer error (invalid status, unknown + * path). `schemaVersion` is stamped from the start so a future Block 4 migration is cheap. + * + * THIN scope (Block 3a): ledger core + roll-up + persistence only — NOT execution, + * orchestration, or a command surface (those are Blocks 3b/3c/4). Zero dependencies. + * See docs/v5.7-optimization-lens-plan.md §Fase 2. + */ + +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { homedir } from 'node:os'; + +/** Ledger schema version — bump + add a migration (Block 4) on any breaking shape change. */ +export const CAMPAIGN_SCHEMA_VERSION = 1; + +/** Per-repo lifecycle, in order. A repo advances through these as the campaign progresses. */ +export const STATUSES = Object.freeze(['pending', 'audited', 'planned', 'implemented']); + +/** Severity buckets aggregated by the machine-wide roll-up. */ +const SEVERITIES = Object.freeze(['critical', 'high', 'medium', 'low']); + +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Validate an injected `now` (required, YYYY-MM-DD). Throws — callers pass today's date. */ +function requireNow(now) { + if (typeof now !== 'string' || !DATE_RE.test(now)) { + throw new TypeError('now must be a YYYY-MM-DD string'); + } + return now; +} + +/** Canonicalize a repo path so the same repo never appears twice under different spellings. */ +function normalizePath(path) { + if (typeof path !== 'string' || path.trim() === '') { + throw new TypeError('repo path is required'); + } + return resolve(path); +} + +/** + * Build an empty, versioned ledger stamped with `now`. + * @param {{now: string}} opts + * @returns {{schemaVersion:number, createdDate:string, updatedDate:string, repos:object[]}} + */ +export function createLedger({ now } = {}) { + requireNow(now); + return { schemaVersion: CAMPAIGN_SCHEMA_VERSION, createdDate: now, updatedDate: now, repos: [] }; +} + +/** + * Add a repo to the campaign (status `pending`). Idempotent on the normalized path — a repo + * already present is left untouched (its progress is NOT reset). Returns a NEW ledger. + * @param {object} ledger + * @param {{path: string, name?: string}} repo + * @param {{now: string}} opts + */ +export function addRepo(ledger, { path, name } = {}, { now } = {}) { + requireNow(now); + const resolved = normalizePath(path); + if (ledger.repos.some((r) => r.path === resolved)) { + return ledger; // idempotent: already tracked, preserve its status + } + const repo = { + path: resolved, + name: typeof name === 'string' && name.trim() !== '' ? name : resolved.split('/').pop(), + status: 'pending', + sessionId: null, + findingsBySeverity: null, + updatedDate: now, + }; + return { ...ledger, updatedDate: now, repos: [...ledger.repos, repo] }; +} + +/** + * Transition a tracked repo to a new status, optionally attaching the audit's + * findings-by-severity and the producing sessionId. Returns a NEW ledger. + * @param {object} ledger + * @param {string} path + * @param {string} status - one of STATUSES + * @param {{now: string, findingsBySeverity?: object|null, sessionId?: string|null}} opts + */ +export function setRepoStatus(ledger, path, status, { now, findingsBySeverity, sessionId } = {}) { + requireNow(now); + if (!STATUSES.includes(status)) { + throw new RangeError(`invalid status "${status}" — must be one of ${STATUSES.join(', ')}`); + } + const resolved = normalizePath(path); + const idx = ledger.repos.findIndex((r) => r.path === resolved); + if (idx === -1) { + throw new Error(`repo "${resolved}" is not in the ledger — addRepo first`); + } + const updated = { ...ledger.repos[idx], status, updatedDate: now }; + if (findingsBySeverity !== undefined) updated.findingsBySeverity = findingsBySeverity; + if (sessionId !== undefined) updated.sessionId = sessionId; + const repos = ledger.repos.slice(); + repos[idx] = updated; + return { ...ledger, updatedDate: now, repos }; +} + +/** + * Machine-wide roll-up: repo counts by status, plus a severity total aggregated across every + * repo that carries `findingsBySeverity`. Pure derivation — never mutates. + * @param {object} ledger + * @returns {{totalRepos:number, byStatus:object, bySeverity:object, reposWithFindings:number}} + */ +export function rollUp(ledger) { + const byStatus = Object.fromEntries(STATUSES.map((s) => [s, 0])); + const bySeverity = Object.fromEntries(SEVERITIES.map((s) => [s, 0])); + let reposWithFindings = 0; + + for (const repo of ledger.repos) { + if (byStatus[repo.status] !== undefined) byStatus[repo.status] += 1; + const f = repo.findingsBySeverity; + if (f && typeof f === 'object') { + reposWithFindings += 1; + for (const sev of SEVERITIES) { + if (typeof f[sev] === 'number') bySeverity[sev] += f[sev]; + } + } + } + return { totalRepos: ledger.repos.length, byStatus, bySeverity, reposWithFindings }; +} + +/** + * Validate a parsed ledger against the schema. Never throws — returns every problem at once + * so the caller (and tests) can inspect them. Soft by design (loaded data may be corrupt). + * @param {unknown} data + * @returns {{valid:boolean, errors:string[]}} + */ +export function validateLedger(data) { + const errors = []; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + return { valid: false, errors: ['ledger must be an object'] }; + } + if (data.schemaVersion !== CAMPAIGN_SCHEMA_VERSION) { + errors.push(`schemaVersion must be ${CAMPAIGN_SCHEMA_VERSION}`); + } + for (const field of ['createdDate', 'updatedDate']) { + if (typeof data[field] !== 'string' || !DATE_RE.test(data[field])) { + errors.push(`${field} must be YYYY-MM-DD`); + } + } + if (!Array.isArray(data.repos)) { + errors.push('repos must be an array'); + return { valid: false, errors }; + } + + const seen = new Set(); + data.repos.forEach((r, i) => { + const at = `repos[${i}]${r && typeof r === 'object' && r.path ? ` (${r.path})` : ''}`; + if (!r || typeof r !== 'object' || Array.isArray(r)) { + errors.push(`${at}: must be an object`); + return; + } + if (typeof r.path !== 'string' || r.path.trim() === '') { + errors.push(`${at}: path is required`); + } else if (seen.has(r.path)) { + errors.push(`${at}: duplicate path`); + } else { + seen.add(r.path); + } + if (!STATUSES.includes(r.status)) { + errors.push(`${at}: status must be one of ${STATUSES.join(', ')}`); + } + }); + + return { valid: errors.length === 0, errors }; +} + +// ── Persistence (thin IO shell) ──────────────────────────────────────────────── + +/** + * Default on-disk location: next to `sessions/`, OUTSIDE the plugin dir, so the campaign + * survives plugin uninstall/reinstall/upgrade. + * @returns {string} + */ +export function defaultLedgerPath() { + return join(homedir(), '.claude', 'config-audit', 'campaign-ledger.json'); +} + +/** + * Load + parse a ledger file. Returns `null` if the file does not exist (graceful first run); + * other read/parse errors propagate so corruption is not silently swallowed. + * @param {string} [path] + * @returns {Promise} + */ +export async function loadLedger(path = defaultLedgerPath()) { + let content; + try { + content = await readFile(path, 'utf8'); + } catch (err) { + if (err && err.code === 'ENOENT') return null; + throw err; + } + return JSON.parse(content); +} + +/** + * Persist a ledger as human-readable JSON, creating parent directories as needed. + * @param {string} path + * @param {object} ledger + * @returns {Promise<{path: string}>} + */ +export async function saveLedger(path = defaultLedgerPath(), ledger) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(ledger, null, 2)}\n`, 'utf8'); + return { path }; +} diff --git a/tests/lib/campaign-ledger.test.mjs b/tests/lib/campaign-ledger.test.mjs new file mode 100644 index 0000000..c1fe43a --- /dev/null +++ b/tests/lib/campaign-ledger.test.mjs @@ -0,0 +1,230 @@ +import { describe, it, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtemp, rm, readFile } from 'node:fs/promises'; +import { tmpdir, homedir } from 'node:os'; +import { + CAMPAIGN_SCHEMA_VERSION, + STATUSES, + createLedger, + addRepo, + setRepoStatus, + rollUp, + validateLedger, + defaultLedgerPath, + loadLedger, + saveLedger, +} from '../../scanners/lib/campaign-ledger.mjs'; + +const NOW = '2026-06-22'; +const LATER = '2026-06-23'; + +describe('constants', () => { + it('schema version is 1', () => { + assert.equal(CAMPAIGN_SCHEMA_VERSION, 1); + }); + it('STATUSES is the frozen four-step lifecycle', () => { + assert.deepEqual([...STATUSES], ['pending', 'audited', 'planned', 'implemented']); + assert.ok(Object.isFrozen(STATUSES)); + }); +}); + +describe('createLedger', () => { + it('builds an empty, versioned ledger stamped with `now`', () => { + const l = createLedger({ now: NOW }); + assert.deepEqual(l, { + schemaVersion: 1, + createdDate: NOW, + updatedDate: NOW, + repos: [], + }); + }); + it('throws on a missing/invalid now', () => { + assert.throws(() => createLedger({}), /now/); + assert.throws(() => createLedger({ now: 'June 22' }), /now/); + }); +}); + +describe('addRepo', () => { + it('adds a pending repo with name, resolved path, and stamps', () => { + const l = addRepo(createLedger({ now: NOW }), { path: '/Users/ktg/repos/foo', name: 'foo' }, { now: NOW }); + assert.equal(l.repos.length, 1); + assert.deepEqual(l.repos[0], { + path: '/Users/ktg/repos/foo', + name: 'foo', + status: 'pending', + sessionId: null, + findingsBySeverity: null, + updatedDate: NOW, + }); + }); + + it('normalizes the path (trailing slash, .. segments)', () => { + const l = addRepo(createLedger({ now: NOW }), { path: '/Users/ktg/repos/foo/../foo/', name: 'foo' }, { now: NOW }); + assert.equal(l.repos[0].path, '/Users/ktg/repos/foo'); + }); + + it('is idempotent on path — no duplicate, status preserved', () => { + let l = addRepo(createLedger({ now: NOW }), { path: '/r/foo', name: 'foo' }, { now: NOW }); + l = setRepoStatus(l, '/r/foo', 'audited', { now: NOW }); + l = addRepo(l, { path: '/r/foo', name: 'foo' }, { now: LATER }); + assert.equal(l.repos.length, 1); + assert.equal(l.repos[0].status, 'audited'); // progress not reset + }); + + it('bumps ledger.updatedDate but not createdDate', () => { + const base = createLedger({ now: NOW }); + const l = addRepo(base, { path: '/r/foo', name: 'foo' }, { now: LATER }); + assert.equal(l.createdDate, NOW); + assert.equal(l.updatedDate, LATER); + }); + + it('does not mutate the input ledger', () => { + const base = createLedger({ now: NOW }); + addRepo(base, { path: '/r/foo', name: 'foo' }, { now: NOW }); + assert.equal(base.repos.length, 0); + }); + + it('throws on a missing path', () => { + assert.throws(() => addRepo(createLedger({ now: NOW }), { name: 'foo' }, { now: NOW }), /path/); + }); +}); + +describe('setRepoStatus', () => { + const seed = () => addRepo(createLedger({ now: NOW }), { path: '/r/foo', name: 'foo' }, { now: NOW }); + + it('updates status and stamps the repo + ledger', () => { + const l = setRepoStatus(seed(), '/r/foo', 'audited', { now: LATER }); + assert.equal(l.repos[0].status, 'audited'); + assert.equal(l.repos[0].updatedDate, LATER); + assert.equal(l.updatedDate, LATER); + }); + + it('attaches findingsBySeverity and sessionId when provided', () => { + const findings = { critical: 0, high: 2, medium: 5, low: 3 }; + const l = setRepoStatus(seed(), '/r/foo', 'audited', { now: NOW, findingsBySeverity: findings, sessionId: '20260404_162210' }); + assert.deepEqual(l.repos[0].findingsBySeverity, findings); + assert.equal(l.repos[0].sessionId, '20260404_162210'); + }); + + it('resolves the path the same way addRepo does', () => { + const l = setRepoStatus(seed(), '/r/foo/', 'planned', { now: NOW }); + assert.equal(l.repos[0].status, 'planned'); + }); + + it('throws on an invalid status', () => { + assert.throws(() => setRepoStatus(seed(), '/r/foo', 'done', { now: NOW }), /status/); + }); + + it('throws on an unknown path', () => { + assert.throws(() => setRepoStatus(seed(), '/r/bar', 'audited', { now: NOW }), /not in the ledger|unknown/i); + }); + + it('does not mutate the input ledger', () => { + const l = seed(); + setRepoStatus(l, '/r/foo', 'audited', { now: NOW }); + assert.equal(l.repos[0].status, 'pending'); + }); +}); + +describe('rollUp', () => { + it('returns all-zero buckets for an empty ledger', () => { + const r = rollUp(createLedger({ now: NOW })); + assert.equal(r.totalRepos, 0); + assert.deepEqual(r.byStatus, { pending: 0, audited: 0, planned: 0, implemented: 0 }); + assert.deepEqual(r.bySeverity, { critical: 0, high: 0, medium: 0, low: 0 }); + assert.equal(r.reposWithFindings, 0); + }); + + it('counts statuses and aggregates severity machine-wide', () => { + let l = createLedger({ now: NOW }); + l = addRepo(l, { path: '/r/a', name: 'a' }, { now: NOW }); + l = addRepo(l, { path: '/r/b', name: 'b' }, { now: NOW }); + l = addRepo(l, { path: '/r/c', name: 'c' }, { now: NOW }); + l = setRepoStatus(l, '/r/a', 'audited', { now: NOW, findingsBySeverity: { critical: 1, high: 2, medium: 0, low: 4 } }); + l = setRepoStatus(l, '/r/b', 'implemented', { now: NOW, findingsBySeverity: { critical: 0, high: 1, medium: 3, low: 0 } }); + // c stays pending, no findings + + const r = rollUp(l); + assert.equal(r.totalRepos, 3); + assert.deepEqual(r.byStatus, { pending: 1, audited: 1, planned: 0, implemented: 1 }); + assert.deepEqual(r.bySeverity, { critical: 1, high: 3, medium: 3, low: 4 }); + assert.equal(r.reposWithFindings, 2); + }); +}); + +describe('validateLedger', () => { + const good = () => setRepoStatus( + addRepo(createLedger({ now: NOW }), { path: '/r/foo', name: 'foo' }, { now: NOW }), + '/r/foo', 'audited', { now: NOW, findingsBySeverity: { critical: 0, high: 0, medium: 1, low: 0 } }, + ); + + it('accepts a well-formed ledger', () => { + const res = validateLedger(good()); + assert.equal(res.valid, true, res.errors.join('; ')); + assert.deepEqual(res.errors, []); + }); + + it('rejects a non-object', () => { + assert.equal(validateLedger(null).valid, false); + assert.equal(validateLedger([]).valid, false); + }); + + it('rejects a bad schemaVersion', () => { + const l = { ...good(), schemaVersion: '1' }; + assert.equal(validateLedger(l).valid, false); + }); + + it('rejects repos that is not an array', () => { + const l = { ...good(), repos: {} }; + assert.equal(validateLedger(l).valid, false); + }); + + it('rejects a repo missing a path', () => { + const l = good(); + l.repos = [{ ...l.repos[0], path: undefined }]; + assert.equal(validateLedger(l).valid, false); + }); + + it('rejects a repo with an out-of-enum status', () => { + const l = good(); + l.repos = [{ ...l.repos[0], status: 'done' }]; + assert.equal(validateLedger(l).valid, false); + }); + + it('rejects duplicate repo paths', () => { + const l = good(); + l.repos = [l.repos[0], { ...l.repos[0] }]; + assert.equal(validateLedger(l).valid, false); + }); +}); + +describe('persistence (IO)', () => { + it('defaultLedgerPath lives under ~/.claude/config-audit (survives uninstall)', () => { + const p = defaultLedgerPath(); + assert.equal(p, join(homedir(), '.claude', 'config-audit', 'campaign-ledger.json')); + }); + + it('round-trips through save/load and creates parent dirs', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ca-ledger-')); + after(() => rm(dir, { recursive: true, force: true })); + const file = join(dir, 'nested', 'campaign-ledger.json'); + const l = good(); + function good() { + return addRepo(createLedger({ now: NOW }), { path: '/r/foo', name: 'foo' }, { now: NOW }); + } + await saveLedger(file, l); + const loaded = await loadLedger(file); + assert.deepEqual(loaded, l); + // human-readable JSON on disk + const raw = await readFile(file, 'utf8'); + assert.match(raw, /\n/); + }); + + it('loadLedger returns null for a missing file (graceful first run)', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ca-ledger-')); + after(() => rm(dir, { recursive: true, force: true })); + const loaded = await loadLedger(join(dir, 'nope.json')); + assert.equal(loaded, null); + }); +}); From 4be7a16788442128f838d13162c5e1d2c5d4c75e Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 22 Jun 2026 14:13:27 +0200 Subject: [PATCH 02/53] feat(campaign): read-only campaign-cli ledger reporter (v5.7 Fase 2 Block 3b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only reporter over the durable campaign-ledger core (Block 3a). Mirrors knowledge-refresh-cli: the deterministic, READ-ONLY half of the campaign motor. - scanners/campaign-cli.mjs (-cli → not an orchestrated scanner): loadLedger + validateLedger + rollUp, emits {status, initialized, ledgerPath, schemaVersion, createdDate, updatedDate, repos, rollUp} JSON. NEVER writes — a missing ledger is reported gracefully (initialized:false), never created. init + status transitions belong to the Block 3c command layer (human-approved writes). - --ledger-file overrides default path (deterministic testing); --output-file mirrors the sibling. Exit: 0 = initialized & valid, 1 = not initialized (advisory), 3 = error (parse/corrupt/invalid). - 8 tests (tests/scanners/campaign-cli.test.mjs). Suite 1119→1127 green hermetic; scanner count stays 15 (-cli), SC-5 byte-stable. - CLAUDE.md: §campaign-cli added; Testing badge corrected 1091/62 → 1127/64 (fixes pre-existing Block 3a drift). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 14 ++- scanners/campaign-cli.mjs | 109 +++++++++++++++++++++ tests/scanners/campaign-cli.test.mjs | 140 +++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 scanners/campaign-cli.mjs create mode 100644 tests/scanners/campaign-cli.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 8b270d0..5e40915 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,7 +112,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -1091 tests across 62 test files (20 lib + 32 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1127 tests across 64 test files (21 lib + 33 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) @@ -435,6 +435,18 @@ command surface (Blocks 3b/3c/4). **Internal plumbing, byte-stable until consume function scan` + lives in `lib/` → scanner count stays 15, no orchestrator wiring, SC-5 unchanged. 28 tests, suite 1091→1119. +### campaign-cli — read-only ledger reporter (v5.7 Fase 2, Block 3b) + +`scanners/campaign-cli.mjs` (`-cli` → NOT an orchestrated scanner → scanner count stays 15, suite +byte-stable; 8 tests): the DETERMINISTIC, READ-ONLY half of the campaign motor, mirroring +`knowledge-refresh-cli`. It `loadLedger`s the durable ledger, `validateLedger`s it, and emits +`{status, initialized, ledgerPath, schemaVersion, createdDate, updatedDate, repos, rollUp}` as JSON. +It NEVER writes — a missing ledger is reported gracefully (`initialized:false`, all-zero roll-up), +**never created**; init + every status transition belong to the Block 3c command layer (human-approved +writes, Verifiseringsplikt). `--ledger-file` overrides the default path (deterministic testing); +`--output-file` mirrors the sibling. Exit codes: **0** = initialized & valid, **1** = not initialized +yet (advisory), **3** = error (parse/corrupt/invalid). suite 1119→1127. + ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage diff --git a/scanners/campaign-cli.mjs b/scanners/campaign-cli.mjs new file mode 100644 index 0000000..fc3846b --- /dev/null +++ b/scanners/campaign-cli.mjs @@ -0,0 +1,109 @@ +#!/usr/bin/env node + +/** + * campaign-cli — read-only reporter for the durable machine-wide campaign ledger + * (v5.7 Fase 2, Block 3b). + * + * Mirrors the knowledge-refresh-cli precedent: it is the DETERMINISTIC, READ-ONLY half + * of the hybrid motor. It loads the campaign ledger (the durable file that sits ABOVE + * individual config-audit sessions), validates it, and emits the repo list + a + * machine-wide roll-up as JSON. It NEVER writes the ledger — initialization and every + * status transition belong to the command layer (Block 3c `/config-audit campaign`), + * which calls the lib's pure transforms + saveLedger only on explicit, human-approved + * action. A missing ledger file is reported gracefully (initialized:false), NEVER created. + * + * Naming: `-cli` suffix → NOT an orchestrated scanner (the scan-orchestrator only loads + * scanner modules), so the scanner count is unchanged and the snapshot suite stays + * byte-stable. + * + * Usage: + * node campaign-cli.mjs [--ledger-file ] [--output-file ] + * + * Exit codes: 0 = initialized & valid, 1 = not initialized yet (advisory), 3 = error. + */ + +import { resolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { + loadLedger, + validateLedger, + rollUp, + defaultLedgerPath, +} from './lib/campaign-ledger.mjs'; + +function fail(message) { + process.stderr.write(`Error: ${message}\n`); + process.exit(3); +} + +async function main() { + const args = process.argv.slice(2); + let ledgerFile = null; + let outputFile = null; + + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === '--ledger-file' && args[i + 1]) ledgerFile = args[++i]; + else if (a === '--output-file' && args[i + 1]) outputFile = args[++i]; + } + + const ledgerPath = resolve(ledgerFile || defaultLedgerPath()); + + let ledger; + try { + // loadLedger returns null on ENOENT (graceful first run) and throws on parse error. + ledger = await loadLedger(ledgerPath); + } catch (err) { + fail(`could not read ledger at ${ledgerPath}: ${err.message}`); + } + + let payload; + let exitCode; + + if (ledger === null) { + // Graceful first run — the ledger does not exist yet. We DO NOT create it; that is + // the command layer's job (Block 3c), on explicit human-approved action. + payload = { + status: 'ok', + initialized: false, + ledgerPath, + schemaVersion: null, + createdDate: null, + updatedDate: null, + repos: [], + rollUp: rollUp({ repos: [] }), + }; + exitCode = 1; // advisory: there is no campaign to report yet + } else { + const { valid, errors } = validateLedger(ledger); + if (!valid) { + fail(`ledger at ${ledgerPath} is invalid:\n - ${errors.join('\n - ')}`); + } + payload = { + status: 'ok', + initialized: true, + ledgerPath, + schemaVersion: ledger.schemaVersion, + createdDate: ledger.createdDate, + updatedDate: ledger.updatedDate, + repos: ledger.repos, + rollUp: rollUp(ledger), + }; + exitCode = 0; + } + + const json = JSON.stringify(payload, null, 2); + if (outputFile) await writeFile(outputFile, json, 'utf-8'); + else process.stdout.write(json + '\n'); + + process.exit(exitCode); +} + +const isDirectRun = + process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname); +if (isDirectRun) { + main().catch((err) => { + process.stderr.write(`Fatal: ${err.message}\n`); + process.exit(3); + }); +} diff --git a/tests/scanners/campaign-cli.test.mjs b/tests/scanners/campaign-cli.test.mjs new file mode 100644 index 0000000..dea642b --- /dev/null +++ b/tests/scanners/campaign-cli.test.mjs @@ -0,0 +1,140 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { readFileSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { + createLedger, + addRepo, + setRepoStatus, + rollUp, + saveLedger, +} from '../../scanners/lib/campaign-ledger.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const CLI = resolve(__dirname, '../../scanners/campaign-cli.mjs'); + +const NOW = '2026-06-22'; + +// Every test passes an explicit --ledger-file in a temp dir, so the CLI never touches +// the real default path under HOME — the suite is hermetic by construction. +function runCli(extraArgs) { + try { + const stdout = execFileSync('node', [CLI, ...extraArgs], { encoding: 'utf-8', timeout: 15000 }); + return { status: 0, stdout }; + } catch (err) { + return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' }; + } +} + +// Round-trip a populated ledger through the real lib + saveLedger so the CLI reads +// exactly what the command layer would persist. +async function seedLedger(dir) { + let l = createLedger({ now: NOW }); + l = addRepo(l, { path: '/r/a', name: 'a' }, { now: NOW }); + l = addRepo(l, { path: '/r/b', name: 'b' }, { now: NOW }); + l = addRepo(l, { path: '/r/c', name: 'c' }, { now: NOW }); + l = setRepoStatus(l, '/r/a', 'audited', { + now: NOW, + findingsBySeverity: { critical: 1, high: 2, medium: 0, low: 4 }, + sessionId: '20260622_120000', + }); + l = setRepoStatus(l, '/r/b', 'implemented', { + now: NOW, + findingsBySeverity: { critical: 0, high: 1, medium: 3, low: 0 }, + }); + // c stays pending, no findings + const file = join(dir, 'campaign-ledger.json'); + await saveLedger(file, l); + return { file, ledger: l }; +} + +const EMPTY_ROLLUP = { + totalRepos: 0, + byStatus: { pending: 0, audited: 0, planned: 0, implemented: 0 }, + bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, + reposWithFindings: 0, +}; + +describe('campaign-cli — exit codes', () => { + it('exits 0 when the ledger exists and is valid', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { file } = await seedLedger(dir); + const { status, stdout } = runCli(['--ledger-file', file]); + assert.equal(status, 0); + assert.equal(JSON.parse(stdout).initialized, true); + }); + + it('exits 1 (advisory) when no ledger file exists yet', () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { status, stdout } = runCli(['--ledger-file', join(dir, 'nope.json')]); + assert.equal(status, 1); + const out = JSON.parse(stdout); + assert.equal(out.initialized, false); + assert.deepEqual(out.repos, []); + }); + + it('exits 3 on a malformed (non-JSON) ledger file', () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const file = join(dir, 'campaign-ledger.json'); + writeFileSync(file, '{ not valid json'); + assert.equal(runCli(['--ledger-file', file]).status, 3); + }); + + it('exits 3 on a schema-invalid ledger', () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const file = join(dir, 'campaign-ledger.json'); + writeFileSync(file, JSON.stringify({ schemaVersion: 99, repos: 'nope' })); + assert.equal(runCli(['--ledger-file', file]).status, 3); + }); +}); + +describe('campaign-cli — payload shape', () => { + it('emits the expected keys and a roll-up matching the lib', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { file, ledger } = await seedLedger(dir); + const out = JSON.parse(runCli(['--ledger-file', file]).stdout); + assert.equal(out.status, 'ok'); + assert.equal(out.initialized, true); + assert.equal(out.ledgerPath, resolve(file)); + assert.equal(out.schemaVersion, 1); + assert.equal(out.createdDate, NOW); + assert.equal(out.updatedDate, NOW); + assert.equal(out.repos.length, 3); + assert.deepEqual(out.rollUp, rollUp(ledger)); + }); + + it('repos carry per-repo provenance (status, findingsBySeverity, sessionId)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { file } = await seedLedger(dir); + const out = JSON.parse(runCli(['--ledger-file', file]).stdout); + const a = out.repos.find((r) => r.name === 'a'); + assert.equal(a.status, 'audited'); + assert.deepEqual(a.findingsBySeverity, { critical: 1, high: 2, medium: 0, low: 4 }); + assert.equal(a.sessionId, '20260622_120000'); + }); + + it('uninitialized payload has empty repos and an all-zero roll-up', () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const out = JSON.parse(runCli(['--ledger-file', join(dir, 'nope.json')]).stdout); + assert.equal(out.initialized, false); + assert.equal(out.schemaVersion, null); + assert.deepEqual(out.repos, []); + assert.deepEqual(out.rollUp, EMPTY_ROLLUP); + }); +}); + +describe('campaign-cli — --output-file', () => { + it('writes JSON to the file and stays silent on stdout', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { file } = await seedLedger(dir); + const out = join(dir, 'report.json'); + const { stdout } = runCli(['--ledger-file', file, '--output-file', out]); + assert.equal(stdout.trim(), '', 'stdout is silent when --output-file is given'); + const written = JSON.parse(readFileSync(out, 'utf-8')); + assert.equal(written.status, 'ok'); + assert.equal(written.repos.length, 3); + }); +}); From ee0c762151ec4f33db2cc4ec31a3a166d91423cf Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 22 Jun 2026 17:25:15 +0200 Subject: [PATCH 03/53] =?UTF-8?q?feat(campaign):=20/config-audit=20campaig?= =?UTF-8?q?n=20=E2=80=94=20human-approved=20write-CLI=20+=20command=20(v5.?= =?UTF-8?q?7=20Fase=202=20Block=203c)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the THIN machine-wide campaign surface (ledger + roll-up + status). The write half of the campaign motor, mirroring how knowledge-refresh gates register writes: - scanners/campaign-write-cli.mjs (-cli → NOT a scanner; 11 tests): init/add/ set-status, each a thin wrapper over the invariant-enforcing lib transforms (createLedger/addRepo/setRepoStatus) + saveLedger — path-normalization/dedup, idempotent add, status-lifecycle guard and updatedDate bump never hand-rolled. init refuses to clobber an existing/corrupt ledger (exit 1, file untouched); add auto-inits + reports added vs skipped; set-status takes --findings/--session. Deterministic: --reference-date is the only clock read, injected as `now`. Exit 0=write, 1=advisory no-op, 3=error. - commands/campaign.md (opus, no Web): thin orchestrator — always reports first (read-only campaign-cli), then proposes init/add/set-status and invokes ONE write-CLI subcommand only on explicit human approval (Verifiseringsplikt; never hand-edits the JSON). add --discover finds git repos under a root to pick from. Not byte-stable (own command, outside snapshot suite) like /config-audit optimize + knowledge-refresh. Both CLIs are -cli → scanner count stays 15, snapshot suite untouched. commands 20→21, suite 1127→1138 (+11), test files 64→65. Docs: CLAUDE.md §campaign-write-cli + command table + Testing badge; README commands badge 20→21 + table row; help.md + router wiring. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 34 +++- README.md | 3 +- commands/campaign.md | 177 +++++++++++++++++ commands/config-audit.md | 3 +- commands/help.md | 1 + scanners/campaign-write-cli.mjs | 196 +++++++++++++++++++ tests/scanners/campaign-write-cli.test.mjs | 211 +++++++++++++++++++++ 7 files changed, 622 insertions(+), 3 deletions(-) create mode 100644 commands/campaign.md create mode 100644 scanners/campaign-write-cli.mjs create mode 100644 tests/scanners/campaign-write-cli.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 5e40915..0a60a44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,6 +35,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — deterministic stale check (sources older than ~90d) + web candidate poll (CC changelog + Anthropic blog); **human-approved writes only** (Verifiseringsplikt). The "living" half of the knowledge base. Web/judgment-driven, **not byte-stable** | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity, resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via a deterministic write-CLI (init/add/set-status). THIN: tracks state, doesn't run audits/fixes. Judgment-driven, **not byte-stable** | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Gather user preferences (opt-in) | @@ -112,7 +113,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -1127 tests across 64 test files (21 lib + 33 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1138 tests across 65 test files (21 lib + 34 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) @@ -447,6 +448,37 @@ writes, Verifiseringsplikt). `--ledger-file` overrides the default path (determi `--output-file` mirrors the sibling. Exit codes: **0** = initialized & valid, **1** = not initialized yet (advisory), **3** = error (parse/corrupt/invalid). suite 1119→1127. +### campaign-write-cli + `/config-audit campaign` — the WRITE half (v5.7 Fase 2, Block 3c, commands 20→21) + +The human-approved mutation half of the campaign motor, completing the THIN campaign surface +(ledger + roll-up + status). Two pieces: + +- **`scanners/campaign-write-cli.mjs`** (`-cli` → NOT an orchestrated scanner → scanner count stays + 15, suite byte-stable; 11 tests): the sibling of `campaign-cli` that *mutates*. Subcommands + `init` / `add ...` / `set-status `, each a thin wrapper over the + invariant-enforcing lib transforms (`createLedger`/`addRepo`/`setRepoStatus`) + `saveLedger` — so + path-normalization/dedup, idempotent add, the status-lifecycle guard, and the `updatedDate` bump + are **never re-implemented by hand**. `init` refuses to clobber an existing (or corrupt) ledger + (**exit 1** advisory, file untouched); `add` **auto-inits** when no ledger exists and reports + `added` vs `skipped`; `set-status` accepts `--findings ''` + `--session `. Determinism + mirrors the lib + `knowledge-refresh-cli`: `--reference-date` is the **only** place the clock is + read (defaults to today), passed to the transforms as the injected `now`. Exit: **0** = write + performed, **1** = advisory no-op (init-clobber), **3** = error (unknown subcommand, bad args, + invalid status, untracked repo, no/corrupt ledger). +- **`commands/campaign.md`** (opus, `allowed-tools: Read/Write/Edit/Bash/Glob` — **no Web**, + judgment-free): a thin orchestrator. It always **reports** first (read-only `campaign-cli`), then + for `init`/`add`/`set-status` it proposes the change and, **only on explicit human approval**, + invokes one write-CLI subcommand (Verifiseringsplikt — it never hand-edits the ledger JSON). + `add --discover ` finds git repos under a root and lets the user pick. When marking a repo + `audited` it attaches findings-by-severity from the repo's session (or user-provided counts) — + **never invented**. + +**Not a new scanner, not byte-stable.** Both CLIs carry the `-cli` suffix (out of the +scan-orchestrator → scanner count stays **15**, snapshot suite untouched); the command's +orchestration is judgment-driven and deliberately outside the snapshot suite, exactly like +`/config-audit optimize` + `knowledge-refresh`. **No new agent** (web/judgment-free, runs in the +command's own context). suite 1127→1138. + ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage diff --git a/README.md b/README.md index ba0bd9f..e138712 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ![Version](https://img.shields.io/badge/version-5.7.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-15-cyan) -![Commands](https://img.shields.io/badge/commands-20-green) +![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) ![Tests](https://img.shields.io/badge/tests-1091+-brightgreen) @@ -293,6 +293,7 @@ Your team configuration changes over time. Track it: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — flag stale entries (sources older than ~90d) + poll for new/changed Claude Code practices; **human-approved writes only** (Verifiseringsplikt). Deterministic stale core + web candidate poll | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity, resumable across sessions; **human-approved writes only** (read-only report + deterministic write-CLI) | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Set preferences for action plan _(optional)_ | diff --git a/commands/campaign.md b/commands/campaign.md new file mode 100644 index 0000000..d3428c5 --- /dev/null +++ b/commands/campaign.md @@ -0,0 +1,177 @@ +--- +name: config-audit:campaign +description: Machine-wide audit campaign — track which repos are pending/audited/planned/implemented across sessions, with a machine-wide roll-up. Human-approved writes only. +argument-hint: "[init | add ... | add --discover | set-status ]" +allowed-tools: Read, Write, Edit, Bash, Glob +model: opus +--- + +# Config-Audit: Campaign + +A single config-audit session audits **one** scope. A **campaign** sits above sessions: a +durable ledger of every repo you mean to bring up to standard, each repo's lifecycle status +(**pending → audited → planned → implemented**), and a machine-wide roll-up of findings by +severity. It persists to `~/.claude/config-audit/campaign-ledger.json` — **outside** the +plugin dir, next to `sessions/` — so it survives plugin uninstall/reinstall/upgrade and +resumes across sessions. + +**The Iron rule (Verifiseringsplikt): nothing is ever auto-written.** Reporting is read-only. +Every mutation — creating the ledger, adding a repo, changing a status — is proposed first +and applied **only on explicit approval**, by invoking one deterministic write-CLI subcommand. +The command never hand-edits the ledger JSON. + +This is the **THIN** campaign surface (ledger + roll-up + status). Cross-repo backlog +prioritization and execution are a later block — this command does not run audits or apply +fixes itself; it tracks where each repo stands. + +## Two CLIs back this command + +- **Read (report):** `scanners/campaign-cli.mjs` — loads + validates the ledger, emits the + repo list + roll-up. Never writes. +- **Write (mutate):** `scanners/campaign-write-cli.mjs` — `init` / `add` / `set-status`, each a + thin wrapper over the invariant-enforcing lib transforms + save. Invoked **only** after the + user approves a specific action. + +Both take `--ledger-file ` (defaults to the durable path) and `--output-file `; +the write-CLI also takes `--reference-date ` (the audit date stamp). + +## Implementation + +### Step 1: Parse arguments + +From `$ARGUMENTS`, pick the mode: + +- *(empty)* or `report` → **report** (read-only). Default. +- `init` → initialize the ledger. +- `add ...` → add one or more repo paths. +- `add --discover ` → find git repos under `` and let the user pick which to add. +- `set-status ` → transition a tracked repo (`status` ∈ pending/audited/planned/implemented). +- `help` → show this surface and stop. + +Set a shared date stamp for any write: `TODAY=$(date +%F)`. + +### Step 2: Always report current state first + +Whatever the mode, start by showing where the campaign stands (read-only): + +```bash +node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-cli.mjs \ + --output-file ~/.claude/config-audit/sessions/campaign-report.json 2>/dev/null; echo $? +``` + +Exit **0** = a campaign exists, **1** = not initialized yet (advisory — normal first run), +**3** = real error → "The campaign ledger couldn't be read — it may be corrupt." (Stop; do not +attempt a write over a corrupt ledger.) + +Read `~/.claude/config-audit/sessions/campaign-report.json` with the Read tool (per the UX +rules — never show the raw JSON). It has `initialized`, `repos[]` (each: `path, name, status, +sessionId, findingsBySeverity, updatedDate`), and `rollUp {totalRepos, byStatus, bySeverity, +reposWithFindings}`. + +Present it as two short tables: + +**Campaign roll-up** + +| Status | Repos | +|--------|-------| +| pending / audited / planned / implemented | … | + +…plus a one-line severity total across audited repos (e.g. "Findings so far: 3 critical, +8 high, 5 medium, 12 low across 4 audited repos"). + +**Repos** + +| Repo | Status | Findings (C/H/M/L) | Last updated | +|------|--------|--------------------|--------------| + +If `initialized` is false, say so plainly: "No campaign yet. Run `/config-audit campaign init` +to start one." Then — if the mode was `init` or `add` — continue to that step (those bootstrap +a campaign); for `report`/`set-status` on an uninitialized ledger, stop after this message. + +### Step 3 (mode `init`): Initialize + +If already initialized, say so and stop (no clobber). Otherwise tell the user what will happen, +then create it: + +```bash +node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-write-cli.mjs init \ + --reference-date "$TODAY" \ + --output-file ~/.claude/config-audit/sessions/campaign-write.json 2>/dev/null; echo $? +``` + +Exit **0** = created, **1** = already initialized (advisory). Confirm: "Campaign ledger created +at `~/.claude/config-audit/campaign-ledger.json`." Then suggest `add`. + +### Step 4 (mode `add`): Add repos — propose, approve, write + +**Gather candidates.** +- Explicit paths: use the paths given after `add`. +- `--discover `: find git repos (depth-limited), e.g. + ```bash + find "" -maxdepth 3 -type d -name .git 2>/dev/null | sed 's:/\.git$::' + ``` + Present the discovered repos as a numbered list and ask **which** to add (and confirm any + that are already tracked will be skipped). Use Glob as a fallback if `find` is unavailable. + +**Confirm, then write.** Show the final list and ask for explicit approval. On approval, add +them in one call (idempotent — already-tracked repos are skipped, not reset): + +```bash +node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-write-cli.mjs add ... \ + --reference-date "$TODAY" \ + --output-file ~/.claude/config-audit/sessions/campaign-write.json 2>/dev/null; echo $? +``` + +(For a single repo with a custom display name, add `--name ""`.) Read the result file and +report what was `added` vs `skipped`, then re-show the repo table. + +### Step 5 (mode `set-status`): Transition a repo — propose, approve, write + +Confirm the repo is tracked (from Step 2's report) and that `status` is one of +pending/audited/planned/implemented. State the transition ("``: pending → audited") and +ask for approval. + +When marking a repo **audited**, optionally attach its findings-by-severity so the machine-wide +roll-up stays meaningful. Two honest sources, in order of preference: +1. If the repo was audited in a config-audit session, read that session's finding counts and + build `{"critical":C,"high":H,"medium":M,"low":L}` — pass `--session ` too. +2. Otherwise, use counts the user provides. **Never invent counts** (Verifiseringsplikt) — if + none are available, transition the status without `--findings`. + +On approval: + +```bash +node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-write-cli.mjs set-status \ + --reference-date "$TODAY" \ + [--findings '{"critical":0,"high":0,"medium":0,"low":0}'] [--session ] \ + --output-file ~/.claude/config-audit/sessions/campaign-write.json 2>/dev/null; echo $? +``` + +Exit **3** = invalid status, untracked repo, or no ledger → report the message plainly and do +not retry blindly. On success, read the result and re-show the updated roll-up + repo row. + +### Step 6: Next steps + +Tailor to where the campaign stands: + +- **Just initialized / few repos:** "`/config-audit campaign add --discover ~/repos` to enroll + your repos." +- **Pending repos exist:** "Run `/config-audit` in a pending repo to audit it, then + `/config-audit campaign set-status audited` to record the result here." +- **Audited but not planned:** "`/config-audit plan` in that repo, then mark it `planned`." +- Always: the campaign survives this session — re-run `/config-audit campaign` anytime to see + the machine-wide picture. + +## Notes + +- **Read-only report, human-approved writes.** `campaign-cli` never writes; every mutation goes + through `campaign-write-cli` and only after explicit approval, exactly mirroring how + `/config-audit knowledge-refresh` gates register writes. +- **Deterministic core, not byte-stable command.** The lib transforms + both CLIs are + unit-tested and deterministic (`--reference-date` injected); this command's orchestration is + judgment-driven and deliberately **not** in the snapshot suite (like `/config-audit optimize` + and `knowledge-refresh`). +- The `-cli` suffix keeps both CLIs out of the scan-orchestrator, so the scanner count and the + byte-stable snapshot suite are unaffected. +- **THIN scope:** ledger + roll-up + status only. Cross-repo backlog prioritization and execution + are a later block — this command tracks state, it does not run audits or apply fixes. diff --git a/commands/config-audit.md b/commands/config-audit.md index bbec033..b699f04 100644 --- a/commands/config-audit.md +++ b/commands/config-audit.md @@ -1,7 +1,7 @@ --- name: config-audit description: Claude Code Configuration Intelligence - audit, analyze, and optimize your configuration -argument-hint: "[posture|tokens|manifest|feature-gap|optimize|fix|rollback|plan|implement|help|discover|analyze|interview|drift|plugin-health|whats-active|status|cleanup]" +argument-hint: "[posture|tokens|manifest|feature-gap|optimize|fix|rollback|plan|implement|help|discover|analyze|interview|drift|plugin-health|whats-active|campaign|status|cleanup]" allowed-tools: Read, Write, Glob, Grep, Bash, Agent, AskUserQuestion model: opus --- @@ -29,6 +29,7 @@ If a subcommand is provided, route to it: - `drift` → `/config-audit:drift` - `plugin-health` → `/config-audit:plugin-health` - `whats-active` → `/config-audit:whats-active` +- `campaign` → `/config-audit:campaign` - `status` → `/config-audit:status` - `cleanup` → `/config-audit:cleanup` diff --git a/commands/help.md b/commands/help.md index 7f0f7e8..5e88745 100644 --- a/commands/help.md +++ b/commands/help.md @@ -54,6 +54,7 @@ if echo "$ARGUMENTS" | grep -q -- "--raw"; then RAW_FLAG="--raw"; fi | `/config-audit drift` | Compare current config against a saved baseline | | `/config-audit plugin-health` | Audit plugin structure and the metadata block at the top of each command/agent file | | `/config-audit whats-active` | Show active plugins/skills/MCP/hooks/CLAUDE.md with token estimates | +| `/config-audit campaign` | Track a machine-wide audit campaign across repos — per-repo status + roll-up, resumable across sessions (human-approved writes) | ### Utility diff --git a/scanners/campaign-write-cli.mjs b/scanners/campaign-write-cli.mjs new file mode 100644 index 0000000..5438112 --- /dev/null +++ b/scanners/campaign-write-cli.mjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node + +/** + * campaign-write-cli — the human-approved WRITE half of the durable campaign ledger + * (v5.7 Fase 2, Block 3c). + * + * Sibling of the read-only `campaign-cli`: where that one only reports, this one mutates. + * Every mutation is routed through the pure, invariant-enforcing lib transforms + * (`createLedger`/`addRepo`/`setRepoStatus`) + `saveLedger` — so path normalization/dedup, + * idempotent add, the status-lifecycle guard, and the `updatedDate` bump are never + * re-implemented by hand. The `/config-audit campaign` command is a thin opus orchestrator: + * it reports (via campaign-cli), proposes a change, and only on explicit human approval + * invokes a single subcommand here (Verifiseringsplikt). It NEVER auto-writes. + * + * Determinism mirrors the lib + the knowledge-refresh CLI: `--reference-date` is the only + * place the clock is read (defaulting to today), and it is passed to the transforms as the + * injected `now`, so the persisted stamps are fully testable. + * + * Naming: `-cli` suffix → NOT an orchestrated scanner (the scan-orchestrator only loads + * scanner modules), so the scanner count is unchanged and the snapshot suite stays + * byte-stable. + * + * Usage: + * node campaign-write-cli.mjs init [--ledger-file

] [--reference-date ] + * node campaign-write-cli.mjs add ... [--name ] [--ledger-file

] [--reference-date ] + * node campaign-write-cli.mjs set-status + * [--findings ''] [--session ] + * [--ledger-file

] [--reference-date ] + * (all accept [--output-file

] to write the result payload to a file instead of stdout) + * + * Exit codes: 0 = write performed, 1 = advisory no-op (init when already initialized), + * 3 = error (unknown subcommand, bad args, invalid status, untracked repo, no/corrupt ledger). + */ + +import { resolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { + createLedger, + addRepo, + setRepoStatus, + rollUp, + loadLedger, + saveLedger, + defaultLedgerPath, +} from './lib/campaign-ledger.mjs'; + +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +function fail(message) { + process.stderr.write(`Error: ${message}\n`); + process.exit(3); +} + +/** Parse argv into a subcommand, positional args, and the flag map. */ +function parseArgs(argv) { + const positionals = []; + const flags = { ledgerFile: null, referenceDate: null, outputFile: null, name: null, findings: null, session: null }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--ledger-file' && argv[i + 1] !== undefined) flags.ledgerFile = argv[++i]; + else if (a === '--reference-date' && argv[i + 1] !== undefined) flags.referenceDate = argv[++i]; + else if (a === '--output-file' && argv[i + 1] !== undefined) flags.outputFile = argv[++i]; + else if (a === '--name' && argv[i + 1] !== undefined) flags.name = argv[++i]; + else if (a === '--findings' && argv[i + 1] !== undefined) flags.findings = argv[++i]; + else if (a === '--session' && argv[i + 1] !== undefined) flags.session = argv[++i]; + else if (a.startsWith('--')) fail(`unknown flag "${a}"`); + else positionals.push(a); + } + return { subcommand: positionals[0], rest: positionals.slice(1), flags }; +} + +/** Load an existing ledger, treating a parse error as a hard failure (never clobber corrupt data). */ +async function loadOrFail(path) { + try { + return await loadLedger(path); // null on ENOENT (no ledger yet) + } catch (err) { + fail(`could not read ledger at ${path}: ${err.message}`); + } +} + +async function emit(payload, outputFile, exitCode) { + const json = JSON.stringify(payload, null, 2); + if (outputFile) await writeFile(outputFile, json, 'utf-8'); + else process.stdout.write(json + '\n'); + process.exit(exitCode); +} + +async function main() { + const { subcommand, rest, flags } = parseArgs(process.argv.slice(2)); + + if (!subcommand) fail('a subcommand is required: init | add | set-status'); + + const ledgerPath = resolve(flags.ledgerFile || defaultLedgerPath()); + if (flags.referenceDate && !DATE_RE.test(flags.referenceDate)) fail('--reference-date must be YYYY-MM-DD'); + // The clock is read here ONLY — the transforms take this injected `now` and stay pure. + const now = flags.referenceDate || new Date().toISOString().slice(0, 10); + + if (subcommand === 'init') { + const existing = await loadOrFail(ledgerPath); + if (existing !== null) { + // Advisory no-op: never wipe an existing campaign. + return emit( + { status: 'ok', action: 'init', written: false, alreadyInitialized: true, ledgerPath }, + flags.outputFile, + 1, + ); + } + const ledger = createLedger({ now }); + await saveLedger(ledgerPath, ledger); + return emit( + { + status: 'ok', action: 'init', written: true, alreadyInitialized: false, ledgerPath, + schemaVersion: ledger.schemaVersion, createdDate: ledger.createdDate, updatedDate: ledger.updatedDate, + repos: ledger.repos, rollUp: rollUp(ledger), + }, + flags.outputFile, + 0, + ); + } + + if (subcommand === 'add') { + const paths = rest; + if (paths.length === 0) fail('add requires at least one repo path'); + const loaded = await loadOrFail(ledgerPath); + const autoInitialized = loaded === null; + let ledger = loaded === null ? createLedger({ now }) : loaded; + + const added = []; + const skipped = []; + for (const p of paths) { + const resolved = resolve(p); + const present = ledger.repos.some((r) => r.path === resolved); + // --name applies only to a lone path; multi-add lets the lib derive each basename. + const name = paths.length === 1 ? flags.name || undefined : undefined; + ledger = addRepo(ledger, { path: p, name }, { now }); + (present ? skipped : added).push(resolved); + } + await saveLedger(ledgerPath, ledger); + return emit( + { + status: 'ok', action: 'add', written: true, autoInitialized, ledgerPath, + added, skipped, repos: ledger.repos, rollUp: rollUp(ledger), + }, + flags.outputFile, + 0, + ); + } + + if (subcommand === 'set-status') { + const [path, status] = rest; + if (!path || !status) fail('set-status requires '); + const ledger = await loadOrFail(ledgerPath); + if (ledger === null) fail(`no ledger at ${ledgerPath} — run "init" or "add" first`); + + let findingsBySeverity; + if (flags.findings !== null) { + try { + findingsBySeverity = JSON.parse(flags.findings); + } catch (err) { + fail(`--findings must be valid JSON: ${err.message}`); + } + } + const opts = { now }; + if (findingsBySeverity !== undefined) opts.findingsBySeverity = findingsBySeverity; + if (flags.session !== null) opts.sessionId = flags.session; + + let next; + try { + next = setRepoStatus(ledger, path, status, opts); + } catch (err) { + // RangeError (bad status) or Error (untracked repo) → caller error. + fail(err.message); + } + await saveLedger(ledgerPath, next); + const resolved = resolve(path); + return emit( + { + status: 'ok', action: 'set-status', written: true, ledgerPath, + repo: next.repos.find((r) => r.path === resolved), repos: next.repos, rollUp: rollUp(next), + }, + flags.outputFile, + 0, + ); + } + + fail(`unknown subcommand "${subcommand}" — expected init | add | set-status`); +} + +const isDirectRun = + process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname); +if (isDirectRun) { + main().catch((err) => { + process.stderr.write(`Fatal: ${err.message}\n`); + process.exit(3); + }); +} diff --git a/tests/scanners/campaign-write-cli.test.mjs b/tests/scanners/campaign-write-cli.test.mjs new file mode 100644 index 0000000..ddf42b5 --- /dev/null +++ b/tests/scanners/campaign-write-cli.test.mjs @@ -0,0 +1,211 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { readFileSync, mkdtempSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { + loadLedger, + validateLedger, + rollUp, +} from '../../scanners/lib/campaign-ledger.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const CLI = resolve(__dirname, '../../scanners/campaign-write-cli.mjs'); + +const NOW = '2026-06-22'; + +// Every test passes an explicit --ledger-file in a temp dir + an injected --reference-date, +// so the write-CLI never touches the real default path under HOME and never reads the clock. +// The suite is hermetic + deterministic by construction. +function runWrite(args) { + try { + const stdout = execFileSync('node', [CLI, ...args], { encoding: 'utf-8', timeout: 15000 }); + return { status: 0, stdout }; + } catch (err) { + return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' }; + } +} + +function newDir() { + return mkdtempSync(join(tmpdir(), 'camp-write-')); +} + +// Read the persisted ledger back through the real lib so a test proves the on-disk file +// is exactly what the command layer would later load. +async function readLedger(file) { + const ledger = await loadLedger(file); + return { ledger, validation: validateLedger(ledger) }; +} + +describe('campaign-write-cli — init', () => { + it('creates a valid, empty, versioned ledger (exit 0)', async () => { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.action, 'init'); + assert.equal(out.written, true); + assert.equal(out.alreadyInitialized, false); + assert.equal(out.ledgerPath, resolve(file)); + + const { ledger, validation } = await readLedger(file); + assert.ok(validation.valid, validation.errors.join('; ')); + assert.equal(ledger.schemaVersion, 1); + assert.equal(ledger.createdDate, NOW); + assert.equal(ledger.updatedDate, NOW); + assert.deepEqual(ledger.repos, []); + }); + + it('refuses to clobber an existing ledger (exit 1, file untouched)', async () => { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + // First init, then add a repo so we can prove a second init does NOT wipe it. + runWrite(['init', '--ledger-file', file, '--reference-date', NOW]); + runWrite(['add', '/r/a', '--ledger-file', file, '--reference-date', NOW]); + + const { status, stdout } = runWrite(['init', '--ledger-file', file, '--reference-date', NOW]); + assert.equal(status, 1, 'advisory: already initialized'); + const out = JSON.parse(stdout); + assert.equal(out.written, false); + assert.equal(out.alreadyInitialized, true); + + const { ledger } = await readLedger(file); + assert.equal(ledger.repos.length, 1, 'existing repo preserved — not clobbered'); + }); +}); + +describe('campaign-write-cli — add', () => { + it('auto-initializes when no ledger exists, then adds repos (exit 0)', async () => { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + assert.ok(!existsSync(file)); + const { status, stdout } = runWrite([ + 'add', '/r/a', '/r/b', '--ledger-file', file, '--reference-date', NOW, + ]); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.action, 'add'); + assert.equal(out.autoInitialized, true); + assert.deepEqual(out.added.sort(), [resolve('/r/a'), resolve('/r/b')].sort()); + + const { ledger, validation } = await readLedger(file); + assert.ok(validation.valid, validation.errors.join('; ')); + assert.equal(ledger.repos.length, 2); + assert.ok(ledger.repos.every((r) => r.status === 'pending')); + }); + + it('appends to an existing ledger and is idempotent on re-add (added vs skipped)', async () => { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + runWrite(['add', '/r/a', '/r/b', '--ledger-file', file, '--reference-date', NOW]); + + const { status, stdout } = runWrite([ + 'add', '/r/a', '/r/c', '--ledger-file', file, '--reference-date', NOW, + ]); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.autoInitialized, false); + assert.deepEqual(out.added, [resolve('/r/c')]); + assert.deepEqual(out.skipped, [resolve('/r/a')]); + + const { ledger } = await readLedger(file); + assert.equal(ledger.repos.length, 3); + }); + + it('uses --name for a single path; derives basenames for multiple', async () => { + const dir = newDir(); + const solo = join(dir, 'solo.json'); + runWrite(['add', '/r/x', '--name', 'custom', '--ledger-file', solo, '--reference-date', NOW]); + const { ledger: l1 } = await readLedger(solo); + assert.equal(l1.repos[0].name, 'custom'); + + const multi = join(dir, 'multi.json'); + runWrite(['add', '/r/alpha', '/r/beta', '--ledger-file', multi, '--reference-date', NOW]); + const { ledger: l2 } = await readLedger(multi); + assert.deepEqual(l2.repos.map((r) => r.name).sort(), ['alpha', 'beta']); + }); +}); + +describe('campaign-write-cli — set-status', () => { + async function seeded() { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + runWrite(['add', '/r/a', '/r/b', '--ledger-file', file, '--reference-date', NOW]); + return file; + } + + it('transitions a tracked repo and attaches findings + session (exit 0)', async () => { + const file = await seeded(); + const findings = { critical: 1, high: 2, medium: 0, low: 4 }; + const { status, stdout } = runWrite([ + 'set-status', '/r/a', 'audited', + '--findings', JSON.stringify(findings), + '--session', '20260622_120000', + '--ledger-file', file, '--reference-date', NOW, + ]); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.action, 'set-status'); + assert.equal(out.repo.status, 'audited'); + assert.deepEqual(out.repo.findingsBySeverity, findings); + assert.equal(out.repo.sessionId, '20260622_120000'); + + const { ledger } = await readLedger(file); + assert.deepEqual(rollUp(ledger).bySeverity, findings); + assert.deepEqual(out.rollUp, rollUp(ledger)); + }); + + it('exits 3 on an invalid status', async () => { + const file = await seeded(); + const { status } = runWrite([ + 'set-status', '/r/a', 'nonsense', '--ledger-file', file, '--reference-date', NOW, + ]); + assert.equal(status, 3); + }); + + it('exits 3 when the repo is not tracked', async () => { + const file = await seeded(); + const { status } = runWrite([ + 'set-status', '/r/ghost', 'audited', '--ledger-file', file, '--reference-date', NOW, + ]); + assert.equal(status, 3); + }); + + it('exits 3 when no ledger exists yet', () => { + const dir = newDir(); + const { status } = runWrite([ + 'set-status', '/r/a', 'audited', + '--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW, + ]); + assert.equal(status, 3); + }); +}); + +describe('campaign-write-cli — determinism + --output-file', () => { + it('stamps updatedDate from --reference-date and writes the payload to --output-file', async () => { + const dir = newDir(); + const file = join(dir, 'ledger.json'); + runWrite(['init', '--ledger-file', file, '--reference-date', '2026-01-01']); + const report = join(dir, 'report.json'); + const { stdout } = runWrite([ + 'add', '/r/a', '--ledger-file', file, '--reference-date', '2026-03-15', + '--output-file', report, + ]); + assert.equal(stdout.trim(), '', 'stdout is silent when --output-file is given'); + const written = JSON.parse(readFileSync(report, 'utf-8')); + assert.equal(written.action, 'add'); + + const { ledger } = await readLedger(file); + assert.equal(ledger.createdDate, '2026-01-01', 'created stamp preserved'); + assert.equal(ledger.updatedDate, '2026-03-15', 'updated stamp from the later --reference-date'); + }); + + it('exits 3 on an unknown subcommand', () => { + const dir = newDir(); + const { status } = runWrite(['frobnicate', '--ledger-file', join(dir, 'x.json')]); + assert.equal(status, 3); + }); +}); From 49833aded80f478e7aa4ccc5bab7b3ad0472c0e9 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 02:44:21 +0200 Subject: [PATCH 04/53] feat(campaign): cross-repo prioritized backlog (v5.7 Fase 2 Block 4b) buildBacklog(ledger) pure transform + read-only campaign-cli payload field + command rendering. The single machine-wide pick-list: per-repo (the ledger tracks severity counts, not individual findings), severity-weighted (SEVERITY_WEIGHTS c1000/h100/m10/l1), deterministic tie-break, excludes implemented/pending/zero-finding repos. No schema change, no new scanner -> scanner count stays 15, snapshot/backcompat byte-stable. suite 1138->1150 (lib +9, campaign-cli +3). README badge 1091+->1150+. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 30 +++++++- README.md | 4 +- commands/campaign.md | 30 ++++++-- scanners/campaign-cli.mjs | 3 + scanners/lib/campaign-ledger.mjs | 67 +++++++++++++++++ tests/lib/campaign-ledger.test.mjs | 105 +++++++++++++++++++++++++++ tests/scanners/campaign-cli.test.mjs | 34 +++++++++ 7 files changed, 263 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0a60a44..9203923 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -113,7 +113,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -1138 tests across 65 test files (21 lib + 34 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1150 tests across 65 test files (21 lib + 34 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) @@ -479,6 +479,34 @@ orchestration is judgment-driven and deliberately outside the snapshot suite, ex `/config-audit optimize` + `knowledge-refresh`. **No new agent** (web/judgment-free, runs in the command's own context). suite 1127→1138. +### campaign backlog — cross-repo prioritized pick-list (v5.7 Fase 2, Block 4b) + +The first half of Block 4 ("one cross-repo prioritized backlog the user picks from"). A pure +lib transform + a read-only CLI-payload field — **no schema change, no new scanner, byte-stable**. + +- **`buildBacklog(ledger)`** (`scanners/lib/campaign-ledger.mjs`, pure, mirrors `rollUp`): the + single machine-wide prioritized work list. The actionable unit is a **repo** (the ledger tracks + per-repo severity *counts*, not individual findings — it tracks state, it does not re-run + audits), so each item is one repo: `{path, name, status, sessionId, findingsBySeverity + (normalized), totalFindings, weightedScore, rank}`. **Inclusion:** `status !== 'implemented'` + AND `totalFindings > 0` (implemented = done; pending / zero-finding repos have nothing known to + fix — they still surface in `rollUp.byStatus`). **Order:** DESC by `weightedScore` (exported + `SEVERITY_WEIGHTS = {critical:1000, high:100, medium:10, low:1}`), tie-broken lexicographically + by critical→high→medium→low count, then ascending `name` — fully deterministic, and the + tie-break keeps "criticals always win" even on a weighted-score collision (1 critical vs 10 high). + `rank` is 1-based after the sort. +- **`campaign-cli`** now emits `backlog: buildBacklog(ledger)` in both branches (uninitialized → + `[]`). Purely additive + read-only → fits the Block 3b read-only contract; the existing CLI + tests use targeted asserts (not full `deepEqual`), so the new field doesn't break them. +- **`commands/campaign.md`** renders the backlog as a "Prioritized backlog" pick-list and points + the user at the top item (still a pick-list, NOT an executor — execution is the later 4c block). + +**Byte-stability.** `-cli`/lib/command only → scanner count stays **15**, snapshot/backcompat +suite untouched. suite 1138→1150 (lib +9, campaign-cli +3). **Deferred to 4c:** per-repo plan +export to each repo's `docs/` + reuse of backup/rollback for execution. **Deferred until the first +breaking schema change:** `migrateLedger` (4a) — backlog needs no schema bump, so building +migration now would be speculative (`schemaVersion` is already stamped for when it's needed). + ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage diff --git a/README.md b/README.md index e138712..5a9d3b5 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1091+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1150+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 15 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, and output-style detection. Zero external dependencies. @@ -293,7 +293,7 @@ Your team configuration changes over time. Track it: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — flag stale entries (sources older than ~90d) + poll for new/changed Claude Code practices; **human-approved writes only** (Verifiseringsplikt). Deterministic stale core + web candidate poll | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity, resumable across sessions; **human-approved writes only** (read-only report + deterministic write-CLI) | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity + a single **cross-repo prioritized backlog** to pick from (severity-weighted), resumable across sessions; **human-approved writes only** (read-only report + deterministic write-CLI) | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Set preferences for action plan _(optional)_ | diff --git a/commands/campaign.md b/commands/campaign.md index d3428c5..ffeb2b4 100644 --- a/commands/campaign.md +++ b/commands/campaign.md @@ -20,9 +20,9 @@ Every mutation — creating the ledger, adding a repo, changing a status — is and applied **only on explicit approval**, by invoking one deterministic write-CLI subcommand. The command never hand-edits the ledger JSON. -This is the **THIN** campaign surface (ledger + roll-up + status). Cross-repo backlog -prioritization and execution are a later block — this command does not run audits or apply -fixes itself; it tracks where each repo stands. +This is the **THIN** campaign surface (ledger + roll-up + status + a cross-repo prioritized +backlog to pick from). Execution is a later block — this command does not run audits or apply +fixes itself; it tracks where each repo stands and shows what to tackle next. ## Two CLIs back this command @@ -65,8 +65,10 @@ attempt a write over a corrupt ledger.) Read `~/.claude/config-audit/sessions/campaign-report.json` with the Read tool (per the UX rules — never show the raw JSON). It has `initialized`, `repos[]` (each: `path, name, status, -sessionId, findingsBySeverity, updatedDate`), and `rollUp {totalRepos, byStatus, bySeverity, -reposWithFindings}`. +sessionId, findingsBySeverity, updatedDate`), `rollUp {totalRepos, byStatus, bySeverity, +reposWithFindings}`, and `backlog[]` — the single cross-repo prioritized work list (each: +`path, name, status, findingsBySeverity, totalFindings, weightedScore, rank`), already sorted +DESC by severity (most critical work first). Present it as two short tables: @@ -84,6 +86,19 @@ Present it as two short tables: | Repo | Status | Findings (C/H/M/L) | Last updated | |------|--------|--------------------|--------------| +**Prioritized backlog** — the one cross-repo list to pick from, highest-severity work first. +Render `backlog[]` in `rank` order (it is already sorted); omit this table entirely when the +backlog is empty (nothing outstanding — say "Backlog clear — no outstanding findings across +tracked repos."). Implemented repos and repos with no known findings are deliberately absent. + +| # | Repo | Status | Findings (C/H/M/L) | Total | +|---|------|--------|--------------------|-------| + +After it, point the user at the top item: "Highest priority: **``** (``) — pick it +with `/config-audit` (audit), `/config-audit plan`, or `/config-audit implement` in that repo, +then record progress here with `set-status`." The backlog is a **pick-list**, not an executor — +this command does not run audits or fixes (that is the later execution block). + If `initialized` is false, say so plainly: "No campaign yet. Run `/config-audit campaign init` to start one." Then — if the mode was `init` or `add` — continue to that step (those bootstrap a campaign); for `report`/`set-status` on an uninitialized ledger, stop after this message. @@ -173,5 +188,6 @@ Tailor to where the campaign stands: and `knowledge-refresh`). - The `-cli` suffix keeps both CLIs out of the scan-orchestrator, so the scanner count and the byte-stable snapshot suite are unaffected. -- **THIN scope:** ledger + roll-up + status only. Cross-repo backlog prioritization and execution - are a later block — this command tracks state, it does not run audits or apply fixes. +- **THIN scope:** ledger + roll-up + status + a read-only cross-repo prioritized backlog. The + backlog is a derived pick-list (`buildBacklog`, severity-weighted); execution is a later block — + this command tracks state and shows what to tackle next, it does not run audits or apply fixes. diff --git a/scanners/campaign-cli.mjs b/scanners/campaign-cli.mjs index fc3846b..913ecda 100644 --- a/scanners/campaign-cli.mjs +++ b/scanners/campaign-cli.mjs @@ -28,6 +28,7 @@ import { loadLedger, validateLedger, rollUp, + buildBacklog, defaultLedgerPath, } from './lib/campaign-ledger.mjs'; @@ -72,6 +73,7 @@ async function main() { updatedDate: null, repos: [], rollUp: rollUp({ repos: [] }), + backlog: buildBacklog({ repos: [] }), }; exitCode = 1; // advisory: there is no campaign to report yet } else { @@ -88,6 +90,7 @@ async function main() { updatedDate: ledger.updatedDate, repos: ledger.repos, rollUp: rollUp(ledger), + backlog: buildBacklog(ledger), }; exitCode = 0; } diff --git a/scanners/lib/campaign-ledger.mjs b/scanners/lib/campaign-ledger.mjs index e8a47c9..12a2dba 100644 --- a/scanners/lib/campaign-ledger.mjs +++ b/scanners/lib/campaign-ledger.mjs @@ -33,6 +33,14 @@ export const STATUSES = Object.freeze(['pending', 'audited', 'planned', 'impleme /** Severity buckets aggregated by the machine-wide roll-up. */ const SEVERITIES = Object.freeze(['critical', 'high', 'medium', 'low']); +/** + * Order-of-magnitude severity weights for the cross-repo backlog priority score. Each tier + * dominates the next so a single higher-severity finding outranks many lower ones; exact + * score collisions are still broken deterministically by the lexicographic + name tie-break + * in `buildBacklog`. Exported so the score is documented, not a magic number. + */ +export const SEVERITY_WEIGHTS = Object.freeze({ critical: 1000, high: 100, medium: 10, low: 1 }); + const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; /** Validate an injected `now` (required, YYYY-MM-DD). Throws — callers pass today's date. */ @@ -135,6 +143,65 @@ export function rollUp(ledger) { return { totalRepos: ledger.repos.length, byStatus, bySeverity, reposWithFindings }; } +/** + * Build the single, machine-wide PRIORITIZED backlog the user picks from. Pure derivation — + * never mutates. The actionable unit is a REPO (the ledger tracks per-repo severity counts, + * not individual findings — it tracks state, it does not re-run audits), so each backlog item + * is one repo with outstanding work. + * + * Inclusion: a repo is in the backlog iff it is NOT yet `implemented` AND has at least one + * outstanding finding (`totalFindings > 0`). `implemented` repos are done; `pending` and + * zero-finding repos have nothing known to fix (they still surface in `rollUp.byStatus`). + * + * Order: DESC by `weightedScore` (SEVERITY_WEIGHTS), tie-broken lexicographically by + * critical→high→medium→low count, then ascending by `name` — fully deterministic, and the + * tie-break preserves "criticals always win" even when two repos share a weighted score. + * + * @param {object} ledger + * @returns {Array<{path:string,name:string,status:string,sessionId:string|null,findingsBySeverity:object,totalFindings:number,weightedScore:number,rank:number}>} + */ +export function buildBacklog(ledger) { + const items = []; + + for (const repo of ledger.repos) { + if (repo.status === 'implemented') continue; + const f = repo.findingsBySeverity; + if (!f || typeof f !== 'object') continue; + + const findingsBySeverity = Object.fromEntries( + SEVERITIES.map((s) => [s, typeof f[s] === 'number' ? f[s] : 0]), + ); + const totalFindings = SEVERITIES.reduce((sum, s) => sum + findingsBySeverity[s], 0); + if (totalFindings === 0) continue; + + const weightedScore = SEVERITIES.reduce( + (score, s) => score + findingsBySeverity[s] * SEVERITY_WEIGHTS[s], + 0, + ); + items.push({ + path: repo.path, + name: repo.name, + status: repo.status, + sessionId: repo.sessionId ?? null, + findingsBySeverity, + totalFindings, + weightedScore, + }); + } + + items.sort( + (x, y) => + y.weightedScore - x.weightedScore || + y.findingsBySeverity.critical - x.findingsBySeverity.critical || + y.findingsBySeverity.high - x.findingsBySeverity.high || + y.findingsBySeverity.medium - x.findingsBySeverity.medium || + y.findingsBySeverity.low - x.findingsBySeverity.low || + x.name.localeCompare(y.name), + ); + + return items.map((item, i) => ({ ...item, rank: i + 1 })); +} + /** * Validate a parsed ledger against the schema. Never throws — returns every problem at once * so the caller (and tests) can inspect them. Soft by design (loaded data may be corrupt). diff --git a/tests/lib/campaign-ledger.test.mjs b/tests/lib/campaign-ledger.test.mjs index c1fe43a..8bc8119 100644 --- a/tests/lib/campaign-ledger.test.mjs +++ b/tests/lib/campaign-ledger.test.mjs @@ -6,10 +6,12 @@ import { tmpdir, homedir } from 'node:os'; import { CAMPAIGN_SCHEMA_VERSION, STATUSES, + SEVERITY_WEIGHTS, createLedger, addRepo, setRepoStatus, rollUp, + buildBacklog, validateLedger, defaultLedgerPath, loadLedger, @@ -153,6 +155,109 @@ describe('rollUp', () => { }); }); +describe('buildBacklog', () => { + // Seed a ledger from a compact spec. A repo with `status` (and optional `findings`) is + // transitioned; a bare repo stays `pending` with null findings. + function ledgerWith(specs) { + let l = createLedger({ now: NOW }); + for (const s of specs) { + l = addRepo(l, { path: s.path, name: s.name }, { now: NOW }); + if (s.status) { + l = setRepoStatus(l, s.path, s.status, { + now: NOW, + findingsBySeverity: s.findings, + sessionId: s.sessionId, + }); + } + } + return l; + } + + it('SEVERITY_WEIGHTS is the frozen order-of-magnitude weighting', () => { + assert.deepEqual({ ...SEVERITY_WEIGHTS }, { critical: 1000, high: 100, medium: 10, low: 1 }); + assert.ok(Object.isFrozen(SEVERITY_WEIGHTS)); + }); + + it('returns [] for an empty ledger', () => { + assert.deepEqual(buildBacklog(createLedger({ now: NOW })), []); + }); + + it('excludes implemented, pending, and zero-finding repos', () => { + const l = ledgerWith([ + { path: '/r/done', name: 'done', status: 'implemented', findings: { critical: 5, high: 0, medium: 0, low: 0 } }, + { path: '/r/pend', name: 'pend' }, // pending, no findings + { path: '/r/clean', name: 'clean', status: 'audited', findings: { critical: 0, high: 0, medium: 0, low: 0 } }, + ]); + assert.deepEqual(buildBacklog(l), []); + }); + + it('includes audited and planned repos that still have findings', () => { + const l = ledgerWith([ + { path: '/r/a', name: 'a', status: 'audited', findings: { critical: 0, high: 1, medium: 0, low: 0 } }, + { path: '/r/p', name: 'p', status: 'planned', findings: { critical: 0, high: 0, medium: 2, low: 0 } }, + ]); + const names = buildBacklog(l).map((i) => i.name).sort(); + assert.deepEqual(names, ['a', 'p']); + }); + + it('ranks by severity (critical outranks high outranks low) and assigns rank 1..n', () => { + const l = ledgerWith([ + { path: '/r/low', name: 'low', status: 'audited', findings: { critical: 0, high: 0, medium: 0, low: 9 } }, + { path: '/r/crit', name: 'crit', status: 'audited', findings: { critical: 1, high: 0, medium: 0, low: 0 } }, + { path: '/r/high', name: 'high', status: 'planned', findings: { critical: 0, high: 5, medium: 0, low: 0 } }, + ]); + const backlog = buildBacklog(l); + assert.deepEqual(backlog.map((i) => i.name), ['crit', 'high', 'low']); + assert.deepEqual(backlog.map((i) => i.rank), [1, 2, 3]); + }); + + it('computes weightedScore and totalFindings, normalizing missing severity keys to 0', () => { + const l = ledgerWith([ + { path: '/r/x', name: 'x', status: 'audited', findings: { high: 2 } }, // critical/medium/low missing + ]); + const [item] = buildBacklog(l); + assert.deepEqual(item.findingsBySeverity, { critical: 0, high: 2, medium: 0, low: 0 }); + assert.equal(item.totalFindings, 2); + assert.equal(item.weightedScore, 200); + }); + + it('breaks a weightedScore tie by critical count, then by name', () => { + const l = ledgerWith([ + // both score 1000: 10 high vs 1 critical → critical wins the tie + { path: '/r/tenHigh', name: 'tenHigh', status: 'audited', findings: { critical: 0, high: 10, medium: 0, low: 0 } }, + { path: '/r/oneCrit', name: 'oneCrit', status: 'audited', findings: { critical: 1, high: 0, medium: 0, low: 0 } }, + ]); + assert.deepEqual(buildBacklog(l).map((i) => i.name), ['oneCrit', 'tenHigh']); + + // fully identical findings → final tie-break is name (ascending) + const l2 = ledgerWith([ + { path: '/r/zebra', name: 'zebra', status: 'audited', findings: { critical: 0, high: 1, medium: 0, low: 0 } }, + { path: '/r/alpha', name: 'alpha', status: 'audited', findings: { critical: 0, high: 1, medium: 0, low: 0 } }, + ]); + assert.deepEqual(buildBacklog(l2).map((i) => i.name), ['alpha', 'zebra']); + }); + + it('carries per-repo provenance on each item', () => { + const l = ledgerWith([ + { path: '/r/a', name: 'a', status: 'audited', findings: { critical: 1, high: 0, medium: 0, low: 0 }, sessionId: '20260622_120000' }, + ]); + const [item] = buildBacklog(l); + assert.equal(item.path, '/r/a'); + assert.equal(item.name, 'a'); + assert.equal(item.status, 'audited'); + assert.equal(item.sessionId, '20260622_120000'); + }); + + it('does not mutate the input ledger', () => { + const l = ledgerWith([ + { path: '/r/a', name: 'a', status: 'audited', findings: { critical: 1, high: 0, medium: 0, low: 0 } }, + ]); + const snapshot = JSON.stringify(l); + buildBacklog(l); + assert.equal(JSON.stringify(l), snapshot); + }); +}); + describe('validateLedger', () => { const good = () => setRepoStatus( addRepo(createLedger({ now: NOW }), { path: '/r/foo', name: 'foo' }, { now: NOW }), diff --git a/tests/scanners/campaign-cli.test.mjs b/tests/scanners/campaign-cli.test.mjs index dea642b..6d784db 100644 --- a/tests/scanners/campaign-cli.test.mjs +++ b/tests/scanners/campaign-cli.test.mjs @@ -10,6 +10,7 @@ import { addRepo, setRepoStatus, rollUp, + buildBacklog, saveLedger, } from '../../scanners/lib/campaign-ledger.mjs'; @@ -126,6 +127,39 @@ describe('campaign-cli — payload shape', () => { }); }); +describe('campaign-cli — backlog', () => { + it('emits a backlog matching the lib, excluding implemented/pending/zero-finding repos', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const { file, ledger } = await seedLedger(dir); + const out = JSON.parse(runCli(['--ledger-file', file]).stdout); + // seed: a=audited(with findings), b=implemented(excluded), c=pending(excluded) + assert.deepEqual(out.backlog, buildBacklog(ledger)); + assert.equal(out.backlog.length, 1); + assert.equal(out.backlog[0].name, 'a'); + assert.equal(out.backlog[0].rank, 1); + }); + + it('ranks multiple actionable repos by severity', async () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + let l = createLedger({ now: NOW }); + l = addRepo(l, { path: '/r/low', name: 'low' }, { now: NOW }); + l = addRepo(l, { path: '/r/crit', name: 'crit' }, { now: NOW }); + l = setRepoStatus(l, '/r/low', 'audited', { now: NOW, findingsBySeverity: { critical: 0, high: 0, medium: 0, low: 9 } }); + l = setRepoStatus(l, '/r/crit', 'planned', { now: NOW, findingsBySeverity: { critical: 1, high: 0, medium: 0, low: 0 } }); + const file = join(dir, 'campaign-ledger.json'); + await saveLedger(file, l); + + const out = JSON.parse(runCli(['--ledger-file', file]).stdout); + assert.deepEqual(out.backlog.map((i) => i.name), ['crit', 'low']); + }); + + it('uninitialized payload has an empty backlog', () => { + const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); + const out = JSON.parse(runCli(['--ledger-file', join(dir, 'nope.json')]).stdout); + assert.deepEqual(out.backlog, []); + }); +}); + describe('campaign-cli — --output-file', () => { it('writes JSON to the file and stays silent on stdout', async () => { const dir = mkdtempSync(join(tmpdir(), 'camp-cli-')); From 319e5541c9ad8313e398c8fc25df23fa426f37b5 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 10:08:04 +0200 Subject: [PATCH 05/53] feat(campaign): plan export + execution-by-reuse (v5.7 Fase 2 Block 4c) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes Block 4 (4b backlog + 4c export/execution). Asymmetric: plan export is new testable code; execution is pure reuse of the existing per-repo implement/rollback (no new execution machinery), per the plan's "reuse existing backup/rollback". Plan export ("planer følger arbeidsstedet"): - scanners/lib/campaign-export.mjs (pure, now injected, 8 tests): planExportPath(repo,sessionId) -> /docs/config-audit-plan-.md (sessionId-keyed so same-day re-audits never collide); buildPlanExportDocument({...,now}) -> provenance header + verbatim plan. - scanners/campaign-export-cli.mjs (-cli, read-only by default, 10 tests): --repo resolves the repo's linked session, reads its action-plan.md, assembles the doc, emits {exportable,problems,targetPath,document}. Two gates -> exit 1 advisory: no-session-linked / no-action-plan. Writes the file ONLY under opt-in --write (byte-faithful copy; the LLM never re-types a 200-line plan). --sessions-dir override for hermetic tests; exit 0/1/3. Command: commands/campaign.md gains an `export ` mode (Step 6: preview -> approve -> --write), then routes the user to the existing /config-audit implement (backup + verify) + rollback + set-status implemented. Nothing auto-written (Verifiseringsplikt). Byte-stable: lib + -cli + command-doc only -> scanner count stays 15, agents 7, commands 21 (export is a mode, not a new command), SC-5 + backcompat suite untouched. suite 1150->1168. Block 4a (migrateLedger) still deferred to the first breaking schema change. Docs: CLAUDE.md section + badge 1150->1168/65->67 files; README badge + campaign row + Testing prose (fixed stale 1055/59 -> true 1168/67). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 30 +++- README.md | 6 +- commands/campaign.md | 92 +++++++++-- scanners/campaign-export-cli.mjs | 165 ++++++++++++++++++++ scanners/lib/campaign-export.mjs | 78 +++++++++ tests/lib/campaign-export.test.mjs | 72 +++++++++ tests/scanners/campaign-export-cli.test.mjs | 154 ++++++++++++++++++ 7 files changed, 575 insertions(+), 22 deletions(-) create mode 100644 scanners/campaign-export-cli.mjs create mode 100644 scanners/lib/campaign-export.mjs create mode 100644 tests/lib/campaign-export.test.mjs create mode 100644 tests/scanners/campaign-export-cli.test.mjs diff --git a/CLAUDE.md b/CLAUDE.md index 9203923..1d9166f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — deterministic stale check (sources older than ~90d) + web candidate poll (CC changelog + Anthropic blog); **human-approved writes only** (Verifiseringsplikt). The "living" half of the knowledge base. Web/judgment-driven, **not byte-stable** | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity, resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via a deterministic write-CLI (init/add/set-status). THIN: tracks state, doesn't run audits/fixes. Judgment-driven, **not byte-stable** | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity + cross-repo prioritized backlog + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via deterministic write/export CLIs. THIN: tracks+routes state, reuses existing implement/rollback for execution. Judgment-driven, **not byte-stable** | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Gather user preferences (opt-in) | @@ -113,7 +113,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| node --test 'tests/**/*.test.mjs' ``` -1150 tests across 65 test files (21 lib + 34 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1168 tests across 67 test files (22 lib + 35 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. ### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) @@ -507,6 +507,32 @@ export to each repo's `docs/` + reuse of backup/rollback for execution. **Deferr breaking schema change:** `migrateLedger` (4a) — backlog needs no schema bump, so building migration now would be speculative (`schemaVersion` is already stamped for when it's needed). +### campaign plan-export + execution-by-reuse (v5.7 Fase 2, Block 4c — the rest of Block 4) + +The second half of Block 4. **Asymmetric:** plan export is the new testable code; execution is +*pure reuse* (no new machinery), per the plan's "reuse existing backup/rollback". + +- **Plan export** (`scanners/lib/campaign-export.mjs`, pure, 8 tests): `planExportPath(repoPath, + sessionId)` → `/docs/config-audit-plan-.md` (keyed on the timestamp-unique + sessionId, not the date, so same-day re-audits don't collide); `buildPlanExportDocument({...,now})` + → provenance header (repo/session/how-to-execute-and-undo) + the verbatim session plan. `now` + injected → deterministic. **CLI** `scanners/campaign-export-cli.mjs` (`-cli`, read-only by + default, 10 tests): `--repo ` resolves the repo's linked session, reads its + `action-plan.md`, assembles the doc, emits `{exportable, problems, targetPath, document, ...}`. + Two gates → exit 1 advisory: `no-session-linked` (repo has no `sessionId`), `no-action-plan` + (linked session has no plan yet). Writes the file **only** under opt-in `--write` — the CLI does + the byte-faithful copy so a 200-line plan is never re-typed/mutated by the LLM. `--sessions-dir` + override for hermetic tests; exit 0/1/3 mirror the sibling CLIs. +- **Execution = reuse.** No campaign-side execution code. The exported `docs/` file is the repo's + durable record; `/config-audit implement` still reads the canonical plan from the session + (backup + apply + verify), `/config-audit rollback` undoes, then `set-status implemented` + records it. The command (`commands/campaign.md`, new `export ` mode) previews → asks → on + approval invokes `--write` → routes the user to that existing machinery. +- **Byte-stable.** lib + `-cli` + command-doc only → scanner count stays **15**, agents **7**, + commands **21** (export is a *mode*, not a new command), snapshot/backcompat suite untouched. + suite 1150→1168 (lib +8, export-cli +10). **Block 4a (`migrateLedger`) still deferred** to the + first breaking schema change (export needs no schema bump). + ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage diff --git a/README.md b/README.md index 5a9d3b5..6d59e16 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1150+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1168+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 15 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, and output-style detection. Zero external dependencies. @@ -293,7 +293,7 @@ Your team configuration changes over time. Track it: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — flag stale entries (sources older than ~90d) + poll for new/changed Claude Code practices; **human-approved writes only** (Verifiseringsplikt). Deterministic stale core + web candidate poll | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity + a single **cross-repo prioritized backlog** to pick from (severity-weighted), resumable across sessions; **human-approved writes only** (read-only report + deterministic write-CLI) | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity + a single **cross-repo prioritized backlog** to pick from (severity-weighted) + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions; **human-approved writes only** (read-only report + deterministic write/export CLIs). Execution reuses the existing `/config-audit implement` + `rollback` | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Set preferences for action plan _(optional)_ | @@ -602,7 +602,7 @@ date, and a `confidence`. It is the source of truth for the optimization lens (O node --test 'tests/**/*.test.mjs' ``` -1055 tests across 59 test files (18 lib + 31 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Requires Node.js 18+ (`node:test`). +1168 tests across 67 test files (22 lib + 35 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Requires Node.js 18+ (`node:test`). --- diff --git a/commands/campaign.md b/commands/campaign.md index ffeb2b4..dbcecf0 100644 --- a/commands/campaign.md +++ b/commands/campaign.md @@ -1,7 +1,7 @@ --- name: config-audit:campaign description: Machine-wide audit campaign — track which repos are pending/audited/planned/implemented across sessions, with a machine-wide roll-up. Human-approved writes only. -argument-hint: "[init | add ... | add --discover | set-status ]" +argument-hint: "[init | add ... | set-status | export ]" allowed-tools: Read, Write, Edit, Bash, Glob model: opus --- @@ -21,19 +21,28 @@ and applied **only on explicit approval**, by invoking one deterministic write-C The command never hand-edits the ledger JSON. This is the **THIN** campaign surface (ledger + roll-up + status + a cross-repo prioritized -backlog to pick from). Execution is a later block — this command does not run audits or apply -fixes itself; it tracks where each repo stands and shows what to tackle next. +backlog to pick from + plan **export**). It does not run audits or apply fixes itself: it tracks +where each repo stands, shows what to tackle next, exports a planned repo's plan into that repo's +own `docs/`, and points at the **existing** per-repo `/config-audit implement` (backup + apply + +verify) and `/config-audit rollback` for execution — Block 4c reuses that machinery, it does not +reinvent it. -## Two CLIs back this command +## Three CLIs back this command - **Read (report):** `scanners/campaign-cli.mjs` — loads + validates the ledger, emits the - repo list + roll-up. Never writes. + repo list + roll-up + backlog. Never writes. - **Write (mutate):** `scanners/campaign-write-cli.mjs` — `init` / `add` / `set-status`, each a thin wrapper over the invariant-enforcing lib transforms + save. Invoked **only** after the user approves a specific action. +- **Export:** `scanners/campaign-export-cli.mjs` — `--repo ` resolves the repo's linked + session, reads its `action-plan.md`, and assembles a `docs/config-audit-plan-.md`. + Read-only (a preview) by default; it writes the file **only** under `--write`, which is + invoked **only** after the user approves. The CLI writes the file byte-faithfully — the plan is + never re-typed. -Both take `--ledger-file ` (defaults to the durable path) and `--output-file `; -the write-CLI also takes `--reference-date ` (the audit date stamp). +All take `--ledger-file ` (defaults to the durable path) and `--output-file `; the +write-CLI + export-CLI also take `--reference-date ` (the date stamp), and the +export-CLI takes `--sessions-dir ` (defaults to `~/.claude/config-audit/sessions`). ## Implementation @@ -46,6 +55,7 @@ From `$ARGUMENTS`, pick the mode: - `add ...` → add one or more repo paths. - `add --discover ` → find git repos under `` and let the user pick which to add. - `set-status ` → transition a tracked repo (`status` ∈ pending/audited/planned/implemented). +- `export ` → export a planned repo's action plan into that repo's own `docs/`. - `help` → show this surface and stop. Set a shared date stamp for any write: `TODAY=$(date +%F)`. @@ -165,7 +175,48 @@ node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-write-cli.mjs set-status " \ + --reference-date "$TODAY" \ + --output-file ~/.claude/config-audit/sessions/campaign-export.json 2>/dev/null; echo $? +``` + +Exit **0** = previewable, **1** = tracked but not exportable yet, **3** = error (untracked repo, +no/corrupt ledger). Read `~/.claude/config-audit/sessions/campaign-export.json` with the Read tool. + +- **Exit 1 — read `problems`** and guide, then stop (nothing to export): + - `no-session-linked` → "`` has no linked audit session. Link one with + `/config-audit campaign set-status --session `, or audit + plan it first." + - `no-action-plan` → "``'s session has no plan yet. Run `/config-audit plan` in that repo + first, mark it `planned`, then export." +- **Exit 0 — show, then ask.** Tell the user the destination (`targetPath`) and a **short** preview + — the first ~12 lines of `document` only, never the whole file, never the raw JSON (UX rules). + Ask for explicit approval to write it. + +**On approval, write it** (the CLI does the faithful copy — do NOT hand-write the file): + +```bash +node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-export-cli.mjs --repo "" --write \ + --reference-date "$TODAY" \ + --output-file ~/.claude/config-audit/sessions/campaign-export.json 2>/dev/null; echo $? +``` + +Confirm: "Plan exported to ``." Then hand off to the **existing** execution machinery +(Block 4c reuses it — this command does not run it for you): + +> To **execute**: run `/config-audit implement` in `` — it backs up every changed file, +> applies the plan, and verifies. To **undo**: `/config-audit rollback`. When done, record it: +> `/config-audit campaign set-status implemented`. + +### Step 7: Next steps Tailor to where the campaign stands: @@ -174,20 +225,27 @@ Tailor to where the campaign stands: - **Pending repos exist:** "Run `/config-audit` in a pending repo to audit it, then `/config-audit campaign set-status audited` to record the result here." - **Audited but not planned:** "`/config-audit plan` in that repo, then mark it `planned`." +- **Planned repos exist:** "`/config-audit campaign export ` to drop the plan into that + repo's own `docs/`, then `/config-audit implement` there to execute it (backup + verify)." +- **Backlog has items:** point at the top backlog repo and the natural next verb for its status + (audit → plan → export → implement). - Always: the campaign survives this session — re-run `/config-audit campaign` anytime to see the machine-wide picture. ## Notes -- **Read-only report, human-approved writes.** `campaign-cli` never writes; every mutation goes - through `campaign-write-cli` and only after explicit approval, exactly mirroring how - `/config-audit knowledge-refresh` gates register writes. -- **Deterministic core, not byte-stable command.** The lib transforms + both CLIs are +- **Read-only report, human-approved writes.** `campaign-cli` never writes; every mutation — + ledger changes via `campaign-write-cli`, plan exports via `campaign-export-cli --write` — happens + only after explicit approval, exactly mirroring how `/config-audit knowledge-refresh` gates + register writes. The export-CLI's default (no `--write`) is a read-only preview. +- **Deterministic core, not byte-stable command.** The lib transforms + all three CLIs are unit-tested and deterministic (`--reference-date` injected); this command's orchestration is judgment-driven and deliberately **not** in the snapshot suite (like `/config-audit optimize` and `knowledge-refresh`). -- The `-cli` suffix keeps both CLIs out of the scan-orchestrator, so the scanner count and the - byte-stable snapshot suite are unaffected. -- **THIN scope:** ledger + roll-up + status + a read-only cross-repo prioritized backlog. The - backlog is a derived pick-list (`buildBacklog`, severity-weighted); execution is a later block — - this command tracks state and shows what to tackle next, it does not run audits or apply fixes. +- The `-cli` suffix keeps all three CLIs out of the scan-orchestrator, so the scanner count and + the byte-stable snapshot suite are unaffected. +- **THIN scope:** ledger + roll-up + status + a cross-repo prioritized backlog + plan export. + Execution is **not** reinvented here — `export` drops a planned repo's plan into its own `docs/` + (a durable record), and the user runs the existing `/config-audit implement` (backup + apply + + verify) + `/config-audit rollback` to execute and undo. This command tracks state and routes the + work; it does not run audits or apply fixes itself. diff --git a/scanners/campaign-export-cli.mjs b/scanners/campaign-export-cli.mjs new file mode 100644 index 0000000..781a8df --- /dev/null +++ b/scanners/campaign-export-cli.mjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +/** + * campaign-export-cli — export a tracked repo's action plan into that repo's own `docs/` + * (v5.7 Fase 2, Block 4c). + * + * Block 4b built the cross-repo prioritized backlog; this is the "plan export" half of Block 4c. + * Given a repo tracked in the campaign ledger, it resolves the repo's linked config-audit + * session, reads that session's `action-plan.md`, and assembles (via the pure + * `campaign-export` lib) a `docs/config-audit-plan-.md` document carrying a + * provenance header + the verbatim plan ("planer følger arbeidsstedet"). + * + * Read-only by DEFAULT (a dry-run preview that returns the assembled `document` + `targetPath` + * so the command can show the user what will be written). The actual write happens ONLY under + * the opt-in `--write` flag — which the `/config-audit campaign` command invokes solely after + * explicit human approval (Verifiseringsplikt — nothing auto-written). Writing the file + * faithfully (a byte-exact copy of the assembled document) is the CLI's job, not the LLM's, so + * a 200-line plan is never re-typed and cannot drift. + * + * Execution is NOT here: Block 4c reuses the existing `/config-audit implement` (backup + + * apply + verify) + `/config-audit rollback`. This CLI only exports the durable record. + * + * Naming: `-cli` suffix → NOT an orchestrated scanner, so the scanner count is unchanged and + * the snapshot suite stays byte-stable. + * + * Usage: + * node campaign-export-cli.mjs --repo [--write] + * [--ledger-file

] [--sessions-dir

] [--reference-date ] [--output-file

] + * + * Exit codes: 0 = exportable (preview ready, or written under --write), + * 1 = advisory: repo tracked but not exportable yet (no linked session / no plan), + * 3 = error (missing --repo, untracked repo, no/corrupt ledger, unreadable plan). + */ + +import { resolve, join, dirname } from 'node:path'; +import { homedir } from 'node:os'; +import { readFile, writeFile, mkdir } from 'node:fs/promises'; +import { + loadLedger, + validateLedger, + defaultLedgerPath, +} from './lib/campaign-ledger.mjs'; +import { planExportPath, buildPlanExportDocument } from './lib/campaign-export.mjs'; + +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +function fail(message) { + process.stderr.write(`Error: ${message}\n`); + process.exit(3); +} + +/** Default session store: next to the ledger, OUTSIDE the plugin dir. */ +function defaultSessionsDir() { + return join(homedir(), '.claude', 'config-audit', 'sessions'); +} + +function parseArgs(argv) { + const flags = { repo: null, ledgerFile: null, sessionsDir: null, referenceDate: null, outputFile: null, write: false }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--repo' && argv[i + 1] !== undefined) flags.repo = argv[++i]; + else if (a === '--ledger-file' && argv[i + 1] !== undefined) flags.ledgerFile = argv[++i]; + else if (a === '--sessions-dir' && argv[i + 1] !== undefined) flags.sessionsDir = argv[++i]; + else if (a === '--reference-date' && argv[i + 1] !== undefined) flags.referenceDate = argv[++i]; + else if (a === '--output-file' && argv[i + 1] !== undefined) flags.outputFile = argv[++i]; + else if (a === '--write') flags.write = true; + else if (a.startsWith('--')) fail(`unknown flag "${a}"`); + else fail(`unexpected argument "${a}"`); + } + return flags; +} + +async function emit(payload, outputFile, exitCode) { + const json = JSON.stringify(payload, null, 2); + if (outputFile) await writeFile(outputFile, json, 'utf-8'); + else process.stdout.write(json + '\n'); + process.exit(exitCode); +} + +async function main() { + const flags = parseArgs(process.argv.slice(2)); + if (!flags.repo) fail('--repo is required'); + if (flags.referenceDate && !DATE_RE.test(flags.referenceDate)) fail('--reference-date must be YYYY-MM-DD'); + + const ledgerPath = resolve(flags.ledgerFile || defaultLedgerPath()); + const sessionsDir = resolve(flags.sessionsDir || defaultSessionsDir()); + const repoPath = resolve(flags.repo); + // The clock is read here ONLY — passed to the pure lib as the injected `now`. + const now = flags.referenceDate || new Date().toISOString().slice(0, 10); + + let ledger; + try { + ledger = await loadLedger(ledgerPath); + } catch (err) { + fail(`could not read ledger at ${ledgerPath}: ${err.message}`); + } + if (ledger === null) fail(`no campaign ledger at ${ledgerPath} — run "/config-audit campaign init" first`); + + const { valid, errors } = validateLedger(ledger); + if (!valid) fail(`ledger at ${ledgerPath} is invalid:\n - ${errors.join('\n - ')}`); + + const repo = ledger.repos.find((r) => r.path === repoPath); + if (!repo) fail(`repo "${repoPath}" is not tracked in the campaign — add it first`); + + const repoInfo = { path: repo.path, name: repo.name, status: repo.status, sessionId: repo.sessionId ?? null }; + + // Gate 1: the repo must have a linked session (set via `set-status … --session `). + if (typeof repo.sessionId !== 'string' || repo.sessionId.trim() === '') { + return emit( + { status: 'ok', action: 'export', repo: repoInfo, exportable: false, problems: ['no-session-linked'], + written: false, targetPath: null, document: null }, + flags.outputFile, + 1, + ); + } + + // Gate 2: that session must carry an action-plan.md (i.e. `/config-audit plan` has run). + const sourcePlanPath = join(sessionsDir, repo.sessionId, 'action-plan.md'); + let planMarkdown; + try { + planMarkdown = await readFile(sourcePlanPath, 'utf-8'); + } catch (err) { + if (err && err.code === 'ENOENT') { + return emit( + { status: 'ok', action: 'export', repo: repoInfo, sessionId: repo.sessionId, sourcePlanPath, + exportable: false, problems: ['no-action-plan'], written: false, targetPath: null, document: null }, + flags.outputFile, + 1, + ); + } + fail(`could not read action plan at ${sourcePlanPath}: ${err.message}`); + } + + const targetPath = planExportPath(repo.path, repo.sessionId); + const document = buildPlanExportDocument({ + repoName: repo.name, + repoPath: repo.path, + sessionId: repo.sessionId, + planMarkdown, + now, + }); + + let written = false; + if (flags.write) { + await mkdir(dirname(targetPath), { recursive: true }); + await writeFile(targetPath, document, 'utf-8'); + written = true; + } + + return emit( + { status: 'ok', action: 'export', repo: repoInfo, sessionId: repo.sessionId, sourcePlanPath, + exportable: true, problems: [], written, targetPath, document }, + flags.outputFile, + 0, + ); +} + +const isDirectRun = + process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname); +if (isDirectRun) { + main().catch((err) => { + process.stderr.write(`Fatal: ${err.message}\n`); + process.exit(3); + }); +} diff --git a/scanners/lib/campaign-export.mjs b/scanners/lib/campaign-export.mjs new file mode 100644 index 0000000..b42e44a --- /dev/null +++ b/scanners/lib/campaign-export.mjs @@ -0,0 +1,78 @@ +/** + * campaign-export — plan-export transforms (v5.7 Fase 2, Block 4c). + * + * The second half of Block 4 ("durable backlog + execution"). Block 4b built the cross-repo + * prioritized backlog the user picks from; this exports a picked repo's per-repo action plan + * into the TARGET repo's OWN `docs/` directory, so the plan gets a durable, human-readable + * home where the work is done ("planer følger arbeidsstedet" — the operator's continuity rule + * that plans live next to the workplace, in `docs/`). + * + * Design mirrors campaign-ledger: PURE, deterministic transforms — `now` is injected as a + * YYYY-MM-DD string, never read from the clock here, so they are fully unit-testable. The IO + * (loading the ledger, reading the session's action-plan.md, writing the exported file) lives + * in the thin `campaign-export-cli` shell. The transforms throw on programmer error + * (missing/blank required field), consistent with the ledger transforms. + * + * NOTE on execution: Block 4c deliberately adds NO new execution machinery. Execution reuses + * the existing per-repo `/config-audit implement` (which backs up every changed file, applies + * the plan from the session, and verifies) + `/config-audit rollback`. The exported `docs/` + * copy is the repo's durable record of the plan, NOT the execution input — `implement` still + * reads the canonical plan from the session directory. See docs/v5.7-optimization-lens-plan.md + * §Fase 2 (Block 4). + */ + +import { join } from 'node:path'; + +/** + * The exported plan's destination inside the TARGET repo's own `docs/`. Keyed on the source + * `sessionId` (timestamp-unique per audit) rather than the calendar date, so two audits of the + * same repo on the same day produce distinct files (history is preserved, never silently + * overwritten) and the filename ties the export back to the audit that produced it. + * + * @param {string} repoPath - absolute path to the target repo (the ledger stores it resolved) + * @param {string} sessionId - the config-audit session that produced the plan + * @returns {string} `/docs/config-audit-plan-.md` + */ +export function planExportPath(repoPath, sessionId) { + if (typeof repoPath !== 'string' || repoPath.trim() === '') { + throw new TypeError('repoPath is required'); + } + if (typeof sessionId !== 'string' || sessionId.trim() === '') { + throw new TypeError('sessionId is required'); + } + return join(repoPath, 'docs', `config-audit-plan-${sessionId}.md`); +} + +/** + * Assemble the exported document: a provenance header (who/when/where this came from + how to + * execute and undo it) followed by the verbatim session plan body. Pure — given the same inputs + * it always produces the same bytes, so it is snapshot-testable. + * + * @param {{repoName:string, repoPath:string, sessionId:string, planMarkdown:string, now:string}} input + * @returns {string} the full markdown to write into the repo's docs/ + */ +export function buildPlanExportDocument({ repoName, repoPath, sessionId, planMarkdown, now } = {}) { + for (const [k, v] of Object.entries({ repoName, repoPath, sessionId, planMarkdown, now })) { + if (typeof v !== 'string' || v.trim() === '') { + throw new TypeError(`${k} is required`); + } + } + + const header = [ + `# Config-Audit Action Plan — ${repoName}`, + '', + `> Exported from the config-audit machine-wide campaign on ${now}.`, + `> **Repo:** \`${repoPath}\``, + `> **Source session:** \`${sessionId}\``, + '>', + '> Generated by `/config-audit plan`. To **execute**: run `/config-audit implement` in this', + '> repo — it backs up every changed file, applies the plan, then verifies the result. To', + '> **undo**: `/config-audit rollback`. Record progress back in the campaign with', + `> \`/config-audit campaign set-status ${repoPath} implemented\`.`, + '', + '---', + '', + ].join('\n'); + + return `${header}${planMarkdown.trimEnd()}\n`; +} diff --git a/tests/lib/campaign-export.test.mjs b/tests/lib/campaign-export.test.mjs new file mode 100644 index 0000000..8290d38 --- /dev/null +++ b/tests/lib/campaign-export.test.mjs @@ -0,0 +1,72 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { planExportPath, buildPlanExportDocument } from '../../scanners/lib/campaign-export.mjs'; + +const NOW = '2026-06-23'; +const SESSION = '20260623_101500'; + +describe('planExportPath', () => { + it('targets /docs/config-audit-plan-.md', () => { + assert.equal( + planExportPath('/Users/ktg/repos/foo', SESSION), + `/Users/ktg/repos/foo/docs/config-audit-plan-${SESSION}.md`, + ); + }); + + it('keys on sessionId (not date) so same-day audits do not collide', () => { + const a = planExportPath('/r/x', '20260623_090000'); + const b = planExportPath('/r/x', '20260623_180000'); + assert.notEqual(a, b); + }); + + it('throws on a missing/blank repoPath or sessionId (programmer error)', () => { + assert.throws(() => planExportPath('', SESSION), TypeError); + assert.throws(() => planExportPath('/r/x', ' '), TypeError); + assert.throws(() => planExportPath('/r/x'), TypeError); + }); +}); + +describe('buildPlanExportDocument', () => { + const base = { + repoName: 'foo', + repoPath: '/Users/ktg/repos/foo', + sessionId: SESSION, + planMarkdown: '## Action 1\nDo the thing.\n', + now: NOW, + }; + + it('prepends a provenance header, then the verbatim plan body', () => { + const doc = buildPlanExportDocument(base); + assert.match(doc, /^# Config-Audit Action Plan — foo\n/); + assert.ok(doc.includes(`on ${NOW}.`)); + assert.ok(doc.includes('`/Users/ktg/repos/foo`')); + assert.ok(doc.includes(`\`${SESSION}\``)); + // body is present verbatim, after the --- separator + const [, body] = doc.split('\n---\n'); + assert.ok(body.includes('## Action 1\nDo the thing.')); + }); + + it('points at the existing execute/undo/record machinery (reuse, not reinvent)', () => { + const doc = buildPlanExportDocument(base); + assert.ok(doc.includes('/config-audit implement')); + assert.ok(doc.includes('/config-audit rollback')); + assert.ok(doc.includes(`/config-audit campaign set-status ${base.repoPath} implemented`)); + }); + + it('is deterministic — same inputs produce identical bytes', () => { + assert.equal(buildPlanExportDocument(base), buildPlanExportDocument({ ...base })); + }); + + it('trims trailing whitespace on the body and ends with exactly one newline', () => { + const doc = buildPlanExportDocument({ ...base, planMarkdown: 'body\n\n\n' }); + assert.ok(doc.endsWith('body\n')); + assert.ok(!doc.endsWith('body\n\n')); + }); + + it('throws when any required field is missing or blank', () => { + for (const k of ['repoName', 'repoPath', 'sessionId', 'planMarkdown', 'now']) { + assert.throws(() => buildPlanExportDocument({ ...base, [k]: '' }), TypeError, `blank ${k}`); + } + assert.throws(() => buildPlanExportDocument(), TypeError); + }); +}); diff --git a/tests/scanners/campaign-export-cli.test.mjs b/tests/scanners/campaign-export-cli.test.mjs new file mode 100644 index 0000000..197648f --- /dev/null +++ b/tests/scanners/campaign-export-cli.test.mjs @@ -0,0 +1,154 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { createLedger, addRepo, setRepoStatus, saveLedger } from '../../scanners/lib/campaign-ledger.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const CLI = resolve(__dirname, '../../scanners/campaign-export-cli.mjs'); + +const NOW = '2026-06-23'; +const SESSION = '20260623_101500'; +const PLAN_BODY = '## Executive summary\n\n2 actions, 1 auto-fixable.\n\n## Action 1\nFix the thing.\n'; + +// Every test points --ledger-file + --sessions-dir at temp dirs, so the CLI never touches the +// real ~/.claude/ — hermetic by construction. +function runCli(extraArgs) { + try { + const stdout = execFileSync('node', [CLI, ...extraArgs], { encoding: 'utf-8', timeout: 15000 }); + return { status: 0, stdout }; + } catch (err) { + return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' }; + } +} + +/** + * Build a hermetic world: a temp dir holding the ledger + a sessions/ store + a repo/ dir. + * `opts.withSession` links a sessionId; `opts.withPlan` writes that session's action-plan.md. + */ +function world({ withSession = true, withPlan = true } = {}) { + const root = mkdtempSync(join(tmpdir(), 'camp-exp-')); + const sessionsDir = join(root, 'sessions'); + const repoDir = join(root, 'repo'); + mkdirSync(sessionsDir, { recursive: true }); + mkdirSync(repoDir, { recursive: true }); + + let l = createLedger({ now: NOW }); + l = addRepo(l, { path: repoDir, name: 'repo' }, { now: NOW }); + l = setRepoStatus(l, repoDir, 'planned', { + now: NOW, + findingsBySeverity: { critical: 0, high: 1, medium: 0, low: 2 }, + sessionId: withSession ? SESSION : undefined, + }); + const ledgerFile = join(root, 'campaign-ledger.json'); + // saveLedger is async via the lib; call synchronously through writeFileSync to keep tests simple. + writeFileSync(ledgerFile, `${JSON.stringify(l, null, 2)}\n`); + + if (withSession && withPlan) { + mkdirSync(join(sessionsDir, SESSION), { recursive: true }); + writeFileSync(join(sessionsDir, SESSION, 'action-plan.md'), PLAN_BODY); + } + return { root, sessionsDir, repoDir, ledgerFile }; +} + +const baseArgs = (w) => ['--ledger-file', w.ledgerFile, '--sessions-dir', w.sessionsDir, '--reference-date', NOW]; + +describe('campaign-export-cli — preview (read-only default)', () => { + it('exits 0 and returns an exportable preview without writing', () => { + const w = world(); + const { status, stdout } = runCli([...baseArgs(w), '--repo', w.repoDir]); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.exportable, true); + assert.equal(out.written, false); + assert.deepEqual(out.problems, []); + assert.equal(out.targetPath, join(w.repoDir, 'docs', `config-audit-plan-${SESSION}.md`)); + assert.ok(out.document.startsWith('# Config-Audit Action Plan — repo')); + assert.ok(out.document.includes('## Action 1\nFix the thing.')); + // preview must NOT have created the file + assert.equal(existsSync(out.targetPath), false); + }); +}); + +describe('campaign-export-cli — --write', () => { + it('writes a byte-exact copy of the assembled document into the repo docs/', () => { + const w = world(); + const preview = JSON.parse(runCli([...baseArgs(w), '--repo', w.repoDir]).stdout); + const { status, stdout } = runCli([...baseArgs(w), '--repo', w.repoDir, '--write']); + assert.equal(status, 0); + const out = JSON.parse(stdout); + assert.equal(out.written, true); + assert.ok(existsSync(out.targetPath)); + // file on disk equals the document the preview reported (no drift) + assert.equal(readFileSync(out.targetPath, 'utf-8'), preview.document); + }); + + it('creates the docs/ dir if missing', () => { + const w = world(); + const out = JSON.parse(runCli([...baseArgs(w), '--repo', w.repoDir, '--write']).stdout); + assert.ok(existsSync(join(w.repoDir, 'docs'))); + assert.ok(existsSync(out.targetPath)); + }); +}); + +describe('campaign-export-cli — gates (advisory exit 1)', () => { + it('exits 1 with no-session-linked when the repo has no sessionId', () => { + const w = world({ withSession: false }); + const { status, stdout } = runCli([...baseArgs(w), '--repo', w.repoDir]); + assert.equal(status, 1); + const out = JSON.parse(stdout); + assert.equal(out.exportable, false); + assert.deepEqual(out.problems, ['no-session-linked']); + assert.equal(out.written, false); + }); + + it('exits 1 with no-action-plan when the linked session has no action-plan.md', () => { + const w = world({ withPlan: false }); + const { status, stdout } = runCli([...baseArgs(w), '--repo', w.repoDir]); + assert.equal(status, 1); + const out = JSON.parse(stdout); + assert.equal(out.exportable, false); + assert.deepEqual(out.problems, ['no-action-plan']); + }); + + it('--write writes nothing when the repo is not exportable', () => { + const w = world({ withPlan: false }); + const { status, stdout } = runCli([...baseArgs(w), '--repo', w.repoDir, '--write']); + assert.equal(status, 1); + assert.equal(JSON.parse(stdout).written, false); + assert.equal(existsSync(join(w.repoDir, 'docs')), false); + }); +}); + +describe('campaign-export-cli — errors (exit 3)', () => { + it('exits 3 when --repo is missing', () => { + const w = world(); + assert.equal(runCli(baseArgs(w)).status, 3); + }); + + it('exits 3 when the repo is not tracked in the ledger', () => { + const w = world(); + assert.equal(runCli([...baseArgs(w), '--repo', join(w.root, 'other')]).status, 3); + }); + + it('exits 3 when no ledger file exists', () => { + const w = world(); + const status = runCli(['--ledger-file', join(w.root, 'nope.json'), '--sessions-dir', w.sessionsDir, '--repo', w.repoDir]).status; + assert.equal(status, 3); + }); +}); + +describe('campaign-export-cli — --output-file', () => { + it('writes the payload JSON to the file and stays silent on stdout', () => { + const w = world(); + const out = join(w.root, 'report.json'); + const { stdout } = runCli([...baseArgs(w), '--repo', w.repoDir, '--output-file', out]); + assert.equal(stdout.trim(), ''); + const written = JSON.parse(readFileSync(out, 'utf-8')); + assert.equal(written.exportable, true); + assert.equal(written.targetPath, join(w.repoDir, 'docs', `config-audit-plan-${SESSION}.md`)); + }); +}); From 817fb8933ff25a746586a9e875de85b585b56e62 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 10:31:05 +0200 Subject: [PATCH 06/53] fix(commands): wire knowledge-refresh into router + help MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit knowledge-refresh shipped as a command file in v5.7 Fase 1 Chunk 3 but was never added to the /config-audit router (argument-hint + routing list) or the help command table, so `/config-audit knowledge-refresh` did not route. Add it in both. (optimize was already fully wired — the STATE note was stale.) Co-Authored-By: Claude Opus 4.8 (1M context) --- commands/config-audit.md | 3 ++- commands/help.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/commands/config-audit.md b/commands/config-audit.md index b699f04..971da1e 100644 --- a/commands/config-audit.md +++ b/commands/config-audit.md @@ -1,7 +1,7 @@ --- name: config-audit description: Claude Code Configuration Intelligence - audit, analyze, and optimize your configuration -argument-hint: "[posture|tokens|manifest|feature-gap|optimize|fix|rollback|plan|implement|help|discover|analyze|interview|drift|plugin-health|whats-active|campaign|status|cleanup]" +argument-hint: "[posture|tokens|manifest|feature-gap|optimize|fix|rollback|plan|implement|help|discover|analyze|interview|drift|plugin-health|whats-active|campaign|knowledge-refresh|status|cleanup]" allowed-tools: Read, Write, Glob, Grep, Bash, Agent, AskUserQuestion model: opus --- @@ -30,6 +30,7 @@ If a subcommand is provided, route to it: - `plugin-health` → `/config-audit:plugin-health` - `whats-active` → `/config-audit:whats-active` - `campaign` → `/config-audit:campaign` +- `knowledge-refresh` → `/config-audit:knowledge-refresh` - `status` → `/config-audit:status` - `cleanup` → `/config-audit:cleanup` diff --git a/commands/help.md b/commands/help.md index 5e88745..68303a0 100644 --- a/commands/help.md +++ b/commands/help.md @@ -55,6 +55,7 @@ if echo "$ARGUMENTS" | grep -q -- "--raw"; then RAW_FLAG="--raw"; fi | `/config-audit plugin-health` | Audit plugin structure and the metadata block at the top of each command/agent file | | `/config-audit whats-active` | Show active plugins/skills/MCP/hooks/CLAUDE.md with token estimates | | `/config-audit campaign` | Track a machine-wide audit campaign across repos — per-repo status + roll-up, resumable across sessions (human-approved writes) | +| `/config-audit knowledge-refresh` | Keep the built-in best-practices knowledge current — flags guidance that's gone stale and checks sources for updates (changes need your approval) | ### Utility From 9c8acec71f32c4a25a7f77ae5ca49e233d3d8702 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 10:31:17 +0200 Subject: [PATCH 07/53] docs(claude-md): trim to lean invariants; move impl notes to scanner-internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md was 540 lines (CML >500 MEDIUM, config-grade B/89). Move the 19 per-scanner/per-block implementation notes (v5.6/v5.7 design rationale + primary-source verification + byte-stability lessons) verbatim into docs/scanner-internals.md under a new "Implementation notes" section, leaving a read-on-demand pointer. Add a real "## Conventions" section pointing at .claude/rules/ — this clears CA-CML-001 (missing recommended section), which the moved detail headings had been masking via incidental rule/style keyword matches. CLAUDE.md 540->134 lines / 8.5k chars; config-grade A/97; CML findings: none; readmeCheck green; full suite 1168 green (hermetic). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 430 ++------------------------------------ docs/scanner-internals.md | 422 +++++++++++++++++++++++++++++++++++++ 2 files changed, 434 insertions(+), 418 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1d9166f..bdb48ad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: ## Reference docs (read on demand) -- **Scanner inventory, lib modules, action engines, knowledge base:** `docs/scanner-internals.md` +- **Scanner inventory, lib modules, action engines, knowledge base, per-scanner/per-block implementation notes:** `docs/scanner-internals.md` - **Plain-language output (v5.1.0), humanizer vocabularies, output modes:** `docs/humanizer.md` ## Plain-Language Output (v5.1.0) — summary @@ -107,6 +107,16 @@ Default: auto-detects scope from git context. Override with `/config-audit full| ### Finding ID Format `CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001`, `CA-SKL-001`, `CA-OST-001`, `CA-OPT-001` +## Conventions + +Enforced project conventions live in `.claude/rules/` (auto-loaded as project instructions): + +- `ux-rules.md` — output/narration/formatting rules for all commands (never dump raw JSON, narrate before each step, space-separated command suggestions) +- `command-development.md` — required command frontmatter + `plugin:action` naming +- `state-management.md` — update `state.yaml` after every workflow phase + +Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{SCANNER}-{NNN}` ID format; byte-stable CLIs are verified against frozen `tests/snapshots/v5.0.0/` baselines. + ## Testing ```bash @@ -115,423 +125,7 @@ node --test 'tests/**/*.test.mjs' 1168 tests across 67 test files (22 lib + 35 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. -### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) - -`scanners/lib/active-config-reader.mjs` now enumerates the three source kinds it previously -missed — **rules** (`enumerateRules`), **agents** (`enumerateAgents`), and **output styles** -(`enumerateOutputStyles`) — alongside the existing CLAUDE.md/plugins/skills/hooks/MCP enumerators. -Each new item, plus a pure `deriveLoadPattern(kind, {scoped})` helper, carries a -`loadPattern ∈ {always, on-demand, external}`, `survivesCompaction ∈ {yes, no, n/a}`, and -`derivationConfidence ∈ {confirmed, inferred}` derived from the published Claude Code loading -model (the V-rows in `docs/v5.5-steering-model-plan.md`). `readActiveConfig` exposes `rules`/ -`agents`/`outputStyles` arrays + `totals` counts/subtotals (folded into `grandTotal`). This is -**internal plumbing** for v5.6 B (manifest/tokens rendering) — no command output changes yet, so -`--json`/`--raw`/SC-5 stay byte-stable. Output-style discovery is done directly (mirroring -`enumerateSkills`), **not** via a new `file-discovery` type, to keep the discovery surface stable. - -The frontmatter parser (`scanners/lib/yaml-parser.mjs`) now also reads **YAML block sequences** -(`paths:\n - a\n - b`), not just inline `paths: "a, b"`. This resolves a pre-existing RUL -false-positive (a block-sequence-scoped rule was misread as unscoped). An empty-valued key with -no following `- ` items still resolves to `null` (backwards-compatible); only a real `- ` item -list becomes an array. - -### manifest — load-pattern accounting (v5.6 B) - -`buildManifest` (`scanners/manifest.mjs`) now consumes the Foundation enumeration. Two changes: - -1. **Component-level sources (plugin roll-up dropped).** The coarse `kind:'plugin'` aggregate is - gone. A plugin contributes via its skills/rules/agents/output-styles/hooks/MCP — each already - enumerated **once** by `readActiveConfig` — so the old roll-up double-counted them (the plugin - aggregate's `estimatedTokens` already summed its components). Source kinds are now - `claude-md`/`skill`/`rule`/`agent`/`output-style`/`mcp-server`/`hook`. -2. **Load-pattern triple on every record + a `summary`.** Each source carries - `loadPattern`/`survivesCompaction`/`derivationConfidence`. Rules/agents/output-styles - **propagate** the foundation-derived values (rules vary by `scoped`); CLAUDE.md maps `scope`→ - kind via `CLAUDE_MD_SCOPE_KIND` (all cascade files walk **up**, so all are always-loaded); - skills are tagged **on-demand** via `deriveLoadPattern('skill-body')` — the measured tokens are - the skill **body** (paid on invoke), not the tiny always-loaded name+desc listing (tracked by - `skill-listing-budget`/posture), so tagging the body always would inflate the headline. The new - `summary` buckets sources into `always`/`onDemand`/`external`/`unknown` `{tokens,count}`; the - **always-loaded subtotal** ("≈X tokens enter context every turn before you type") is the headline. - -**Byte-stability.** manifest is an **environment-aware CLI** → SC-6/SC-7 verify it by -**mode-equivalence** (`--json == --raw`), not byte-equal against a frozen snapshot, and it is not in -SC-5 default-output. Adding fields in place therefore keeps all snapshots green with **no regen** -(verified). `total` changes (de-duped, component-level) — that is the intended correctness fix. - -### token-hotspots — load-pattern column (v5.6 B2) - -TOK now annotates every ranked hotspot with the same load-pattern triple (`hotspotLoadPattern` -maps each discovery `type`→a `deriveLoadPattern` kind; rules reuse `activeConfig.rules` for precise -`scoped` handling; `claude-md` maps by scope). Two new `deriveLoadPattern` kinds back this: -**`command`** (on-demand — body loads on `/invoke`) and **`harness-config`** (external — settings/ -keybindings/`.mcp.json`/hooks.json/plugin.json configure the CLI, **not** the model context, so they -cost no per-turn context tokens). Note the honest split: the `.mcp.json` **file** is `external`, -while the MCP **server**'s tool schemas are a separate `always` hotspot. - -**Byte-stability — the opposite of manifest.** token-hotspots **is** a byte-equal SC-6/SC-7 CLI, -**and** its hotspots ride inside the scan-orchestrator + posture payloads, so the change broke -**six** frozen-v5.0.0 comparisons across five test files (json/raw-backcompat + the three Step 5/6/7 -humanizer tests). Resolved by **preserving the frozen v5.0.0 baselines**: a shared -`tests/helpers/strip-hotspot-load-pattern.mjs` strips the additive triple before each byte-equal -compare (proves the original schema is byte-identical), and the **SC-5 default-output** snapshots -(scan-orchestrator + token-hotspots) were **regenerated** (`UPDATE_SNAPSHOT=1`) since their job is to -track current output — diff reviewed as additive-only. **Lesson for any future hotspot/scanner-output -field:** grep every frozen-v5.0.0 comparator (it is 5 files, not 2) before assuming the blast radius. - -### CML scanner — context-window-scaled char budget - -Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors -Claude Code's own startup warning — *"Large CLAUDE.md will impact performance -(X chars > 40.0k)"* — as a `char`-based finding: - -- **Char budget** — flags a CLAUDE.md over **~40.0k chars** (CC's startup-warning - figure at a 200k-context model). CC 2.1.169 scales that threshold with the model's - context window, so the finding anchors on the conservative 200k window (we cannot - observe the user's window; the anchor fires earliest) and discloses the relaxed - ~200,000-char figure at 1M context. Severity MEDIUM (token cost, not an adherence - cliff). New `CA-CML` finding. - -It keys on chars, not lines, so it is complementary to the line checks: a file can be -long by lines yet under budget (short lines), or short by lines yet over it (long lines). -The 200k/1M window constants live in the shared `scanners/lib/context-window.mjs` -(single source of truth, also re-exported by `skill-listing-budget.mjs`). The 40.0k -figure and context-window scaling are verified against the CC changelog (2.1.169) and -the live startup-warning text. - -### DIS scanner — permission-rule hygiene - -Beyond deny/allow overlap, the DIS scanner now also flags: - -- **Ineffective allow wildcards** — unanchored tool-name globs in `permissions.allow` - (`*`, `B*`, `mcp__*`) that Claude Code silently skips (auto-approve nothing). Valid - only as a glob-free `mcp____*`. New `CA-DIS` finding, severity low. -- **`Tool(*)` deny-all glob** — treated as equivalent to a bare deny (`Bash(*)` ≡ `Bash`), - so a bare allow killed by it is correctly reported as dead config. -- **Forbidden-param rules** — `Tool(param:value)` whose key is the tool's own canonicalizing - field (`command` for Bash/PowerShell, `file_path` for Read/Edit/Write, `path` for - Grep/Glob, `notebook_path` for NotebookEdit, `url` for WebFetch). CC ignores these and - emits a startup warning. Severity follows intent: **deny/ask = false security (medium)** — - the block never applies; **allow = dead config (low)** — `param:value` matching is - deny/ask-only. Valid forms (`Bash(npm:*)`, `WebFetch(domain:host)`, `Agent(model:opus)`) - are never flagged. Predicate `forbiddenParamRule` in `permission-rules.mjs`. - -These predicates live in `scanners/lib/permission-rules.mjs` (shared with the CNF -conflict-detector). Behavior verified against `code.claude.com/docs/en/permissions`. - -### PLH scanner — plugin namespace collision - -The standalone PLH scanner (cross-plugin checks in `scan()`) flags **plugin namespace -collisions**: two or more discovered plugins that declare the **same `name`** in -`plugin.json`. The search-first finding that shaped this check: Claude Code namespaces -every plugin component by the declared `name` — `/name:command`, `name:skill`, agent -`name` (verified against `code.claude.com/docs/en/plugins`, and observable in any session's -namespaced skill listing). A plugin component therefore can **never** shadow a user- or -project-level one; the only shadow that loses components is a same-`name` collision, where -the namespaces collapse into one and CC must pick a winner. Resolution between two installed -same-name plugins is **undocumented**, so the loser's commands/skills/agents go silently -unreachable — hence severity **MEDIUM** (dead config), `category: 'plugin-hygiene'`, with a -COL-shaped `details.namespaces` payload (`{ source: 'plugin:

', name, path }`). - -Two design notes: (1) the check keys on the declared `name` field, **not** `basename(dir)` — -the folder name is irrelevant to the namespace; `scanSinglePlugin` now returns `declaredName` -for this. (2) Name-less plugins are excluded from the collision map (they are flagged by the -missing-field check and must never group on an `undefined` key). - -The sibling cross-plugin **command-name** check was corrected to match the same model. Because -commands are namespaced (`/name:command`), a command name shared by two **differently-named** -plugins is ambiguity — not a hard conflict — so it now mirrors COL's plugin-vs-plugin skill -finding: severity **LOW**, `category: 'plugin-hygiene'`, COL-shaped `details.namespaces`, and a -group-first shape (one finding per command name listing every namespace, not pairwise). It keys -on the declared namespace and fires only when a name spans **2+ distinct** namespaces; when two -plugins share the same declared name, the namespace-collision finding above is the right (more -severe) signal, so the command check stays silent there to avoid a redundant `"dup, dup"` report. -The earlier HIGH `Cross-plugin command name conflict` finding (basename-keyed, "only one wins") -is gone, along with its now-inaccurate humanizer entry. - -### PLH scanner — plugin-folder shadowing (`CA-PLH-015`) - -Per-plugin check (in `scanSinglePlugin`, right after the required-field loop): a `plugin.json` -component-path key that **replaces** its default folder while that folder still exists on disk → -the folder is silently ignored (dead config). Severity **MEDIUM**, `category: 'plugin-hygiene'`, -`details: { field, ignoredDir, customPaths }`. Mirrors Claude Code's own warning in `/doctor`, -`claude plugin list`, and the `/plugin` detail view (v2.1.140+). - -The field set is **primary-source-pinned** to the *replaces* category only — -`SHADOWING_PATH_FIELDS` = `commands`/`agents`/`outputStyles` (defaults `commands/`, `agents/`, -`output-styles/`). Deliberately excluded: **`skills`** (per -`code.claude.com/docs/.../path-behavior-rules` it *adds to* the default `skills/` scan — both -load, never a shadow), and **`hooks`/`mcpServers`/`lspServers`** (own merge rules, not a -folder-shadow). Experimental `themes`/`monitors` are omitted because the docs warn their manifest -schema may change between releases. The check also honors the doc's explicit-address exception: a -custom path that resolves *into* the default folder (`"commands": ["./commands/x.md"]`) is not -flagged, because Claude Code keeps scanning the folder in that case (`addressesDefaultDir` -predicate). The v5.4.0 plan originally listed `commands/agents/skills/hooks`; that set was -corrected here against the live docs (Verifiseringsplikt). - -### PLH scanner — skills:-array validation (`CA-PLH-016`) - -Per-plugin check (in `scanSinglePlugin`, after the shadow check): when `plugin.json` has a -`skills` field (string or array), each entry must resolve to an **existing directory inside the -plugin root**. The value is normalized `Array.isArray(v) ? v : [v]`, so a single string is one -entry — and a non-string top-level value (e.g. `42`) is naturally caught as a single non-string -entry (no separate top-level check needed). One finding per bad entry, severity **MEDIUM**, -`category: 'plugin-hygiene'`, `details: { field: 'skills', entry, problem }` where `problem` is -one of `non-string` / `escapes-root` / `not-found` / `not-a-directory`. Mirrors -`claude plugin validate` (~2.1.145). - -Escape detection uses `skillsEntryEscapesRoot` (resolve + `startsWith(pluginDir + sep)` -containment — robust against a literal `..foo` dir name), backed by the docs' path-traversal rule -(*"Installed plugins cannot reference files outside their directory … such as `../shared-utils`"*). -`statOrNull` distinguishes missing from file-vs-dir. **Verifiseringsplikt note:** the v5.4.0 plan -claimed CC "suggests the parent directory when an entry points at a file"; that exact error text is -**not** in the primary docs, so it was dropped — the finding asserts only the four -primary-source-verified conditions. `skills` is deliberately *not* in `SHADOWING_PATH_FIELDS` -(it adds to the default scan, never shadows). - -### SET scanner — autoMode validation (`CA-SET`) - -Per-file check in `settings-validator.mjs` (`autoMode` was in `KNOWN_KEYS` but had no nested -validation). Two sub-checks, both primary-source-verified against -`code.claude.com/docs/en/auto-mode-config`: - -1. **Structure** (severity **MEDIUM**): `autoMode`, if present, must be an object whose only keys - are `environment`/`allow`/`soft_deny`/`hard_deny` (`AUTO_MODE_SUBKEYS`), each a **string - array** (the literal `"$defaults"` is a valid entry, so it passes the string check for free). - `problem` ∈ `not-an-object` / `unknown-subkey` / `not-string-array` in `details`. -2. **Dead-config** (severity **LOW**): Claude Code does **not** read `autoMode` from *shared* - project settings — verbatim: *"The classifier does not read `autoMode` from shared project - settings in `.claude/settings.json`, so a checked-in repo cannot inject its own allow rules."* - The check keys on **`file.scope === 'project'`** (file-discovery's `classifyScope` returns - `'project'` for a committed `.claude/settings.json`; `'local'`/`'user'`/`'managed'` are read and - not flagged). `problem: 'shared-project-scope'`. This is why the plan's "test per-file scope - first" gate passed — `ConfigFile` already carries `scope`. - -The two sub-checks are independent (a malformed autoMode in shared scope yields both). SET is in the -orchestrator, so SC-5 was re-checked after this change — byte-equal (the snapshot fixture has no -`autoMode`, so the block never fires there). - -### OST scanner — output-style validation (`CA-OST`, v5.6 C, count 13→14) - -New orchestrated scanner `output-style-scanner.mjs` — the first new scanner family since SKL -(v5.2.0). It reads the active config (`readActiveConfig`) and each output-style file's frontmatter -(via `parseFrontmatter`, keys hyphen→underscore-normalized, so it reads `keep_coding_instructions` / -`force_for_plugin`). Three findings, every claim pinned to a CONFIRMED row of -`docs/v5.5-steering-model-plan.md` (V9/V10/V11/V12), re-verified against -`code.claude.com/docs/en/output-styles` + `.../plugins-reference`: - -- **`CA-OST-001`** (medium) — a **user/project** custom style not setting `keep-coding-instructions: - true`. The flag defaults to **false**, so the style silently **removes** Claude Code's built-in - software-engineering instructions when active (V10). Scoped to user/project (the styles the user - authors); a plugin author's choice is out of scope. -- **`CA-OST-002`** (low) — a **plugin** style with `force-for-plugin: true`, which auto-applies and - **overrides** the user's selected `outputStyle` (V11). **Verifiseringsplikt correction:** the v5.5+ - plan's CA-OST-002 bullet said "in a project/user style," but `force-for-plugin` is - **plugin-styles-only** per the docs (its own cited V11 + `output-styles.md`), so the check keys on - `source === 'plugin'` — a user/project style with the flag is simply ignored, not an override. -- **`CA-OST-003`** (medium) — a settings `outputStyle` value resolving to **no** built-in - (`Default`/`Explanatory`/`Learning`/`Proactive`, matched case-insensitively) and **no** discovered - custom style → dead config (CC falls back to default; the configured behavior never applies). - -**Byte-stability — a scanner addition, NOT a field addition.** Adding the 14th scanner grows -`envelope.scanners` by one entry and bumps `aggregate.scanners_ok` 12→13 on the deterministic -fixture **regardless of findings** — a field-strip helper cannot paper this over. The SKL precedent -(`7bb2547`) re-seeded the frozen v5.0.0 snapshots, but that predates B2's strip-preservation regime; -re-seeding now would **bake in** B2's hotspot triple + `claudeMdEstimatedTokens` drift (verified by -inspecting the seed diff). So, consistent with the B2 lesson ("preserve frozen via strip-helper; -regen ONLY SC-5"), C **preserves** the frozen v5.0.0 snapshots and **strips the OST entry at compare -time**: shared `tests/helpers/strip-added-scanner.mjs` (`stripAddedScanners` removes OST entries + -decrements `scanners_ok`; `stripAddedScannerStderr` drops the `[OST]` progress line) is wired into -json/raw-backcompat + the Step 5/6 humanizer wiring tests (cli-humanizer did **not** break — its -v5.0.0 compares don't grow a scanners array). Only **SC-5 default-output** (scan-orchestrator + -posture) is regenerated (additive OST entry only — diff reviewed). OST is fixture-gated: the -`marketplace-medium` fixture and the hermetic HOME have no output styles, so it emits nothing there. - -Wiring: orchestrator import + `SCANNERS` entry; `humanizer.mjs` `SCANNER_TO_CATEGORY` -(`OST: 'Configuration mistake'`); `humanizer-data.mjs` OST family (title-coupled to the three exact -finding titles); `scoring.mjs` `SCANNER_AREA_MAP` (`OST: 'Settings'` — keeps the 10 quality areas, -byte-stable on zero-finding projects). Count badges: self-audit scanner count 13→14; humanizer-data -TRANSLATIONS families 14→15 (PLH is a translation family but not orchestrated). - -### best-practices register — machine-readable knowledge layer (v5.7 Fase 1 Chunk 1) - -`knowledge/best-practices.json`: provenance-stamped, schema-validated register (entry = -`id`/`claim`/`confidence`/`source` + optional `mechanism`/`lensCheck`/…). First runtime-consumed -file in `knowledge/` (the `*.md` stay human-only); source of truth for the v5.7 optimization lens -(`CA-OPT`); seeded from the v5.5 V-rows + the Anthropic "Steering Claude Code" blog. Only -**confirmed** entries are user-facing (Verifiseringsplikt). Loaded/validated by -`scanners/lib/best-practices-register.mjs` (`loadRegister`/`validateRegister`/`getEntry`; zero-dep -JSON, **not** YAML — `yaml-parser.mjs` can't do arrays-of-objects). Byte-stable until a scanner -consumes it (Chunk 2). Full design: `docs/v5.7-optimization-lens-plan.md`. - -### OPT scanner — optimization lens / mechanism-fit (`CA-OPT`, v5.7 Fase 1 Chunk 2a, count 14→15) - -First detector of the «optimal?» axis (vs «correct?»). `optimization-lens-scanner.mjs` reads the -best-practices register and flags config that works but fits a better mechanism. **`CA-OPT-001`** -(low, *Missed opportunity*): a CLAUDE.md procedure (≥6 consecutive numbered steps) that belongs in a -skill — recommendation/provenance from register `BP-MECH-003`. Conservative (negative corpus = null -false-positive); prose-judgment cases (lifecycle→hook, unscoped path→rule, «never»→permission) are -handled by the Chunk 2b opus analyzer (below). Wiring mirrors OST: orchestrator entry, humanizer -`OPT:'Missed opportunity'` + family, scoring `OPT:'CLAUDE.md'` (existing area → no new posture row → -byte-stable), strip-helper `OPT`, SC-5 regenerated (additive). - -### Optimization lens Chunk 2b — opus analyzer (prose-judgment half, `/config-audit optimize`) - -The hybrid motor's recall + precision halves for the three cases the deterministic OPT scanner skips. -**Pre-filter** (`scanners/lib/lens-prefilter.mjs`, pure + tested): cheap, recall-oriented line scan -of CLAUDE.md body for lifecycle phrasing (`BP-MECH-001`→hook), unscoped path-specific instructions -(`BP-MECH-002`→rule), and absolute «never» prohibitions (`BP-MECH-004`→permission); skips fenced -code, gates the path class on an instruction verb. Detector names = the register `lensCheck` fields. -**CLI** (`optimize-lens-cli.mjs`, `-cli` → not a scanner): runs discovery + OPT scanner + pre-filter, -attaches the **confirmed** register entry to each candidate (unverifiable → dropped, Verifiseringsplikt), -emits `{deterministic, candidates, register, counts}`. **Agent** (`optimization-lens-agent`, opus, -orange — the 7th agent, **precision gate**): reads the real CLAUDE.md, drops low-confidence candidates, -keeps only genuine opportunities, cites register id + source. **Command** `/config-audit optimize` -orchestrates pre-filter→agent→report. **Agent-driven → deliberately NOT byte-stable** (own command, -outside the snapshot suite); the pre-filter lib *is* unit-tested (13 tests). No new orchestrated -scanner → scanner count stays 15; agents 6→7, commands 18→19, suite 1055→1068. - -**Test-isolation fix (this session):** `token-hotspots.test.mjs` `runScanner` now wraps `scan()` in -the shared `withHermeticHome` helper — the suite is green on BOTH a real and a clean `HOME` (the OPT -section's old «run with clean HOME» caveat is resolved). Snapshot/byte tests were already hermetic. - -### knowledge-refresh — the "living" half of the register (v5.7 Fase 1 Chunk 3, commands 19→20) - -Keeps `knowledge/best-practices.json` current so the optimization lens never reads stale rules. -Same hybrid split as Chunk 2b — a deterministic, byte-stable, unit-tested core + a web/judgment shell: - -- **Deterministic core** (`scanners/lib/knowledge-refresh.mjs`, pure, 15 tests): `assessFreshness(register, - {referenceDate, staleAfterDays})` classifies each entry `fresh`/`stale` by the age of its - `source.verified` stamp. `referenceDate` is **injected** (not read from the clock) so the function is - fully deterministic; default threshold `STALE_AFTER_DAYS_DEFAULT = 90` (quarterly re-verify cadence). - An unparseable/missing `verified` → stale with `ageDays: null` (defensive; the schema-validated bundle - never hits this, but the command's hand-built candidates might). «Source changed» detection is a **web - responsibility** (command layer), **not** in this core. -- **CLI** (`scanners/knowledge-refresh-cli.mjs`, `-cli` → **NOT** an orchestrated scanner → scanner count - stays 15, suite byte-stable; 8 tests): read-only — it NEVER writes the register and NEVER hits the - network. `--reference-date` (defaults to today; the **only** place the clock is read) makes it - deterministically testable against the bundled register. `--stale-after N`, `--dry-run` (implicit + only - mode, echoed as `requestedDryRun`). Exit **0** = all fresh, **1** = some stale (advisory), **3** = error. -- **Command** (`commands/knowledge-refresh.md`, opus): orchestrates CLI stale-report → re-verify each stale - entry by re-reading its `source.url` (WebFetch) → poll CC changelog + Anthropic blog for new/changed - practices (WebSearch) → present everything → **apply ONLY human-approved writes**, then re-run the - register schema test before declaring done. **No unverified claim is ever auto-written** (Verifiseringsplikt). - Web/judgment-driven → **deliberately NOT byte-stable** (own command, outside the snapshot suite), exactly - like `/config-audit optimize`. **No new agent** (web poll runs in the command's own context), **no new - orchestrated scanner**. suite 1068→1091. - -### campaign-ledger — durable machine-wide campaign core (v5.7 Fase 2, Block 3a THIN) - -`scanners/lib/campaign-ledger.mjs`: the durable ledger that sits ABOVE individual sessions for a -machine-wide audit campaign — repo list + per-repo lifecycle (`STATUSES` = pending→audited→planned -→implemented) + a machine-wide `rollUp` (counts by status + severity aggregated across repos). It -persists to a single JSON file **outside** the plugin dir (`~/.claude/config-audit/campaign-ledger -.json`, next to `sessions/`) so it survives uninstall/reinstall/upgrade. Same hybrid split as -knowledge-refresh: PURE transforms (`createLedger`/`addRepo`/`setRepoStatus`/`rollUp`) with `now` -**injected** (YYYY-MM-DD, never the clock) + soft `validateLedger` (returns `{valid,errors}`, never -throws) + a thin IO shell (`defaultLedgerPath`/`loadLedger`→null-on-ENOENT/`saveLedger`). Transforms -throw on programmer error (invalid status, unknown path); `schemaVersion` stamped from the start so a -Block 4 migration is cheap. **THIN**: ledger + roll-up + persistence only — NO execution, CLI, or -command surface (Blocks 3b/3c/4). **Internal plumbing, byte-stable until consumed**: no `export async -function scan` + lives in `lib/` → scanner count stays 15, no orchestrator wiring, SC-5 unchanged. -28 tests, suite 1091→1119. - -### campaign-cli — read-only ledger reporter (v5.7 Fase 2, Block 3b) - -`scanners/campaign-cli.mjs` (`-cli` → NOT an orchestrated scanner → scanner count stays 15, suite -byte-stable; 8 tests): the DETERMINISTIC, READ-ONLY half of the campaign motor, mirroring -`knowledge-refresh-cli`. It `loadLedger`s the durable ledger, `validateLedger`s it, and emits -`{status, initialized, ledgerPath, schemaVersion, createdDate, updatedDate, repos, rollUp}` as JSON. -It NEVER writes — a missing ledger is reported gracefully (`initialized:false`, all-zero roll-up), -**never created**; init + every status transition belong to the Block 3c command layer (human-approved -writes, Verifiseringsplikt). `--ledger-file` overrides the default path (deterministic testing); -`--output-file` mirrors the sibling. Exit codes: **0** = initialized & valid, **1** = not initialized -yet (advisory), **3** = error (parse/corrupt/invalid). suite 1119→1127. - -### campaign-write-cli + `/config-audit campaign` — the WRITE half (v5.7 Fase 2, Block 3c, commands 20→21) - -The human-approved mutation half of the campaign motor, completing the THIN campaign surface -(ledger + roll-up + status). Two pieces: - -- **`scanners/campaign-write-cli.mjs`** (`-cli` → NOT an orchestrated scanner → scanner count stays - 15, suite byte-stable; 11 tests): the sibling of `campaign-cli` that *mutates*. Subcommands - `init` / `add ...` / `set-status `, each a thin wrapper over the - invariant-enforcing lib transforms (`createLedger`/`addRepo`/`setRepoStatus`) + `saveLedger` — so - path-normalization/dedup, idempotent add, the status-lifecycle guard, and the `updatedDate` bump - are **never re-implemented by hand**. `init` refuses to clobber an existing (or corrupt) ledger - (**exit 1** advisory, file untouched); `add` **auto-inits** when no ledger exists and reports - `added` vs `skipped`; `set-status` accepts `--findings ''` + `--session `. Determinism - mirrors the lib + `knowledge-refresh-cli`: `--reference-date` is the **only** place the clock is - read (defaults to today), passed to the transforms as the injected `now`. Exit: **0** = write - performed, **1** = advisory no-op (init-clobber), **3** = error (unknown subcommand, bad args, - invalid status, untracked repo, no/corrupt ledger). -- **`commands/campaign.md`** (opus, `allowed-tools: Read/Write/Edit/Bash/Glob` — **no Web**, - judgment-free): a thin orchestrator. It always **reports** first (read-only `campaign-cli`), then - for `init`/`add`/`set-status` it proposes the change and, **only on explicit human approval**, - invokes one write-CLI subcommand (Verifiseringsplikt — it never hand-edits the ledger JSON). - `add --discover ` finds git repos under a root and lets the user pick. When marking a repo - `audited` it attaches findings-by-severity from the repo's session (or user-provided counts) — - **never invented**. - -**Not a new scanner, not byte-stable.** Both CLIs carry the `-cli` suffix (out of the -scan-orchestrator → scanner count stays **15**, snapshot suite untouched); the command's -orchestration is judgment-driven and deliberately outside the snapshot suite, exactly like -`/config-audit optimize` + `knowledge-refresh`. **No new agent** (web/judgment-free, runs in the -command's own context). suite 1127→1138. - -### campaign backlog — cross-repo prioritized pick-list (v5.7 Fase 2, Block 4b) - -The first half of Block 4 ("one cross-repo prioritized backlog the user picks from"). A pure -lib transform + a read-only CLI-payload field — **no schema change, no new scanner, byte-stable**. - -- **`buildBacklog(ledger)`** (`scanners/lib/campaign-ledger.mjs`, pure, mirrors `rollUp`): the - single machine-wide prioritized work list. The actionable unit is a **repo** (the ledger tracks - per-repo severity *counts*, not individual findings — it tracks state, it does not re-run - audits), so each item is one repo: `{path, name, status, sessionId, findingsBySeverity - (normalized), totalFindings, weightedScore, rank}`. **Inclusion:** `status !== 'implemented'` - AND `totalFindings > 0` (implemented = done; pending / zero-finding repos have nothing known to - fix — they still surface in `rollUp.byStatus`). **Order:** DESC by `weightedScore` (exported - `SEVERITY_WEIGHTS = {critical:1000, high:100, medium:10, low:1}`), tie-broken lexicographically - by critical→high→medium→low count, then ascending `name` — fully deterministic, and the - tie-break keeps "criticals always win" even on a weighted-score collision (1 critical vs 10 high). - `rank` is 1-based after the sort. -- **`campaign-cli`** now emits `backlog: buildBacklog(ledger)` in both branches (uninitialized → - `[]`). Purely additive + read-only → fits the Block 3b read-only contract; the existing CLI - tests use targeted asserts (not full `deepEqual`), so the new field doesn't break them. -- **`commands/campaign.md`** renders the backlog as a "Prioritized backlog" pick-list and points - the user at the top item (still a pick-list, NOT an executor — execution is the later 4c block). - -**Byte-stability.** `-cli`/lib/command only → scanner count stays **15**, snapshot/backcompat -suite untouched. suite 1138→1150 (lib +9, campaign-cli +3). **Deferred to 4c:** per-repo plan -export to each repo's `docs/` + reuse of backup/rollback for execution. **Deferred until the first -breaking schema change:** `migrateLedger` (4a) — backlog needs no schema bump, so building -migration now would be speculative (`schemaVersion` is already stamped for when it's needed). - -### campaign plan-export + execution-by-reuse (v5.7 Fase 2, Block 4c — the rest of Block 4) - -The second half of Block 4. **Asymmetric:** plan export is the new testable code; execution is -*pure reuse* (no new machinery), per the plan's "reuse existing backup/rollback". - -- **Plan export** (`scanners/lib/campaign-export.mjs`, pure, 8 tests): `planExportPath(repoPath, - sessionId)` → `/docs/config-audit-plan-.md` (keyed on the timestamp-unique - sessionId, not the date, so same-day re-audits don't collide); `buildPlanExportDocument({...,now})` - → provenance header (repo/session/how-to-execute-and-undo) + the verbatim session plan. `now` - injected → deterministic. **CLI** `scanners/campaign-export-cli.mjs` (`-cli`, read-only by - default, 10 tests): `--repo ` resolves the repo's linked session, reads its - `action-plan.md`, assembles the doc, emits `{exportable, problems, targetPath, document, ...}`. - Two gates → exit 1 advisory: `no-session-linked` (repo has no `sessionId`), `no-action-plan` - (linked session has no plan yet). Writes the file **only** under opt-in `--write` — the CLI does - the byte-faithful copy so a 200-line plan is never re-typed/mutated by the LLM. `--sessions-dir` - override for hermetic tests; exit 0/1/3 mirror the sibling CLIs. -- **Execution = reuse.** No campaign-side execution code. The exported `docs/` file is the repo's - durable record; `/config-audit implement` still reads the canonical plan from the session - (backup + apply + verify), `/config-audit rollback` undoes, then `set-status implemented` - records it. The command (`commands/campaign.md`, new `export ` mode) previews → asks → on - approval invokes `--write` → routes the user to that existing machinery. -- **Byte-stable.** lib + `-cli` + command-doc only → scanner count stays **15**, agents **7**, - commands **21** (export is a *mode*, not a new command), snapshot/backcompat suite untouched. - suite 1150→1168 (lib +8, export-cli +10). **Block 4a (`migrateLedger`) still deferred** to the - first breaking schema change (export needs no schema bump). +Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. ## Gotchas diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index 5183a9f..0a91879 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -77,3 +77,425 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `gap-closure-templates.md` | Config-specific templates for closing gaps | | `prompt-cache-patterns.md` | Token-cost dynamics (prompt-cache patterns) — patterns powering the TOK scanner | | `cache-telemetry-recipe.md` | Manual `jq` recipe for verifying prompt-cache hit rate from session transcripts (v5 M7) | + +## Implementation notes (per scanner / build block) + +Detailed design rationale, primary-source verification, and byte-stability lessons for each scanner family and v5.6/v5.7 build block. Moved out of `CLAUDE.md` (kept lean per the "invariants only" rule); each note records why a change is correct and which frozen baselines it touched. Read on demand when working on the named scanner/block. + +### active-config-reader — load-pattern model + rule/agent/output-style enumeration (v5.6 Foundation) + +`scanners/lib/active-config-reader.mjs` now enumerates the three source kinds it previously +missed — **rules** (`enumerateRules`), **agents** (`enumerateAgents`), and **output styles** +(`enumerateOutputStyles`) — alongside the existing CLAUDE.md/plugins/skills/hooks/MCP enumerators. +Each new item, plus a pure `deriveLoadPattern(kind, {scoped})` helper, carries a +`loadPattern ∈ {always, on-demand, external}`, `survivesCompaction ∈ {yes, no, n/a}`, and +`derivationConfidence ∈ {confirmed, inferred}` derived from the published Claude Code loading +model (the V-rows in `docs/v5.5-steering-model-plan.md`). `readActiveConfig` exposes `rules`/ +`agents`/`outputStyles` arrays + `totals` counts/subtotals (folded into `grandTotal`). This is +**internal plumbing** for v5.6 B (manifest/tokens rendering) — no command output changes yet, so +`--json`/`--raw`/SC-5 stay byte-stable. Output-style discovery is done directly (mirroring +`enumerateSkills`), **not** via a new `file-discovery` type, to keep the discovery surface stable. + +The frontmatter parser (`scanners/lib/yaml-parser.mjs`) now also reads **YAML block sequences** +(`paths:\n - a\n - b`), not just inline `paths: "a, b"`. This resolves a pre-existing RUL +false-positive (a block-sequence-scoped rule was misread as unscoped). An empty-valued key with +no following `- ` items still resolves to `null` (backwards-compatible); only a real `- ` item +list becomes an array. + +### manifest — load-pattern accounting (v5.6 B) + +`buildManifest` (`scanners/manifest.mjs`) now consumes the Foundation enumeration. Two changes: + +1. **Component-level sources (plugin roll-up dropped).** The coarse `kind:'plugin'` aggregate is + gone. A plugin contributes via its skills/rules/agents/output-styles/hooks/MCP — each already + enumerated **once** by `readActiveConfig` — so the old roll-up double-counted them (the plugin + aggregate's `estimatedTokens` already summed its components). Source kinds are now + `claude-md`/`skill`/`rule`/`agent`/`output-style`/`mcp-server`/`hook`. +2. **Load-pattern triple on every record + a `summary`.** Each source carries + `loadPattern`/`survivesCompaction`/`derivationConfidence`. Rules/agents/output-styles + **propagate** the foundation-derived values (rules vary by `scoped`); CLAUDE.md maps `scope`→ + kind via `CLAUDE_MD_SCOPE_KIND` (all cascade files walk **up**, so all are always-loaded); + skills are tagged **on-demand** via `deriveLoadPattern('skill-body')` — the measured tokens are + the skill **body** (paid on invoke), not the tiny always-loaded name+desc listing (tracked by + `skill-listing-budget`/posture), so tagging the body always would inflate the headline. The new + `summary` buckets sources into `always`/`onDemand`/`external`/`unknown` `{tokens,count}`; the + **always-loaded subtotal** ("≈X tokens enter context every turn before you type") is the headline. + +**Byte-stability.** manifest is an **environment-aware CLI** → SC-6/SC-7 verify it by +**mode-equivalence** (`--json == --raw`), not byte-equal against a frozen snapshot, and it is not in +SC-5 default-output. Adding fields in place therefore keeps all snapshots green with **no regen** +(verified). `total` changes (de-duped, component-level) — that is the intended correctness fix. + +### token-hotspots — load-pattern column (v5.6 B2) + +TOK now annotates every ranked hotspot with the same load-pattern triple (`hotspotLoadPattern` +maps each discovery `type`→a `deriveLoadPattern` kind; rules reuse `activeConfig.rules` for precise +`scoped` handling; `claude-md` maps by scope). Two new `deriveLoadPattern` kinds back this: +**`command`** (on-demand — body loads on `/invoke`) and **`harness-config`** (external — settings/ +keybindings/`.mcp.json`/hooks.json/plugin.json configure the CLI, **not** the model context, so they +cost no per-turn context tokens). Note the honest split: the `.mcp.json` **file** is `external`, +while the MCP **server**'s tool schemas are a separate `always` hotspot. + +**Byte-stability — the opposite of manifest.** token-hotspots **is** a byte-equal SC-6/SC-7 CLI, +**and** its hotspots ride inside the scan-orchestrator + posture payloads, so the change broke +**six** frozen-v5.0.0 comparisons across five test files (json/raw-backcompat + the three Step 5/6/7 +humanizer tests). Resolved by **preserving the frozen v5.0.0 baselines**: a shared +`tests/helpers/strip-hotspot-load-pattern.mjs` strips the additive triple before each byte-equal +compare (proves the original schema is byte-identical), and the **SC-5 default-output** snapshots +(scan-orchestrator + token-hotspots) were **regenerated** (`UPDATE_SNAPSHOT=1`) since their job is to +track current output — diff reviewed as additive-only. **Lesson for any future hotspot/scanner-output +field:** grep every frozen-v5.0.0 comparator (it is 5 files, not 2) before assuming the blast radius. + +### CML scanner — context-window-scaled char budget + +Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors +Claude Code's own startup warning — *"Large CLAUDE.md will impact performance +(X chars > 40.0k)"* — as a `char`-based finding: + +- **Char budget** — flags a CLAUDE.md over **~40.0k chars** (CC's startup-warning + figure at a 200k-context model). CC 2.1.169 scales that threshold with the model's + context window, so the finding anchors on the conservative 200k window (we cannot + observe the user's window; the anchor fires earliest) and discloses the relaxed + ~200,000-char figure at 1M context. Severity MEDIUM (token cost, not an adherence + cliff). New `CA-CML` finding. + +It keys on chars, not lines, so it is complementary to the line checks: a file can be +long by lines yet under budget (short lines), or short by lines yet over it (long lines). +The 200k/1M window constants live in the shared `scanners/lib/context-window.mjs` +(single source of truth, also re-exported by `skill-listing-budget.mjs`). The 40.0k +figure and context-window scaling are verified against the CC changelog (2.1.169) and +the live startup-warning text. + +### DIS scanner — permission-rule hygiene + +Beyond deny/allow overlap, the DIS scanner now also flags: + +- **Ineffective allow wildcards** — unanchored tool-name globs in `permissions.allow` + (`*`, `B*`, `mcp__*`) that Claude Code silently skips (auto-approve nothing). Valid + only as a glob-free `mcp____*`. New `CA-DIS` finding, severity low. +- **`Tool(*)` deny-all glob** — treated as equivalent to a bare deny (`Bash(*)` ≡ `Bash`), + so a bare allow killed by it is correctly reported as dead config. +- **Forbidden-param rules** — `Tool(param:value)` whose key is the tool's own canonicalizing + field (`command` for Bash/PowerShell, `file_path` for Read/Edit/Write, `path` for + Grep/Glob, `notebook_path` for NotebookEdit, `url` for WebFetch). CC ignores these and + emits a startup warning. Severity follows intent: **deny/ask = false security (medium)** — + the block never applies; **allow = dead config (low)** — `param:value` matching is + deny/ask-only. Valid forms (`Bash(npm:*)`, `WebFetch(domain:host)`, `Agent(model:opus)`) + are never flagged. Predicate `forbiddenParamRule` in `permission-rules.mjs`. + +These predicates live in `scanners/lib/permission-rules.mjs` (shared with the CNF +conflict-detector). Behavior verified against `code.claude.com/docs/en/permissions`. + +### PLH scanner — plugin namespace collision + +The standalone PLH scanner (cross-plugin checks in `scan()`) flags **plugin namespace +collisions**: two or more discovered plugins that declare the **same `name`** in +`plugin.json`. The search-first finding that shaped this check: Claude Code namespaces +every plugin component by the declared `name` — `/name:command`, `name:skill`, agent +`name` (verified against `code.claude.com/docs/en/plugins`, and observable in any session's +namespaced skill listing). A plugin component therefore can **never** shadow a user- or +project-level one; the only shadow that loses components is a same-`name` collision, where +the namespaces collapse into one and CC must pick a winner. Resolution between two installed +same-name plugins is **undocumented**, so the loser's commands/skills/agents go silently +unreachable — hence severity **MEDIUM** (dead config), `category: 'plugin-hygiene'`, with a +COL-shaped `details.namespaces` payload (`{ source: 'plugin:', name, path }`). + +Two design notes: (1) the check keys on the declared `name` field, **not** `basename(dir)` — +the folder name is irrelevant to the namespace; `scanSinglePlugin` now returns `declaredName` +for this. (2) Name-less plugins are excluded from the collision map (they are flagged by the +missing-field check and must never group on an `undefined` key). + +The sibling cross-plugin **command-name** check was corrected to match the same model. Because +commands are namespaced (`/name:command`), a command name shared by two **differently-named** +plugins is ambiguity — not a hard conflict — so it now mirrors COL's plugin-vs-plugin skill +finding: severity **LOW**, `category: 'plugin-hygiene'`, COL-shaped `details.namespaces`, and a +group-first shape (one finding per command name listing every namespace, not pairwise). It keys +on the declared namespace and fires only when a name spans **2+ distinct** namespaces; when two +plugins share the same declared name, the namespace-collision finding above is the right (more +severe) signal, so the command check stays silent there to avoid a redundant `"dup, dup"` report. +The earlier HIGH `Cross-plugin command name conflict` finding (basename-keyed, "only one wins") +is gone, along with its now-inaccurate humanizer entry. + +### PLH scanner — plugin-folder shadowing (`CA-PLH-015`) + +Per-plugin check (in `scanSinglePlugin`, right after the required-field loop): a `plugin.json` +component-path key that **replaces** its default folder while that folder still exists on disk → +the folder is silently ignored (dead config). Severity **MEDIUM**, `category: 'plugin-hygiene'`, +`details: { field, ignoredDir, customPaths }`. Mirrors Claude Code's own warning in `/doctor`, +`claude plugin list`, and the `/plugin` detail view (v2.1.140+). + +The field set is **primary-source-pinned** to the *replaces* category only — +`SHADOWING_PATH_FIELDS` = `commands`/`agents`/`outputStyles` (defaults `commands/`, `agents/`, +`output-styles/`). Deliberately excluded: **`skills`** (per +`code.claude.com/docs/.../path-behavior-rules` it *adds to* the default `skills/` scan — both +load, never a shadow), and **`hooks`/`mcpServers`/`lspServers`** (own merge rules, not a +folder-shadow). Experimental `themes`/`monitors` are omitted because the docs warn their manifest +schema may change between releases. The check also honors the doc's explicit-address exception: a +custom path that resolves *into* the default folder (`"commands": ["./commands/x.md"]`) is not +flagged, because Claude Code keeps scanning the folder in that case (`addressesDefaultDir` +predicate). The v5.4.0 plan originally listed `commands/agents/skills/hooks`; that set was +corrected here against the live docs (Verifiseringsplikt). + +### PLH scanner — skills:-array validation (`CA-PLH-016`) + +Per-plugin check (in `scanSinglePlugin`, after the shadow check): when `plugin.json` has a +`skills` field (string or array), each entry must resolve to an **existing directory inside the +plugin root**. The value is normalized `Array.isArray(v) ? v : [v]`, so a single string is one +entry — and a non-string top-level value (e.g. `42`) is naturally caught as a single non-string +entry (no separate top-level check needed). One finding per bad entry, severity **MEDIUM**, +`category: 'plugin-hygiene'`, `details: { field: 'skills', entry, problem }` where `problem` is +one of `non-string` / `escapes-root` / `not-found` / `not-a-directory`. Mirrors +`claude plugin validate` (~2.1.145). + +Escape detection uses `skillsEntryEscapesRoot` (resolve + `startsWith(pluginDir + sep)` +containment — robust against a literal `..foo` dir name), backed by the docs' path-traversal rule +(*"Installed plugins cannot reference files outside their directory … such as `../shared-utils`"*). +`statOrNull` distinguishes missing from file-vs-dir. **Verifiseringsplikt note:** the v5.4.0 plan +claimed CC "suggests the parent directory when an entry points at a file"; that exact error text is +**not** in the primary docs, so it was dropped — the finding asserts only the four +primary-source-verified conditions. `skills` is deliberately *not* in `SHADOWING_PATH_FIELDS` +(it adds to the default scan, never shadows). + +### SET scanner — autoMode validation (`CA-SET`) + +Per-file check in `settings-validator.mjs` (`autoMode` was in `KNOWN_KEYS` but had no nested +validation). Two sub-checks, both primary-source-verified against +`code.claude.com/docs/en/auto-mode-config`: + +1. **Structure** (severity **MEDIUM**): `autoMode`, if present, must be an object whose only keys + are `environment`/`allow`/`soft_deny`/`hard_deny` (`AUTO_MODE_SUBKEYS`), each a **string + array** (the literal `"$defaults"` is a valid entry, so it passes the string check for free). + `problem` ∈ `not-an-object` / `unknown-subkey` / `not-string-array` in `details`. +2. **Dead-config** (severity **LOW**): Claude Code does **not** read `autoMode` from *shared* + project settings — verbatim: *"The classifier does not read `autoMode` from shared project + settings in `.claude/settings.json`, so a checked-in repo cannot inject its own allow rules."* + The check keys on **`file.scope === 'project'`** (file-discovery's `classifyScope` returns + `'project'` for a committed `.claude/settings.json`; `'local'`/`'user'`/`'managed'` are read and + not flagged). `problem: 'shared-project-scope'`. This is why the plan's "test per-file scope + first" gate passed — `ConfigFile` already carries `scope`. + +The two sub-checks are independent (a malformed autoMode in shared scope yields both). SET is in the +orchestrator, so SC-5 was re-checked after this change — byte-equal (the snapshot fixture has no +`autoMode`, so the block never fires there). + +### OST scanner — output-style validation (`CA-OST`, v5.6 C, count 13→14) + +New orchestrated scanner `output-style-scanner.mjs` — the first new scanner family since SKL +(v5.2.0). It reads the active config (`readActiveConfig`) and each output-style file's frontmatter +(via `parseFrontmatter`, keys hyphen→underscore-normalized, so it reads `keep_coding_instructions` / +`force_for_plugin`). Three findings, every claim pinned to a CONFIRMED row of +`docs/v5.5-steering-model-plan.md` (V9/V10/V11/V12), re-verified against +`code.claude.com/docs/en/output-styles` + `.../plugins-reference`: + +- **`CA-OST-001`** (medium) — a **user/project** custom style not setting `keep-coding-instructions: + true`. The flag defaults to **false**, so the style silently **removes** Claude Code's built-in + software-engineering instructions when active (V10). Scoped to user/project (the styles the user + authors); a plugin author's choice is out of scope. +- **`CA-OST-002`** (low) — a **plugin** style with `force-for-plugin: true`, which auto-applies and + **overrides** the user's selected `outputStyle` (V11). **Verifiseringsplikt correction:** the v5.5+ + plan's CA-OST-002 bullet said "in a project/user style," but `force-for-plugin` is + **plugin-styles-only** per the docs (its own cited V11 + `output-styles.md`), so the check keys on + `source === 'plugin'` — a user/project style with the flag is simply ignored, not an override. +- **`CA-OST-003`** (medium) — a settings `outputStyle` value resolving to **no** built-in + (`Default`/`Explanatory`/`Learning`/`Proactive`, matched case-insensitively) and **no** discovered + custom style → dead config (CC falls back to default; the configured behavior never applies). + +**Byte-stability — a scanner addition, NOT a field addition.** Adding the 14th scanner grows +`envelope.scanners` by one entry and bumps `aggregate.scanners_ok` 12→13 on the deterministic +fixture **regardless of findings** — a field-strip helper cannot paper this over. The SKL precedent +(`7bb2547`) re-seeded the frozen v5.0.0 snapshots, but that predates B2's strip-preservation regime; +re-seeding now would **bake in** B2's hotspot triple + `claudeMdEstimatedTokens` drift (verified by +inspecting the seed diff). So, consistent with the B2 lesson ("preserve frozen via strip-helper; +regen ONLY SC-5"), C **preserves** the frozen v5.0.0 snapshots and **strips the OST entry at compare +time**: shared `tests/helpers/strip-added-scanner.mjs` (`stripAddedScanners` removes OST entries + +decrements `scanners_ok`; `stripAddedScannerStderr` drops the `[OST]` progress line) is wired into +json/raw-backcompat + the Step 5/6 humanizer wiring tests (cli-humanizer did **not** break — its +v5.0.0 compares don't grow a scanners array). Only **SC-5 default-output** (scan-orchestrator + +posture) is regenerated (additive OST entry only — diff reviewed). OST is fixture-gated: the +`marketplace-medium` fixture and the hermetic HOME have no output styles, so it emits nothing there. + +Wiring: orchestrator import + `SCANNERS` entry; `humanizer.mjs` `SCANNER_TO_CATEGORY` +(`OST: 'Configuration mistake'`); `humanizer-data.mjs` OST family (title-coupled to the three exact +finding titles); `scoring.mjs` `SCANNER_AREA_MAP` (`OST: 'Settings'` — keeps the 10 quality areas, +byte-stable on zero-finding projects). Count badges: self-audit scanner count 13→14; humanizer-data +TRANSLATIONS families 14→15 (PLH is a translation family but not orchestrated). + +### best-practices register — machine-readable knowledge layer (v5.7 Fase 1 Chunk 1) + +`knowledge/best-practices.json`: provenance-stamped, schema-validated register (entry = +`id`/`claim`/`confidence`/`source` + optional `mechanism`/`lensCheck`/…). First runtime-consumed +file in `knowledge/` (the `*.md` stay human-only); source of truth for the v5.7 optimization lens +(`CA-OPT`); seeded from the v5.5 V-rows + the Anthropic "Steering Claude Code" blog. Only +**confirmed** entries are user-facing (Verifiseringsplikt). Loaded/validated by +`scanners/lib/best-practices-register.mjs` (`loadRegister`/`validateRegister`/`getEntry`; zero-dep +JSON, **not** YAML — `yaml-parser.mjs` can't do arrays-of-objects). Byte-stable until a scanner +consumes it (Chunk 2). Full design: `docs/v5.7-optimization-lens-plan.md`. + +### OPT scanner — optimization lens / mechanism-fit (`CA-OPT`, v5.7 Fase 1 Chunk 2a, count 14→15) + +First detector of the «optimal?» axis (vs «correct?»). `optimization-lens-scanner.mjs` reads the +best-practices register and flags config that works but fits a better mechanism. **`CA-OPT-001`** +(low, *Missed opportunity*): a CLAUDE.md procedure (≥6 consecutive numbered steps) that belongs in a +skill — recommendation/provenance from register `BP-MECH-003`. Conservative (negative corpus = null +false-positive); prose-judgment cases (lifecycle→hook, unscoped path→rule, «never»→permission) are +handled by the Chunk 2b opus analyzer (below). Wiring mirrors OST: orchestrator entry, humanizer +`OPT:'Missed opportunity'` + family, scoring `OPT:'CLAUDE.md'` (existing area → no new posture row → +byte-stable), strip-helper `OPT`, SC-5 regenerated (additive). + +### Optimization lens Chunk 2b — opus analyzer (prose-judgment half, `/config-audit optimize`) + +The hybrid motor's recall + precision halves for the three cases the deterministic OPT scanner skips. +**Pre-filter** (`scanners/lib/lens-prefilter.mjs`, pure + tested): cheap, recall-oriented line scan +of CLAUDE.md body for lifecycle phrasing (`BP-MECH-001`→hook), unscoped path-specific instructions +(`BP-MECH-002`→rule), and absolute «never» prohibitions (`BP-MECH-004`→permission); skips fenced +code, gates the path class on an instruction verb. Detector names = the register `lensCheck` fields. +**CLI** (`optimize-lens-cli.mjs`, `-cli` → not a scanner): runs discovery + OPT scanner + pre-filter, +attaches the **confirmed** register entry to each candidate (unverifiable → dropped, Verifiseringsplikt), +emits `{deterministic, candidates, register, counts}`. **Agent** (`optimization-lens-agent`, opus, +orange — the 7th agent, **precision gate**): reads the real CLAUDE.md, drops low-confidence candidates, +keeps only genuine opportunities, cites register id + source. **Command** `/config-audit optimize` +orchestrates pre-filter→agent→report. **Agent-driven → deliberately NOT byte-stable** (own command, +outside the snapshot suite); the pre-filter lib *is* unit-tested (13 tests). No new orchestrated +scanner → scanner count stays 15; agents 6→7, commands 18→19, suite 1055→1068. + +**Test-isolation fix (this session):** `token-hotspots.test.mjs` `runScanner` now wraps `scan()` in +the shared `withHermeticHome` helper — the suite is green on BOTH a real and a clean `HOME` (the OPT +section's old «run with clean HOME» caveat is resolved). Snapshot/byte tests were already hermetic. + +### knowledge-refresh — the "living" half of the register (v5.7 Fase 1 Chunk 3, commands 19→20) + +Keeps `knowledge/best-practices.json` current so the optimization lens never reads stale rules. +Same hybrid split as Chunk 2b — a deterministic, byte-stable, unit-tested core + a web/judgment shell: + +- **Deterministic core** (`scanners/lib/knowledge-refresh.mjs`, pure, 15 tests): `assessFreshness(register, + {referenceDate, staleAfterDays})` classifies each entry `fresh`/`stale` by the age of its + `source.verified` stamp. `referenceDate` is **injected** (not read from the clock) so the function is + fully deterministic; default threshold `STALE_AFTER_DAYS_DEFAULT = 90` (quarterly re-verify cadence). + An unparseable/missing `verified` → stale with `ageDays: null` (defensive; the schema-validated bundle + never hits this, but the command's hand-built candidates might). «Source changed» detection is a **web + responsibility** (command layer), **not** in this core. +- **CLI** (`scanners/knowledge-refresh-cli.mjs`, `-cli` → **NOT** an orchestrated scanner → scanner count + stays 15, suite byte-stable; 8 tests): read-only — it NEVER writes the register and NEVER hits the + network. `--reference-date` (defaults to today; the **only** place the clock is read) makes it + deterministically testable against the bundled register. `--stale-after N`, `--dry-run` (implicit + only + mode, echoed as `requestedDryRun`). Exit **0** = all fresh, **1** = some stale (advisory), **3** = error. +- **Command** (`commands/knowledge-refresh.md`, opus): orchestrates CLI stale-report → re-verify each stale + entry by re-reading its `source.url` (WebFetch) → poll CC changelog + Anthropic blog for new/changed + practices (WebSearch) → present everything → **apply ONLY human-approved writes**, then re-run the + register schema test before declaring done. **No unverified claim is ever auto-written** (Verifiseringsplikt). + Web/judgment-driven → **deliberately NOT byte-stable** (own command, outside the snapshot suite), exactly + like `/config-audit optimize`. **No new agent** (web poll runs in the command's own context), **no new + orchestrated scanner**. suite 1068→1091. + +### campaign-ledger — durable machine-wide campaign core (v5.7 Fase 2, Block 3a THIN) + +`scanners/lib/campaign-ledger.mjs`: the durable ledger that sits ABOVE individual sessions for a +machine-wide audit campaign — repo list + per-repo lifecycle (`STATUSES` = pending→audited→planned +→implemented) + a machine-wide `rollUp` (counts by status + severity aggregated across repos). It +persists to a single JSON file **outside** the plugin dir (`~/.claude/config-audit/campaign-ledger +.json`, next to `sessions/`) so it survives uninstall/reinstall/upgrade. Same hybrid split as +knowledge-refresh: PURE transforms (`createLedger`/`addRepo`/`setRepoStatus`/`rollUp`) with `now` +**injected** (YYYY-MM-DD, never the clock) + soft `validateLedger` (returns `{valid,errors}`, never +throws) + a thin IO shell (`defaultLedgerPath`/`loadLedger`→null-on-ENOENT/`saveLedger`). Transforms +throw on programmer error (invalid status, unknown path); `schemaVersion` stamped from the start so a +Block 4 migration is cheap. **THIN**: ledger + roll-up + persistence only — NO execution, CLI, or +command surface (Blocks 3b/3c/4). **Internal plumbing, byte-stable until consumed**: no `export async +function scan` + lives in `lib/` → scanner count stays 15, no orchestrator wiring, SC-5 unchanged. +28 tests, suite 1091→1119. + +### campaign-cli — read-only ledger reporter (v5.7 Fase 2, Block 3b) + +`scanners/campaign-cli.mjs` (`-cli` → NOT an orchestrated scanner → scanner count stays 15, suite +byte-stable; 8 tests): the DETERMINISTIC, READ-ONLY half of the campaign motor, mirroring +`knowledge-refresh-cli`. It `loadLedger`s the durable ledger, `validateLedger`s it, and emits +`{status, initialized, ledgerPath, schemaVersion, createdDate, updatedDate, repos, rollUp}` as JSON. +It NEVER writes — a missing ledger is reported gracefully (`initialized:false`, all-zero roll-up), +**never created**; init + every status transition belong to the Block 3c command layer (human-approved +writes, Verifiseringsplikt). `--ledger-file` overrides the default path (deterministic testing); +`--output-file` mirrors the sibling. Exit codes: **0** = initialized & valid, **1** = not initialized +yet (advisory), **3** = error (parse/corrupt/invalid). suite 1119→1127. + +### campaign-write-cli + `/config-audit campaign` — the WRITE half (v5.7 Fase 2, Block 3c, commands 20→21) + +The human-approved mutation half of the campaign motor, completing the THIN campaign surface +(ledger + roll-up + status). Two pieces: + +- **`scanners/campaign-write-cli.mjs`** (`-cli` → NOT an orchestrated scanner → scanner count stays + 15, suite byte-stable; 11 tests): the sibling of `campaign-cli` that *mutates*. Subcommands + `init` / `add ...` / `set-status `, each a thin wrapper over the + invariant-enforcing lib transforms (`createLedger`/`addRepo`/`setRepoStatus`) + `saveLedger` — so + path-normalization/dedup, idempotent add, the status-lifecycle guard, and the `updatedDate` bump + are **never re-implemented by hand**. `init` refuses to clobber an existing (or corrupt) ledger + (**exit 1** advisory, file untouched); `add` **auto-inits** when no ledger exists and reports + `added` vs `skipped`; `set-status` accepts `--findings ''` + `--session `. Determinism + mirrors the lib + `knowledge-refresh-cli`: `--reference-date` is the **only** place the clock is + read (defaults to today), passed to the transforms as the injected `now`. Exit: **0** = write + performed, **1** = advisory no-op (init-clobber), **3** = error (unknown subcommand, bad args, + invalid status, untracked repo, no/corrupt ledger). +- **`commands/campaign.md`** (opus, `allowed-tools: Read/Write/Edit/Bash/Glob` — **no Web**, + judgment-free): a thin orchestrator. It always **reports** first (read-only `campaign-cli`), then + for `init`/`add`/`set-status` it proposes the change and, **only on explicit human approval**, + invokes one write-CLI subcommand (Verifiseringsplikt — it never hand-edits the ledger JSON). + `add --discover ` finds git repos under a root and lets the user pick. When marking a repo + `audited` it attaches findings-by-severity from the repo's session (or user-provided counts) — + **never invented**. + +**Not a new scanner, not byte-stable.** Both CLIs carry the `-cli` suffix (out of the +scan-orchestrator → scanner count stays **15**, snapshot suite untouched); the command's +orchestration is judgment-driven and deliberately outside the snapshot suite, exactly like +`/config-audit optimize` + `knowledge-refresh`. **No new agent** (web/judgment-free, runs in the +command's own context). suite 1127→1138. + +### campaign backlog — cross-repo prioritized pick-list (v5.7 Fase 2, Block 4b) + +The first half of Block 4 ("one cross-repo prioritized backlog the user picks from"). A pure +lib transform + a read-only CLI-payload field — **no schema change, no new scanner, byte-stable**. + +- **`buildBacklog(ledger)`** (`scanners/lib/campaign-ledger.mjs`, pure, mirrors `rollUp`): the + single machine-wide prioritized work list. The actionable unit is a **repo** (the ledger tracks + per-repo severity *counts*, not individual findings — it tracks state, it does not re-run + audits), so each item is one repo: `{path, name, status, sessionId, findingsBySeverity + (normalized), totalFindings, weightedScore, rank}`. **Inclusion:** `status !== 'implemented'` + AND `totalFindings > 0` (implemented = done; pending / zero-finding repos have nothing known to + fix — they still surface in `rollUp.byStatus`). **Order:** DESC by `weightedScore` (exported + `SEVERITY_WEIGHTS = {critical:1000, high:100, medium:10, low:1}`), tie-broken lexicographically + by critical→high→medium→low count, then ascending `name` — fully deterministic, and the + tie-break keeps "criticals always win" even on a weighted-score collision (1 critical vs 10 high). + `rank` is 1-based after the sort. +- **`campaign-cli`** now emits `backlog: buildBacklog(ledger)` in both branches (uninitialized → + `[]`). Purely additive + read-only → fits the Block 3b read-only contract; the existing CLI + tests use targeted asserts (not full `deepEqual`), so the new field doesn't break them. +- **`commands/campaign.md`** renders the backlog as a "Prioritized backlog" pick-list and points + the user at the top item (still a pick-list, NOT an executor — execution is the later 4c block). + +**Byte-stability.** `-cli`/lib/command only → scanner count stays **15**, snapshot/backcompat +suite untouched. suite 1138→1150 (lib +9, campaign-cli +3). **Deferred to 4c:** per-repo plan +export to each repo's `docs/` + reuse of backup/rollback for execution. **Deferred until the first +breaking schema change:** `migrateLedger` (4a) — backlog needs no schema bump, so building +migration now would be speculative (`schemaVersion` is already stamped for when it's needed). + +### campaign plan-export + execution-by-reuse (v5.7 Fase 2, Block 4c — the rest of Block 4) + +The second half of Block 4. **Asymmetric:** plan export is the new testable code; execution is +*pure reuse* (no new machinery), per the plan's "reuse existing backup/rollback". + +- **Plan export** (`scanners/lib/campaign-export.mjs`, pure, 8 tests): `planExportPath(repoPath, + sessionId)` → `/docs/config-audit-plan-.md` (keyed on the timestamp-unique + sessionId, not the date, so same-day re-audits don't collide); `buildPlanExportDocument({...,now})` + → provenance header (repo/session/how-to-execute-and-undo) + the verbatim session plan. `now` + injected → deterministic. **CLI** `scanners/campaign-export-cli.mjs` (`-cli`, read-only by + default, 10 tests): `--repo ` resolves the repo's linked session, reads its + `action-plan.md`, assembles the doc, emits `{exportable, problems, targetPath, document, ...}`. + Two gates → exit 1 advisory: `no-session-linked` (repo has no `sessionId`), `no-action-plan` + (linked session has no plan yet). Writes the file **only** under opt-in `--write` — the CLI does + the byte-faithful copy so a 200-line plan is never re-typed/mutated by the LLM. `--sessions-dir` + override for hermetic tests; exit 0/1/3 mirror the sibling CLIs. +- **Execution = reuse.** No campaign-side execution code. The exported `docs/` file is the repo's + durable record; `/config-audit implement` still reads the canonical plan from the session + (backup + apply + verify), `/config-audit rollback` undoes, then `set-status implemented` + records it. The command (`commands/campaign.md`, new `export ` mode) previews → asks → on + approval invokes `--write` → routes the user to that existing machinery. +- **Byte-stable.** lib + `-cli` + command-doc only → scanner count stays **15**, agents **7**, + commands **21** (export is a *mode*, not a new command), snapshot/backcompat suite untouched. + suite 1150→1168 (lib +8, export-cli +10). **Block 4a (`migrateLedger`) still deferred** to the + first breaking schema change (export needs no schema bump). From 759daa720174030066f07fce0868710cc4c05135 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 12:03:36 +0200 Subject: [PATCH 08/53] =?UTF-8?q?chore(release):=20v5.8.0=20=E2=80=94=20ca?= =?UTF-8?q?mpaign=20motor=20(Fase=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump plugin.json 5.7.0 -> 5.8.0, README version badge + version-history row, and CHANGELOG [5.8.0]. Covers the full Fase 2 campaign motor (durable ledger, read-only campaign-cli, human-approved campaign-write-cli + /config-audit campaign, cross-repo backlog, per-repo plan export with execution-by-reuse) plus the pre-release cleanup (knowledge-refresh wiring, CLAUDE.md trim). Scanner count 15, agents 7, commands 20 -> 21, tests 1091 -> 1168. All counts verified green via self-audit --check-readme. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 51 ++++++++++++++++++++++++++++++++++++++ README.md | 3 ++- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 79d4ba3..e875504 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.7.0", + "version": "5.8.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 81034c3..4ba1820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,57 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.8.0] - 2026-06-23 + +### Summary +"Campaign motor" — a durable, machine-wide audit **campaign** that sits ABOVE individual sessions. +A single audited repo is one session; a fleet of repos is a campaign. This release adds a durable +ledger (per-repo lifecycle + machine-wide roll-up), a read-only reporter and a human-approved write +surface behind one `/config-audit campaign` command, a cross-repo prioritized backlog, and per-repo +plan export — with execution deliberately **reusing** the existing implement/rollback engines rather +than adding new machinery. The whole campaign surface is `-cli`/lib only, so the scanner count stays +**15**, agents **7**, and the snapshot suite is untouched; commands 20 → **21**. + +### Added +- **Durable campaign ledger (`scanners/lib/campaign-ledger.mjs`).** The ledger that sits above + sessions: a repo list + per-repo lifecycle (`pending → audited → planned → implemented`) + a + machine-wide roll-up (counts by status and by severity, aggregated across repos). Persists to a + single JSON file **outside** the plugin dir (`~/.claude/config-audit/campaign-ledger.json`, next + to `sessions/`) so it survives uninstall / reinstall / upgrade. Pure transforms + (`createLedger` / `addRepo` / `setRepoStatus` / `rollUp`) with `now` injected (never the clock) + + a soft `validateLedger` + a thin IO shell; `schemaVersion` stamped from the start. +- **Cross-repo prioritized backlog (`buildBacklog`).** One machine-wide work list — each item is a + repo (the ledger tracks per-repo severity *counts*, not individual findings), ordered by a + weighted severity score (`critical:1000 / high:100 / medium:10 / low:1`) with a deterministic + tie-break so criticals always win. Includes only repos not yet `implemented` that have known + findings. +- **`/config-audit campaign` (commands 20 → 21).** A thin orchestrator that always **reports** first + (read-only `campaign-cli`: status, roll-up, backlog) and, for `init` / `add` / `set-status`, + proposes the change and mutates **only on explicit human approval** via `campaign-write-cli` — it + never hand-edits the ledger JSON (verification duty). `add --discover ` finds git repos + under a root to pick from; marking a repo `audited` attaches findings-by-severity from its session + (or user-provided counts), never invented. +- **Per-repo plan export (`campaign-export-cli`).** `--repo ` resolves the repo's linked + session, reads its `action-plan.md`, and (under opt-in `--write`) drops a provenance-stamped, + byte-faithful copy into that repo's own `docs/config-audit-plan-.md` — a durable record + next to the code it changes. Advisory gates for "no session linked" / "no action plan yet". + +### Changed +- **Execution by reuse.** A campaign adds no execution engine of its own: the exported `docs/` file + is the durable record, while `/config-audit implement` (backup + apply + verify) and + `/config-audit rollback` do the work, then `campaign set-status implemented` records it. +- **`knowledge-refresh` wired into the surface.** The v5.7.0 `knowledge-refresh` command shipped but + was never routed; it is now in the `/config-audit` router (argument-hint + routing) and the `help` + command table. +- **CLAUDE.md trimmed to lean invariants.** The 19 per-scanner / per-block implementation notes + moved verbatim into `docs/scanner-internals.md` ("Implementation notes"); CLAUDE.md drops + 540 → 134 lines (raising its own config grade B → A). A real "Conventions" section now points at + `.claude/rules/`. + +### Tests +- 1091 → **1168** (campaign ledger +28, read-only campaign-cli +8, campaign-write-cli +11, + cross-repo backlog +12, plan-export +18). + ## [5.7.0] - 2026-06-21 ### Summary diff --git a/README.md b/README.md index 6d59e16..8e03ea0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.7.0-blue) +![Version](https://img.shields.io/badge/version-5.8.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-15-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) @@ -661,6 +661,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.8.0** | 2026-06-23 | "Campaign motor" — a durable, machine-wide audit **campaign** that sits ABOVE individual sessions (one repo = one session; a fleet of repos = a campaign). **Ledger:** `~/.claude/config-audit/campaign-ledger.json` (outside the plugin dir → survives uninstall/upgrade) tracks a repo list + per-repo lifecycle (pending→audited→planned→implemented) + a machine-wide roll-up by status & severity; pure transforms with injected `now`. **`/config-audit campaign` (commands 20→21):** read-only report (`campaign-cli`) + human-approved writes (`campaign-write-cli`: init / add / set-status) — reports first, mutates only on explicit approval, never hand-edits the ledger. **Cross-repo backlog:** one severity-weighted prioritized pick-list (`buildBacklog`, `critical:1000/high:100/medium:10/low:1`). **Plan export + execution-by-reuse:** `campaign-export-cli --write` drops a planned repo's plan verbatim into its own `docs/`; execution reuses the existing `/config-audit implement` + `rollback` (no new execution machinery). All campaign code is `-cli`/lib → scanner count stays **15**, agents **7**, byte-stable. Plus pre-release cleanup: `knowledge-refresh` wired into the router + help; CLAUDE.md trimmed 540→134 lines (impl notes → `docs/scanner-internals.md`, config grade B→A). 1168 tests | | **5.7.0** | 2026-06-21 | "Optimization lens" — first detector of the «optimally shaped?» axis (vs «correct?»), plus a living knowledge layer. **Register:** `knowledge/best-practices.json`, a provenance-stamped, schema-validated best-practices register (first runtime-consumed `knowledge/` file). **OPT scanner (count 14→15):** `CA-OPT-001` (LOW) a ≥6-step CLAUDE.md procedure that would fit better as a skill, citing register entry `BP-MECH-003`. **`/config-audit optimize` + `optimization-lens-agent` (opus, agents 6→7):** prose-judgment lens for lifecycle→hook (`BP-MECH-001`), unscoped path→rule (`BP-MECH-002`), "never"→permission (`BP-MECH-004`); pre-filter recall + opus precision gate. **`/config-audit knowledge-refresh` (commands 19→20):** deterministic stale-check (injected reference date, 90-day cadence) + web re-verify/poll, human-approved writes only. Last two are agent/web-driven (not byte-stable). 1091 tests | | **5.6.0** | 2026-06-20 | "Steering-model II" — the load-pattern / compaction-survival model lands end-to-end. **Foundation:** `active-config-reader` now enumerates rules, agents, and output styles (alongside CLAUDE.md/plugins/skills/hooks/MCP), each tagged `loadPattern` (always / on-demand / external) + `survivesCompaction` from the published loading model; the frontmatter parser also reads YAML block sequences (`paths:` lists). **B (load-pattern accounting):** `manifest` reports component-level sources (the double-counting plugin roll-up is gone), tags every source with the load-pattern triple, and leads with an **always-loaded subtotal** ("tokens that enter context every turn"); `token-hotspots` annotates each ranked hotspot with its load pattern. **C (output styles):** new orchestrated **OST** scanner (count 13→**14**) — `CA-OST-001` a custom style stripping built-in coding instructions (missing `keep-coding-instructions: true`, V10), `CA-OST-002` a plugin style with `force-for-plugin: true` overriding the user's `outputStyle` (V11), `CA-OST-003` a settings `outputStyle` resolving to no known style (dead config). Doc-verified; frozen v5.0.0 snapshots preserved via strip-helpers, SC-5 regenerated. 1023 tests | | **5.5.0** | 2026-06-20 | "Steering-model I" — two additive compaction-durability / dead-config findings (count stays **13**, `--json`/`--raw` byte-stable). Per the official "what survives compaction" model: RUL flags a large (>50-line) **path-scoped** rule not re-injected after compaction (LOW); CML flags a **nested** (subdir) CLAUDE.md not re-injected after compaction (LOW). PLH flags a plugin agent setting `hooks`/`mcpServers`/`permissionMode` — Claude Code ignores these for plugin subagents, so it's dead config (`permissionMode` = MEDIUM false-security, `hooks`/`mcpServers` = LOW). Known limitation: the frontmatter parser reads inline `paths:` but not YAML block sequences (deferred to v5.6 Foundation). 961 tests | From fcfb2979efa39fbd07655b0cf1654d9945aaa888 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 15:02:59 +0200 Subject: [PATCH 09/53] feat(scanner): add AGT agent-listing always-loaded budget finding (v5.9 B1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New AGT scanner flags the aggregate always-loaded cost of the agent listing (every active agent's name+description is injected each turn so the model knows what it can delegate to). Mirrors the SKL skill-listing pattern but encodes the key honesty caveat: the agent-listing mechanism is INFERRED (agents are absent from Claude Code's documented context breakdown), so the token figure is an UPPER-BOUND estimate and the aggregate budget is a config-audit heuristic anchored on 200k — not a CC-documented allotment. - scanners/agent-listing-scanner.mjs + scanners/lib/agent-listing-budget.mjs - 8 TDD tests (tests/scanners/agent-listing-scanner.test.mjs) - AGT folds into the Token Efficiency health area (scoring.mjs) - byte-stability: AGT added to strip-added-scanner (frozen v5.0.0 baselines left untouched, same as OST/OPT); SC-5 default-output snapshot refreshed - suite 1168 -> 1176 green Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/agent-listing-scanner.mjs | 71 +++++++ scanners/lib/agent-listing-budget.mjs | 123 +++++++++++++ scanners/lib/scoring.mjs | 1 + scanners/scan-orchestrator.mjs | 2 + tests/helpers/strip-added-scanner.mjs | 21 ++- tests/scanners/agent-listing-scanner.test.mjs | 174 ++++++++++++++++++ .../scan-orchestrator-humanizer.test.mjs | 2 +- tests/snapshots/default-output/posture.json | 2 +- .../default-output/scan-orchestrator.json | 16 +- 9 files changed, 399 insertions(+), 13 deletions(-) create mode 100644 scanners/agent-listing-scanner.mjs create mode 100644 scanners/lib/agent-listing-budget.mjs create mode 100644 tests/scanners/agent-listing-scanner.test.mjs diff --git a/scanners/agent-listing-scanner.mjs b/scanners/agent-listing-scanner.mjs new file mode 100644 index 0000000..b91a872 --- /dev/null +++ b/scanners/agent-listing-scanner.mjs @@ -0,0 +1,71 @@ +/** + * AGT Scanner — Agent-listing always-loaded token budget + * + * Claude Code injects a listing of every active agent's name+description into the + * system prompt so it knows which subagents it can delegate to. That listing is + * re-sent on EVERY turn, whether or not a delegation happens — so with many + * installed agents it is a large always-loaded cost (on a heavily-plugged machine + * the dominant single always-loaded source). + * + * Detection: + * CA-AGT-NNN aggregate agent-listing estimate exceeds the listing budget (low) + * + * INTELLECTUAL-HONESTY CONTRACT (the reason this is `low`, not a hard finding): + * unlike the skill listing, the agent-listing mechanism is NOT documented (agents + * are absent from Claude Code's published context breakdown). So the finding is an + * INFERRED, UPPER-BOUND ESTIMATE, the budget is a config-audit heuristic (no + * documented agent allotment) anchored on a conservative 200k window, and the + * evidence discloses all of that. The cap, budget, and enumerate-and-measure step + * live in `lib/agent-listing-budget.mjs` — AGT only constructs the finding. + * + * Zero external dependencies. + */ + +import { finding, scannerResult } from './lib/output.mjs'; +import { SEVERITY } from './lib/severity.mjs'; +import { + AGGREGATE_BUDGET_TOKENS, + BUDGET_CALIBRATION_NOTE, + measureActiveAgentListing, +} from './lib/agent-listing-budget.mjs'; + +const SCANNER = 'AGT'; + +/** + * Main scanner entry point. + * + * @param {string} _targetPath unused (agent listing is HOME-scoped) + * @param {object} _discovery unused (ignores project discovery) + */ +export async function scan(_targetPath, _discovery) { + const start = Date.now(); + const findings = []; + + const { aggregate } = await measureActiveAgentListing(); + + if (aggregate.overBudget) { + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Aggregate agent listing may exceed the always-loaded budget', + description: + `The ${aggregate.scanned} active agents carry about ${aggregate.aggregateTokens} tokens of ` + + 'name+description text that Claude Code injects every turn so it knows which subagents it can ' + + `delegate to — above the ${AGGREGATE_BUDGET_TOKENS}-token budget this scanner anchors on a 200k ` + + 'context window. Every one of those tokens is re-sent on every turn whether or not you delegate. ' + + 'Note: unlike the skill listing, the agent-listing always-loaded mechanism is inferred (not ' + + 'documented), so this is an upper-bound estimate (see evidence).', + evidence: + `active_agents_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars}; ` + + `description_tokens~${aggregate.aggregateTokens}; budget@200k=${AGGREGATE_BUDGET_TOKENS} tok; ` + + `over_by~${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, + recommendation: + 'Shrink the always-loaded agent listing: disable plugins whose agents you do not use (the whole ' + + 'agent block leaves the listing), remove dead user agents from ~/.claude/agents/, and trim long ' + + 'agent descriptions toward their trigger phrases.', + category: 'token-efficiency', + })); + } + + return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start); +} diff --git a/scanners/lib/agent-listing-budget.mjs b/scanners/lib/agent-listing-budget.mjs new file mode 100644 index 0000000..c3c1eba --- /dev/null +++ b/scanners/lib/agent-listing-budget.mjs @@ -0,0 +1,123 @@ +/** + * Agent-listing budget — single source of truth for the AGT scanner. + * + * Claude Code injects a listing of every active agent's name+description into the + * system prompt so the model knows which subagents it can delegate to. With many + * installed agents that listing is a large always-loaded cost: re-sent every turn + * whether or not a delegation actually happens. + * + * CRUCIAL HONESTY CAVEAT — this is why AGT differs from SKL. Unlike the skill + * listing (documented ~2% allotment, CC 2.1.32; verified 1,536-char truncation + * cap, CC 2.1.105), the agent-listing mechanism is NOT documented — agents are + * absent from Claude Code's published context breakdown. Therefore: + * - "always-loaded" is INFERRED (reasoned from the skill analogue + agents' + * absence from the deferred/on-demand list), not documented. + * - the token figure is an UPPER-BOUND ESTIMATE, not measured telemetry. + * - there is no documented per-listing allotment, so the aggregate budget is a + * config-audit heuristic anchored — by analogy to the skill listing — on a + * conservative 200k window, and the evidence says so loudly. + * - there is no verified per-description truncation cap, so (unlike CA-SKL-001) + * each description contributes its FULL length to the aggregate. + * + * Zero external dependencies. + */ + +import { estimateTokens, enumeratePlugins, enumerateAgents } from './active-config-reader.mjs'; +import { readTextFile } from './file-discovery.mjs'; +import { parseFrontmatter } from './yaml-parser.mjs'; +import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas } from './context-window.mjs'; + +// Heuristic budget by analogy to the skill listing's documented ~2% allotment. +// Agents have NO documented allotment of their own — disclosed loudly in +// BUDGET_CALIBRATION_NOTE so the number is never mistaken for a CC guarantee. +export const BUDGET_FRACTION = 0.02; +export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WINDOW_ANCHOR); // 4000 +export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000 +export { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas }; + +// The honest framing required because (a) the mechanism is inferred and (b) the +// budget depends on a context window we cannot observe. Appended to overflow evidence. +export const BUDGET_CALIBRATION_NOTE = + 'the agent-listing always-loaded mechanism is INFERRED, not documented (agents are absent from ' + + "Claude Code's published context breakdown), so this token figure is an UPPER-BOUND ESTIMATE, not " + + 'measured telemetry. unlike the skill listing there is no documented per-listing allotment, so this ' + + 'budget is a config-audit heuristic anchored on a conservative 200k window; at ' + + `${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` + + 'tok and you are likely within it'; + +/** + * @typedef {object} AgentBudgetAssessment + * @property {number} scanned - number of agent descriptions assessed + * @property {number} aggregateChars - sum of description lengths (no cap; no verified truncation) + * @property {number} aggregateTokens - estimateTokens(aggregateChars, 'markdown') + * @property {number} budgetTokens - AGGREGATE_BUDGET_TOKENS (the 200k-anchored heuristic) + * @property {boolean} overBudget - aggregateTokens strictly greater than budgetTokens + * @property {number} overBy - tokens over budget (0 when not over) + */ + +/** + * Pure aggregate-budget assessment. Each description contributes its full length: + * agents have no verified truncation cap, so nothing is dropped from the estimate. + * + * @param {number[]} descLengths - one entry per active agent (description char count) + * @returns {AgentBudgetAssessment} + */ +export function assessAgentListingBudget(descLengths) { + let aggregateChars = 0; + for (const len of descLengths) { + const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0; + aggregateChars += safe; + } + const aggregateTokens = estimateTokens(aggregateChars, 'markdown'); + const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS; + return { + scanned: descLengths.length, + aggregateChars, + aggregateTokens, + budgetTokens: AGGREGATE_BUDGET_TOKENS, + overBudget, + overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0, + }; +} + +/** + * @typedef {object} ActiveAgentEntry + * @property {string} name + * @property {'user'|'plugin'} source + * @property {string|null} pluginName + * @property {string} path + * @property {number} descLength + */ + +/** + * Enumerate every active agent (user + plugin) and measure the listing budget. + * HOME-scoped (mirrors the skill listing): resolves ~/.claude via process.env.HOME + * and excludes project-scoped agents — those load only inside their own repo. + * Callers that run under test MUST override HOME (runScannerWithHome pattern). + * + * @returns {Promise<{ agents: ActiveAgentEntry[], aggregate: AgentBudgetAssessment }>} + */ +export async function measureActiveAgentListing() { + const plugins = await enumeratePlugins(); + // enumerateAgents yields project + user + plugin; drop project (repo-local). + const allAgents = await enumerateAgents('', plugins); + + const agents = []; + for (const agent of allAgents) { + if (!agent || agent.source === 'project' || typeof agent.path !== 'string') continue; + const content = await readTextFile(agent.path); + if (!content) continue; + const fm = parseFrontmatter(content)?.frontmatter || null; + const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + agents.push({ + name: agent.name, + source: agent.source, + pluginName: agent.pluginName, + path: agent.path, + descLength: desc.length, + }); + } + + const aggregate = assessAgentListingBudget(agents.map((a) => a.descLength)); + return { agents, aggregate }; +} diff --git a/scanners/lib/scoring.mjs b/scanners/lib/scoring.mjs index cb2ec31..e8bc64f 100644 --- a/scanners/lib/scoring.mjs +++ b/scanners/lib/scoring.mjs @@ -166,6 +166,7 @@ const SCANNER_AREA_MAP = { TOK: 'Token Efficiency', CPS: 'Token Efficiency', SKL: 'Token Efficiency', + AGT: 'Token Efficiency', DIS: 'Settings', COL: 'Plugin Hygiene', OST: 'Settings', diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 039eec3..b3e9929 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -29,6 +29,7 @@ import { scan as scanCachePrefix } from './cache-prefix-scanner.mjs'; import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs'; import { scan as scanCollision } from './collision-scanner.mjs'; import { scan as scanSkillListing } from './skill-listing-scanner.mjs'; +import { scan as scanAgentListing } from './agent-listing-scanner.mjs'; import { scan as scanOutputStyle } from './output-style-scanner.mjs'; import { scan as scanOptimizationLens } from './optimization-lens-scanner.mjs'; @@ -66,6 +67,7 @@ const SCANNERS = [ { name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' }, { name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' }, { name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' }, + { name: 'AGT', fn: scanAgentListing, label: 'Agent-Listing Budget' }, { name: 'OST', fn: scanOutputStyle, label: 'Output-Style Validation' }, { name: 'OPT', fn: scanOptimizationLens, label: 'Optimization Lens' }, ]; diff --git a/tests/helpers/strip-added-scanner.mjs b/tests/helpers/strip-added-scanner.mjs index 6e135bc..fd4e70b 100644 --- a/tests/helpers/strip-added-scanner.mjs +++ b/tests/helpers/strip-added-scanner.mjs @@ -1,12 +1,13 @@ /** - * v5.6 C added the OST (Output-Style Validation) scanner — a 14th orchestrated - * scanner. The frozen v5.0.0 byte-equal baselines predate it (13 scanners), so - * this strips the additive OST scanner entry before comparison, the same way - * strip-hotspot-load-pattern.mjs handles B2's additive hotspot fields. Stripping - * (rather than re-seeding the frozen snapshots) keeps the strongest invariant: - * the ORIGINAL 13 scanners still emit byte-identical output, and the frozen - * snapshots stay free of post-v5.0.0 drift (the hotspot triple, ancestor token - * counts) that re-seeding would bake in. + * Post-v5.0.0 additive scanners that the frozen v5.0.0 byte-equal baselines + * predate: OST (Output-Style Validation, v5.6 C), OPT (Optimization Lens, v5.7), + * and AGT (Agent-Listing Budget, v5.9 B1). The baselines froze the original + * scanner set, so this strips these additive scanner entries before comparison, + * the same way strip-hotspot-load-pattern.mjs handles B2's additive hotspot + * fields. Stripping (rather than re-seeding the frozen snapshots) keeps the + * strongest invariant: the ORIGINAL scanners still emit byte-identical output, + * and the frozen snapshots stay free of post-v5.0.0 drift (the hotspot triple, + * ancestor token counts) that re-seeding would bake in. * * Removing a zero-finding scanner entry only moves `aggregate.scanners_ok` * (OST is always status 'ok' and emits nothing on the deterministic fixture, so @@ -17,7 +18,7 @@ * return stripAddedScanners(stripHotspotLoadPattern(out)); */ -const ADDED_SCANNERS = new Set(['OST', 'OPT']); +const ADDED_SCANNERS = new Set(['OST', 'OPT', 'AGT']); function stripFromEnvelope(env) { if (!env || typeof env !== 'object' || !Array.isArray(env.scanners)) return; @@ -58,5 +59,5 @@ export function stripAddedScanners(payload) { */ export function stripAddedScannerStderr(text) { if (typeof text !== 'string') return text; - return text.replace(/^[ \t]*\[(OST|OPT)\][^\n]*\n/gm, ''); + return text.replace(/^[ \t]*\[(OST|OPT|AGT)\][^\n]*\n/gm, ''); } diff --git a/tests/scanners/agent-listing-scanner.test.mjs b/tests/scanners/agent-listing-scanner.test.mjs new file mode 100644 index 0000000..cfb2e12 --- /dev/null +++ b/tests/scanners/agent-listing-scanner.test.mjs @@ -0,0 +1,174 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resetCounter } from '../../scanners/lib/output.mjs'; +import { scan } from '../../scanners/agent-listing-scanner.mjs'; + +/** + * AGT scanner — agent-listing always-loaded token budget. + * + * Claude Code injects a listing of every active agent's name+description into the + * system prompt so the model knows which subagents it can delegate to. With ~100 + * agents that listing is a large always-loaded cost. UNLIKE the skill listing, + * the agent-listing mechanism is NOT documented (agents are absent from CC's + * deferred / always-loaded context breakdown), so AGT findings are an INFERRED + * upper-bound estimate, not a verified truncation like SKL-001. The aggregate + * budget is a config-audit heuristic anchored on a conservative 200k window. + * + * Token heuristic mirrors estimateTokens('markdown'): ceil(chars/4). + * 16 agents * 1000 desc-chars = 16000 chars -> 4000 tok == budget (NOT over). + * 17 agents * 1000 desc-chars = 17000 chars -> 4250 tok > budget (over). + */ +const AGGREGATE_BUDGET_TOKENS = 4000; // heuristic: 0.02 * 200_000 (anchored, disclosed) + +function uniqueDir(suffix) { + return join(tmpdir(), `config-audit-agt-${suffix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); +} + +/** + * The AGT scanner enumerates active agents via process.env.HOME. Tests must + * override HOME, run, restore — never rely on the developer's real ~/.claude. + */ +async function runScannerWithHome(home) { + resetCounter(); + const original = process.env.HOME; + process.env.HOME = home; + try { + return await scan('/unused', { files: [] }); + } finally { + process.env.HOME = original; + } +} + +/** Write one user agent (name + description of `descLen` chars) into a HOME. */ +async function addUserAgent(home, name, descLen) { + const dir = join(home, '.claude', 'agents'); + await mkdir(dir, { recursive: true }); + const desc = 'a'.repeat(descLen); + await writeFile( + join(dir, `${name}.md`), + `---\nname: ${name}\ndescription: ${desc}\n---\nYou are the ${name} agent. Body of the agent prompt.\n`, + ); +} + +/** Build a fake HOME with `count` user agents, each description `descLen` chars. */ +async function homeWithNUserAgents(count, descLen, prefix = 'agg') { + const home = uniqueDir(`${prefix}-${count}x${descLen}`); + for (let i = 0; i < count; i++) { + await addUserAgent(home, `${prefix}${i}`, descLen); + } + return home; +} + +const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title)); + +describe('AGT scanner — basic structure', () => { + it('reports scanner prefix AGT', async () => { + const home = uniqueDir('empty'); + try { + await mkdir(join(home, '.claude'), { recursive: true }); + const result = await runScannerWithHome(home); + assert.equal(result.scanner, 'AGT'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('finding IDs match CA-AGT-NNN pattern', async () => { + const home = await homeWithNUserAgents(17, 1000); + try { + const result = await runScannerWithHome(home); + for (const f of result.findings) assert.match(f.id, /^CA-AGT-\d{3}$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('AGT scanner — aggregate always-loaded budget', () => { + it('fires a low-severity finding when the agent listing exceeds the 200k-anchored budget', async () => { + const home = await homeWithNUserAgents(17, 1000); // 17000 chars -> ~4250 tok > 4000 + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg, `expected an aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); + assert.equal(agg.severity, 'low', `aggregate should be low severity, got ${agg.severity}`); + assert.match(agg.id, /^CA-AGT-\d{3}$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('does NOT fire when only a few small agents are active', async () => { + const home = await homeWithNUserAgents(3, 1000); // 3000 chars -> 750 tok < 4000 + try { + const result = await runScannerWithHome(home); + assert.equal(findAggregate(result.findings), undefined, + `expected no aggregate finding; got: ${result.findings.map(f => f.title).join(' | ')}`); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('treats the budget as a strict boundary: 16x1000 (at budget) no fire, 17x1000 (over) fires', async () => { + const atBudget = await homeWithNUserAgents(16, 1000, 'at'); // 16000 chars -> 4000 tok == budget + try { + const result = await runScannerWithHome(atBudget); + assert.equal(findAggregate(result.findings), undefined, + 'aggregate exactly at the budget must not fire (strictly-greater rule)'); + } finally { + await rm(atBudget, { recursive: true, force: true }); + } + const overBudget = await homeWithNUserAgents(17, 1000, 'over'); // 17000 chars -> 4250 tok + try { + const result = await runScannerWithHome(overBudget); + assert.ok(findAggregate(result.findings), 'aggregate one step over the budget must fire'); + } finally { + await rm(overBudget, { recursive: true, force: true }); + } + }); + + it('DISCLOSES the inferred mechanism + upper-bound estimate + anchor (intellectual honesty)', async () => { + const home = await homeWithNUserAgents(17, 1000); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg, 'expected an aggregate finding'); + const ev = String(agg.evidence); + assert.match(ev, /infer/i, 'evidence must disclose the always-loaded mechanism is INFERRED, not documented'); + assert.match(ev, /estimate|upper.?bound/i, 'evidence must flag the figure as an estimate / upper bound'); + assert.match(ev, new RegExp(String(AGGREGATE_BUDGET_TOKENS)), 'evidence must state the budget'); + assert.match(ev, /200k/, 'evidence must anchor the budget on a 200k window'); + assert.match(ev, /1,000,000|1M|20,000/, 'evidence must note the 1M-context scaling'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('recommends the reduction levers (disable unused plugins / trim descriptions)', async () => { + const home = await homeWithNUserAgents(17, 1000); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg, 'expected an aggregate finding to carry remediation'); + const rec = String(agg.recommendation); + assert.match(rec, /disable|plugin/i, 'recommendation should mention disabling unused plugins'); + assert.match(rec, /trim|description/i, 'recommendation should mention trimming descriptions'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('an empty HOME (no agents) yields zero findings', async () => { + const home = uniqueDir('noagents'); + try { + await mkdir(join(home, '.claude'), { recursive: true }); + const result = await runScannerWithHome(home); + assert.equal(result.findings.length, 0); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/scanners/scan-orchestrator-humanizer.test.mjs b/tests/scanners/scan-orchestrator-humanizer.test.mjs index a009b07..e1f02eb 100644 --- a/tests/scanners/scan-orchestrator-humanizer.test.mjs +++ b/tests/scanners/scan-orchestrator-humanizer.test.mjs @@ -103,7 +103,7 @@ describe('scan-orchestrator humanizer wiring (Step 5)', () => { assert.ok(actual.meta, 'meta present'); assert.ok(Array.isArray(actual.scanners), 'scanners array present'); assert.ok(actual.aggregate, 'aggregate present'); - assert.equal(actual.scanners.length, 15, 'all 15 scanners present'); + assert.equal(actual.scanners.length, 16, 'all 16 scanners present'); }); it('preserves scanner shape (scanner/status/findings/counts)', async () => { diff --git a/tests/snapshots/default-output/posture.json b/tests/snapshots/default-output/posture.json index 8708097..1de34a9 100644 --- a/tests/snapshots/default-output/posture.json +++ b/tests/snapshots/default-output/posture.json @@ -1,4 +1,4 @@ { "kind": "text", - "payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n `[OST] Output-Style Validation`: 0 finding(s) (0ms)\n `[OPT] Optimization Lens`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + "payload": "`[CML] CLAUDE.md Linter`: 1 finding(s) (0ms)\n `[SET] Settings Validator`: 0 finding(s) (0ms)\n `[HKV] Hook Validator`: 0 finding(s) (0ms)\n `[RUL] Rules Validator`: 0 finding(s) (0ms)\n `[MCP] MCP Config Validator`: 0 finding(s) (0ms)\n `[IMP] Import Resolver`: 0 finding(s) (0ms)\n `[CNF] Conflict Detector`: 0 finding(s) (0ms)\n `[GAP] Feature Gap Scanner`: 17 finding(s) (0ms)\n `[TOK] Token Hotspots`: 1 finding(s) (0ms)\n `[CPS] Cache-Prefix Stability`: 0 finding(s) (0ms)\n `[DIS] Disabled-In-Schema`: 0 finding(s) (0ms)\n `[COL] Plugin Skill Collision`: 0 finding(s) (0ms)\n `[SKL] Skill-Listing Budget`: 0 finding(s) (0ms)\n `[AGT] Agent-Listing Budget`: 0 finding(s) (0ms)\n `[OST] Output-Style Validation`: 0 finding(s) (0ms)\n `[OPT] Optimization Lens`: 0 finding(s) (0ms)\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n Configuration health\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n Health: A (98/100) — Healthy setup, only minor polish needed\n 9 areas reviewed\n\n Area scores\n ───────────\n `CLAUDE.md` ........... A (90) `Settings` ............ A (100)\n `Hooks` ............... A (100) `Rules` ............... A (100)\n `MCP` ................. A (100) `Imports` ............. A (100)\n `Conflicts` ........... A (100) `Token Efficiency` .... A (90)\n `Plugin Hygiene` ...... A (100)\n\n 17 ways you could get more out of Claude Code — see /config-audit feature-gap\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" } diff --git a/tests/snapshots/default-output/scan-orchestrator.json b/tests/snapshots/default-output/scan-orchestrator.json index 19b8442..6b23edc 100644 --- a/tests/snapshots/default-output/scan-orchestrator.json +++ b/tests/snapshots/default-output/scan-orchestrator.json @@ -569,6 +569,20 @@ "info": 0 } }, + { + "scanner": "AGT", + "status": "ok", + "files_scanned": 0, + "duration_ms": 0, + "findings": [], + "counts": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "info": 0 + } + }, { "scanner": "OST", "status": "ok", @@ -610,7 +624,7 @@ "risk_score": 11, "risk_band": "Medium", "verdict": "PASS", - "scanners_ok": 14, + "scanners_ok": 15, "scanners_error": 0, "scanners_skipped": 1 } From 2a3cb537f96b39e5c653859809edc400ba0ac273 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 16:27:24 +0200 Subject: [PATCH 10/53] feat(scanner): add AGT per-agent description bloat advisory (v5.9 B1-rest) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flag any single active agent whose description exceeds a 500-char soft cap (the same bloat threshold TOK pattern F applies to SKILL.md descriptions). Every char of an agent description re-enters context in the always-loaded agent listing on every turn, so a long one is a per-turn cost. Framing is deliberately ADVISORY, severity low: unlike CA-SKL-001 agents have NO verified per-description cap, so nothing is dropped — this is a bloat advisory, never a truncation claim. Per-agent advisories are emitted BEFORE the aggregate roll-up (mirrors SKL 001->002). - PER_AGENT_DESC_SOFT_CAP (=500) added to lib/agent-listing-budget.mjs as the single source of truth, disclosed as a heuristic. - Per-agent loop reuses measureActiveAgentListing()'s agents[] (name, source, pluginName, path, descLength) — no new enumeration. - 5 new tests (fire/strict-boundary/no-truncation-claim/recommendation/order). Byte-stable: AGT already stripped from frozen v5.0.0 baselines; SC-5 unchanged (HOME-scoped, hermetic HOME has no agents). Suite green: 1181 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/agent-listing-scanner.mjs | 47 +++++++- scanners/lib/agent-listing-budget.mjs | 9 ++ tests/scanners/agent-listing-scanner.test.mjs | 109 ++++++++++++++++++ 3 files changed, 164 insertions(+), 1 deletion(-) diff --git a/scanners/agent-listing-scanner.mjs b/scanners/agent-listing-scanner.mjs index b91a872..24dc434 100644 --- a/scanners/agent-listing-scanner.mjs +++ b/scanners/agent-listing-scanner.mjs @@ -8,8 +8,15 @@ * the dominant single always-loaded source). * * Detection: + * CA-AGT-NNN per-agent description over the soft bloat cap (low, advisory) * CA-AGT-NNN aggregate agent-listing estimate exceeds the listing budget (low) * + * Per-agent advisories are emitted FIRST (mirroring SKL 001→002). They are a + * soft bloat heuristic (the 500-char threshold TOK pattern F uses for SKILL.md + * descriptions), NOT a truncation finding — agents have no verified per- + * description cap, so nothing is dropped; the description is simply re-sent in + * full every turn. + * * INTELLECTUAL-HONESTY CONTRACT (the reason this is `low`, not a hard finding): * unlike the skill listing, the agent-listing mechanism is NOT documented (agents * are absent from Claude Code's published context breakdown). So the finding is an @@ -26,6 +33,7 @@ import { SEVERITY } from './lib/severity.mjs'; import { AGGREGATE_BUDGET_TOKENS, BUDGET_CALIBRATION_NOTE, + PER_AGENT_DESC_SOFT_CAP, measureActiveAgentListing, } from './lib/agent-listing-budget.mjs'; @@ -41,7 +49,44 @@ export async function scan(_targetPath, _discovery) { const start = Date.now(); const findings = []; - const { aggregate } = await measureActiveAgentListing(); + const { agents, aggregate } = await measureActiveAgentListing(); + + // Per-agent advisory (emitted FIRST so the common "long agent + aggregate" + // case reads 001=per-agent, 002=aggregate, mirroring SKL). This is a soft + // heuristic, NOT a truncation finding — agents have no verified per-description + // cap, so the framing is "this is large and re-sent every turn", never "Claude + // Code drops the tail". + for (const agent of agents) { + if (agent.descLength <= PER_AGENT_DESC_SOFT_CAP) continue; + + const sourceLabel = agent.source === 'plugin' + ? `plugin:${agent.pluginName}` + : 'user'; + + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Agent description is long (re-sent every turn in the always-loaded listing)', + description: + `Agent "${agent.name}" (${sourceLabel}) has a description of ${agent.descLength} ` + + `characters (>${PER_AGENT_DESC_SOFT_CAP}). Claude Code injects every active agent's ` + + 'name+description into the agent listing on every turn so it knows which subagents it ' + + 'can delegate to, so every character of this description re-enters context each turn ' + + 'whether or not you delegate. Unlike the skill listing there is no verified ' + + 'per-description cap, so nothing is dropped — this is a bloat advisory, not a ' + + 'hard-cap finding.', + file: agent.path, + evidence: + `description_chars=${agent.descLength}; soft_cap=${PER_AGENT_DESC_SOFT_CAP} ` + + `(heuristic, same bloat threshold TOK pattern F uses for SKILL.md descriptions; agents ` + + `have NO verified per-description cap); agent="${agent.name}"; source=${sourceLabel}`, + recommendation: + 'Trim the description toward its trigger phrases / "when to use this agent" cues and move ' + + 'long examples into the agent body, or disable the plugin if you never delegate to this ' + + 'agent (the whole agent block then leaves the always-loaded listing).', + category: 'token-efficiency', + })); + } if (aggregate.overBudget) { findings.push(finding({ diff --git a/scanners/lib/agent-listing-budget.mjs b/scanners/lib/agent-listing-budget.mjs index c3c1eba..c062abf 100644 --- a/scanners/lib/agent-listing-budget.mjs +++ b/scanners/lib/agent-listing-budget.mjs @@ -35,6 +35,15 @@ export const AGGREGATE_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * CONTEXT_WIND export const LARGE_CONTEXT_BUDGET_TOKENS = Math.round(BUDGET_FRACTION * LARGE_CONTEXT_WINDOW); // 20000 export { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, withCommas }; +// Per-agent soft cap for the description-bloat advisory. This is a HEURISTIC, +// NOT a truncation cap: agents have no verified per-description limit (unlike +// the skill listing's 1,536-char cap, CC 2.1.105). We reuse the 500-char +// bloat threshold TOK pattern F already applies to SKILL.md descriptions, so a +// long agent description is flagged for the same reason — every char re-enters +// context in the always-loaded agent listing on every turn — without ever +// claiming Claude Code drops the tail. +export const PER_AGENT_DESC_SOFT_CAP = 500; + // The honest framing required because (a) the mechanism is inferred and (b) the // budget depends on a context window we cannot observe. Appended to overflow evidence. export const BUDGET_CALIBRATION_NOTE = diff --git a/tests/scanners/agent-listing-scanner.test.mjs b/tests/scanners/agent-listing-scanner.test.mjs index cfb2e12..76efee1 100644 --- a/tests/scanners/agent-listing-scanner.test.mjs +++ b/tests/scanners/agent-listing-scanner.test.mjs @@ -5,6 +5,7 @@ import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/agent-listing-scanner.mjs'; +import { PER_AGENT_DESC_SOFT_CAP } from '../../scanners/lib/agent-listing-budget.mjs'; /** * AGT scanner — agent-listing always-loaded token budget. @@ -62,7 +63,16 @@ async function homeWithNUserAgents(count, descLen, prefix = 'agg') { return home; } +/** Build a fake HOME with a single user agent whose description is `descLen` chars. */ +async function homeWithOneAgent(descLen, name = 'solo') { + const home = uniqueDir(`one-${name}-${descLen}`); + await addUserAgent(home, name, descLen); + return home; +} + const findAggregate = (findings) => findings.find(f => /agent listing/i.test(f.title)); +// Per-agent advisories are every finding that is NOT the aggregate roll-up. +const findPerAgent = (findings) => findings.filter(f => !/agent listing/i.test(f.title)); describe('AGT scanner — basic structure', () => { it('reports scanner prefix AGT', async () => { @@ -172,3 +182,102 @@ describe('AGT scanner — aggregate always-loaded budget', () => { } }); }); + +/** + * Per-agent description advisory (the B1-rest companion to the aggregate). + * + * A single agent with a long description is flagged on its own: every char of + * that description re-enters context in the always-loaded agent listing on every + * turn. UNLIKE CA-SKL-001, this is NOT a truncation finding — agents have no + * verified per-description cap — so the advisory is a soft heuristic (low, the + * same 500-char bloat threshold TOK pattern F uses for SKILL.md descriptions), + * not a hard "Claude Code drops the tail" claim. + */ +describe('AGT scanner — per-agent description advisory', () => { + it('fires a low-severity advisory when one agent description exceeds the soft cap', async () => { + // One agent, description over the cap; aggregate stays well under budget so + // the per-agent advisory is the only finding (clean isolation). + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + assert.equal(findAggregate(result.findings), undefined, + 'a single small-aggregate agent must not trip the aggregate budget'); + const perAgent = findPerAgent(result.findings); + assert.equal(perAgent.length, 1, + `expected exactly one per-agent advisory; got: ${result.findings.map(f => f.title).join(' | ')}`); + assert.equal(perAgent[0].severity, 'low', `per-agent advisory should be low, got ${perAgent[0].severity}`); + assert.match(perAgent[0].id, /^CA-AGT-\d{3}$/); + assert.match(`${perAgent[0].description} ${perAgent[0].evidence}`, /solo/, + 'the advisory should name the offending agent'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('treats the soft cap as a strict boundary: at-cap no fire, one-over fires', async () => { + const atCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP, 'atcap'); + try { + const result = await runScannerWithHome(atCap); + assert.equal(findPerAgent(result.findings).length, 0, + 'a description exactly at the soft cap must not fire (strictly-greater rule)'); + } finally { + await rm(atCap, { recursive: true, force: true }); + } + const overCap = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 1, 'overcap'); + try { + const result = await runScannerWithHome(overCap); + assert.equal(findPerAgent(result.findings).length, 1, + 'a description one char over the soft cap must fire'); + } finally { + await rm(overCap, { recursive: true, force: true }); + } + }); + + it('does NOT claim truncation and DISCLOSES the advisory/heuristic framing (intellectual honesty)', async () => { + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + const advisory = findPerAgent(result.findings)[0]; + assert.ok(advisory, 'expected a per-agent advisory'); + const text = `${advisory.description} ${advisory.evidence}`; + assert.doesNotMatch(text, /truncat/i, + 'agents have no verified cap — the advisory must NOT claim Claude Code truncates the description'); + assert.match(text, /every turn|always.?loaded/i, + 'the advisory must explain the description re-enters context every turn'); + assert.match(String(advisory.evidence), new RegExp(String(PER_AGENT_DESC_SOFT_CAP)), + 'evidence must state the soft-cap threshold so it reads as a heuristic'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('recommends trimming the description toward its trigger phrases', async () => { + const home = await homeWithOneAgent(PER_AGENT_DESC_SOFT_CAP + 100); + try { + const result = await runScannerWithHome(home); + const advisory = findPerAgent(result.findings)[0]; + assert.ok(advisory, 'expected a per-agent advisory to carry remediation'); + assert.match(String(advisory.recommendation), /trim|trigger|shorten/i, + 'recommendation should point at trimming the description'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('emits per-agent advisories BEFORE the aggregate roll-up', async () => { + // 17 large agents → both per-agent advisories AND the aggregate fire. + const home = await homeWithNUserAgents(17, 1000); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg, 'expected the aggregate to fire alongside per-agent advisories'); + const aggIndex = result.findings.indexOf(agg); + assert.equal(aggIndex, result.findings.length - 1, + 'the aggregate roll-up must be emitted last (per-agent advisories precede it, mirroring SKL 001→002)'); + assert.equal(findPerAgent(result.findings).length, 17, + 'each of the 17 over-cap agents should carry its own advisory'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); From cefa751990aa6736727ac99fb9f9c5de71b56810 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 16:57:31 +0200 Subject: [PATCH 11/53] =?UTF-8?q?feat(campaign):=20machine-wide=20always-l?= =?UTF-8?q?oaded=20token=20roll-up=20=E2=80=94=20pure=20layer=20(v5.9=20B2?= =?UTF-8?q?a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the campaign ledger with a machine-wide always-loaded token bill, the pure data-model + aggregation half of B2. Deliberately does NOT do live I/O: the cross-repo readActiveConfig sweep that populates real numbers is B2b. Design — shared layer counted ONCE, structurally: - Ledger root gets an optional `sharedGlobal` summary (the always-loaded layer paid in every repo: global CLAUDE.md + agent listing + global MCP + unscoped global rules + active plugins' always components), stored ONCE via the new setSharedGlobal(). - Each repo entry gets an optional `tokens` summary (its PER-REPO delta only), set via the new setRepoTokens(); addRepo now seeds `tokens: null` (mirrors findingsBySeverity). - rollUp() stays PURE and gains a `tokens` aggregate: machineWide = sharedGlobal + Σ(per-repo deltas), so the shared layer is counted exactly once by construction — the structural guard against the historic double-count bug. Plus a `byRepo` table ranked DESC by always-loaded cost ("most expensive repos") with a deterministic name tie-break. Summary shape mirrors manifest's summarizeByLoadPattern exactly ({always|onDemand|external|unknown: {tokens,count}}) so B2b wires in trivially. Keeps rollUp pure (no filesystem I/O) → byte-stable, respects the THIN campaign motor invariant. Chosen over the plan's literal "run readActiveConfig inside rollUp" which would break both (operator-approved deviation). - 8 new ledger tests incl. the counted-once regression guard + byRepo ranking. - Updated addRepo shape test (+tokens:null) and campaign-cli EMPTY_ROLLUP. - No frozen snapshot affected (campaign is v5.7, absent from v5.0.0 baselines). Suite green: 1189 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/lib/campaign-ledger.mjs | 88 ++++++++++++++++++++++++-- tests/lib/campaign-ledger.test.mjs | 94 ++++++++++++++++++++++++++++ tests/scanners/campaign-cli.test.mjs | 7 +++ 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/scanners/lib/campaign-ledger.mjs b/scanners/lib/campaign-ledger.mjs index 12a2dba..2104456 100644 --- a/scanners/lib/campaign-ledger.mjs +++ b/scanners/lib/campaign-ledger.mjs @@ -88,6 +88,7 @@ export function addRepo(ledger, { path, name } = {}, { now } = {}) { status: 'pending', sessionId: null, findingsBySeverity: null, + tokens: null, updatedDate: now, }; return { ...ledger, updatedDate: now, repos: [...ledger.repos, repo] }; @@ -119,17 +120,76 @@ export function setRepoStatus(ledger, path, status, { now, findingsBySeverity, s return { ...ledger, updatedDate: now, repos }; } +/** Load-pattern buckets carried by a token summary (manifest `summarizeByLoadPattern` shape). */ +const LOAD_PATTERNS = Object.freeze(['always', 'onDemand', 'external', 'unknown']); + /** - * Machine-wide roll-up: repo counts by status, plus a severity total aggregated across every - * repo that carries `findingsBySeverity`. Pure derivation — never mutates. + * Set the machine-wide SHARED global always-loaded layer (global CLAUDE.md + agent listing + + * global MCP + unscoped global rules + active plugins' always-loaded components). Stored ONCE + * at the ledger root — never per repo — so the machine-wide roll-up counts it exactly once + * (the structural guard against the historic double-count). The `summary` is the shape + * manifest's `summarizeByLoadPattern` emits: `{always|onDemand|external|unknown: {tokens,count}}`. + * Returns a NEW ledger. (B2b populates this from a live cross-repo sweep.) * @param {object} ledger - * @returns {{totalRepos:number, byStatus:object, bySeverity:object, reposWithFindings:number}} + * @param {object} summary + * @param {{now: string}} opts + */ +export function setSharedGlobal(ledger, summary, { now } = {}) { + requireNow(now); + return { ...ledger, updatedDate: now, sharedGlobal: summary }; +} + +/** + * Attach a tracked repo's PER-REPO always-loaded token delta (its project-scoped contribution + * beyond the shared global layer — project CLAUDE.md / rules / agents / MCP). Same `summary` + * shape as `setSharedGlobal`. Mirrors `setRepoStatus`: throws if the repo is untracked. Returns + * a NEW ledger. + * @param {object} ledger + * @param {string} path + * @param {object} tokens + * @param {{now: string}} opts + */ +export function setRepoTokens(ledger, path, tokens, { now } = {}) { + requireNow(now); + const resolved = normalizePath(path); + const idx = ledger.repos.findIndex((r) => r.path === resolved); + if (idx === -1) { + throw new Error(`repo "${resolved}" is not in the ledger — addRepo first`); + } + const repos = ledger.repos.slice(); + repos[idx] = { ...ledger.repos[idx], tokens, updatedDate: now }; + return { ...ledger, updatedDate: now, repos }; +} + +/** Tolerantly read the four load-pattern token numbers from a stored summary (or null/old data). */ +function bucketTokens(summary) { + const out = {}; + for (const k of LOAD_PATTERNS) { + const v = summary && summary[k]; + out[k] = v && typeof v.tokens === 'number' ? v.tokens : 0; + } + return out; +} + +/** + * Machine-wide roll-up: repo counts by status, a severity total across every repo carrying + * `findingsBySeverity`, AND a machine-wide always-loaded token bill. The token aggregate adds + * the SHARED global layer (counted once, from the ledger root) to the SUM of per-repo deltas, + * so `tokens.machineWide` is the honest "context spent every turn across the whole machine". + * Pure derivation — never mutates; tolerant of old ledgers without token fields. + * @param {object} ledger + * @returns {{totalRepos:number, byStatus:object, bySeverity:object, reposWithFindings:number, tokens:object}} */ export function rollUp(ledger) { const byStatus = Object.fromEntries(STATUSES.map((s) => [s, 0])); const bySeverity = Object.fromEntries(SEVERITIES.map((s) => [s, 0])); let reposWithFindings = 0; + const sharedGlobal = bucketTokens(ledger.sharedGlobal); + const perRepoDelta = Object.fromEntries(LOAD_PATTERNS.map((k) => [k, 0])); + const byRepo = []; + let reposWithTokens = 0; + for (const repo of ledger.repos) { if (byStatus[repo.status] !== undefined) byStatus[repo.status] += 1; const f = repo.findingsBySeverity; @@ -139,8 +199,28 @@ export function rollUp(ledger) { if (typeof f[sev] === 'number') bySeverity[sev] += f[sev]; } } + if (repo.tokens && typeof repo.tokens === 'object') { + reposWithTokens += 1; + const b = bucketTokens(repo.tokens); + for (const k of LOAD_PATTERNS) perRepoDelta[k] += b[k]; + byRepo.push({ name: repo.name, path: repo.path, always: b.always, onDemand: b.onDemand, external: b.external }); + } } - return { totalRepos: ledger.repos.length, byStatus, bySeverity, reposWithFindings }; + + // DESC by always-loaded cost ("most expensive repos"); deterministic name tie-break. + byRepo.sort((x, y) => y.always - x.always || x.name.localeCompare(y.name)); + + const machineWide = Object.fromEntries( + LOAD_PATTERNS.map((k) => [k, sharedGlobal[k] + perRepoDelta[k]]), + ); + + return { + totalRepos: ledger.repos.length, + byStatus, + bySeverity, + reposWithFindings, + tokens: { sharedGlobal, perRepoDelta, machineWide, reposWithTokens, byRepo }, + }; } /** diff --git a/tests/lib/campaign-ledger.test.mjs b/tests/lib/campaign-ledger.test.mjs index 8bc8119..6d50355 100644 --- a/tests/lib/campaign-ledger.test.mjs +++ b/tests/lib/campaign-ledger.test.mjs @@ -10,6 +10,8 @@ import { createLedger, addRepo, setRepoStatus, + setSharedGlobal, + setRepoTokens, rollUp, buildBacklog, validateLedger, @@ -21,6 +23,17 @@ import { const NOW = '2026-06-22'; const LATER = '2026-06-23'; +/** + * Build a load-pattern summary in the exact shape manifest's `summarizeByLoadPattern` + * emits (and the shape B2b will store): one {tokens,count} bucket per load pattern. + */ +const tokSummary = (always, onDemand = 0, external = 0, unknown = 0) => ({ + always: { tokens: always, count: 1 }, + onDemand: { tokens: onDemand, count: 1 }, + external: { tokens: external, count: 1 }, + unknown: { tokens: unknown, count: 1 }, +}); + describe('constants', () => { it('schema version is 1', () => { assert.equal(CAMPAIGN_SCHEMA_VERSION, 1); @@ -57,6 +70,7 @@ describe('addRepo', () => { status: 'pending', sessionId: null, findingsBySeverity: null, + tokens: null, updatedDate: NOW, }); }); @@ -155,6 +169,86 @@ describe('rollUp', () => { }); }); +describe('machine-wide token roll-up (B2a)', () => { + it('addRepo seeds a null tokens field (mirrors findingsBySeverity)', () => { + const l = addRepo(createLedger({ now: NOW }), { path: '/r/a', name: 'a' }, { now: NOW }); + assert.equal(l.repos[0].tokens, null); + }); + + it('setSharedGlobal stores the shared layer at the ledger root, immutably + stamps', () => { + const l0 = createLedger({ now: NOW }); + const l1 = setSharedGlobal(l0, tokSummary(1000, 50, 7), { now: LATER }); + assert.deepEqual(l1.sharedGlobal, tokSummary(1000, 50, 7)); + assert.equal(l1.updatedDate, LATER); + // immutable: original untouched, no sharedGlobal leaked onto it + assert.equal(l0.sharedGlobal, undefined); + assert.equal(l0.updatedDate, NOW); + }); + + it('setRepoTokens attaches per-repo tokens, immutably + stamps', () => { + let l = addRepo(createLedger({ now: NOW }), { path: '/r/a', name: 'a' }, { now: NOW }); + const before = l; + l = setRepoTokens(l, '/r/a', tokSummary(120), { now: LATER }); + assert.deepEqual(l.repos[0].tokens, tokSummary(120)); + assert.equal(l.repos[0].updatedDate, LATER); + assert.equal(l.updatedDate, LATER); + assert.equal(before.repos[0].tokens, null); // input ledger not mutated + }); + + it('setRepoTokens throws when the repo is not tracked', () => { + const l = createLedger({ now: NOW }); + assert.throws(() => setRepoTokens(l, '/r/missing', tokSummary(1), { now: NOW }), /not in the ledger/); + }); + + it('rollUp returns an all-zero token aggregate for a fresh ledger', () => { + const r = rollUp(createLedger({ now: NOW })); + assert.deepEqual(r.tokens, { + sharedGlobal: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + perRepoDelta: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + machineWide: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + reposWithTokens: 0, + byRepo: [], + }); + }); + + it('counts the shared-global layer ONCE, not per repo (double-count regression guard)', () => { + let l = createLedger({ now: NOW }); + l = setSharedGlobal(l, tokSummary(1000), { now: NOW }); // shared always-loaded = 1000 + for (const name of ['a', 'b', 'c']) { + l = addRepo(l, { path: `/r/${name}`, name }, { now: NOW }); + l = setRepoTokens(l, `/r/${name}`, tokSummary(100), { now: NOW }); // each per-repo delta = 100 + } + const t = rollUp(l).tokens; + assert.equal(t.sharedGlobal.always, 1000, 'shared layer stored once'); + assert.equal(t.perRepoDelta.always, 300, 'sum of three per-repo deltas'); + // THE invariant: shared layer counted ONCE (1000 + 300), NOT once per repo (3*1000 + 300) + assert.equal(t.machineWide.always, 1300); + assert.equal(t.reposWithTokens, 3); + }); + + it('ranks byRepo DESC by always-tokens with a deterministic name tie-break', () => { + let l = createLedger({ now: NOW }); + // deltas out of order + a tie at 100 to exercise the name tie-break + const spec = [['a', 100], ['z', 300], ['m', 100], ['q', 50]]; + for (const [name, always] of spec) { + l = addRepo(l, { path: `/r/${name}`, name }, { now: NOW }); + l = setRepoTokens(l, `/r/${name}`, tokSummary(always), { now: NOW }); + } + const byRepo = rollUp(l).tokens.byRepo; + assert.deepEqual(byRepo.map((r) => r.name), ['z', 'a', 'm', 'q']); + assert.deepEqual(byRepo[0], { name: 'z', path: '/r/z', always: 300, onDemand: 0, external: 0 }); + }); + + it('tolerates old ledgers: no sharedGlobal + repos without tokens → all zero', () => { + let l = createLedger({ now: NOW }); + l = addRepo(l, { path: '/r/a', name: 'a' }, { now: NOW }); // no tokens set + const t = rollUp(l).tokens; + assert.equal(t.machineWide.always, 0); + assert.equal(t.reposWithTokens, 0); + assert.deepEqual(t.byRepo, []); + }); +}); + describe('buildBacklog', () => { // Seed a ledger from a compact spec. A repo with `status` (and optional `findings`) is // transitioned; a bare repo stays `pending` with null findings. diff --git a/tests/scanners/campaign-cli.test.mjs b/tests/scanners/campaign-cli.test.mjs index 6d784db..457c558 100644 --- a/tests/scanners/campaign-cli.test.mjs +++ b/tests/scanners/campaign-cli.test.mjs @@ -57,6 +57,13 @@ const EMPTY_ROLLUP = { byStatus: { pending: 0, audited: 0, planned: 0, implemented: 0 }, bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, reposWithFindings: 0, + tokens: { + sharedGlobal: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + perRepoDelta: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + machineWide: { always: 0, onDemand: 0, external: 0, unknown: 0 }, + reposWithTokens: 0, + byRepo: [], + }, }; describe('campaign-cli — exit codes', () => { From 872b8ac2819f3e95498480283a5f78ca93fefbd2 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:10:03 +0200 Subject: [PATCH 12/53] feat(manifest): ownership split for machine-wide token roll-up (v5.9 B2b-1) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure classifier splitManifestByOwnership(sources) → {shared, delta}, the FS-free core that B2b's live cross-repo sweep will feed into the campaign ledger's setSharedGlobal/setRepoTokens (B2a). classifyOwnership maps each source string to its layer: shared : user | managed | plugin:* | ~/.claude.json:projects (global MCP) delta : project | local | .mcp.json | @import | unrecognized Anything not positively global falls to delta, so a source is never silently folded into the once-counted shared layer (a wrong fold HIDES machine-wide cost; a wrong delta is at worst visibly attributed to a repo). Both layers carry the canonical summarizeByLoadPattern shape so the ledger setters consume them verbatim. TDD: 6 unit tests first (RED → GREEN). buildManifest/CLI output unchanged → manifest snapshot byte-identical. Suite 1195 green (1189 + 6). [skip-docs]: internal pure export only, no user-facing surface yet. User-facing docs (commands/campaign.md + CLAUDE.md manifest/campaign rows) land in B2b-3 when refresh-tokens + the rendered token-bill ship. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/manifest.mjs | 50 +++++++++++++++++++++ tests/scanners/manifest.test.mjs | 77 +++++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 1 deletion(-) diff --git a/scanners/manifest.mjs b/scanners/manifest.mjs index da9c32b..6bafe2f 100644 --- a/scanners/manifest.mjs +++ b/scanners/manifest.mjs @@ -168,6 +168,56 @@ export function summarizeByLoadPattern(sources) { return summary; } +/** + * Source strings (the `source` field buildManifest stamps) that belong to the + * SHARED GLOBAL layer — config paid once per machine and identical in every + * repo: the global ~/.claude CLAUDE.md, managed enterprise policy, and the + * global MCP slice of ~/.claude.json. Installed plugins are also shared but are + * matched by the `plugin:` prefix below, not by this set. + */ +const SHARED_GLOBAL_SOURCES = Object.freeze(new Set(['user', 'managed', '~/.claude.json:projects'])); + +/** + * Classify one manifest source as part of the once-counted shared global layer + * or a per-repo delta (v5.9 B2b). Anything not positively identified as global + * (project / local / .mcp.json / @import / unrecognized) falls to `delta`, so a + * source is never silently folded into the shared layer — a wrong fold would + * HIDE machine-wide cost, whereas a wrong delta is at worst attributed visibly + * to a repo. + * @param {string} source + * @returns {'shared'|'delta'} + */ +export function classifyOwnership(source) { + if (typeof source === 'string') { + if (source.startsWith('plugin:')) return 'shared'; // installed plugins are machine-global + if (SHARED_GLOBAL_SOURCES.has(source)) return 'shared'; + } + return 'delta'; +} + +/** + * Partition manifest sources by ownership for the machine-wide token roll-up, + * returning two load-pattern summaries in the exact shape `summarizeByLoadPattern` + * emits ({always,onDemand,external,unknown:{tokens,count}}), so the campaign + * ledger setters (`setSharedGlobal` / `setRepoTokens`) consume them verbatim. + * + * - `shared`: the global layer, identical across repos — set ONCE on the ledger + * root so the roll-up counts it exactly once (the structural double-count guard). + * - `delta`: this repo's own project/local contribution beyond the shared layer. + * + * The split is total: every source lands in exactly one layer. + * @param {Array<{source:string, loadPattern:string, estimated_tokens:number}>} sources + * @returns {{shared:object, delta:object}} + */ +export function splitManifestByOwnership(sources) { + const shared = []; + const delta = []; + for (const s of sources || []) { + (classifyOwnership(s.source) === 'shared' ? shared : delta).push(s); + } + return { shared: summarizeByLoadPattern(shared), delta: summarizeByLoadPattern(delta) }; +} + /** * Distribute the cascade-level estimated tokens across the individual files * proportional to their byte size. claudeMd.estimatedTokens is computed for diff --git a/tests/scanners/manifest.test.mjs b/tests/scanners/manifest.test.mjs index 519f075..d2ca618 100644 --- a/tests/scanners/manifest.test.mjs +++ b/tests/scanners/manifest.test.mjs @@ -5,7 +5,7 @@ import { resolve, join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { buildManifest } from '../../scanners/manifest.mjs'; +import { buildManifest, splitManifestByOwnership, classifyOwnership } from '../../scanners/manifest.mjs'; import { deriveLoadPattern } from '../../scanners/lib/active-config-reader.mjs'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -193,6 +193,81 @@ describe('buildManifest — load-pattern accounting (unit)', () => { }); }); +describe('splitManifestByOwnership — shared-global vs per-repo-delta (unit, v5.9 B2b)', () => { + // Minimal source records — only the fields the splitter reads (source, loadPattern, + // estimated_tokens), matching the shape buildManifest stamps onto every source. + const src = (source, loadPattern, tokens) => ({ source, loadPattern, estimated_tokens: tokens }); + + it('classifyOwnership routes each source string to the right layer', () => { + // Shared: paid once per machine, identical in every repo. + assert.equal(classifyOwnership('user'), 'shared'); + assert.equal(classifyOwnership('managed'), 'shared'); + assert.equal(classifyOwnership('plugin:foo'), 'shared'); + assert.equal(classifyOwnership('~/.claude.json:projects'), 'shared'); + // Per-repo delta: the repo's own project/local contribution. + assert.equal(classifyOwnership('project'), 'delta'); + assert.equal(classifyOwnership('local'), 'delta'); + assert.equal(classifyOwnership('.mcp.json'), 'delta'); + // import + anything unrecognized → delta (never silently folded into the shared layer). + assert.equal(classifyOwnership('import'), 'delta'); + assert.equal(classifyOwnership('mystery'), 'delta'); + assert.equal(classifyOwnership(undefined), 'delta'); + }); + + const sources = [ + src('user', 'always', 100), // global CLAUDE.md → shared + src('managed', 'always', 50), // managed policy → shared + src('plugin:foo', 'always', 30), // plugin agent/rule → shared + src('~/.claude.json:projects', 'always', 20), // global MCP → shared + src('project', 'always', 10), // project CLAUDE.md/rule → delta + src('local', 'always', 5), // local → delta + src('.mcp.json', 'always', 7), // project MCP → delta + src('import', 'always', 3), // conservative → delta + ]; + const { shared, delta } = splitManifestByOwnership(sources); + + it('folds global/user/managed/plugin/global-MCP into the shared layer', () => { + assert.equal(shared.always.tokens, 100 + 50 + 30 + 20); // 200 + assert.equal(shared.always.count, 4); + }); + + it('attributes project/local/.mcp.json/import to the per-repo delta', () => { + assert.equal(delta.always.tokens, 10 + 5 + 7 + 3); // 25 + assert.equal(delta.always.count, 4); + }); + + it('preserves load-pattern buckets within each layer', () => { + const mixed = [ + src('user', 'always', 100), + src('plugin:foo', 'on-demand', 40), // plugin skill body → shared, onDemand + src('plugin:foo', 'external', 0), // plugin hook → shared, external + src('project', 'on-demand', 8), // delta, onDemand + ]; + const r = splitManifestByOwnership(mixed); + assert.equal(r.shared.always.tokens, 100); + assert.equal(r.shared.onDemand.tokens, 40); + assert.equal(r.shared.external.count, 1); + assert.equal(r.delta.onDemand.tokens, 8); + }); + + it('is a total partition — every source lands in exactly one layer (no loss, no dup)', () => { + const wholeTokens = sources.reduce((a, s) => a + s.estimated_tokens, 0); + const buckets = (sum) => sum.always.tokens + sum.onDemand.tokens + sum.external.tokens + sum.unknown.tokens; + const counts = (sum) => sum.always.count + sum.onDemand.count + sum.external.count + sum.unknown.count; + assert.equal(buckets(shared) + buckets(delta), wholeTokens); + assert.equal(counts(shared) + counts(delta), sources.length); + }); + + it('emits the canonical {always,onDemand,external,unknown:{tokens,count}} shape for both layers', () => { + for (const summary of [shared, delta]) { + for (const bucket of ['always', 'onDemand', 'external', 'unknown']) { + assert.equal(typeof summary[bucket].tokens, 'number', `${bucket}.tokens`); + assert.equal(typeof summary[bucket].count, 'number', `${bucket}.count`); + } + } + }); +}); + describe('manifest CLI — fixture path (rich-repo with patched HOME)', () => { let fixture; From 82f881afc4d04f75e09553f68c3cb6edb82446a2 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:14:28 +0200 Subject: [PATCH 13/53] fix(manifest): classify ~/.claude.json:projects MCP as per-repo delta, not shared [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readClaudeJsonProjectSlice(repoPath) returns the slice keyed to the SPECIFIC repo path (exact / longest-prefix match), so MCP servers under ~/.claude.json:projects are per-repo: they load only in their own project and differ across repos. B2b-1 wrongly grouped them with the shared global layer (reasoning from file location, not the keyed slice). The live sweep captures the shared layer ONCE from the first repo — folding a per-repo slice into it would silently drop every other repo's claude.json MCP servers. Corrected: SHARED_GLOBAL_SOURCES = {user, managed}; the only machine-global MCP is plugin-provided (plugin: prefix). Per-repo MCP (.mcp.json AND the ~/.claude.json project slice) → delta. Tests updated. [skip-docs]: internal classifier correction, no user-facing surface yet. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/manifest.mjs | 23 +++++++++++++++-------- tests/scanners/manifest.test.mjs | 17 +++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/scanners/manifest.mjs b/scanners/manifest.mjs index 6bafe2f..1940076 100644 --- a/scanners/manifest.mjs +++ b/scanners/manifest.mjs @@ -171,19 +171,26 @@ export function summarizeByLoadPattern(sources) { /** * Source strings (the `source` field buildManifest stamps) that belong to the * SHARED GLOBAL layer — config paid once per machine and identical in every - * repo: the global ~/.claude CLAUDE.md, managed enterprise policy, and the - * global MCP slice of ~/.claude.json. Installed plugins are also shared but are - * matched by the `plugin:` prefix below, not by this set. + * repo: the global ~/.claude CLAUDE.md (`user`) and managed enterprise policy + * (`managed`). Installed plugins are also shared but are matched by the + * `plugin:` prefix below, not by this set. + * + * Deliberately NOT here: `~/.claude.json:projects`. Although that file lives in + * HOME, `readClaudeJsonProjectSlice` returns the slice keyed to the SPECIFIC + * repo path — those MCP servers are per-repo, load only in their own project, + * and differ across repos, so they are a delta (folding them into the + * once-counted shared layer would drop every repo's slice but the first). The + * only machine-global MCP is plugin-provided (caught by the `plugin:` prefix). */ -const SHARED_GLOBAL_SOURCES = Object.freeze(new Set(['user', 'managed', '~/.claude.json:projects'])); +const SHARED_GLOBAL_SOURCES = Object.freeze(new Set(['user', 'managed'])); /** * Classify one manifest source as part of the once-counted shared global layer * or a per-repo delta (v5.9 B2b). Anything not positively identified as global - * (project / local / .mcp.json / @import / unrecognized) falls to `delta`, so a - * source is never silently folded into the shared layer — a wrong fold would - * HIDE machine-wide cost, whereas a wrong delta is at worst attributed visibly - * to a repo. + * (project / local / .mcp.json / ~/.claude.json:projects / @import / unrecognized) + * falls to `delta`, so a source is never silently folded into the shared layer — + * a wrong fold would HIDE machine-wide cost, whereas a wrong delta is at worst + * attributed visibly to a repo. * @param {string} source * @returns {'shared'|'delta'} */ diff --git a/tests/scanners/manifest.test.mjs b/tests/scanners/manifest.test.mjs index d2ca618..cd0da1a 100644 --- a/tests/scanners/manifest.test.mjs +++ b/tests/scanners/manifest.test.mjs @@ -203,11 +203,12 @@ describe('splitManifestByOwnership — shared-global vs per-repo-delta (unit, v5 assert.equal(classifyOwnership('user'), 'shared'); assert.equal(classifyOwnership('managed'), 'shared'); assert.equal(classifyOwnership('plugin:foo'), 'shared'); - assert.equal(classifyOwnership('~/.claude.json:projects'), 'shared'); // Per-repo delta: the repo's own project/local contribution. assert.equal(classifyOwnership('project'), 'delta'); assert.equal(classifyOwnership('local'), 'delta'); assert.equal(classifyOwnership('.mcp.json'), 'delta'); + // ~/.claude.json:projects is a per-repo MCP slice (keyed by repo path), NOT shared. + assert.equal(classifyOwnership('~/.claude.json:projects'), 'delta'); // import + anything unrecognized → delta (never silently folded into the shared layer). assert.equal(classifyOwnership('import'), 'delta'); assert.equal(classifyOwnership('mystery'), 'delta'); @@ -218,7 +219,7 @@ describe('splitManifestByOwnership — shared-global vs per-repo-delta (unit, v5 src('user', 'always', 100), // global CLAUDE.md → shared src('managed', 'always', 50), // managed policy → shared src('plugin:foo', 'always', 30), // plugin agent/rule → shared - src('~/.claude.json:projects', 'always', 20), // global MCP → shared + src('~/.claude.json:projects', 'always', 20), // per-repo MCP slice → delta src('project', 'always', 10), // project CLAUDE.md/rule → delta src('local', 'always', 5), // local → delta src('.mcp.json', 'always', 7), // project MCP → delta @@ -226,14 +227,14 @@ describe('splitManifestByOwnership — shared-global vs per-repo-delta (unit, v5 ]; const { shared, delta } = splitManifestByOwnership(sources); - it('folds global/user/managed/plugin/global-MCP into the shared layer', () => { - assert.equal(shared.always.tokens, 100 + 50 + 30 + 20); // 200 - assert.equal(shared.always.count, 4); + it('folds global/user/managed/plugin into the shared layer', () => { + assert.equal(shared.always.tokens, 100 + 50 + 30); // 180 + assert.equal(shared.always.count, 3); }); - it('attributes project/local/.mcp.json/import to the per-repo delta', () => { - assert.equal(delta.always.tokens, 10 + 5 + 7 + 3); // 25 - assert.equal(delta.always.count, 4); + it('attributes project/local/.mcp.json/claude.json-slice/import to the per-repo delta', () => { + assert.equal(delta.always.tokens, 20 + 10 + 5 + 7 + 3); // 45 + assert.equal(delta.always.count, 5); }); it('preserves load-pattern buckets within each layer', () => { From d664b7052013582f68ee287a6bb3750dc701b12d Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:23:15 +0200 Subject: [PATCH 14/53] feat(campaign): refresh-tokens live cross-repo token sweep (v5.9 B2b-2) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The IO half of the machine-wide token roll-up (B2a shipped the pure data model). New campaign-write-cli subcommand 'refresh-tokens': for every tracked repo it runs readActiveConfig → buildManifest → splitManifestByOwnership, stores each repo's per-repo delta via setRepoTokens, and captures the HOME-derived shared global layer ONCE (from the first repo that reads cleanly) via setSharedGlobal. Capturing shared once is the structural counted-once guard: a repo that fails to read is recorded in 'skipped' and its delta omitted, never aborting the sweep. Idempotent — a re-sweep replaces (setters overwrite), never accumulates. Determinism unchanged: the clock is still read only via --reference-date. TDD: 3 integration tests first (RED → GREEN) against a fixture with a dominant global CLAUDE.md + two repos. Counted-once guard asserts each stored delta stays a small fraction of the shared layer (a double-fold would make each delta EXCEED shared). Caught a per-repo-MCP misclassification (fixed in 82f881a) + a test-side shape slip (rollUp flattens its token buckets to numbers; stored summaries keep {tokens,count}). Suite 1198 green (1195 + 3). Manifest + all frozen snapshots untouched. [skip-docs]: internal sweep CLI (campaign plumbing), no user-facing surface yet. The rendered token-bill in commands/campaign.md + CLAUDE.md rows land in B2b-3. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/campaign-write-cli.mjs | 64 +++++++++++- tests/scanners/campaign-write-cli.test.mjs | 113 ++++++++++++++++++++- 2 files changed, 170 insertions(+), 7 deletions(-) diff --git a/scanners/campaign-write-cli.mjs b/scanners/campaign-write-cli.mjs index 5438112..45ff2ef 100644 --- a/scanners/campaign-write-cli.mjs +++ b/scanners/campaign-write-cli.mjs @@ -26,10 +26,20 @@ * node campaign-write-cli.mjs set-status * [--findings ''] [--session ] * [--ledger-file

] [--reference-date ] + * node campaign-write-cli.mjs refresh-tokens [--ledger-file

] [--reference-date ] * (all accept [--output-file

] to write the result payload to a file instead of stdout) * - * Exit codes: 0 = write performed, 1 = advisory no-op (init when already initialized), - * 3 = error (unknown subcommand, bad args, invalid status, untracked repo, no/corrupt ledger). + * `refresh-tokens` is the live cross-repo token sweep (v5.9 B2b): for every tracked + * repo it runs the manifest's always-loaded accounting (readActiveConfig → buildManifest) + * and splits each source into the shared global layer vs the repo's per-repo delta + * (splitManifestByOwnership). The shared layer is HOME-derived and identical across + * repos, so it is captured ONCE (from the first successful read) and stored at the + * ledger root; each repo gets only its delta. This is the IO half of the machine-wide + * token roll-up whose pure data model shipped in B2a. + * + * Exit codes: 0 = write performed (or no repos to sweep, a benign no-op), 1 = advisory + * no-op (init when already initialized), 3 = error (unknown subcommand, bad args, + * invalid status, untracked repo, no/corrupt ledger). */ import { resolve } from 'node:path'; @@ -38,11 +48,15 @@ import { createLedger, addRepo, setRepoStatus, + setSharedGlobal, + setRepoTokens, rollUp, loadLedger, saveLedger, defaultLedgerPath, } from './lib/campaign-ledger.mjs'; +import { readActiveConfig } from './lib/active-config-reader.mjs'; +import { buildManifest, splitManifestByOwnership } from './manifest.mjs'; const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; @@ -88,7 +102,7 @@ async function emit(payload, outputFile, exitCode) { async function main() { const { subcommand, rest, flags } = parseArgs(process.argv.slice(2)); - if (!subcommand) fail('a subcommand is required: init | add | set-status'); + if (!subcommand) fail('a subcommand is required: init | add | set-status | refresh-tokens'); const ledgerPath = resolve(flags.ledgerFile || defaultLedgerPath()); if (flags.referenceDate && !DATE_RE.test(flags.referenceDate)) fail('--reference-date must be YYYY-MM-DD'); @@ -183,7 +197,49 @@ async function main() { ); } - fail(`unknown subcommand "${subcommand}" — expected init | add | set-status`); + if (subcommand === 'refresh-tokens') { + const ledger0 = await loadOrFail(ledgerPath); + if (ledger0 === null) fail(`no ledger at ${ledgerPath} — run "init" or "add" first`); + + let ledger = ledger0; + const swept = []; + const skipped = []; + // The shared global layer is HOME-derived and identical across every repo, so it + // is captured ONCE (from the first repo that reads cleanly) and stored at the + // ledger root — the structural guard against the historic shared-layer double-count. + let sharedSummary = null; + + for (const repo of ledger0.repos) { + let split; + try { + const activeConfig = await readActiveConfig(repo.path, { verbose: false }); + const { sources } = buildManifest(activeConfig); + split = splitManifestByOwnership(sources); + } catch (err) { + skipped.push({ path: repo.path, reason: err.message }); + continue; + } + if (sharedSummary === null) sharedSummary = split.shared; + ledger = setRepoTokens(ledger, repo.path, split.delta, { now }); + swept.push(repo.path); + } + + if (sharedSummary !== null) ledger = setSharedGlobal(ledger, sharedSummary, { now }); + + const written = swept.length > 0; + if (written) await saveLedger(ledgerPath, ledger); + + return emit( + { + status: 'ok', action: 'refresh-tokens', written, ledgerPath, + swept, skipped, sharedGlobal: ledger.sharedGlobal ?? null, rollUp: rollUp(ledger), + }, + flags.outputFile, + 0, + ); + } + + fail(`unknown subcommand "${subcommand}" — expected init | add | set-status | refresh-tokens`); } const isDirectRun = diff --git a/tests/scanners/campaign-write-cli.test.mjs b/tests/scanners/campaign-write-cli.test.mjs index ddf42b5..97a6566 100644 --- a/tests/scanners/campaign-write-cli.test.mjs +++ b/tests/scanners/campaign-write-cli.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; -import { readFileSync, mkdtempSync, existsSync } from 'node:fs'; +import { readFileSync, mkdtempSync, existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { loadLedger, @@ -19,9 +19,13 @@ const NOW = '2026-06-22'; // Every test passes an explicit --ledger-file in a temp dir + an injected --reference-date, // so the write-CLI never touches the real default path under HOME and never reads the clock. // The suite is hermetic + deterministic by construction. -function runWrite(args) { +function runWrite(args, env = {}) { try { - const stdout = execFileSync('node', [CLI, ...args], { encoding: 'utf-8', timeout: 15000 }); + const stdout = execFileSync('node', [CLI, ...args], { + encoding: 'utf-8', + timeout: 30000, + env: { ...process.env, ...env }, + }); return { status: 0, stdout }; } catch (err) { return { status: err.status, stdout: err.stdout || '', stderr: err.stderr || '' }; @@ -184,6 +188,109 @@ describe('campaign-write-cli — set-status', () => { }); }); +describe('campaign-write-cli — refresh-tokens (live cross-repo sweep, v5.9 B2b)', () => { + // Fixture: a fake HOME with a LARGE global CLAUDE.md (the shared layer, made to + // dominate so the counted-once guard is meaningful) + two repos OUTSIDE that HOME, + // each with a small project CLAUDE.md (the per-repo delta). repos sit beside the + // fake home so the cascade walk never picks up the global file as a project file. + function buildSweepFixture() { + const root = mkdtempSync(join(tmpdir(), 'camp-sweep-')); + const fakeHome = join(root, 'home'); + mkdirSync(join(fakeHome, '.claude'), { recursive: true }); + writeFileSync( + join(fakeHome, '.claude', 'CLAUDE.md'), + `# Global user config\n\n${'Shared instruction line that every repo pays for. '.repeat(120)}\n`, + ); + const repoA = join(root, 'repo-a'); + const repoB = join(root, 'repo-b'); + const bodies = [ + [repoA, '# Repo A\n\nProject A note.\n'], + [repoB, '# Repo B\n\nProject B has a little more local text here.\n'], + ]; + for (const [repo, body] of bodies) { + mkdirSync(join(repo, '.git'), { recursive: true }); + writeFileSync(join(repo, '.git', 'HEAD'), 'ref: refs/heads/main\n'); + writeFileSync(join(repo, 'CLAUDE.md'), body); + } + return { root, fakeHome, repoA, repoB }; + } + + it('sweeps every tracked repo, counts the shared layer ONCE, attributes deltas per repo', async () => { + const { root, fakeHome, repoA, repoB } = buildSweepFixture(); + try { + const file = join(root, 'ledger.json'); + runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }); + + const { status, stdout } = runWrite( + ['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], + { HOME: fakeHome }, + ); + assert.equal(status, 0, stdout); + const out = JSON.parse(stdout); + assert.equal(out.action, 'refresh-tokens'); + assert.equal(out.written, true); + assert.deepEqual(out.swept.slice().sort(), [resolve(repoA), resolve(repoB)].sort()); + assert.deepEqual(out.skipped, []); + + // NOTE on shapes (B2a contract): the STORED summaries — out.sharedGlobal and each + // repo.tokens — carry {always:{tokens,count}}, but rollUp().tokens flattens its + // sharedGlobal/perRepoDelta/machineWide buckets to {always:} (bucketTokens). + const t = out.rollUp.tokens; + assert.ok(t.sharedGlobal.always > 0, 'shared global layer should be non-empty'); + assert.equal(t.reposWithTokens, 2); + assert.equal(t.byRepo.length, 2); + + // Persisted ledger carries the new token fields and stays schema-valid. + const ledgerOut = await loadLedger(file); + assert.ok(validateLedger(ledgerOut).valid, validateLedger(ledgerOut).errors.join('; ')); + assert.deepEqual(ledgerOut.sharedGlobal, out.sharedGlobal); + + // Counted-ONCE guard: each repo's stored delta must NOT contain the dominant + // shared global CLAUDE.md. Had the sweep folded shared into the per-repo tokens, + // each delta would EXCEED the shared layer instead of being a small fraction. + for (const repo of ledgerOut.repos) { + assert.ok(repo.tokens, `${repo.name} should have tokens`); + assert.ok( + repo.tokens.always.tokens < t.sharedGlobal.always, + `${repo.name} delta (${repo.tokens.always.tokens}) must be < shared (${t.sharedGlobal.always})`, + ); + } + + // rollUp invariant: machineWide = shared (once) + sum of per-repo deltas. + const deltaSum = ledgerOut.repos.reduce((s, r) => s + r.tokens.always.tokens, 0); + assert.equal(t.perRepoDelta.always, deltaSum); + assert.equal(t.machineWide.always, t.sharedGlobal.always + deltaSum); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('is idempotent — a second sweep replaces, never accumulates', async () => { + const { root, fakeHome, repoA, repoB } = buildSweepFixture(); + try { + const file = join(root, 'ledger.json'); + runWrite(['add', repoA, repoB, '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }); + const first = JSON.parse( + runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout, + ); + const second = JSON.parse( + runWrite(['refresh-tokens', '--ledger-file', file, '--reference-date', NOW], { HOME: fakeHome }).stdout, + ); + assert.deepEqual(second.rollUp.tokens, first.rollUp.tokens); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('exits 3 when no ledger exists yet', () => { + const dir = newDir(); + const { status } = runWrite([ + 'refresh-tokens', '--ledger-file', join(dir, 'nope.json'), '--reference-date', NOW, + ]); + assert.equal(status, 3); + }); +}); + describe('campaign-write-cli — determinism + --output-file', () => { it('stamps updatedDate from --reference-date and writes the payload to --output-file', async () => { const dir = newDir(); From a17823a9af42ee3a4a370842706bfbc1633964fc Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:28:05 +0200 Subject: [PATCH 15/53] docs(campaign): render machine-wide token bill + refresh-tokens surface (v5.9 B2b-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the user-facing surface of the machine-wide token roll-up (executable CLI shipped in B2b-2, d664b70). The campaign report (Step 2) now renders rollUp.tokens: the headline machine-wide always-loaded total split into the shared global layer (paid once, every repo) + per-repo deltas, plus a ranked 'most expensive repos' table. New mode 'refresh-tokens' (Step 6) documents the human-approved live sweep — idempotent, skips unreadable repos, names any skipped so the bill's coverage stays honest (Verifiseringsplikt). Shapes documented carefully: rollUp.tokens.{sharedGlobal,perRepoDelta,machineWide} are flat number maps; byRepo[] is the ranked {name,path,always,...} list. campaign-cli already emitted rollUp.tokens (B2a) — no reader change; this is rendering only. commands/campaign.md (73 lines) is the primary doc; README + CLAUDE.md campaign rows get the matching one-line summary. Markdown-only → suite unchanged at 1198 green; campaign command stays judgment-driven (not byte-stable). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- README.md | 2 +- commands/campaign.md | 73 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bdb48ad..f9e5474 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — deterministic stale check (sources older than ~90d) + web candidate poll (CC changelog + Anthropic blog); **human-approved writes only** (Verifiseringsplikt). The "living" half of the knowledge base. Web/judgment-driven, **not byte-stable** | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity + cross-repo prioritized backlog + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via deterministic write/export CLIs. THIN: tracks+routes state, reuses existing implement/rollback for execution. Judgment-driven, **not byte-stable** | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity + **machine-wide always-loaded token bill** (`refresh-tokens` live cross-repo sweep — shared global layer counted once + per-repo deltas) + cross-repo prioritized backlog + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via deterministic write/export CLIs. THIN: tracks+routes state, reuses existing implement/rollback for execution. Judgment-driven, **not byte-stable** | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Gather user preferences (opt-in) | diff --git a/README.md b/README.md index 8e03ea0..628407e 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ Your team configuration changes over time. Track it: | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | | `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | | `/config-audit knowledge-refresh` | Keep the best-practices register fresh — flag stale entries (sources older than ~90d) + poll for new/changed Claude Code practices; **human-approved writes only** (Verifiseringsplikt). Deterministic stale core + web candidate poll | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity + a single **cross-repo prioritized backlog** to pick from (severity-weighted) + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions; **human-approved writes only** (read-only report + deterministic write/export CLIs). Execution reuses the existing `/config-audit implement` + `rollback` | +| `/config-audit campaign` | Machine-wide audit campaign — durable ledger above sessions tracking each repo's lifecycle (pending → audited → planned → implemented) + a machine-wide roll-up by severity + a **machine-wide always-loaded token bill** (`refresh-tokens` live cross-repo sweep — the shared global layer counted once + per-repo deltas, ranked "most expensive repos") + a single **cross-repo prioritized backlog** to pick from (severity-weighted) + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions; **human-approved writes only** (read-only report + deterministic write/export CLIs). Execution reuses the existing `/config-audit implement` + `rollback` | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Set preferences for action plan _(optional)_ | diff --git a/commands/campaign.md b/commands/campaign.md index dbcecf0..134e5a1 100644 --- a/commands/campaign.md +++ b/commands/campaign.md @@ -1,7 +1,7 @@ --- name: config-audit:campaign description: Machine-wide audit campaign — track which repos are pending/audited/planned/implemented across sessions, with a machine-wide roll-up. Human-approved writes only. -argument-hint: "[init | add ... | set-status | export ]" +argument-hint: "[init | add ... | set-status | refresh-tokens | export ]" allowed-tools: Read, Write, Edit, Bash, Glob model: opus --- @@ -31,9 +31,12 @@ reinvent it. - **Read (report):** `scanners/campaign-cli.mjs` — loads + validates the ledger, emits the repo list + roll-up + backlog. Never writes. -- **Write (mutate):** `scanners/campaign-write-cli.mjs` — `init` / `add` / `set-status`, each a - thin wrapper over the invariant-enforcing lib transforms + save. Invoked **only** after the - user approves a specific action. +- **Write (mutate):** `scanners/campaign-write-cli.mjs` — `init` / `add` / `set-status` / + `refresh-tokens`, each a thin wrapper over the invariant-enforcing lib transforms + save. + Invoked **only** after the user approves a specific action. `refresh-tokens` is the live + cross-repo token sweep: it runs the manifest's always-loaded accounting across every tracked + repo and folds the result into the machine-wide token bill (shared global layer counted once + + per-repo deltas). - **Export:** `scanners/campaign-export-cli.mjs` — `--repo ` resolves the repo's linked session, reads its `action-plan.md`, and assembles a `docs/config-audit-plan-.md`. Read-only (a preview) by default; it writes the file **only** under `--write`, which is @@ -55,6 +58,7 @@ From `$ARGUMENTS`, pick the mode: - `add ...` → add one or more repo paths. - `add --discover ` → find git repos under `` and let the user pick which to add. - `set-status ` → transition a tracked repo (`status` ∈ pending/audited/planned/implemented). +- `refresh-tokens` → live cross-repo token sweep: compute the machine-wide always-loaded bill. - `export ` → export a planned repo's action plan into that repo's own `docs/`. - `help` → show this surface and stop. @@ -75,8 +79,8 @@ attempt a write over a corrupt ledger.) Read `~/.claude/config-audit/sessions/campaign-report.json` with the Read tool (per the UX rules — never show the raw JSON). It has `initialized`, `repos[]` (each: `path, name, status, -sessionId, findingsBySeverity, updatedDate`), `rollUp {totalRepos, byStatus, bySeverity, -reposWithFindings}`, and `backlog[]` — the single cross-repo prioritized work list (each: +sessionId, findingsBySeverity, tokens, updatedDate`), `rollUp {totalRepos, byStatus, bySeverity, +reposWithFindings, tokens}`, and `backlog[]` — the single cross-repo prioritized work list (each: `path, name, status, findingsBySeverity, totalFindings, weightedScore, rank`), already sorted DESC by severity (most critical work first). @@ -109,6 +113,30 @@ with `/config-audit` (audit), `/config-audit plan`, or `/config-audit implement` then record progress here with `set-status`." The backlog is a **pick-list**, not an executor — this command does not run audits or fixes (that is the later execution block). +**Machine-wide token bill** — render from `rollUp.tokens` (the whole-machine always-loaded +accounting). Note the shape: `sharedGlobal`, `perRepoDelta`, and `machineWide` are flat +`{always, onDemand, external, unknown}` number maps; `byRepo[]` is `{name, path, always, +onDemand, external}` already sorted DESC by always-loaded cost; `reposWithTokens` is the count. + +If `reposWithTokens` is `0`, no sweep has run yet — say: "No token bill yet — run +`/config-audit campaign refresh-tokens` to compute the machine-wide always-loaded cost." and +omit the table. Otherwise lead with the headline and the once-vs-delta split: + +> **Always-loaded every turn, machine-wide: ~`machineWide.always` tokens** — `sharedGlobal.always` +> paid once (global config + installed plugins, in *every* repo) + `perRepoDelta.always` across +> `reposWithTokens` repos' own project config. + +Then the **most expensive repos** (their per-repo delta — what each adds beyond the shared layer): + +| # | Repo | Always-loaded delta | +|---|------|---------------------| +| 1 | `` | `` | + +Add one plain-language line so the number is actionable, e.g. "The shared global layer is the +biggest lever — trim `~/.claude/CLAUDE.md`, the global agent listing, or rarely-used plugins +to cut cost in every repo at once." The bill reflects the **last** sweep; re-run +`refresh-tokens` after config changes. + If `initialized` is false, say so plainly: "No campaign yet. Run `/config-audit campaign init` to start one." Then — if the mode was `init` or `add` — continue to that step (those bootstrap a campaign); for `report`/`set-status` on an uninitialized ledger, stop after this message. @@ -175,7 +203,30 @@ node ${CLAUDE_PLUGIN_ROOT}/scanners/campaign-write-cli.mjs set-status /dev/null; echo $? +``` + +Exit **0** = swept (or nothing to sweep — a benign no-op on an empty campaign), **3** = no/corrupt +ledger. Read the result: `swept[]` (repos accounted for), `skipped[]` (each `{path, reason}`), and +the refreshed `rollUp.tokens`. Re-render the **Machine-wide token bill** (Step 2) with the new +numbers. If anything was skipped, name those repos plainly so the user knows the bill omits them +(honest coverage — Verifiseringsplikt). + +### Step 7 (mode `export`): Export a repo's plan to its own `docs/` — preview, approve, write "Planer følger arbeidsstedet": a planned repo's action plan belongs in **that repo's** `docs/`, not buried in a session dir. This step copies it there, byte-faithfully. @@ -216,7 +267,7 @@ Confirm: "Plan exported to ``." Then hand off to the **existing** ex > applies the plan, and verifies. To **undo**: `/config-audit rollback`. When done, record it: > `/config-audit campaign set-status implemented`. -### Step 7: Next steps +### Step 8: Next steps Tailor to where the campaign stands: @@ -229,6 +280,8 @@ Tailor to where the campaign stands: repo's own `docs/`, then `/config-audit implement` there to execute it (backup + verify)." - **Backlog has items:** point at the top backlog repo and the natural next verb for its status (audit → plan → export → implement). +- **No token bill yet (or config changed):** "`/config-audit campaign refresh-tokens` to compute + the machine-wide always-loaded cost — the shared global layer is the biggest lever." - Always: the campaign survives this session — re-run `/config-audit campaign` anytime to see the machine-wide picture. @@ -244,7 +297,9 @@ Tailor to where the campaign stands: and `knowledge-refresh`). - The `-cli` suffix keeps all three CLIs out of the scan-orchestrator, so the scanner count and the byte-stable snapshot suite are unaffected. -- **THIN scope:** ledger + roll-up + status + a cross-repo prioritized backlog + plan export. +- **THIN scope:** ledger + roll-up (findings + machine-wide token bill) + status + a cross-repo + prioritized backlog + plan export. The token sweep reuses the manifest's existing always-loaded + accounting per repo — it does not reinvent measurement, only aggregates it machine-wide. Execution is **not** reinvented here — `export` drops a planned repo's plan into its own `docs/` (a durable record), and the user runs the existing `/config-audit implement` (backup + apply + verify) + `/config-audit rollback` to execute and undo. This command tracks state and routes the From a37183268849c64c0af1b69a86e102195d5ace54 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 17:57:22 +0200 Subject: [PATCH 16/53] feat(discovery): version-aware cache filtering + --exclude-cache flag (v5.9 B3a) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stale ~/.claude/plugins/cache versions polluted token-hotspots ranking and inflated CNF duplicate-hook findings with config that loads on zero turns. installPaths point INTO the cache, so a blunt "skip all of plugins/cache" would drop ACTIVE plugins — the filter is therefore version-aware: it reads the adjacent installed_plugins.json, keeps active version dirs, drops only stale ones (and exposes them via discovery.staleCacheVersions for B3b). - file-discovery: cacheVersionKey() + applyCacheFilter() (active vs stale via installed_plugins.json; HOME-independent, derives the manifest from the cache path); discoverConfigFiles/Multi gain { excludeCache } + staleCacheVersions. Absent/unparseable manifest -> no filtering (never silently drop live config). - token-hotspots-cli + scan-orchestrator: --exclude-cache (default ON for these live-cost scans) / --no-exclude-cache restores the full walk. - Tests: cacheVersionKey unit cases; stale-dropped/active-kept/no-manifest discovery cases; CNF-drop proof (soft-spot verified, not assumed: include=1 -> exclude=0 duplicate-hook findings). Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/lib/file-discovery.mjs | 147 +++++++++++++++++++++- scanners/scan-orchestrator.mjs | 12 +- scanners/token-hotspots-cli.mjs | 8 +- tests/lib/file-discovery.test.mjs | 120 ++++++++++++++++++ tests/scanners/conflict-detector.test.mjs | 56 ++++++++- 5 files changed, 335 insertions(+), 8 deletions(-) diff --git a/scanners/lib/file-discovery.mjs b/scanners/lib/file-discovery.mjs index f36dfb6..7f32bad 100644 --- a/scanners/lib/file-discovery.mjs +++ b/scanners/lib/file-discovery.mjs @@ -13,6 +13,141 @@ const SKIP_DIRS = new Set([ 'vendor', 'venv', '.venv', '.tox', ]); +// Path marker for the plugin install cache (~/.claude/plugins/cache). +// Structure: <...>/plugins/cache////... +// installed_plugins.json's installPath points INTO this tree at the ACTIVE +// version; any other version dir is a stale leftover (superseded install). +const PLUGIN_CACHE_MARKER = `plugins${sep}cache${sep}`; + +/** + * Extract the `//` key for a path inside + * ~/.claude/plugins/cache. Returns null when the path is not under + * plugins/cache, or is shallower than the version directory. + * @param {string} absPath + * @returns {string | null} + */ +export function cacheVersionKey(absPath) { + const i = absPath.indexOf(PLUGIN_CACHE_MARKER); + if (i === -1) return null; + const rest = absPath.slice(i + PLUGIN_CACHE_MARKER.length); + const segs = rest.split(sep).filter(Boolean); + if (segs.length < 3) return null; // need marketplace/plugin/version + return segs.slice(0, 3).join('/'); +} + +/** + * Given a path inside plugins/cache, return the absolute `<...>/plugins` + * directory that owns it (where installed_plugins.json lives). + * @param {string} absPath + * @returns {string | null} + */ +function pluginsDirForCachePath(absPath) { + const i = absPath.indexOf(PLUGIN_CACHE_MARKER); + if (i === -1) return null; + return absPath.slice(0, i + 'plugins'.length); +} + +/** + * Read the set of ACTIVE cache version-keys from a `<...>/plugins` directory's + * installed_plugins.json. Each record's installPath points at the version + * Claude Code loads. Returns null when the manifest is absent or unparseable — + * callers must then NOT filter (we cannot safely tell active from stale, and + * silently dropping active config is the worse failure). + * @param {string} pluginsDir + * @returns {Promise | null>} + */ +async function readActiveCacheVersions(pluginsDir) { + let raw; + try { + raw = await readFile(join(pluginsDir, 'installed_plugins.json'), 'utf-8'); + } catch { + return null; + } + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + const active = new Set(); + const plugins = parsed && parsed.plugins; + if (plugins && typeof plugins === 'object') { + for (const recs of Object.values(plugins)) { + if (!Array.isArray(recs)) continue; + for (const r of recs) { + const key = r && r.installPath ? cacheVersionKey(resolve(r.installPath)) : null; + if (key) active.add(key); + } + } + } + return active; +} + +/** + * Identify stale plugin-cache versions among discovered files and (optionally) + * filter them out. "Stale" = a cache version-key not referenced by the owning + * installed_plugins.json. Active versions are always kept — installPaths point + * INTO the cache, so a blunt "skip all of plugins/cache" would drop live config. + * + * @param {Array} files + * @param {boolean} excludeCache - when true, stale-version files are removed + * @returns {Promise<{ files: Array, staleCacheVersions: Array<{key:string, fileCount:number, estimatedBytes:number}> }>} + */ +async function applyCacheFilter(files, excludeCache) { + const cacheFiles = []; + for (const f of files) { + const key = cacheVersionKey(f.absPath); + if (key) cacheFiles.push({ f, key }); + } + if (cacheFiles.length === 0) return { files, staleCacheVersions: [] }; + + // Union active version-keys across every installed_plugins.json adjacent to + // the cache (a full-machine sweep only ever sees one, but be robust). + const pluginsDirs = new Set(); + for (const { f } of cacheFiles) { + const pd = pluginsDirForCachePath(f.absPath); + if (pd) pluginsDirs.add(pd); + } + let active = null; + for (const pd of pluginsDirs) { + const keys = await readActiveCacheVersions(pd); + if (keys) { + if (active === null) active = new Set(); + for (const k of keys) active.add(k); + } + } + + // No readable manifest → cannot distinguish active from stale → do nothing. + if (active === null) return { files, staleCacheVersions: [] }; + + const byKey = new Map(); + for (const { f, key } of cacheFiles) { + if (!byKey.has(key)) byKey.set(key, []); + byKey.get(key).push(f); + } + const staleCacheVersions = []; + for (const [key, group] of byKey) { + if (!active.has(key)) { + staleCacheVersions.push({ + key, + fileCount: group.length, + estimatedBytes: group.reduce((s, f) => s + (f.size || 0), 0), + }); + } + } + staleCacheVersions.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + + let outFiles = files; + if (excludeCache && staleCacheVersions.length > 0) { + const staleKeys = new Set(staleCacheVersions.map(s => s.key)); + outFiles = files.filter(f => { + const k = cacheVersionKey(f.absPath); + return k === null || !staleKeys.has(k); + }); + } + return { files: outFiles, staleCacheVersions }; +} + /** Config file patterns to discover */ const CONFIG_PATTERNS = { claudeMd: /^CLAUDE\.md$|^CLAUDE\.local\.md$/i, @@ -34,7 +169,8 @@ const CONFIG_PATTERNS = { * @param {object} [opts] * @param {number} [opts.maxFiles=500] - max files to return * @param {boolean} [opts.includeGlobal=false] - also scan ~/.claude/ - * @returns {Promise<{ files: ConfigFile[], skipped: number }>} + * @param {boolean} [opts.excludeCache=false] - drop stale ~/.claude/plugins/cache versions (B3) + * @returns {Promise<{ files: ConfigFile[], skipped: number, staleCacheVersions: Array }>} * * @typedef {{ absPath: string, relPath: string, type: string, scope: string, size: number }} ConfigFile */ @@ -68,7 +204,8 @@ export async function discoverConfigFiles(targetPath, opts = {}) { } catch { /* doesn't exist */ } } - return { files, skipped: skippedRef.count }; + const { files: outFiles, staleCacheVersions } = await applyCacheFilter(files, opts.excludeCache || false); + return { files: outFiles, skipped: skippedRef.count, staleCacheVersions }; } /** @@ -245,7 +382,8 @@ export async function discoverFullMachinePaths() { * @param {Array<{ path: string, maxDepth: number }>} roots * @param {object} [opts] * @param {number} [opts.maxFiles=2000] - global max across all roots - * @returns {Promise<{ files: ConfigFile[], skipped: number }>} + * @param {boolean} [opts.excludeCache=false] - drop stale ~/.claude/plugins/cache versions (B3) + * @returns {Promise<{ files: ConfigFile[], skipped: number, staleCacheVersions: Array }>} */ export async function discoverConfigFilesMulti(roots, opts = {}) { const maxFiles = opts.maxFiles || 2000; @@ -287,7 +425,8 @@ export async function discoverConfigFilesMulti(roots, opts = {}) { } catch { /* doesn't exist */ } } - return { files: allFiles, skipped: totalSkipped }; + const { files: outFiles, staleCacheVersions } = await applyCacheFilter(allFiles, opts.excludeCache || false); + return { files: outFiles, skipped: totalSkipped, staleCacheVersions }; } /** diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index b3e9929..096eb12 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -80,6 +80,7 @@ const SCANNERS = [ * @param {boolean} [opts.fullMachine=false] - Scan all known locations across the machine * @param {boolean} [opts.suppress=true] - Apply suppressions from .config-audit-ignore * @param {boolean} [opts.filterFixtures=true] - Exclude findings from test/example paths + * @param {boolean} [opts.excludeCache=true] - Drop stale ~/.claude/plugins/cache versions so findings reflect live config (B3) * @returns {Promise} Full envelope with all results */ // Exported for testing @@ -89,14 +90,19 @@ export async function runAllScanners(targetPath, opts = {}) { const start = Date.now(); const resolvedPath = resolve(targetPath); + // Default ON: stale cached plugin versions otherwise inflate token hotspots + // and CNF duplicate-hook findings with config that loads on zero turns. (B3) + const excludeCache = opts.excludeCache !== false; + // Shared file discovery — scanners reuse this let discovery; if (opts.fullMachine) { const roots = await discoverFullMachinePaths(); - discovery = await discoverConfigFilesMulti(roots); + discovery = await discoverConfigFilesMulti(roots, { excludeCache }); } else { discovery = await discoverConfigFiles(resolvedPath, { includeGlobal: opts.includeGlobal || false, + excludeCache, }); } @@ -216,6 +222,8 @@ async function main() { // handled below } else if (args[i] === '--include-fixtures') { // handled below + } else if (args[i] === '--exclude-cache' || args[i] === '--no-exclude-cache') { + // handled below } else if (args[i] === '--json') { // handled below — explicit machine-readable mode (bypass humanizer) } else if (args[i] === '--raw') { @@ -229,6 +237,7 @@ async function main() { const fullMachine = args.includes('--full-machine'); const suppress = !args.includes('--no-suppress'); const filterFixtures = !args.includes('--include-fixtures'); + const excludeCache = !args.includes('--no-exclude-cache'); const jsonMode = args.includes('--json'); const rawMode = args.includes('--raw'); @@ -243,6 +252,7 @@ async function main() { fullMachine, suppress, filterFixtures, + excludeCache, humanizedProgress, }); diff --git a/scanners/token-hotspots-cli.mjs b/scanners/token-hotspots-cli.mjs index ff0c542..d09ab2c 100755 --- a/scanners/token-hotspots-cli.mjs +++ b/scanners/token-hotspots-cli.mjs @@ -56,6 +56,10 @@ async function main() { let includeGlobal = false; let withTelemetryRecipe = false; let accurateTokens = false; + // Default ON for this live-cost scan: stale plugin-cache versions pollute the + // hotspot ranking with config that loads on zero turns. --no-exclude-cache + // restores the full walk. (B3) + let excludeCache = true; for (let i = 0; i < args.length; i++) { if (args[i] === '--json') jsonMode = true; @@ -63,6 +67,8 @@ async function main() { else if (args[i] === '--global') includeGlobal = true; else if (args[i] === '--with-telemetry-recipe') withTelemetryRecipe = true; else if (args[i] === '--accurate-tokens') accurateTokens = true; + else if (args[i] === '--exclude-cache') excludeCache = true; + else if (args[i] === '--no-exclude-cache') excludeCache = false; else if (args[i] === '--output-file' && args[i + 1]) outputFile = args[++i]; else if (!args[i].startsWith('-')) targetPath = args[i]; } @@ -80,7 +86,7 @@ async function main() { } resetCounter(); - const discovery = await discoverConfigFiles(absPath, { includeGlobal }); + const discovery = await discoverConfigFiles(absPath, { includeGlobal, excludeCache }); const result = await scan(absPath, discovery); const payload = { diff --git a/tests/lib/file-discovery.test.mjs b/tests/lib/file-discovery.test.mjs index cd2ac74..036fe4f 100644 --- a/tests/lib/file-discovery.test.mjs +++ b/tests/lib/file-discovery.test.mjs @@ -8,6 +8,7 @@ import { discoverConfigFilesMulti, discoverFullMachinePaths, readTextFile, + cacheVersionKey, } from '../../scanners/lib/file-discovery.mjs'; /** @@ -17,6 +18,39 @@ function tempDir(suffix) { return join(tmpdir(), `config-audit-fd-test-${suffix}-${Date.now()}`); } +/** + * Build a realistic ~/.claude/plugins/cache fixture under `root`. + * `versions` is a list of 'marketplace/plugin/version' keys to materialize on + * disk (each gets a CLAUDE.md + hooks/hooks.json). `active` is the subset of + * those keys that installed_plugins.json references via installPath — i.e. the + * versions Claude Code actually loads. Versions NOT in `active` are stale. + */ +async function buildCacheFixture(root, { versions, active }) { + const pluginsDir = join(root, 'plugins'); + for (const key of versions) { + const verDir = join(pluginsDir, 'cache', ...key.split('/')); + await mkdir(join(verDir, 'hooks'), { recursive: true }); + await writeFile(join(verDir, 'CLAUDE.md'), `# ${key}`); + await writeFile( + join(verDir, 'hooks', 'hooks.json'), + JSON.stringify({ hooks: { PreToolUse: [{ matcher: 'Edit' }] } }) + ); + } + const plugins = {}; + for (const key of active) { + const [mkt, plug, version] = key.split('/'); + plugins[`${plug}@${mkt}`] = [{ + scope: 'user', + installPath: join(pluginsDir, 'cache', mkt, plug, version), + version, + }]; + } + await writeFile( + join(pluginsDir, 'installed_plugins.json'), + JSON.stringify({ version: 2, plugins }) + ); +} + // ─────────────────────────────────────────────────────────────── // Group 1: discoverConfigFiles — single path // ─────────────────────────────────────────────────────────────── @@ -389,3 +423,89 @@ describe('readTextFile', () => { assert.equal(content, null); }); }); + +// ─────────────────────────────────────────────────────────────── +// Group 7: cache-aware filtering (B3) — version-aware exclusion of +// stale ~/.claude/plugins/cache versions; active versions kept. +// ─────────────────────────────────────────────────────────────── + +describe('cacheVersionKey (pure)', () => { + it('extracts marketplace/plugin/version from a plugins/cache path', () => { + const p = '/home/u/.claude/plugins/cache/mkt/voyage/5.6.0/commands/x.md'; + assert.equal(cacheVersionKey(p), 'mkt/voyage/5.6.0'); + }); + + it('works for the version directory itself', () => { + assert.equal(cacheVersionKey('/x/plugins/cache/m/p/1.0.0'), 'm/p/1.0.0'); + }); + + it('returns null for paths outside plugins/cache', () => { + assert.equal(cacheVersionKey('/home/u/repos/proj/cache/foo/bar/baz'), null); + assert.equal(cacheVersionKey('/home/u/.claude/agents/x.md'), null); + }); + + it('returns null when shallower than version level', () => { + assert.equal(cacheVersionKey('/x/plugins/cache/m/p'), null); + }); +}); + +describe('discoverConfigFiles — cache-aware filtering', () => { + let dir; + const ACTIVE = 'mkt/voyage/5.6.0'; + const STALE1 = 'mkt/voyage/5.1.1'; + const STALE2 = 'mkt/config-audit/5.1.0'; + + before(async () => { + dir = tempDir('cache'); + await buildCacheFixture(dir, { + versions: [ACTIVE, STALE1, STALE2], + active: [ACTIVE], + }); + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('default (no excludeCache): keeps every cached version', async () => { + const { files } = await discoverConfigFiles(dir); + const keys = new Set(files.map(f => cacheVersionKey(f.absPath)).filter(Boolean)); + assert.ok(keys.has(ACTIVE) && keys.has(STALE1) && keys.has(STALE2), + 'all version keys present without excludeCache'); + }); + + it('excludeCache: drops stale versions, keeps active', async () => { + const { files } = await discoverConfigFiles(dir, { excludeCache: true }); + const keys = new Set(files.map(f => cacheVersionKey(f.absPath)).filter(Boolean)); + assert.ok(keys.has(ACTIVE), 'active version retained'); + assert.ok(!keys.has(STALE1), 'stale version 1 dropped'); + assert.ok(!keys.has(STALE2), 'stale version 2 dropped'); + }); + + it('reports staleCacheVersions regardless of excludeCache', async () => { + const { staleCacheVersions } = await discoverConfigFiles(dir, { excludeCache: true }); + const staleKeys = staleCacheVersions.map(s => s.key).sort(); + assert.deepEqual(staleKeys, [STALE2, STALE1].sort()); + const s1 = staleCacheVersions.find(s => s.key === STALE1); + assert.ok(s1.fileCount >= 2, 'counts the cached config files (CLAUDE.md + hooks.json)'); + }); + + it('does NOT filter when installed_plugins.json is absent (cannot tell active from stale)', async () => { + const noManifest = tempDir('cache-nomanifest'); + // materialize cache versions but NO installed_plugins.json + await buildCacheFixture(noManifest, { versions: [ACTIVE, STALE1], active: [] }); + await rm(join(noManifest, 'plugins', 'installed_plugins.json'), { force: true }); + const { files, staleCacheVersions } = await discoverConfigFiles(noManifest, { excludeCache: true }); + const keys = new Set(files.map(f => cacheVersionKey(f.absPath)).filter(Boolean)); + assert.ok(keys.has(ACTIVE) && keys.has(STALE1), 'no filtering without a manifest'); + assert.equal(staleCacheVersions.length, 0, 'no stale claims without a manifest'); + await rm(noManifest, { recursive: true, force: true }); + }); + + it('discoverConfigFilesMulti applies the same cache filter', async () => { + const { files } = await discoverConfigFilesMulti([{ path: dir, maxDepth: 10 }], { excludeCache: true }); + const keys = new Set(files.map(f => cacheVersionKey(f.absPath)).filter(Boolean)); + assert.ok(keys.has(ACTIVE), 'active retained via Multi'); + assert.ok(!keys.has(STALE1) && !keys.has(STALE2), 'stale dropped via Multi'); + }); +}); diff --git a/tests/scanners/conflict-detector.test.mjs b/tests/scanners/conflict-detector.test.mjs index 7ab59d5..f0c3226 100644 --- a/tests/scanners/conflict-detector.test.mjs +++ b/tests/scanners/conflict-detector.test.mjs @@ -1,7 +1,9 @@ -import { describe, it, beforeEach } from 'node:test'; +import { describe, it, beforeEach, before, after } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve } from 'node:path'; +import { resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; import { scan } from '../../scanners/conflict-detector.mjs'; @@ -156,3 +158,53 @@ describe('CNF scanner — param-qualified cross-scope conflicts', () => { assert.equal(perm.length, 1); }); }); + +// B3 soft-spot proof: stale ~/.claude/plugins/cache versions ship hooks.json +// that the SAME plugin also ships in its active version. CNF groups hooks by +// event:matcher across sources, so multiple cached versions inflate the +// "Duplicate hook" count. Excluding stale cache must measurably DROP CNF +// findings. This verifies the mechanism rather than assuming it. +describe('CNF scanner — cache exclusion drops duplicate-hook count (B3)', () => { + let dir, includeCount, excludeCount; + + before(async () => { + dir = join(tmpdir(), `config-audit-cnf-cache-${Date.now()}`); + const pluginsDir = join(dir, 'plugins'); + const versions = ['mkt/voyage/5.6.0', 'mkt/voyage/5.1.1', 'mkt/voyage/5.0.0']; + for (const key of versions) { + const verDir = join(pluginsDir, 'cache', ...key.split('/'), 'hooks'); + await mkdir(verDir, { recursive: true }); + await writeFile(join(verDir, 'hooks.json'), + JSON.stringify({ hooks: { PreToolUse: [{ matcher: 'Edit' }] } })); + } + // Only 5.6.0 is active; 5.1.1 + 5.0.0 are stale. + await writeFile(join(pluginsDir, 'installed_plugins.json'), JSON.stringify({ + version: 2, + plugins: { + 'voyage@mkt': [{ scope: 'user', version: '5.6.0', + installPath: join(pluginsDir, 'cache', 'mkt', 'voyage', '5.6.0') }], + }, + })); + + resetCounter(); + const dIncl = await discoverConfigFiles(dir); + includeCount = (await scan(dir, dIncl)).findings.filter(f => f.title.includes('Duplicate hook')).length; + + resetCounter(); + const dExcl = await discoverConfigFiles(dir, { excludeCache: true }); + excludeCount = (await scan(dir, dExcl)).findings.filter(f => f.title.includes('Duplicate hook')).length; + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('full walk surfaces ≥1 duplicate-hook finding from cached versions', () => { + assert.ok(includeCount >= 1, `expected duplicate hooks with cache, got ${includeCount}`); + }); + + it('excluding cache measurably drops the duplicate-hook count', () => { + assert.ok(excludeCount < includeCount, + `cache exclusion must drop CNF count: include=${includeCount} exclude=${excludeCount}`); + }); +}); From ba9f82f95232a581d4472fcd03fd6dc763b5cb66 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 18:17:55 +0200 Subject: [PATCH 17/53] feat(tokens): stale plugin-cache disk-cleanup finding, honestly labelled Dead config (v5.9 B3b) The B3a cache filter already exposes discovery.staleCacheVersions; surface them as a finding so the user knows the superseded plugin versions are safe to delete. Honesty (Verifiseringsplikt): the finding loads on ZERO turns, so it must NOT read as a per-turn token cost. TOK normally humanizes to "Wasted tokens"; a new per-finding category override ('plugin-cache-hygiene' -> "Dead config") plus a dedicated humanizer translation ("Old plugin versions are sitting on disk (safe to delete) ... cost zero tokens per turn ... housekeeping, not a performance problem") keep the prose accurate instead of the generic "using more space" default. evidence carries the explicit "zero live-context impact" note. - token-hotspots: Pattern H emits CA-TOK (low) from discovery.staleCacheVersions when stale versions exist (`--global`); silent otherwise. - humanizer: CATEGORY_TO_IMPACT lets a finding's category override the scanner-default impact label (raw `category` field unchanged -> --json/--raw byte-stable). humanizer-data: honest static translation for the finding. - Tests: finding fires/severity/category, lists stale keys + zero-impact note, silent when none; humanizer override -> Dead config; honest translation locked. - Docs: tokens command render note (disk-cleanup, not a token problem) + --no-exclude-cache flag; README + CLAUDE.md rows (7 patterns, cache-aware). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- README.md | 4 +-- commands/tokens.md | 7 +++-- scanners/lib/humanizer-data.mjs | 5 ++++ scanners/lib/humanizer.mjs | 14 ++++++++- scanners/token-hotspots.mjs | 36 ++++++++++++++++++++++ tests/lib/humanizer.test.mjs | 21 +++++++++++++ tests/scanners/token-hotspots.test.mjs | 41 ++++++++++++++++++++++++++ 8 files changed, 124 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f9e5474..647cdda 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: |---------|-------------| | `/config-audit` | Full audit with auto-scope detection (no setup needed) | | `/config-audit posture` | Quick health scorecard (A-F grades, 10 quality areas incl. Token Efficiency, Plugin Hygiene) | -| `/config-audit tokens` | prompt-cache-aware token hotspots (6 patterns: cache-breaking, redundant perms, deep imports, oversized cascade, bloated SKILL.md desc, MCP tool-schema budget), each ranked hotspot tagged with its load pattern (always / on-demand / external) — optional `--accurate-tokens` API calibration, `--with-telemetry-recipe` cache-hit recipe pointer | +| `/config-audit tokens` | prompt-cache-aware token hotspots (7 patterns: cache-breaking, redundant perms, deep imports, oversized cascade, bloated SKILL.md desc, MCP tool-schema budget, stale plugin-cache disk-cleanup), each ranked hotspot tagged with its load pattern (always / on-demand / external) — **cache-aware** (stale `~/.claude/plugins/cache` versions excluded by default; only each plugin's active version counts; `--no-exclude-cache` for the full walk), optional `--accurate-tokens` API calibration, `--with-telemetry-recipe` cache-hit recipe pointer | | `/config-audit manifest` | Ranked table of every token source (CLAUDE.md, rules, agents, skills, output styles, MCP, hooks) sorted by estimated tokens, each tagged with its load pattern (always-loaded / on-demand / external) + an always-loaded subtotal ("tokens that enter context every turn") | | `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact (incl. a conditional `disableBundledSkills` lever when the active skill listing is over budget — remediation companion to SKL `CA-SKL-002`) | | `/config-audit optimize` | Optimization lens (mechanism-fit) — config that works but fits a better mechanism: procedure→skill (CA-OPT-001, deterministic), lifecycle→hook / unscoped path→rule / "never"→permission (prose-judgment via opus `optimization-lens-agent`). Hybrid motor; every finding cites a best-practices-register rule. Agent-driven, **not byte-stable** | diff --git a/README.md b/README.md index 628407e..11ee166 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Your team configuration changes over time. Track it: |---------|-------------| | `/config-audit` | Full audit with auto-scope detection (no setup needed) | | `/config-audit posture` | Quick health scorecard: A-F grades across 10 quality areas (incl. Token Efficiency, Plugin Hygiene) | -| `/config-audit tokens` | prompt-cache-aware token hotspots — ranked by estimated waste, each tagged with its load pattern (always / on-demand / external); 6 patterns + optional `--accurate-tokens` API calibration | +| `/config-audit tokens` | prompt-cache-aware token hotspots — ranked by estimated waste, each tagged with its load pattern (always / on-demand / external); 7 patterns + optional `--accurate-tokens` API calibration. **Cache-aware:** stale `~/.claude/plugins/cache` versions (superseded installs that load on zero turns) are excluded from the ranking by default — only each plugin's active version is counted; `--no-exclude-cache` restores the full walk. Stale versions surface as a separate **Dead config** disk-cleanup finding | | `/config-audit manifest` | Ranked table of every token source (CLAUDE.md, rules, agents, skills, output styles, MCP, hooks) sorted by estimated tokens — each tagged with its **load pattern** (always-loaded / on-demand / external) plus an **always-loaded subtotal** ("≈X tokens enter context every turn before you type"). Component-level: no coarse plugin roll-up (it would double-count) | | `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact | | `/config-audit optimize` | Optimization lens (mechanism-fit): config that works but fits a better mechanism — procedure→skill, lifecycle→hook, unscoped path→rule, "never"→permission. Hybrid motor (deterministic pre-filter + opus precision gate), every finding cites a best-practices-register rule | @@ -415,7 +415,7 @@ All tools work standalone — no Claude Code session needed: | **Posture** | `node scanners/posture.mjs [--json] [--global] [--full-machine] [--output-file path]` | | **Fix** | `node scanners/fix-cli.mjs [--apply] [--json] [--global]` | | **Drift** | `node scanners/drift-cli.mjs [--save] [--baseline name] [--json]` | -| **Tokens** | `node scanners/token-hotspots-cli.mjs [--json] [--global] [--output-file path] [--accurate-tokens] [--with-telemetry-recipe]` | +| **Tokens** | `node scanners/token-hotspots-cli.mjs [--json] [--global] [--no-exclude-cache] [--output-file path] [--accurate-tokens] [--with-telemetry-recipe]` | | **Manifest** | `node scanners/manifest.mjs [--json]` — ranked component-level source table with per-source load pattern + always-loaded subtotal | | **What's active** | `node scanners/whats-active.mjs [--json] [--verbose] [--suggest-disables]` | | **Self-audit** | `node scanners/self-audit.mjs [--json] [--fix] [--check-readme]` | diff --git a/commands/tokens.md b/commands/tokens.md index ffce6ca..8a54540 100644 --- a/commands/tokens.md +++ b/commands/tokens.md @@ -28,6 +28,7 @@ Complementary to `/config-audit whats-active`: Split `$ARGUMENTS` into a path and flags. Path is the first non-flag argument. Default to `.` (current working directory). Recognized flags: - `--global` — also include the user-level `~/.claude/` cascade +- `--no-exclude-cache` — include stale `~/.claude/plugins/cache` versions in the ranking. **By default they are excluded** (cache-aware filtering, default ON): the cache holds superseded plugin versions that load on *zero* turns, and counting them used to crowd the top-10 with dead config. The active version of each plugin (per `installed_plugins.json`) is always kept — only stale versions are filtered. Use `--no-exclude-cache` to see the full on-disk walk. - `--json` — emit raw JSON instead of rendered tables (power-user mode; bypasses the humanizer for byte-stable v5.0.0 output) - `--raw` — pass-through to the scanner; produces v5.0.0 verbatim JSON (bypasses the humanizer). Use when piping into v5.0.0-baseline diff tooling. - `--with-telemetry-recipe` — include `telemetry_recipe_path` in the JSON output, pointing to `knowledge/cache-telemetry-recipe.md`. Use this when you want to verify a structural fix actually improved cache hit rate (manual jq recipe, opt-in) @@ -42,7 +43,7 @@ Default mode (no `--json`, no `--raw`) emits a humanized JSON envelope: each fin TMPFILE="/tmp/config-audit-tokens-$$.json" RAW_FLAG="" if echo "$ARGUMENTS" | grep -q -- "--raw"; then RAW_FLAG="--raw"; fi -node ${CLAUDE_PLUGIN_ROOT}/scanners/token-hotspots-cli.mjs --output-file "$TMPFILE" [--global] $RAW_FLAG 2>/dev/null; echo $? +node ${CLAUDE_PLUGIN_ROOT}/scanners/token-hotspots-cli.mjs --output-file "$TMPFILE" [--global] [--no-exclude-cache] $RAW_FLAG 2>/dev/null; echo $? ``` **Exit code handling:** @@ -63,11 +64,13 @@ Use the Read tool on `$TMPFILE`. Extract: - `total_estimated_tokens` — top-line number - `hotspots[]` — top 10 ranked sources; each carries a **load pattern** (`loadPattern` ∈ always / on-demand / external, plus `survivesCompaction` / `derivationConfidence`) -- `findings[]` — prompt-cache pattern findings (CA-TOK-001..003); each finding in default mode carries humanizer fields (`userImpactCategory`, `userActionLanguage`, `relevanceContext`) alongside the v5.0.0 fields +- `findings[]` — prompt-cache pattern findings; each finding in default mode carries humanizer fields (`userImpactCategory`, `userActionLanguage`, `relevanceContext`) alongside the v5.0.0 fields - `counts` — severity breakdown A hotspot's **load pattern** matters as much as its size: an **always**-loaded source (CLAUDE.md, MCP tool schemas) is paid on *every* turn, an **on-demand** one (skill body, path-scoped rule) only when invoked/matched, and an **external** one (hooks, harness-config files like settings.json/.mcp.json) costs no per-turn context tokens at all. A big always-loaded hotspot is the most worth trimming. +**The stale plugin-cache finding is different from the rest.** All other TOK findings are about per-turn token cost. The *"Old plugin versions are sitting on disk"* finding (category `plugin-cache-hygiene`, impact **Dead config**, `--global` only) is a pure **disk-cleanup** item with **zero live-context impact** — the listed versions are never loaded. Render it as housekeeping, not a token problem: don't conflate its disk bytes with the per-turn token numbers above it. + Render as markdown. Group findings by `userImpactCategory` (e.g., "Wasted tokens" vs "Configuration mistake") rather than re-deriving severity prose; lead each line with `userActionLanguage` ("Fix this now", "Fix soon", "Optional cleanup", etc.) so the urgency phrasing stays consistent with the rest of the toolchain. The humanizer already replaced jargon-heavy `title`/`description`/`recommendation` strings with plain-language equivalents — render them verbatim. ```markdown diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index 05ee84f..2356547 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -569,6 +569,11 @@ export const TRANSLATIONS = { description: 'Skill descriptions load on every turn whether you use the skill or not. Long descriptions add up.', recommendation: 'Trim the description to one short sentence and move details into the skill body.', }, + 'Stale plugin-cache versions (disk cleanup, zero live-context impact)': { + title: 'Old plugin versions are sitting on disk (safe to delete)', + description: 'Your plugin cache holds older versions that newer installs have replaced. They take up disk space but are never loaded into a conversation — so they cost zero tokens per turn. This is housekeeping, not a performance problem.', + recommendation: 'Delete the old version folders to reclaim disk. The details list exactly which ones; the active version of each plugin stays untouched.', + }, }, patterns: [ { diff --git a/scanners/lib/humanizer.mjs b/scanners/lib/humanizer.mjs index 283ccd7..be4f788 100644 --- a/scanners/lib/humanizer.mjs +++ b/scanners/lib/humanizer.mjs @@ -45,6 +45,17 @@ const SCANNER_TO_CATEGORY = { OPT: 'Missed opportunity', }; +/** + * Per-finding `category` values that override the scanner-default impact label. + * Needed when one finding inside a scanner means something different from the + * scanner's usual bucket — e.g. stale plugin-cache versions are emitted by TOK + * (normally "Wasted tokens") but load on ZERO turns, so their honest impact is + * "Dead config" (present on disk, never loaded), not wasted per-turn tokens. + */ +const CATEGORY_TO_IMPACT = { + 'plugin-cache-hygiene': 'Dead config', +}; + /** * Map severity to one-line action-language phrase (research/02 line 134). */ @@ -125,7 +136,8 @@ export function humanizeFinding(finding) { } const translation = lookupTranslation(finding.scanner, finding.title); - const category = SCANNER_TO_CATEGORY[finding.scanner] || 'Other'; + const category = + CATEGORY_TO_IMPACT[finding.category] || SCANNER_TO_CATEGORY[finding.scanner] || 'Other'; const action = SEVERITY_TO_ACTION[finding.severity] || 'FYI'; const relevance = computeRelevanceContext(finding.file); diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index 0ac42b0..fc90e7f 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -544,6 +544,42 @@ export async function scan(targetPath, discovery) { })); } + // ── Pattern H: stale plugin-cache versions (v5.9 B3 — disk hygiene) ── + // file-discovery populates discovery.staleCacheVersions when walking + // ~/.claude/plugins/cache: version dirs on disk NOT referenced by + // installed_plugins.json (superseded installs). They occupy disk but load on + // ZERO turns — already excluded from the hotspot ranking + CNF by + // --exclude-cache. Honest framing (category 'plugin-cache-hygiene' → + // "Dead config", not "Wasted tokens"): a disk-cleanup opportunity, NOT a + // per-turn token cost. + const stale = Array.isArray(discovery.staleCacheVersions) ? discovery.staleCacheVersions : []; + if (stale.length > 0) { + const totalFiles = stale.reduce((s, v) => s + (v.fileCount || 0), 0); + const totalBytes = stale.reduce((s, v) => s + (v.estimatedBytes || 0), 0); + const keys = stale.map(v => v.key); + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Stale plugin-cache versions (disk cleanup, zero live-context impact)', + description: + `${stale.length} plugin version${stale.length === 1 ? '' : 's'} under ` + + `~/.claude/plugins/cache ${stale.length === 1 ? 'is' : 'are'} not referenced by ` + + 'installed_plugins.json — superseded by newer installs. They occupy disk but load on ' + + 'ZERO turns: excluded from the hotspot ranking and conflict detection by default ' + + '(--exclude-cache). This is disk hygiene, not a per-turn token cost.', + file: null, + evidence: + `stale_versions=${keys.join(', ')}; config_files=${totalFiles}; ` + + `approx_disk_bytes=${totalBytes}; note=zero live-context impact ` + + '(installed_plugins.json points at newer versions)', + recommendation: + 'Delete the listed stale version directories under ~/.claude/plugins/cache to reclaim ' + + 'disk (reinstall/prune via the plugin manager, or remove the dirs directly). Re-run ' + + 'with --no-exclude-cache to include cached versions in the token/conflict scan.', + category: 'plugin-cache-hygiene', + })); + } + // ── Hotspots ranking ── const hotspots = await buildHotspots(discovery, targetPath, activeConfig); diff --git a/tests/lib/humanizer.test.mjs b/tests/lib/humanizer.test.mjs index 8263e30..c049a70 100644 --- a/tests/lib/humanizer.test.mjs +++ b/tests/lib/humanizer.test.mjs @@ -95,6 +95,19 @@ test('humanizeFinding rewrites title for known static title', () => { `humanized title should mention instructions or claude, got: ${out.title}`); }); +test('humanizeFinding: stale plugin-cache title humanizes to honest disk-cleanup prose, not the generic TOK default (B3)', () => { + const input = makeFinding({ + scanner: 'TOK', + title: 'Stale plugin-cache versions (disk cleanup, zero live-context impact)', + category: 'plugin-cache-hygiene', + }); + const out = humanizeFinding(input); + assert.match(out.title, /disk/i, 'humanized title must name disk, not generic "space"'); + assert.match(out.description, /zero tokens per turn|never loaded/i, + 'humanized description must keep the zero-context-impact honesty'); + assert.equal(out.userImpactCategory, 'Dead config'); +}); + test('humanizeFinding falls back to _default when title unknown', () => { const input = makeFinding({ scanner: 'CML', title: 'Unrecognized brand-new finding title' }); const out = humanizeFinding(input); @@ -183,6 +196,14 @@ test('humanizeFinding sets category Dead config for DIS', () => { assert.equal(out.userImpactCategory, 'Dead config'); }); +test('humanizeFinding: per-finding category plugin-cache-hygiene overrides scanner default to Dead config (B3)', () => { + // TOK normally maps to "Wasted tokens", but a stale-cache finding loads on + // zero turns — its honest impact is "Dead config", not wasted per-turn tokens. + const out = humanizeFinding(makeFinding({ scanner: 'TOK', category: 'plugin-cache-hygiene' })); + assert.equal(out.userImpactCategory, 'Dead config'); + assert.equal(out.category, 'plugin-cache-hygiene', 'raw category field preserved unchanged'); +}); + test('humanizeFinding sets category Missed opportunity for GAP', () => { const out = humanizeFinding(makeFinding({ scanner: 'GAP', title: 'No CLAUDE.md file' })); assert.equal(out.userImpactCategory, 'Missed opportunity'); diff --git a/tests/scanners/token-hotspots.test.mjs b/tests/scanners/token-hotspots.test.mjs index 4b19523..039103b 100644 --- a/tests/scanners/token-hotspots.test.mjs +++ b/tests/scanners/token-hotspots.test.mjs @@ -354,3 +354,44 @@ describe('TOK scanner — F7 severity recalibration (v5)', () => { }); } }); + +// ── Pattern H: stale plugin-cache versions (v5.9 B3) ────────────────────── +describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => { + const staleDiscovery = { + files: [], + staleCacheVersions: [ + { key: 'mkt/voyage/5.1.1', fileCount: 35, estimatedBytes: 416237 }, + { key: 'mkt/config-audit/5.6.0', fileCount: 149, estimatedBytes: 301330 }, + ], + }; + + it('emits a low-severity stale-cache finding when stale versions exist', async () => { + resetCounter(); + const result = await withHermeticHome(() => scan('/tmp/x', staleDiscovery)); + const f = result.findings.find(x => /stale plugin-cache/i.test(x.title || '')); + assert.ok(f, 'expected a stale plugin-cache finding'); + assert.equal(f.severity, 'low'); + assert.equal(f.category, 'plugin-cache-hygiene'); + }); + + it('states zero live-context impact and lists the stale keys', async () => { + resetCounter(); + const result = await withHermeticHome(() => scan('/tmp/x', staleDiscovery)); + const f = result.findings.find(x => /stale plugin-cache/i.test(x.title || '')); + assert.match(f.evidence, /zero live-context impact/i); + assert.match(f.evidence, /mkt\/voyage\/5\.1\.1/); + assert.match(f.evidence, /mkt\/config-audit\/5\.6\.0/); + }); + + it('does NOT fire when there are no stale versions', async () => { + resetCounter(); + const result = await withHermeticHome(() => scan('/tmp/x', { files: [], staleCacheVersions: [] })); + assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || ''))); + }); + + it('does NOT fire when discovery omits staleCacheVersions entirely', async () => { + resetCounter(); + const result = await withHermeticHome(() => scan('/tmp/x', { files: [] })); + assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || ''))); + }); +}); From 42e48514e40ad1a4518ef2dc2ea169d42e72faac Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 18:30:09 +0200 Subject: [PATCH 18/53] =?UTF-8?q?release:=20v5.9.0=20=E2=80=94=20"Machine-?= =?UTF-8?q?wide=20token=20lens"=20(B1+B2+B3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cuts the v5.9.0 release bundling the three hardening gaps shipped since v5.8.0: B1 (agent-listing budget — new orchestrated scanner AGT, count 15→16), B2 (machine-wide always-loaded token roll-up in the campaign ledger), and B3 (cache-aware filtering + stale plugin-cache disk-cleanup finding). Version sync: plugin.json 5.8.0→5.9.0; README badges version 5.9.0 / scanners 15→16 / tests 1168→1215; README intro + Health prose + scanner table (new AGT row, TOK 7th pattern) + self-audit prose 15→16; new version-history row; CHANGELOG [5.9.0]; CLAUDE.md finding-ID example (+CA-AGT) and test count 1168→1215 / 67→68 files (scanner dir 35→36). Gate: `self-audit --json --check-readme` -> readmeCheck.passed:true, mismatches:[]. Full suite green (1215, 0 fail). Tag + catalog ref-bump follow. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 4 +-- README.md | 16 ++++++----- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e875504..132f2bd 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.8.0", + "version": "5.9.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba1820..ac95a4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,62 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.9.0] - 2026-06-23 + +### Summary +"Machine-wide token lens" — the three highest-impact hardening gaps toward whole-machine token +tuning (the shift from *correct?* to *best-practice-tuned?*). **B1** measures the always-loaded +agent listing (a new orchestrated scanner), **B2** gives the campaign ledger a machine-wide +always-loaded token bill that counts the shared global layer once, and **B3** makes the token and +conflict scans cache-aware so stale plugin-cache versions stop polluting them. Every new token +figure carries an honesty caveat where the load mechanism is inferred or the number is an +upper-bound estimate. `--json`/`--raw` stay byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. +Scanner count 15 → **16**, agents **7**, commands **21**; **1215** tests. + +### Added +- **B1 — Agent-listing budget (new orchestrated scanner `AGT`, count 15 → 16).** Claude Code injects + every active agent's name+description into the system prompt on every turn; on a heavily-plugged + machine this is often the dominant single always-loaded source, yet nothing measured it. New + `scanners/agent-listing-scanner.mjs` + `scanners/lib/agent-listing-budget.mjs`: + - `CA-AGT-001` — per-agent description over the soft bloat cap (advisory; mirrors the TOK 500-char + SKILL.md heuristic — agents have no verified per-description cap, so nothing is truncated, the + description is simply re-sent in full every turn). + - `CA-AGT-002` — the summed agent-listing estimate exceeding the listing budget. + - **Intellectual-honesty contract:** both findings are LOW and explicitly **inferred / + upper-bound** — the agent-listing mechanism is undocumented (agents are absent from Claude Code's + published context breakdown), the per-agent estimate is the frontmatter cap not a measurement, + and the budget is a config-audit heuristic anchored on a conservative 200k window. The evidence + discloses all three caveats rather than overstating certainty. +- **B2 — Machine-wide always-loaded token roll-up (campaign).** The campaign ledger now carries a + token bill alongside the severity roll-up. `campaign refresh-tokens` runs a live cross-repo sweep + (`readActiveConfig` → `buildManifest` → ownership split) that factors out the **shared global + always-loaded layer** (global CLAUDE.md + agent listing + global MCP + unscoped global rules) and + counts it **once**, then adds each repo's delta — surfacing one machine-wide always-loaded total + plus a ranked "most expensive repos" table. The shared layer is written once from the first + readable repo (counted-once guard against the `whats-active` double-count). +- **B3 — Stale plugin-cache disk-cleanup finding (`TOK`).** Stale `~/.claude/plugins/cache` versions + surface as a finding (`--global`) categorized **Dead config** (not "Wasted tokens"): they occupy + disk but load on zero turns. A per-finding category override (`plugin-cache-hygiene`) plus a + dedicated humanizer translation keep the prose honest ("safe to delete … zero tokens per turn … + housekeeping, not a performance problem"). + +### Changed +- **B3 — Cache-aware filtering (folds in B0): `--exclude-cache` (default ON) for token-hotspots and + the orchestrator.** `~/.claude/plugins/cache` holds *both* active and stale plugin versions — + `installed_plugins.json`'s `installPath` points INTO the cache — so a blunt "skip all of + plugins/cache" would drop *active* config. Discovery is now **version-aware**: it reads the + adjacent `installed_plugins.json`, keeps each plugin's active version dir, and drops only stale + ones. Result: stale versions no longer pollute the token-hotspot ranking, and stale cached + `hooks.json` no longer inflate CNF "duplicate hook" findings (verified: cache exclusion measurably + drops the count). `--no-exclude-cache` restores the full walk. When `installed_plugins.json` is + absent or unparseable, nothing is filtered (never silently drop live config). +- **B2 — `manifest` ownership split.** `splitManifestByOwnership` separates the shared-global layer + from per-repo deltas; `~/.claude.json:projects` MCP is correctly classified as a per-repo delta + (its slice is keyed on the repo path), not a shared-global source. +- Humanizer: a finding's `category` can now override the scanner-default impact label + (`CATEGORY_TO_IMPACT`), used so a TOK finding can read as **Dead config** when that is the honest + bucket. The raw `category` field is unchanged, so `--json`/`--raw` stay byte-stable. + ## [5.8.0] - 2026-06-23 ### Summary diff --git a/CLAUDE.md b/CLAUDE.md index 647cdda..c86d5b3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,7 +105,7 @@ Default: auto-detects scope from git context. Override with `/config-audit full| ``` ### Finding ID Format -`CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001`, `CA-SKL-001`, `CA-OST-001`, `CA-OPT-001` +`CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001`, `CA-SKL-001`, `CA-OST-001`, `CA-OPT-001`, `CA-AGT-001` ## Conventions @@ -123,7 +123,7 @@ Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{S node --test 'tests/**/*.test.mjs' ``` -1168 tests across 67 test files (22 lib + 35 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1215 tests across 68 test files (22 lib + 36 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. diff --git a/README.md b/README.md index 11ee166..6aa108a 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,16 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.8.0-blue) +![Version](https://img.shields.io/badge/version-5.9.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) -![Scanners](https://img.shields.io/badge/scanners-15-cyan) +![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1168+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1215+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) -A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 15 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, and output-style detection. Zero external dependencies. +A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. --- @@ -76,7 +76,7 @@ Claude Code reads instructions from at least 7 different file types across multi This plugin provides three layers of configuration intelligence: -- **Health** — 15 deterministic scanners verify correctness across every configuration file, catching broken imports, deprecated settings, conflicting rules, format errors, permission contradictions, prompt-cache token waste, cache-prefix instability, dead tool grants, cross-plugin skill collisions, output styles that silently strip Claude Code's coding instructions, and procedures in CLAUDE.md that would fit better as a skill +- **Health** — 16 deterministic scanners verify correctness across every configuration file, catching broken imports, deprecated settings, conflicting rules, format errors, permission contradictions, prompt-cache token waste, cache-prefix instability, dead tool grants, cross-plugin skill collisions, output styles that silently strip Claude Code's coding instructions, an oversized always-loaded agent listing, and procedures in CLAUDE.md that would fit better as a skill - **Opportunities** — context-aware recommendations for Claude Code features that could benefit your specific project, backed by Anthropic's official guidance - **Action** — auto-fix with mandatory backups, syntax validation, rollback support, and a human-in-the-loop workflow for anything non-trivial @@ -322,13 +322,14 @@ By default, `/config-audit` auto-detects scope from your git context. Override w | `import-resolver.mjs` | IMP | Broken @imports, circular references, deep chains, tilde path issues | | `conflict-detector.mjs` | CNF | Settings contradictions across scopes, permission conflicts, hook duplicates | | `feature-gap-scanner.mjs` | GAP | 25 feature checks shown as opportunities, not grades — plus a conditional `disableBundledSkills` recommendation when the active skill listing is over budget | -| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascades, bloated skill descriptions, MCP tool-schema budget | +| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascades, bloated skill descriptions, MCP tool-schema budget, and stale `~/.claude/plugins/cache` versions (disk-cleanup, zero live-context impact) — cache-aware ranking excludes superseded plugin versions by default (`--no-exclude-cache` to include) | | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of the CLAUDE.md cascade — beyond the cache-prefix window but still re-loaded every turn | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries: (1) tools in BOTH `permissions.deny` and `permissions.allow` — deny wins (incl. the `Tool(*)` deny-all glob, equivalent to a bare deny); (2) unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — valid only as `mcp____*`; (3) `Tool(param:value)` rules whose key is the tool's own canonicalizing field (`command`/`file_path`/`path`/`notebook_path`/`url`) — CC ignores these and emits a startup warning | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions; user-vs-plugin overlaps | | `skill-listing-scanner.mjs` | SKL | Skill-listing token budget: a single skill description over the ~1,536-char listing cap Claude Code truncates (`CA-SKL-001`), and the summed active-skill descriptions exceeding the ~2%-of-context listing budget (`CA-SKL-002`) | | `output-style-scanner.mjs` | OST | Output-style validation: a custom (user/project) style missing `keep-coding-instructions: true` that silently strips built-in software-engineering instructions (`CA-OST-001`), a plugin style with `force-for-plugin: true` overriding the user's selected `outputStyle` (`CA-OST-002`), and a settings `outputStyle` resolving to no built-in or custom style — dead config (`CA-OST-003`) | | `optimization-lens-scanner.mjs` | OPT | Optimization lens (mechanism-fit): a multi-step procedure in CLAUDE.md that would fit better as a skill (`CA-OPT-001`) — reads the machine-readable best-practices register, framed as an opportunity, not a failure. The deterministic half of the lens; prose-judgment cases (lifecycle→hook, unscoped path→rule, "never"→permission) are judged by the opus `optimization-lens-agent` via `/config-audit optimize` | +| `agent-listing-scanner.mjs` | AGT | Always-loaded agent-listing budget: a per-agent description over the soft bloat cap (`CA-AGT-001`, advisory) and the summed active-agent name+description listing — re-sent every turn — exceeding the listing budget (`CA-AGT-002`). Both LOW and explicitly **inferred / upper-bound**: the agent-listing mechanism is undocumented, so the evidence discloses the estimate and heuristic budget rather than overstating certainty | > **Cross-scanner remediation — diagnosis meets the fix.** SKL diagnoses an over-budget > skill listing (`CA-SKL-002`); GAP prescribes the remedy. When the active skill listing @@ -529,7 +530,7 @@ node scanners/posture.mjs examples/optimal-setup/ ### Self-Audit: Scanning the Scanner -The plugin runs all 15 scanners + the standalone plugin-health scanner on itself via `self-audit.mjs`. Test fixtures and example files are automatically excluded from scoring — a configuration plugin that ships deliberately broken examples shouldn't fail its own audit. Use `--check-readme` to verify badge counts are in sync with the filesystem. +The plugin runs all 16 scanners + the standalone plugin-health scanner on itself via `self-audit.mjs`. Test fixtures and example files are automatically excluded from scoring — a configuration plugin that ships deliberately broken examples shouldn't fail its own audit. Use `--check-readme` to verify badge counts are in sync with the filesystem. ```bash node scanners/self-audit.mjs @@ -661,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.9.0** | 2026-06-23 | "Machine-wide token lens" — the three highest-impact hardening gaps toward whole-machine token tuning. **B1 — agent-listing budget (new orchestrated scanner AGT, count 15→16):** the always-loaded agent listing (name+description re-sent every turn) is now measured — `CA-AGT-001` per-agent description bloat (advisory), `CA-AGT-002` aggregate listing over budget; both LOW and explicitly **inferred / upper-bound** (the mechanism is undocumented — the evidence discloses it rather than overstating). **B2 — machine-wide always-loaded token roll-up:** the campaign ledger now carries a token bill — `campaign refresh-tokens` does a live cross-repo sweep that counts the **shared global always-loaded layer once** + per-repo deltas, with a ranked "most expensive repos" table (the `whats-active` double-count, avoided by construction). **B3 — cache-aware filtering (folds in B0):** `~/.claude/plugins/cache` holds *both* active and stale plugin versions (installPaths point INTO it), so token-hotspots + CNF are now **version-aware** — `--exclude-cache` (default ON) keeps each plugin's active version and drops only stale ones (`installed_plugins.json`-driven), so stale versions stop polluting the hotspot ranking and inflating duplicate-hook conflicts; stale versions surface as a separate **Dead config** disk-cleanup finding (zero live-context impact). `--json`/`--raw` byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. 1215 tests | | **5.8.0** | 2026-06-23 | "Campaign motor" — a durable, machine-wide audit **campaign** that sits ABOVE individual sessions (one repo = one session; a fleet of repos = a campaign). **Ledger:** `~/.claude/config-audit/campaign-ledger.json` (outside the plugin dir → survives uninstall/upgrade) tracks a repo list + per-repo lifecycle (pending→audited→planned→implemented) + a machine-wide roll-up by status & severity; pure transforms with injected `now`. **`/config-audit campaign` (commands 20→21):** read-only report (`campaign-cli`) + human-approved writes (`campaign-write-cli`: init / add / set-status) — reports first, mutates only on explicit approval, never hand-edits the ledger. **Cross-repo backlog:** one severity-weighted prioritized pick-list (`buildBacklog`, `critical:1000/high:100/medium:10/low:1`). **Plan export + execution-by-reuse:** `campaign-export-cli --write` drops a planned repo's plan verbatim into its own `docs/`; execution reuses the existing `/config-audit implement` + `rollback` (no new execution machinery). All campaign code is `-cli`/lib → scanner count stays **15**, agents **7**, byte-stable. Plus pre-release cleanup: `knowledge-refresh` wired into the router + help; CLAUDE.md trimmed 540→134 lines (impl notes → `docs/scanner-internals.md`, config grade B→A). 1168 tests | | **5.7.0** | 2026-06-21 | "Optimization lens" — first detector of the «optimally shaped?» axis (vs «correct?»), plus a living knowledge layer. **Register:** `knowledge/best-practices.json`, a provenance-stamped, schema-validated best-practices register (first runtime-consumed `knowledge/` file). **OPT scanner (count 14→15):** `CA-OPT-001` (LOW) a ≥6-step CLAUDE.md procedure that would fit better as a skill, citing register entry `BP-MECH-003`. **`/config-audit optimize` + `optimization-lens-agent` (opus, agents 6→7):** prose-judgment lens for lifecycle→hook (`BP-MECH-001`), unscoped path→rule (`BP-MECH-002`), "never"→permission (`BP-MECH-004`); pre-filter recall + opus precision gate. **`/config-audit knowledge-refresh` (commands 19→20):** deterministic stale-check (injected reference date, 90-day cadence) + web re-verify/poll, human-approved writes only. Last two are agent/web-driven (not byte-stable). 1091 tests | | **5.6.0** | 2026-06-20 | "Steering-model II" — the load-pattern / compaction-survival model lands end-to-end. **Foundation:** `active-config-reader` now enumerates rules, agents, and output styles (alongside CLAUDE.md/plugins/skills/hooks/MCP), each tagged `loadPattern` (always / on-demand / external) + `survivesCompaction` from the published loading model; the frontmatter parser also reads YAML block sequences (`paths:` lists). **B (load-pattern accounting):** `manifest` reports component-level sources (the double-counting plugin roll-up is gone), tags every source with the load-pattern triple, and leads with an **always-loaded subtotal** ("tokens that enter context every turn"); `token-hotspots` annotates each ranked hotspot with its load pattern. **C (output styles):** new orchestrated **OST** scanner (count 13→**14**) — `CA-OST-001` a custom style stripping built-in coding instructions (missing `keep-coding-instructions: true`, V10), `CA-OST-002` a plugin style with `force-for-plugin: true` overriding the user's `outputStyle` (V11), `CA-OST-003` a settings `outputStyle` resolving to no known style (dead config). Doc-verified; frozen v5.0.0 snapshots preserved via strip-helpers, SC-5 regenerated. 1023 tests | From 8f7e19604692fe7223ef34bbe7cfd867fc06522c Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 19:59:14 +0200 Subject: [PATCH 19/53] feat(tokens): MCP tool-schema deferral check + CLI-over-MCP lever (v5.10 B4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By default Claude Code defers MCP tool schemas (names-only, ~120 tok; full schemas on demand via tool search). CA-TOK-006 detects config-file signals that force the FULL schemas into the always-loaded prefix every turn: - settings.json env.ENABLE_TOOL_SEARCH="false" (high) - "ToolSearch" in permissions.deny (high) - configured model is a Haiku model (medium) - per-server .mcp.json alwaysLoad:true (CC v2.1.121+) (high) auto[:N] is threshold mode (info, not a trigger). New engine lib/mcp-deferral.mjs: pure assessMcpDeferral({settings,mcpServers}) (unit-tested, no IO) + thin IO wrapper assessMcpDeferralForRepo shared by TOK and GAP. Severity scales with aggregate forced-upfront tokens (medium-confidence reasons cap at medium). feature-gap cliOverMcpLeverFinding fires only as a companion to CA-TOK-006 (prefer gh/aws/gcloud over MCP for common ops). Honest scoping (Verifiseringsplikt): triggers on config files ONLY — never process.env shell vars. Vertex / custom ANTHROPIC_BASE_URL / runtime /model switch are launch state (would flap snapshots machine-dependently), so they are DISCLOSED in every finding, not triggered. Tool-level anthropic/alwaysLoad and claude.ai connectors likewise disclosed. Mechanism verified 2026-06-23 against code.claude.com/docs (context-window.md, mcp.md#configure-tool-search + #exempt-a-server-from-deferral, costs.md); the prefix-cache invalidation claim was NOT-CONFIRMED in docs and is not asserted. alwaysLoad added to CA-MCP VALID_SERVER_FIELDS (no longer flagged as unknown). active-config-reader surfaces per-server alwaysLoad. Byte-stable: CA-TOK-006 fires only on new conditions; frozen v5.0.0 + SC-5 snapshots untouched. Tests 1215 -> 1239 (engine 16, integration 5, lever 2, mcp-field guard 1). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/scanner-internals.md | 41 +++- scanners/feature-gap-scanner.mjs | 48 ++++ scanners/lib/active-config-reader.mjs | 2 + scanners/lib/mcp-deferral.mjs | 206 ++++++++++++++++++ scanners/mcp-config-validator.mjs | 3 + scanners/token-hotspots.mjs | 60 +++++ .../mcp-deferral/alwaysload/.mcp.json | 6 + .../fixtures/mcp-deferral/deferred/.mcp.json | 5 + .../mcp-deferral/disabled-settings/.mcp.json | 5 + tests/scanners/feature-gap-scanner.test.mjs | 26 ++- tests/scanners/mcp-config-validator.test.mjs | 30 +++ tests/scanners/mcp-deferral.test.mjs | 192 ++++++++++++++++ .../scanners/token-hotspots-deferral.test.mjs | 70 ++++++ 13 files changed, 692 insertions(+), 2 deletions(-) create mode 100644 scanners/lib/mcp-deferral.mjs create mode 100644 tests/fixtures/mcp-deferral/alwaysload/.mcp.json create mode 100644 tests/fixtures/mcp-deferral/deferred/.mcp.json create mode 100644 tests/fixtures/mcp-deferral/disabled-settings/.mcp.json create mode 100644 tests/scanners/mcp-deferral.test.mjs create mode 100644 tests/scanners/token-hotspots-deferral.test.mjs diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index 0a91879..1c0745d 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -18,7 +18,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `import-resolver.mjs` | IMP | Broken @imports, circular refs, deep chains, tilde paths | | `conflict-detector.mjs` | CNF | Settings conflicts, permission contradictions, hook duplicates | | `feature-gap-scanner.mjs` | GAP | 25 feature checks across 4 tiers — shown as opportunities, not grades | -| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascade, bloated SKILL.md descriptions, MCP tool-schema budget (prompt-cache patterns) | +| `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascade, bloated SKILL.md descriptions, MCP tool-schema budget, MCP tool-schema deferral (CA-TOK-006), stale plugin-cache disk-cleanup (prompt-cache patterns) | | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of CLAUDE.md cascade (beyond Pattern A's top-30 window) | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries (low). (1) Tools in BOTH `permissions.deny` AND `permissions.allow` — deny wins; dominance is param-aware and treats the `Tool(*)` deny-all glob as equivalent to a bare deny (covers a bare allow). (2) Unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — CC accepts allow globs only after a literal glob-free `mcp____` prefix. Predicates shared with CNF live in `lib/permission-rules.mjs` | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions (low); user-vs-plugin overlaps (medium); `details.namespaces` payload | @@ -146,6 +146,45 @@ compare (proves the original schema is byte-identical), and the **SC-5 default-o track current output — diff reviewed as additive-only. **Lesson for any future hotspot/scanner-output field:** grep every frozen-v5.0.0 comparator (it is 5 files, not 2) before assuming the blast radius. +### token-hotspots — MCP tool-schema deferral (v5.10 B4, CA-TOK-006) + +By default Claude Code **defers** MCP tool schemas: only tool *names* enter the always-loaded prefix +(~120 tokens total) and full schemas load on demand via tool search. Several signals force the FULL +schemas into the prefix every turn instead. CA-TOK-006 detects them from **config files only**, so the +finding is deterministic and hermetic-safe (mirrors Pattern G's project-local scoping): + +| Signal | Source | Confidence | +|--------|--------|------------| +| `ENABLE_TOOL_SEARCH: "false"` | merged project+local settings.json `env` block | high | +| `"ToolSearch"` in `permissions.deny` | settings.json | high | +| configured `model` matches `/haiku/` | settings.json (Haiku lacks `tool_reference` support) | medium | +| per-server `alwaysLoad: true` | project `.mcp.json` (CC v2.1.121+) | high | +| `ENABLE_TOOL_SEARCH: "auto[:N]"` | settings `env` | threshold mode — **info, not a trigger** | + +The engine (`lib/mcp-deferral.mjs`) splits a **pure** `assessMcpDeferral({settings, mcpServers})` +(fully unit-tested, no IO) from a thin IO wrapper `assessMcpDeferralForRepo(repoPath, {mcpServers})` +shared by TOK and GAP. Severity scales with the aggregate forced-upfront token cost +(`severityForForcedSchemas`: ≥5000→high, ≥1500→medium; **medium-confidence reasons cap at medium**). + +**Honest scoping decision (Verifiseringsplikt).** The detector deliberately does **NOT** read +`process.env` shell vars. Tool search is also disabled on **Vertex AI**, with a custom +**`ANTHROPIC_BASE_URL`** (non-first-party host), or after a runtime **`/model`** switch to Haiku — but +those are launch/runtime state, not config files, so triggering on them would make the finding +machine-dependent (the marketplace-medium snapshot has MCP servers; an ambient `ANTHROPIC_BASE_URL` +would flap it). They are **disclosed** in every finding (`DEFERRAL_DISCLOSURE`), never triggered. +Tool-level `anthropic/alwaysLoad` (set server-side in the `tools/list` `_meta`) and claude.ai +connectors are likewise invisible to a static scan and disclosed. Mechanism verified 2026-06-23 +against `code.claude.com/docs`: `context-window.md` (MCP deferred, ~120 tok), +`mcp.md#configure-tool-search` + `#exempt-a-server-from-deferral`, `costs.md`. The prefix-cache +connect/disconnect-invalidation claim from the raw research was **`[NOT CONFIRMED]`** in docs and is +NOT asserted by this finding. + +**feature-gap companion.** `cliOverMcpLeverFinding` (GAP) fires **only** when CA-TOK-006's assessment +shows schemas forced upfront — recommends preferring CLI (`gh`/`aws`/`gcloud`) over MCP for common +operations (CLI adds zero context tokens until invoked). Deferred MCP is effectively free, so the +lever stays silent in the default case (opportunity, not noise — mirrors the bundledSkills lever). +`alwaysLoad` was added to CA-MCP's `VALID_SERVER_FIELDS` so it is never flagged as an unknown field. + ### CML scanner — context-window-scaled char budget Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors diff --git a/scanners/feature-gap-scanner.mjs b/scanners/feature-gap-scanner.mjs index 9e4c528..e6b4451 100644 --- a/scanners/feature-gap-scanner.mjs +++ b/scanners/feature-gap-scanner.mjs @@ -13,6 +13,7 @@ import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { measureActiveSkillListing, isBundledSkillsDisabled, BUDGET_CALIBRATION_NOTE } from './lib/skill-listing-budget.mjs'; +import { assessMcpDeferralForRepo } from './lib/mcp-deferral.mjs'; const SCANNER = 'GAP'; @@ -134,6 +135,46 @@ export function bundledSkillsLeverFinding({ leverPulled, aggregate }) { }); } +/** + * CLI-over-MCP lever — remediation companion to CA-TOK-006 (v5.10 B4). + * + * Fires ONLY when MCP tool schemas are forced into the always-loaded prefix + * (tool search disabled, or a per-server alwaysLoad), i.e. when MCP is actually + * costing always-loaded tokens. When schemas are deferred (the default), MCP is + * effectively free until used, so there is nothing to recommend and we stay + * silent — opportunity, not noise (mirrors the bundledSkills lever's "fire only + * under measured pressure" contract). CLI tools (gh / aws / gcloud) add zero + * context tokens until invoked, so they are the lever the deferral mechanism + * cannot reach for the forced-upfront servers. + * + * Pure and exported for unit testing. + * + * @param {{ assessment: (import('./lib/mcp-deferral.mjs').assessMcpDeferral)|null }} args + * @returns {object|null} a GAP finding, or null when nothing is forced upfront + */ +export function cliOverMcpLeverFinding({ assessment } = {}) { + if (!assessment || !assessment.forcedUpfront) return null; + const names = (assessment.affectedServers || []).map((m) => m.name).join(', '); + return finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Prefer CLI over MCP for common operations', + description: + `Your active project MCP tool schemas (~${assessment.aggregateTokens} tokens) are forced into the ` + + 'always-loaded prefix every turn rather than deferred (see CA-TOK-006). CLI tools (gh, aws, gcloud, …) ' + + 'add ZERO context tokens until you actually call them, so moving common operations off MCP and onto a ' + + 'CLI reclaims always-loaded budget the deferral mechanism cannot.', + evidence: + `forced_schema_tokens~${assessment.aggregateTokens}; servers=${names}; ` + + `reason=${assessment.reason || 'alwaysLoad'} (companion to CA-TOK-006)`, + recommendation: + 'For operations a CLI already covers (GitHub → gh, AWS → aws, GCP → gcloud), prefer the CLI over an ' + + 'MCP server — CLI output enters context only when invoked. Keep MCP for capabilities with no CLI ' + + 'equivalent, disable unused servers via /mcp, and re-enable tool-search deferral so the rest stay names-only.', + category: 'token-efficiency', + }); +} + /** @type {GapCheck[]} */ const GAP_CHECKS = [ // --- Tier 1: Foundation --- @@ -441,6 +482,13 @@ export async function scan(targetPath, sharedDiscovery) { const leverFinding = bundledSkillsLeverFinding({ leverPulled, aggregate }); if (leverFinding) findings.push(leverFinding); + // CLI-over-MCP lever — companion to CA-TOK-006: fires only when project-local + // MCP tool schemas are forced into the always-loaded prefix (tool search + // disabled or a per-server alwaysLoad). Reuses the same static assessment. + const mcpAssessment = await assessMcpDeferralForRepo(ctx.targetPath); + const cliLever = cliOverMcpLeverFinding({ assessment: mcpAssessment }); + if (cliLever) findings.push(cliLever); + const filesScanned = discovery.files.length; return scannerResult(SCANNER, 'ok', findings, filesScanned, Date.now() - start); } diff --git a/scanners/lib/active-config-reader.mjs b/scanners/lib/active-config-reader.mjs index a74f8cd..5fa017e 100644 --- a/scanners/lib/active-config-reader.mjs +++ b/scanners/lib/active-config-reader.mjs @@ -788,6 +788,7 @@ export async function readActiveMcpServers(repoPath, claudeJsonSlice = null, plu toolCount, toolCountUnknown: detected.toolCountUnknown, estimatedTokens: estimateTokens(0, 'mcp', { toolCount: toolCount ?? 0 }), + alwaysLoad: def?.alwaysLoad === true, }); } @@ -817,6 +818,7 @@ async function collectMcpFromFile(path, source, disabled, out, repoPath) { toolCount, toolCountUnknown: detected.toolCountUnknown, estimatedTokens: estimateTokens(0, 'mcp', { toolCount: toolCount ?? 0 }), + alwaysLoad: def?.alwaysLoad === true, }); } } diff --git a/scanners/lib/mcp-deferral.mjs b/scanners/lib/mcp-deferral.mjs new file mode 100644 index 0000000..528d140 --- /dev/null +++ b/scanners/lib/mcp-deferral.mjs @@ -0,0 +1,206 @@ +/** + * MCP tool-schema deferral assessment (v5.10 B4). + * + * By default Claude Code DEFERS MCP tool schemas: only tool *names* enter the + * always-loaded prefix (~120 tokens total) and full schemas load on demand via + * tool search. Certain conditions force ALL full schemas into the always-loaded + * prefix instead — paid on every turn whether or not a tool is used. + * + * This module is a STATIC, CONFIG-FILE assessment. It triggers only on signals + * that live in config files a scanner can read deterministically: + * - settings.json `env.ENABLE_TOOL_SEARCH` = "false" (HIGH confidence) + * - settings.json `permissions.deny` contains "ToolSearch" (HIGH confidence) + * - settings.json `model` is a Haiku model (MEDIUM confidence) + * - a per-server `.mcp.json` `alwaysLoad: true` (HIGH confidence) + * + * It deliberately does NOT read process.env shell variables. Tool search is also + * disabled on Vertex AI, with a custom ANTHROPIC_BASE_URL (non-first-party host), + * or after a runtime `/model` switch to Haiku — but those are launch/runtime + * state, not config files, so a static scan cannot see them without becoming + * machine-dependent. They are disclosed (DEFERRAL_DISCLOSURE), never triggered. + * + * Mechanism verified 2026-06-23 against code.claude.com/docs: + * context-window.md (MCP tools deferred, ~120 tok), mcp.md#configure-tool-search + * + #exempt-a-server-from-deferral, costs.md (reduce MCP server overhead). + * + * The pure `assessMcpDeferral` takes already-parsed inputs so it is fully + * unit-testable without file IO. `assessMcpDeferralForRepo` is the thin IO + * wrapper shared by token-hotspots and feature-gap. + */ + +import { resolve } from 'node:path'; +import { readTextFile } from './file-discovery.mjs'; +import { parseJson } from './yaml-parser.mjs'; +import { readActiveMcpServers } from './active-config-reader.mjs'; + +// Aggregate forced-upfront schema cost (tokens) → severity ladder. MCP token +// estimates are base 500 + ~200/tool (active-config-reader estimateTokens), so +// these anchor on a couple of small servers (medium) vs a large one / several +// (high). Heuristic, not measured — disclosed in every finding. +export const FORCED_SCHEMA_TOKENS_MEDIUM = 1500; +export const FORCED_SCHEMA_TOKENS_HIGH = 5000; + +// Appended to every CA-TOK-006 finding: the launch/runtime conditions a static +// config scan cannot see, so the user knows to check them manually. +export const DEFERRAL_DISCLOSURE = + 'static config-file check: tool search is ALSO disabled (all MCP schemas forced ' + + 'upfront) on Vertex AI, with a custom ANTHROPIC_BASE_URL (non-first-party host), ' + + 'or after a runtime /model switch to a Haiku model — none visible to a static ' + + 'scan, so verify at launch. Tool-level "anthropic/alwaysLoad" set server-side is ' + + 'likewise invisible. Per-server alwaysLoad requires Claude Code v2.1.121+.'; + +/** + * Does a permissions.deny list disable the ToolSearch tool? Matches a bare + * "ToolSearch" tool name (with or without an argument suffix), same shape Claude + * Code uses for built-in tool denies. + */ +function denyListDisablesToolSearch(deny) { + if (!Array.isArray(deny)) return false; + return deny.some((entry) => { + if (typeof entry !== 'string') return false; + const tool = entry.replace(/\(.*\)$/, '').trim(); + return tool === 'ToolSearch'; + }); +} + +function isHaikuModel(model) { + return typeof model === 'string' && /haiku/i.test(model); +} + +/** + * Map severity for the forced-upfront aggregate. High-confidence reasons scale + * with the token cost; medium-confidence reasons (inferred from a configured + * model) are capped at medium so the finding never overstates certainty. + * + * @param {number} aggregateTokens + * @param {'high'|'medium'} confidence + * @returns {'high'|'medium'|'low'} + */ +export function severityForForcedSchemas(aggregateTokens, confidence) { + const tok = typeof aggregateTokens === 'number' ? aggregateTokens : 0; + if (confidence === 'high') { + if (tok >= FORCED_SCHEMA_TOKENS_HIGH) return 'high'; + if (tok >= FORCED_SCHEMA_TOKENS_MEDIUM) return 'medium'; + return 'low'; + } + // medium confidence: cap at medium + if (tok >= FORCED_SCHEMA_TOKENS_HIGH) return 'medium'; + return 'low'; +} + +/** + * Assess whether MCP tool schemas are forced into the always-loaded prefix. + * + * @param {object} args + * @param {{ env?: object, permissions?: { deny?: string[] }, model?: string }} [args.settings] + * merged settings.json view (env block, permissions, model). + * @param {Array<{name:string, source?:string, enabled?:boolean, toolCount?:number, + * estimatedTokens?:number, alwaysLoad?:boolean}>} [args.mcpServers] + * @returns {{ + * toolSearchDisabled: boolean, reason: string|null, + * confidence: 'high'|'medium'|null, thresholdMode: boolean, + * alwaysLoadServers: object[], affectedServers: object[], + * aggregateTokens: number, forcedUpfront: boolean, + * }} + */ +export function assessMcpDeferral({ settings = {}, mcpServers = [] } = {}) { + const s = settings || {}; + const tsRaw = s.env && typeof s.env === 'object' ? s.env.ENABLE_TOOL_SEARCH : undefined; + const ts = String(tsRaw ?? '').trim().toLowerCase(); + const denyTS = denyListDisablesToolSearch(s.permissions?.deny); + const haiku = isHaikuModel(s.model); + + let toolSearchDisabled = false; + let reason = null; + let confidence = null; + let thresholdMode = false; + + if (ts === 'false') { + toolSearchDisabled = true; + reason = 'enable-tool-search-false'; + confidence = 'high'; + } else if (denyTS) { + toolSearchDisabled = true; + reason = 'deny-tool-search'; + confidence = 'high'; + } else if (haiku) { + // Haiku lacks tool_reference support, so tool search cannot run even when + // ENABLE_TOOL_SEARCH=true. Medium confidence: the configured model can be + // switched at runtime (/model), which a static scan cannot observe. + toolSearchDisabled = true; + reason = 'haiku-model'; + confidence = 'medium'; + } else if (ts === 'true') { + toolSearchDisabled = false; + } else if (ts.startsWith('auto')) { + // Threshold mode (auto / auto:N): schemas load upfront only when they fit a + // percentage of the context window — not a clear always-load. Info, not a + // forced-upfront trigger. + thresholdMode = true; + } + + const active = (Array.isArray(mcpServers) ? mcpServers : []).filter( + (m) => m && m.enabled !== false, + ); + const alwaysLoadServers = active.filter((m) => m.alwaysLoad === true); + const affectedServers = toolSearchDisabled ? active : alwaysLoadServers; + const aggregateTokens = affectedServers.reduce( + (sum, m) => sum + (typeof m.estimatedTokens === 'number' ? m.estimatedTokens : 0), + 0, + ); + + return { + toolSearchDisabled, + reason, + confidence, + thresholdMode, + alwaysLoadServers, + affectedServers, + aggregateTokens, + forcedUpfront: affectedServers.length > 0, + }; +} + +/** + * Merge project + local settings.json for the deferral check. Scoped to the + * audited path (NOT the user cascade) so the result stays deterministic and free + * of ambient HOME leakage — mirrors Pattern G's project-local scoping. Local + * overrides project; env/permissions shallow-merge, deny lists concatenate. + */ +async function readMergedProjectSettings(repoPath) { + const merged = { env: {}, permissions: {}, model: undefined }; + for (const rel of ['.claude/settings.json', '.claude/settings.local.json']) { + const content = await readTextFile(resolve(repoPath, rel)); + if (!content) continue; + const parsed = parseJson(content); + if (!parsed || typeof parsed !== 'object') continue; + if (parsed.env && typeof parsed.env === 'object') Object.assign(merged.env, parsed.env); + if (parsed.permissions && typeof parsed.permissions === 'object') { + const deny = [ + ...(Array.isArray(merged.permissions.deny) ? merged.permissions.deny : []), + ...(Array.isArray(parsed.permissions.deny) ? parsed.permissions.deny : []), + ]; + merged.permissions = { ...merged.permissions, ...parsed.permissions, deny }; + } + if (typeof parsed.model === 'string') merged.model = parsed.model; + } + return merged; +} + +/** + * IO wrapper: assess MCP deferral for a repo path. Scopes MCP servers to active + * project-local `.mcp.json` (plugin / ~/.claude.json servers are the manifest's + * concern). Pass `mcpServers` (e.g. an already-loaded activeConfig.mcpServers) to + * avoid a re-read; otherwise they are read via readActiveMcpServers. + * + * @param {string} repoPath + * @param {{ mcpServers?: object[] }} [opts] + */ +export async function assessMcpDeferralForRepo(repoPath, opts = {}) { + const all = Array.isArray(opts.mcpServers) + ? opts.mcpServers + : await readActiveMcpServers(repoPath); + const projectLocal = all.filter((m) => m && m.enabled && m.source === '.mcp.json'); + const settings = await readMergedProjectSettings(repoPath); + return assessMcpDeferral({ settings, mcpServers: projectLocal }); +} diff --git a/scanners/mcp-config-validator.mjs b/scanners/mcp-config-validator.mjs index 010dfa3..5e4b742 100644 --- a/scanners/mcp-config-validator.mjs +++ b/scanners/mcp-config-validator.mjs @@ -18,6 +18,9 @@ const VALID_SERVER_TYPES = new Set(['stdio', 'http', 'sse']); // not a per-server .mcp.json field. Verified against code.claude.com/docs 2026-06-18. const VALID_SERVER_FIELDS = new Set([ 'type', 'command', 'args', 'env', 'url', 'headers', 'timeout', + // alwaysLoad: exempt a server from MCP tool-schema deferral (CC v2.1.121+). + // Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral 2026-06-23. + 'alwaysLoad', ]); // Match only bare ${IDENTIFIER} references. POSIX expansions like ${VAR%pattern} diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index fc90e7f..1dc9eae 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -28,6 +28,11 @@ import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; +import { + assessMcpDeferralForRepo, + severityForForcedSchemas, + DEFERRAL_DISCLOSURE, +} from './lib/mcp-deferral.mjs'; const SCANNER = 'TOK'; @@ -580,6 +585,61 @@ export async function scan(targetPath, discovery) { })); } + // ── Pattern I: MCP tool-schema deferral (v5.10 B4, CA-TOK-006) ── + // By default MCP tool schemas are DEFERRED (names-only, ~120 tok); certain + // config signals force the FULL schemas into the always-loaded prefix every + // turn. Scope: project-local .mcp.json servers (mirrors Pattern G — plugin / + // global servers are the manifest's concern). Static config-file check only; + // runtime conditions (Vertex / ANTHROPIC_BASE_URL / runtime /model switch) are + // DISCLOSED (DEFERRAL_DISCLOSURE), not triggered, so the finding is deterministic. + const mcpForDeferral = (activeConfig && Array.isArray(activeConfig.mcpServers)) + ? activeConfig.mcpServers.filter(m => m && m.enabled && m.source === '.mcp.json') + : []; + if (mcpForDeferral.length > 0) { + const a = await assessMcpDeferralForRepo(targetPath, { mcpServers: activeConfig.mcpServers }); + if (a.forcedUpfront) { + const conf = a.confidence || 'high'; + const severity = severityForForcedSchemas(a.aggregateTokens, conf); + const names = a.affectedServers.map(m => m.name).join(', '); + const reasonText = { + 'enable-tool-search-false': 'ENABLE_TOOL_SEARCH is set to "false" in settings', + 'deny-tool-search': '"ToolSearch" is listed in permissions.deny', + 'haiku-model': 'the configured model is a Haiku model (Haiku has no tool-search support)', + }[a.reason]; + const description = a.toolSearchDisabled + ? `Tool search is disabled (${reasonText}), so the full tool schemas of ` + + `${a.affectedServers.length} active project MCP server${a.affectedServers.length === 1 ? '' : 's'} ` + + `(~${a.aggregateTokens} tokens) load into the always-loaded prefix on every turn instead of ` + + 'being deferred (tool names only, ~120 tokens total). Every schema token is re-sent each turn ' + + 'whether or not a tool is used.' + : `${a.alwaysLoadServers.length} project MCP server${a.alwaysLoadServers.length === 1 ? '' : 's'} ` + + `marked alwaysLoad (${names}) load their full tool schemas (~${a.aggregateTokens} tokens) into ` + + 'the always-loaded prefix on every turn regardless of tool search, instead of deferring them ' + + '(names only, ~120 tokens).'; + const evidence = + `reason=${a.reason || 'alwaysLoad'}; confidence=${conf}; servers=${names}; ` + + `forced_schema_tokens~${a.aggregateTokens} — ${CALIBRATION_NOTE}. ${DEFERRAL_DISCLOSURE}`; + const recommendation = a.toolSearchDisabled + ? 'Re-enable tool search so MCP schemas defer (names-only) by default: remove ' + + 'ENABLE_TOOL_SEARCH="false" / the "ToolSearch" deny, or stop defaulting to a Haiku model. ' + + 'Also disable unused servers via /mcp, and prefer CLI tools (gh / aws / gcloud) over MCP for ' + + 'common operations — CLI adds zero context tokens until used.' + : 'Drop alwaysLoad on large-schema servers so they defer (names-only) until a tool is needed; ' + + 'keep alwaysLoad only for small servers you call on most turns. Prefer CLI tools ' + + '(gh / aws / gcloud) over MCP for common operations.'; + findings.push(finding({ + scanner: SCANNER, + severity, + title: 'MCP tool schemas forced into the always-loaded prefix', + file: null, + evidence, + description, + recommendation, + category: 'token-efficiency', + })); + } + } + // ── Hotspots ranking ── const hotspots = await buildHotspots(discovery, targetPath, activeConfig); diff --git a/tests/fixtures/mcp-deferral/alwaysload/.mcp.json b/tests/fixtures/mcp-deferral/alwaysload/.mcp.json new file mode 100644 index 0000000..83df111 --- /dev/null +++ b/tests/fixtures/mcp-deferral/alwaysload/.mcp.json @@ -0,0 +1,6 @@ +{ + "mcpServers": { + "always-srv": { "command": "npx", "args": ["fake-pkg"], "alwaysLoad": true, "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] }, + "deferred-srv": { "command": "npx", "args": ["other-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/fixtures/mcp-deferral/deferred/.mcp.json b/tests/fixtures/mcp-deferral/deferred/.mcp.json new file mode 100644 index 0000000..7be9be3 --- /dev/null +++ b/tests/fixtures/mcp-deferral/deferred/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "plain-srv": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json b/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json new file mode 100644 index 0000000..10a1000 --- /dev/null +++ b/tests/fixtures/mcp-deferral/disabled-settings/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "srv-a": { "command": "npx", "args": ["fake-pkg"], "tools": [{"name": "t_0", "description": "tool 0"},{"name": "t_1", "description": "tool 1"},{"name": "t_2", "description": "tool 2"},{"name": "t_3", "description": "tool 3"},{"name": "t_4", "description": "tool 4"},{"name": "t_5", "description": "tool 5"},{"name": "t_6", "description": "tool 6"},{"name": "t_7", "description": "tool 7"},{"name": "t_8", "description": "tool 8"},{"name": "t_9", "description": "tool 9"}] } + } +} diff --git a/tests/scanners/feature-gap-scanner.test.mjs b/tests/scanners/feature-gap-scanner.test.mjs index 91294bb..1e0066a 100644 --- a/tests/scanners/feature-gap-scanner.test.mjs +++ b/tests/scanners/feature-gap-scanner.test.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; -import { scan, opportunitySummary, bundledSkillsLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; +import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; import { withHermeticHome } from '../helpers/hermetic-home.mjs'; @@ -340,3 +340,27 @@ describe('GAP scanner — disableBundledSkills lever wiring (HOME-scoped)', () = } }); }); + +describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => { + it('returns null when nothing is forced upfront', () => { + assert.equal(cliOverMcpLeverFinding({ assessment: { forcedUpfront: false } }), null); + assert.equal(cliOverMcpLeverFinding({ assessment: null }), null); + assert.equal(cliOverMcpLeverFinding({}), null); + }); + + it('fires a low-severity opportunity when MCP schemas are forced upfront', () => { + resetCounter(); + const f = cliOverMcpLeverFinding({ + assessment: { + forcedUpfront: true, + aggregateTokens: 2500, + affectedServers: [{ name: 'srv-a' }], + reason: 'enable-tool-search-false', + }, + }); + assert.ok(f, 'expected a lever finding'); + assert.equal(f.severity, 'low'); + assert.equal(f.category, 'token-efficiency'); + assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/); + }); +}); diff --git a/tests/scanners/mcp-config-validator.test.mjs b/tests/scanners/mcp-config-validator.test.mjs index 95b971b..1f4440c 100644 --- a/tests/scanners/mcp-config-validator.test.mjs +++ b/tests/scanners/mcp-config-validator.test.mjs @@ -136,6 +136,36 @@ describe('MCP scanner — stray `trust` field is an unknown field (verify-first, }); }); +describe('MCP scanner — `alwaysLoad` is a valid field (v5.10 B4, verify-first 2026-06-23)', () => { + let result; + let tmpRoot; + + beforeEach(async () => { + resetCounter(); + tmpRoot = await mkdtemp(join(tmpdir(), 'ca-mcp-alwaysload-')); + // alwaysLoad exempts a server from MCP tool-schema deferral (CC v2.1.121+). + // Verified against code.claude.com/docs/en/mcp.md#exempt-a-server-from-deferral. + const mcp = { + mcpServers: { + core: { type: 'http', url: 'https://mcp.example.com/mcp', alwaysLoad: true }, + }, + }; + await writeFile(join(tmpRoot, '.mcp.json'), JSON.stringify(mcp, null, 2) + '\n', 'utf8'); + const discovery = await discoverConfigFiles(tmpRoot); + result = await scan(tmpRoot, discovery); + }); + + afterEach(async () => { + if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true }); + }); + + it('does NOT flag `alwaysLoad` as an unknown MCP server field', () => { + const f = result.findings.find(x => x.title.includes('Unknown MCP server field') + && /alwaysLoad/.test(x.description || '')); + assert.equal(f, undefined, 'alwaysLoad is a valid field and must not be flagged'); + }); +}); + describe('MCP scanner — env-var false positives (CC 2.1.139/2.1.142, Batch 1)', () => { let tmpRoot; let envFindings; diff --git a/tests/scanners/mcp-deferral.test.mjs b/tests/scanners/mcp-deferral.test.mjs new file mode 100644 index 0000000..d804526 --- /dev/null +++ b/tests/scanners/mcp-deferral.test.mjs @@ -0,0 +1,192 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + assessMcpDeferral, + severityForForcedSchemas, + FORCED_SCHEMA_TOKENS_HIGH, + FORCED_SCHEMA_TOKENS_MEDIUM, +} from '../../scanners/lib/mcp-deferral.mjs'; + +// A small helper to build server shapes the way active-config-reader exposes them. +const srv = (over = {}) => ({ + name: 'srv', + source: '.mcp.json', + enabled: true, + toolCount: 10, + estimatedTokens: 2500, + alwaysLoad: false, + ...over, +}); + +describe('assessMcpDeferral — default (deferred) case', () => { + it('no settings + a plain server → tool search NOT disabled, nothing forced upfront', () => { + const a = assessMcpDeferral({ settings: {}, mcpServers: [srv()] }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.reason, null); + assert.equal(a.confidence, null); + assert.equal(a.forcedUpfront, false); + assert.deepEqual(a.affectedServers, []); + assert.equal(a.aggregateTokens, 0); + }); + + it('empty inputs are tolerated', () => { + const a = assessMcpDeferral({}); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, false); + }); +}); + +describe('assessMcpDeferral — config-file disabling signals (HIGH confidence)', () => { + it('settings.env.ENABLE_TOOL_SEARCH="false" → disabled, high, all servers affected', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [srv({ name: 'a', estimatedTokens: 1000 }), srv({ name: 'b', estimatedTokens: 2000 })], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'enable-tool-search-false'); + assert.equal(a.confidence, 'high'); + assert.equal(a.forcedUpfront, true); + assert.equal(a.affectedServers.length, 2); + assert.equal(a.aggregateTokens, 3000); + }); + + it('permissions.deny including bare "ToolSearch" → disabled, high', () => { + const a = assessMcpDeferral({ + settings: { permissions: { deny: ['Read(./.env)', 'ToolSearch'] } }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'deny-tool-search'); + assert.equal(a.confidence, 'high'); + }); + + it('explicit ENABLE_TOOL_SEARCH="false" wins over a haiku model for the reason label', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' }, model: 'claude-haiku-4-5' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'enable-tool-search-false'); + assert.equal(a.confidence, 'high'); + }); +}); + +describe('assessMcpDeferral — configured Haiku model (MEDIUM confidence)', () => { + it('settings.model matching /haiku/ → disabled, medium, haiku-model', () => { + const a = assessMcpDeferral({ + settings: { model: 'claude-haiku-4-5-20251001' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'haiku-model'); + assert.equal(a.confidence, 'medium'); + assert.equal(a.forcedUpfront, true); + }); + + it('haiku disables even when ENABLE_TOOL_SEARCH="true" (Haiku lacks tool_reference support)', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'haiku' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.reason, 'haiku-model'); + }); +}); + +describe('assessMcpDeferral — non-disabling values', () => { + it('ENABLE_TOOL_SEARCH="true" (non-haiku) → NOT disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'true' }, model: 'claude-sonnet-4-6' }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, false); + }); + + it('ENABLE_TOOL_SEARCH="auto" → threshold mode, NOT disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'auto' } }, + mcpServers: [srv()], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.thresholdMode, true); + assert.equal(a.forcedUpfront, false); + }); + + it('ENABLE_TOOL_SEARCH="auto:5" → threshold mode', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'auto:5' } }, + mcpServers: [srv()], + }); + assert.equal(a.thresholdMode, true); + assert.equal(a.toolSearchDisabled, false); + }); +}); + +describe('assessMcpDeferral — per-server alwaysLoad', () => { + it('alwaysLoad:true server with tool search enabled → that server forced upfront only', () => { + const a = assessMcpDeferral({ + settings: {}, + mcpServers: [ + srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }), + srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 9000 }), + ], + }); + assert.equal(a.toolSearchDisabled, false); + assert.equal(a.forcedUpfront, true); + assert.equal(a.alwaysLoadServers.length, 1); + assert.equal(a.affectedServers.length, 1); + assert.equal(a.affectedServers[0].name, 'always'); + assert.equal(a.aggregateTokens, 1500); + }); + + it('when tool search is disabled, ALL active servers are affected (not just alwaysLoad)', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [ + srv({ name: 'always', alwaysLoad: true, estimatedTokens: 1500 }), + srv({ name: 'deferred', alwaysLoad: false, estimatedTokens: 2500 }), + ], + }); + assert.equal(a.affectedServers.length, 2); + assert.equal(a.aggregateTokens, 4000); + }); +}); + +describe('assessMcpDeferral — server filtering', () => { + it('disabled servers (enabled:false) are excluded from affected + aggregate', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [ + srv({ name: 'on', enabled: true, estimatedTokens: 1000 }), + srv({ name: 'off', enabled: false, estimatedTokens: 5000 }), + ], + }); + assert.equal(a.affectedServers.length, 1); + assert.equal(a.affectedServers[0].name, 'on'); + assert.equal(a.aggregateTokens, 1000); + }); + + it('no active servers → not forced upfront even when tool search disabled', () => { + const a = assessMcpDeferral({ + settings: { env: { ENABLE_TOOL_SEARCH: 'false' } }, + mcpServers: [], + }); + assert.equal(a.toolSearchDisabled, true); + assert.equal(a.forcedUpfront, false); + assert.equal(a.aggregateTokens, 0); + }); +}); + +describe('severityForForcedSchemas', () => { + it('high confidence scales with aggregate tokens', () => { + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH, 'high'), 'high'); + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_MEDIUM, 'high'), 'medium'); + assert.equal(severityForForcedSchemas(100, 'high'), 'low'); + }); + + it('medium confidence is capped at medium', () => { + assert.equal(severityForForcedSchemas(FORCED_SCHEMA_TOKENS_HIGH * 10, 'medium'), 'medium'); + assert.equal(severityForForcedSchemas(100, 'medium'), 'low'); + }); +}); diff --git a/tests/scanners/token-hotspots-deferral.test.mjs b/tests/scanners/token-hotspots-deferral.test.mjs new file mode 100644 index 0000000..1b19fb7 --- /dev/null +++ b/tests/scanners/token-hotspots-deferral.test.mjs @@ -0,0 +1,70 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resetCounter } from '../../scanners/lib/output.mjs'; +import { scan } from '../../scanners/token-hotspots.mjs'; +import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; +import { withHermeticHome } from '../helpers/hermetic-home.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const FIXTURES = resolve(__dirname, '../fixtures'); + +// Hermetic HOME so readActiveConfig does not leak the developer's real +// ~/.claude.json plugin MCP servers into the fixture result (mirrors the other +// TOK tests). The deferral check reads project+local settings + project .mcp.json. +async function runScanner(fixtureName) { + resetCounter(); + const path = resolve(FIXTURES, fixtureName); + const discovery = await discoverConfigFiles(path); + return withHermeticHome(() => scan(path, discovery)); +} + +const TITLE = 'MCP tool schemas forced into the always-loaded prefix'; +const findDeferral = (result) => result.findings.find(f => f.title === TITLE); + +describe('TOK scanner — MCP deferral (v5.10 B4, CA-TOK-006)', () => { + it('deferred (default) project → NO deferral finding', async () => { + const result = await runScanner('mcp-deferral/deferred'); + assert.equal(findDeferral(result), undefined, + 'a plain server with tool search on must not be flagged as forced-upfront'); + }); + + it('alwaysLoad:true server → deferral finding fires (medium severity)', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f, `expected a deferral finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'medium', `expected medium for ~2500 forced tokens, got ${f.severity}`); + assert.equal(f.category, 'token-efficiency'); + // Only the alwaysLoad server is named, not the deferred sibling. + assert.match(String(f.evidence || ''), /always-srv/); + assert.doesNotMatch(String(f.description || ''), /deferred-srv/); + assert.match(String(f.evidence || ''), /alwaysLoad/); + }); + + it('ENABLE_TOOL_SEARCH="false" in settings → deferral finding (all servers affected)', async () => { + const result = await runScanner('mcp-deferral/disabled-settings'); + const f = findDeferral(result); + assert.ok(f, `expected a deferral finding for tool-search-disabled fixture`); + assert.match(String(f.evidence || ''), /reason=enable-tool-search-false/); + assert.match(String(f.description || ''), /Tool search is disabled/); + }); + + it('every deferral finding discloses runtime conditions + calibration', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f); + // DEFERRAL_DISCLOSURE names the launch/runtime conditions a static scan can't see. + assert.match(String(f.evidence || ''), /Vertex AI/); + assert.match(String(f.evidence || ''), /ANTHROPIC_BASE_URL/); + // CALIBRATION_NOTE shared by all TOK pattern findings. + assert.match(String(f.evidence || ''), /severity reflects estimated tokens\/turn/i); + }); + + it('finding ID matches CA-TOK-NNN format', async () => { + const result = await runScanner('mcp-deferral/alwaysload'); + const f = findDeferral(result); + assert.ok(f); + assert.match(f.id, /^CA-TOK-\d{3}$/); + }); +}); From d2c45a3bb8bd99e8e0b423f738c655ba85bea3cd Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 20:28:33 +0200 Subject: [PATCH 20/53] feat(hooks): additionalContext injection advisory + filter-before lever (v5.10 B5) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HKV now flags hooks that build hookSpecificOutput.additionalContext from un-grepped command output as an INFO advisory (weight 0, never severity-bearing — excluded from the self-audit nonInfo set). That field enters Claude's context every time the hook fires (plain stdout on exit 0 does not), so an unfiltered payload is a recurring per-turn token cost. - New lib scanners/lib/hook-additional-context.mjs: pure assessHookAdditionalContext (unit-tested, no IO) + IO wrapper assessHookContextForRepo (walk hooks->scripts). Heuristic: additionalContext + verbose-prone capture (cat/git log/execSync/…) && no filter (grep/head/jq/.slice). Deliberately low precision -> advisory only. - HKV: emits the advisory inline on scripts it already reads (after the M5 verbose check). Additive, info severity -> frozen v5.0.0 + SC-5 snapshots untouched. - feature-gap: new filterHookLeverFinding companion, fires ONLY when >=1 chatty hook is detected — surfaces the filter-before-Claude-reads lever (CC filter-test-output.sh). Silent otherwise (opportunity, not noise). - docs: README HKV + GAP scanner rows; scanner-internals.md HKV row + full B5 implementation note. CLAUDE.md kept lean ([skip-docs]); B5 fully documented in README + scanner-internals. Mechanism verified 2026-06-23 against code.claude.com/docs context-window.md (additionalContext enters context; plain stdout does not). Suite 1239 -> 1254 green. Version/badges/CHANGELOG wait for the v5.10 release cut. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 4 +- docs/scanner-internals.md | 36 +++- scanners/feature-gap-scanner.mjs | 42 +++++ scanners/hook-validator.mjs | 30 ++++ scanners/lib/hook-additional-context.mjs | 160 ++++++++++++++++++ .../hooks-additional-context/hooks/hooks.json | 10 ++ .../hooks/scripts/chatty.sh | 6 + .../hooks/scripts/filtered.sh | 5 + tests/scanners/feature-gap-scanner.test.mjs | 49 +++++- .../scanners/hook-additional-context.test.mjs | 90 ++++++++++ tests/scanners/hook-validator.test.mjs | 40 +++++ 11 files changed, 468 insertions(+), 4 deletions(-) create mode 100644 scanners/lib/hook-additional-context.mjs create mode 100644 tests/fixtures/hooks-additional-context/hooks/hooks.json create mode 100644 tests/fixtures/hooks-additional-context/hooks/scripts/chatty.sh create mode 100644 tests/fixtures/hooks-additional-context/hooks/scripts/filtered.sh create mode 100644 tests/scanners/hook-additional-context.test.mjs diff --git a/README.md b/README.md index 6aa108a..01f3b68 100644 --- a/README.md +++ b/README.md @@ -316,12 +316,12 @@ By default, `/config-audit` auto-detects scope from your git context. Override w |---------|--------|-----------------| | `claude-md-linter.mjs` | CML | Oversized files (line count **plus** a context-window-scaled char budget mirroring Claude Code's ~40.0k-char startup warning), missing sections, broken @imports, duplicates, stale TODOs | | `settings-validator.mjs` | SET | Schema violations, unknown/deprecated keys, type mismatches, permission issues | -| `hook-validator.mjs` | HKV | Invalid format, missing scripts, wrong event names, timeout risks | +| `hook-validator.mjs` | HKV | Invalid format, missing scripts, wrong event names, timeout risks, verbose-stdout scripts, and a low-precision **advisory** (info) when a hook injects un-grepped command output into `hookSpecificOutput.additionalContext` — that payload enters context on every fire (plain stdout does not) | | `rules-validator.mjs` | RUL | Bad glob patterns, orphaned rules, deprecated fields, unscoped rules | | `mcp-config-validator.mjs` | MCP | Invalid server types, exposed env vars, unknown fields | | `import-resolver.mjs` | IMP | Broken @imports, circular references, deep chains, tilde path issues | | `conflict-detector.mjs` | CNF | Settings contradictions across scopes, permission conflicts, hook duplicates | -| `feature-gap-scanner.mjs` | GAP | 25 feature checks shown as opportunities, not grades — plus a conditional `disableBundledSkills` recommendation when the active skill listing is over budget | +| `feature-gap-scanner.mjs` | GAP | 25 feature checks shown as opportunities, not grades — plus a conditional `disableBundledSkills` recommendation when the active skill listing is over budget, and a conditional **filter-before-Claude-reads** lever when a hook injects unfiltered output into `additionalContext` (companion to the HKV advisory; cites the documented `filter-test-output.sh` pattern) | | `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascades, bloated skill descriptions, MCP tool-schema budget, and stale `~/.claude/plugins/cache` versions (disk-cleanup, zero live-context impact) — cache-aware ranking excludes superseded plugin versions by default (`--no-exclude-cache` to include) | | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of the CLAUDE.md cascade — beyond the cache-prefix window but still re-loaded every turn | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries: (1) tools in BOTH `permissions.deny` and `permissions.allow` — deny wins (incl. the `Tool(*)` deny-all glob, equivalent to a bare deny); (2) unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — valid only as `mcp____*`; (3) `Tool(param:value)` rules whose key is the tool's own canonicalizing field (`command`/`file_path`/`path`/`notebook_path`/`url`) — CC ignores these and emits a startup warning | diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index 1c0745d..a1a5893 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -12,7 +12,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach |---------|--------|---------| | `claude-md-linter.mjs` | CML | Structure, length, sections, @imports, duplicates, TODOs | | `settings-validator.mjs` | SET | Schema, unknown/deprecated keys, type mismatches, permissions | -| `hook-validator.mjs` | HKV | Format, script existence, event validity, timeouts | +| `hook-validator.mjs` | HKV | Format, script existence, event validity, timeouts, verbose-stdout (low), unfiltered `additionalContext` injection (info advisory, v5.10 B5) | | `rules-validator.mjs` | RUL | Glob matching, orphan rules, deprecated fields, unscoped rules | | `mcp-config-validator.mjs` | MCP | Server types, env vars, unknown fields | | `import-resolver.mjs` | IMP | Broken @imports, circular refs, deep chains, tilde paths | @@ -185,6 +185,40 @@ operations (CLI adds zero context tokens until invoked). Deferred MCP is effecti lever stays silent in the default case (opportunity, not noise — mirrors the bundledSkills lever). `alwaysLoad` was added to CA-MCP's `VALID_SERVER_FIELDS` so it is never flagged as an unknown field. +### hook-validator — unfiltered additionalContext advisory (v5.10 B5) + +A hook that emits `hookSpecificOutput.additionalContext` has that payload injected into Claude's +context **every time it fires** — plain stdout on exit 0 does NOT (it goes to the debug log only). A +hook that dumps large, un-grepped command output into `additionalContext` is therefore a recurring, +compaction-sensitive per-turn token cost. HKV flags it as an **`info` advisory** (weight 0 — never +severity-bearing, excluded from the self-audit `nonInfo` set), paired with a feature-gap lever. + +The heuristic lives in `lib/hook-additional-context.mjs` as a **pure** `assessHookAdditionalContext({scriptContent})` +(unit-tested, no IO) plus a thin IO wrapper `assessHookContextForRepo(discovery)` (walk hooks → scripts +→ assess) used by GAP; HKV calls the pure function inline on scripts it already reads. The signal: + +| Condition | Detected by | Effect | +|-----------|-------------|--------| +| references `additionalContext` | `/additionalContext/` | gate (else not applicable) | +| captures verbose-prone output | `cat`/`find`/`ls`/`git log\|diff\|status\|show`/`npm`/`pytest`/`jest`/`curl`/`execSync`/`readFileSync`… | `hasVerboseCapture` | +| applies any truncating filter | `grep`/`head`/`tail`/`sed`/`awk`/`jq`/`cut`/`wc`/`uniq`/`sort` or `.slice`/`.substring` | suppresses (assumed bounded) | + +`flagged = buildsAdditionalContext && hasVerboseCapture && !hasFilter`. A filtered capture (e.g. +`cat … | grep ERROR`) or a cheap-only capture (`$(date)`) is not flagged. + +**Why `info`, not a hard finding (Verifiseringsplikt).** This is deliberately **low precision** — a +static scan cannot run the hook or measure the real payload, and a filter we don't recognise would be +a false positive. So it ships as an advisory with the precision caveat in its own description, never a +graded/severity-bearing finding. Mechanism verified 2026-06-23 against `code.claude.com/docs`: +`context-window.md` — *"A PostToolUse hook … reports back via `hookSpecificOutput.additionalContext`. +That field enters Claude's context. Plain stdout on exit 0 does not."* + the tip to keep output concise +(it enters context without truncation). + +**feature-gap companion.** `filterHookLeverFinding` (GAP) fires **only** when `assessHookContextForRepo` +returns ≥1 chatty hook — surfaces the documented **filter-before-Claude-reads** lever (`filter-test-output.sh`: +grep ERROR and return only matches instead of a 10,000-line log). No chatty hook → silent (opportunity, +not noise — same contract as the cliOverMcp / bundledSkills levers). + ### CML scanner — context-window-scaled char budget Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors diff --git a/scanners/feature-gap-scanner.mjs b/scanners/feature-gap-scanner.mjs index e6b4451..00a5c23 100644 --- a/scanners/feature-gap-scanner.mjs +++ b/scanners/feature-gap-scanner.mjs @@ -14,6 +14,7 @@ import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; import { measureActiveSkillListing, isBundledSkillsDisabled, BUDGET_CALIBRATION_NOTE } from './lib/skill-listing-budget.mjs'; import { assessMcpDeferralForRepo } from './lib/mcp-deferral.mjs'; +import { assessHookContextForRepo } from './lib/hook-additional-context.mjs'; const SCANNER = 'GAP'; @@ -175,6 +176,41 @@ export function cliOverMcpLeverFinding({ assessment } = {}) { }); } +/** + * filter-before-Claude-reads lever — remediation companion to HKV's B5 advisory + * (v5.10). Fires ONLY when ≥1 active hook was detected injecting unfiltered + * command output into additionalContext, i.e. when there is a measured chatty + * hook to fix. When no such hook exists there is nothing to recommend and we + * stay silent — opportunity, not noise (same "fire only under measured pressure" + * contract as the cliOverMcp / bundledSkills levers). + * + * Pure and exported for unit testing. + * + * @param {{ flaggedHooks: Array<{event:string, scriptPath:string}> }} args + * @returns {object|null} a GAP finding, or null when no chatty hook was detected + */ +export function filterHookLeverFinding({ flaggedHooks } = {}) { + const hooks = Array.isArray(flaggedHooks) ? flaggedHooks : []; + if (hooks.length === 0) return null; + const scripts = hooks.map((h) => h.scriptPath.split('/').slice(-1)[0]).join(', '); + return finding({ + scanner: SCANNER, + severity: SEVERITY.info, + title: 'Filter hook output before it enters context', + description: + `${hooks.length} active hook${hooks.length === 1 ? '' : 's'} build hookSpecificOutput.additionalContext ` + + "from un-grepped command output (see HKV advisory). That field enters Claude's context on every fire, " + + 'so filtering verbose output down to what matters BEFORE Claude reads it reclaims per-turn tokens — the ' + + 'documented filter-test-output.sh pattern (grep ERROR and return only matches instead of a 10,000-line log).', + evidence: `chatty_hooks=${hooks.length}; scripts=${scripts} (companion to HKV additionalContext advisory)`, + recommendation: + 'In each flagged hook, pipe the command output through grep/head/jq to keep only the actionable lines ' + + 'before assigning additionalContext. Reserve additionalContext for concise signals; leave bulk diagnostics ' + + 'on plain stdout (exit 0) so they go to the debug log, not context.', + category: 'token-efficiency', + }); +} + /** @type {GapCheck[]} */ const GAP_CHECKS = [ // --- Tier 1: Foundation --- @@ -489,6 +525,12 @@ export async function scan(targetPath, sharedDiscovery) { const cliLever = cliOverMcpLeverFinding({ assessment: mcpAssessment }); if (cliLever) findings.push(cliLever); + // filter-before-Claude-reads lever — companion to HKV's B5 advisory: fires + // only when an active hook injects unfiltered output into additionalContext. + const flaggedHooks = await assessHookContextForRepo(discovery); + const hookLever = filterHookLeverFinding({ flaggedHooks }); + if (hookLever) findings.push(hookLever); + const filesScanned = discovery.files.length; return scannerResult(SCANNER, 'ok', findings, filesScanned, Date.now() - start); } diff --git a/scanners/hook-validator.mjs b/scanners/hook-validator.mjs index f3811a5..c521277 100644 --- a/scanners/hook-validator.mjs +++ b/scanners/hook-validator.mjs @@ -8,6 +8,7 @@ import { readTextFile, discoverConfigFiles } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { parseJson } from './lib/yaml-parser.mjs'; +import { assessHookAdditionalContext } from './lib/hook-additional-context.mjs'; import { stat } from 'node:fs/promises'; import { resolve, dirname } from 'node:path'; @@ -248,6 +249,35 @@ async function validateHooksObject(hooks, file, findings, baseDir) { autoFixable: false, })); } + + // v5.10 B5: advisory (info) — a hook that injects unfiltered + // command output into hookSpecificOutput.additionalContext pays + // that whole payload into Claude's context on every fire (plain + // stdout does not). Low-precision static heuristic, so info only. + const scriptContent = await readTextFile(scriptPath); + const ac = assessHookAdditionalContext({ scriptContent }); + if (ac.flagged) { + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.info, + title: 'Hook injects unfiltered output into context', + description: + `${file.relPath}: "${event}" runs ${scriptPath.split('/').slice(-2).join('/')} ` + + 'which builds hookSpecificOutput.additionalContext from un-grepped command ' + + "output. That field enters Claude's context every time the hook fires (plain " + + 'stdout does not), so an unfiltered payload is a recurring per-turn token cost. ' + + 'Advisory only — low-precision static heuristic; verify the real payload size.', + file: scriptPath, + evidence: + 'additional_context_unfiltered=true; ' + + `verbose_capture=${ac.hasVerboseCapture}; filter_applied=${ac.hasFilter}`, + recommendation: + 'Filter before Claude reads: grep/head the command output down to what matters ' + + 'before putting it in additionalContext (the documented filter-test-output.sh ' + + 'pattern), or keep large diagnostics on plain stdout so they stay out of context.', + autoFixable: false, + })); + } } } } diff --git a/scanners/lib/hook-additional-context.mjs b/scanners/lib/hook-additional-context.mjs new file mode 100644 index 0000000..70dbcf9 --- /dev/null +++ b/scanners/lib/hook-additional-context.mjs @@ -0,0 +1,160 @@ +/** + * Hook additionalContext injection advisory (v5.10 B5). + * + * A hook that emits `hookSpecificOutput.additionalContext` has that content + * injected into Claude's context EVERY time the hook fires. Plain stdout on + * exit 0 does NOT enter context (it goes to the debug log only). So a hook that + * dumps large, unfiltered command output into additionalContext is a recurring + * per-turn token cost that compounds and is compaction-sensitive. + * + * This module is a STATIC heuristic over hook SCRIPT SOURCE. It is deliberately + * LOW PRECISION — it cannot run the script or measure the real payload — so it + * ships as an INFO advisory (weight 0, never severity-bearing), paired with a + * feature-gap "filter-before-Claude-reads" lever. The signal: + * + * the script references `additionalContext` + * AND captures output from a verbose-prone command (cat/find/git log/test/curl…) + * AND applies no truncating/filtering tool anywhere (grep/head/jq/.slice…). + * + * A script that pipes through a filter, or only captures cheap output ($(date)), + * is assumed bounded and is not flagged. Conversely a verbose capture with no + * filter is the un-grepped pattern worth surfacing. + * + * Mechanism verified 2026-06-23 against code.claude.com/docs: + * context-window.md — "A PostToolUse hook … reports back via + * hookSpecificOutput.additionalContext. That field enters Claude's context. + * Plain stdout on exit 0 does not." + tip: keep output concise; it enters + * context without truncation. The remediation lever is the documented + * filter-test-output.sh pattern (grep before Claude reads). + * + * The pure `assessHookAdditionalContext` takes already-read script text so it is + * fully unit-testable without IO. `assessHookContextForRepo` is the thin IO + * wrapper (walk hooks → scripts → assess) shared by feature-gap; the HKV scanner + * calls the pure function inline on scripts it already reads. + */ + +import { readTextFile } from './file-discovery.mjs'; +import { parseJson } from './yaml-parser.mjs'; +import { stat } from 'node:fs/promises'; +import { resolve, dirname } from 'node:path'; + +// The field that actually enters Claude's context (vs. plain stdout / debug log). +const ADDITIONAL_CONTEXT_RX = /additionalContext/; + +// Commands whose UNfiltered output can be large. A capture invoking one of these +// with no filter anywhere is the low-precision "un-grepped output" signal. +// Shell substitution AND node child_process / file reads are both covered. +const VERBOSE_CAPTURE_RX = + /\b(?:cat|find|ls|git\s+(?:log|diff|status|show)|npm|yarn|pnpm|pytest|jest|go\s+test|cargo\s+test|curl|wget|env|printenv|dmesg|journalctl|execSync|spawnSync|readFileSync)\b/; + +// Truncating / filtering tools that BOUND a payload before it reaches context. +// Their presence anywhere in the script suppresses the advisory (assumed bounded). +// Shell filters + the common node-side bounding operations. +const FILTER_RX = + /\b(?:grep|egrep|rg|head|tail|sed|awk|jq|cut|wc|uniq|sort)\b|\.(?:slice|substring|substr)\s*\(/; + +/** + * Assess one hook script's source for unfiltered additionalContext injection. + * + * @param {{ scriptContent?: string }} [args] + * @returns {{ + * buildsAdditionalContext: boolean, + * hasVerboseCapture: boolean, + * hasFilter: boolean, + * capturesUnfiltered: boolean, + * flagged: boolean, + * }} + */ +export function assessHookAdditionalContext({ scriptContent } = {}) { + const content = typeof scriptContent === 'string' ? scriptContent : ''; + const buildsAdditionalContext = ADDITIONAL_CONTEXT_RX.test(content); + const hasVerboseCapture = VERBOSE_CAPTURE_RX.test(content); + const hasFilter = FILTER_RX.test(content); + const capturesUnfiltered = hasVerboseCapture && !hasFilter; + return { + buildsAdditionalContext, + hasVerboseCapture, + hasFilter, + capturesUnfiltered, + flagged: buildsAdditionalContext && capturesUnfiltered, + }; +} + +/** + * Extract a filesystem script path from a hook command string. + * Mirrors hook-validator's extractScriptPath (kept local so the shared lib has + * no upward dependency on a scanner). Handles ${CLAUDE_PLUGIN_ROOT}. + */ +function extractScriptPath(command, baseDir) { + const match = command.match(/(?:bash|node|sh)\s+(.+?)(?:\s|$)/); + if (!match) return null; + let scriptPath = match[1].trim(); + scriptPath = scriptPath.replace(/\$\{CLAUDE_PLUGIN_ROOT\}/g, resolve(baseDir, '..')); + scriptPath = scriptPath.replace(/\$CLAUDE_PLUGIN_ROOT/g, resolve(baseDir, '..')); + if (scriptPath.includes('$')) return null; + return resolve(baseDir, scriptPath); +} + +/** Yield every command-hook { event, command } from a hooks object. */ +function* iterateCommandHooks(hooks) { + if (!hooks || typeof hooks !== 'object' || Array.isArray(hooks)) return; + for (const [event, handlers] of Object.entries(hooks)) { + if (!Array.isArray(handlers)) continue; + for (const group of handlers) { + const hookList = group && Array.isArray(group.hooks) ? group.hooks : []; + for (const hook of hookList) { + if (hook && hook.type === 'command' && typeof hook.command === 'string') { + yield { event, command: hook.command }; + } + } + } + } +} + +/** + * IO wrapper: walk discovered hooks (hooks.json + settings.json hooks), resolve + * each command hook's script, and return the ones flagged by the heuristic. + * Shared by feature-gap (the HKV scanner assesses inline on scripts it reads). + * + * @param {{ files: import('./file-discovery.mjs').ConfigFile[] }} discovery + * @returns {Promise }>>} + */ +export async function assessHookContextForRepo(discovery) { + const flagged = []; + const files = (discovery && Array.isArray(discovery.files)) ? discovery.files : []; + + const hooksObjects = []; + for (const file of files.filter((f) => f.type === 'hooks-json')) { + const content = await readTextFile(file.absPath); + const parsed = content ? parseJson(content) : null; + if (parsed) hooksObjects.push({ hooks: parsed.hooks || parsed, file }); + } + for (const file of files.filter((f) => f.type === 'settings-json')) { + const content = await readTextFile(file.absPath); + const parsed = content ? parseJson(content) : null; + if (parsed && parsed.hooks && !Array.isArray(parsed.hooks)) { + hooksObjects.push({ hooks: parsed.hooks, file }); + } + } + + for (const { hooks, file } of hooksObjects) { + const baseDir = dirname(file.absPath); + for (const { event, command } of iterateCommandHooks(hooks)) { + const scriptPath = extractScriptPath(command, baseDir); + if (!scriptPath) continue; + try { + await stat(scriptPath); + } catch { + continue; // missing script — HKV reports that separately + } + const scriptContent = await readTextFile(scriptPath); + const assessment = assessHookAdditionalContext({ scriptContent }); + if (assessment.flagged) { + flagged.push({ event, scriptPath, file: file.absPath, assessment }); + } + } + } + + return flagged; +} diff --git a/tests/fixtures/hooks-additional-context/hooks/hooks.json b/tests/fixtures/hooks-additional-context/hooks/hooks.json new file mode 100644 index 0000000..8f94a28 --- /dev/null +++ b/tests/fixtures/hooks-additional-context/hooks/hooks.json @@ -0,0 +1,10 @@ +{ + "hooks": { + "SessionStart": [ + { "hooks": [{ "type": "command", "command": "bash ./scripts/chatty.sh", "timeout": 5000 }] } + ], + "PostToolUse": [ + { "matcher": "Bash", "hooks": [{ "type": "command", "command": "bash ./scripts/filtered.sh", "timeout": 5000 }] } + ] + } +} diff --git a/tests/fixtures/hooks-additional-context/hooks/scripts/chatty.sh b/tests/fixtures/hooks-additional-context/hooks/scripts/chatty.sh new file mode 100644 index 0000000..bf15e53 --- /dev/null +++ b/tests/fixtures/hooks-additional-context/hooks/scripts/chatty.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Chatty hook: dumps unfiltered command output straight into additionalContext. +# Every SessionStart this whole payload enters Claude's context (not just stdout). +NOTES="$(cat ./STATE.md)" +LOG="$(git log --oneline)" +printf '{"hookSpecificOutput":{"hookEventName":"SessionStart","additionalContext":"%s\n%s"}}' "$NOTES" "$LOG" diff --git a/tests/fixtures/hooks-additional-context/hooks/scripts/filtered.sh b/tests/fixtures/hooks-additional-context/hooks/scripts/filtered.sh new file mode 100644 index 0000000..97ea9cb --- /dev/null +++ b/tests/fixtures/hooks-additional-context/hooks/scripts/filtered.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Bounded hook: greps the log down to only failures before injecting context. +# This is the filter-before-Claude-reads pattern — should NOT be flagged. +ERRORS="$(cat ./test.log | grep -E 'FAIL|ERROR' | head -20)" +printf '{"hookSpecificOutput":{"hookEventName":"PostToolUse","additionalContext":"%s"}}' "$ERRORS" diff --git a/tests/scanners/feature-gap-scanner.test.mjs b/tests/scanners/feature-gap-scanner.test.mjs index 1e0066a..ed9cf45 100644 --- a/tests/scanners/feature-gap-scanner.test.mjs +++ b/tests/scanners/feature-gap-scanner.test.mjs @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; -import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; +import { scan, opportunitySummary, bundledSkillsLeverFinding, cliOverMcpLeverFinding, filterHookLeverFinding } from '../../scanners/feature-gap-scanner.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; import { withHermeticHome } from '../helpers/hermetic-home.mjs'; @@ -364,3 +364,50 @@ describe('cliOverMcpLeverFinding (CLI-over-MCP lever, v5.10 B4)', () => { assert.match(f.recommendation || '', /\bgh\b|\baws\b|\bgcloud\b/); }); }); + +describe('filterHookLeverFinding (filter-before-Claude-reads lever, v5.10 B5)', () => { + it('returns null when no chatty hook was detected', () => { + assert.equal(filterHookLeverFinding({ flaggedHooks: [] }), null); + assert.equal(filterHookLeverFinding({}), null); + assert.equal(filterHookLeverFinding(), null); + }); + + it('fires an info opportunity when ≥1 chatty hook is detected', () => { + resetCounter(); + const f = filterHookLeverFinding({ + flaggedHooks: [{ event: 'SessionStart', scriptPath: '/x/hooks/scripts/chatty.sh' }], + }); + assert.ok(f, 'expected a lever finding'); + assert.equal(f.severity, 'info'); + assert.equal(f.category, 'token-efficiency'); + assert.match(f.description || '', /filter-test-output\.sh|filter/i); + assert.match(f.evidence || '', /chatty_hooks=1/); + }); +}); + +describe('GAP scanner — filter-before lever wiring (chatty hook fixture)', () => { + it('emits the filter-before lever when scanning a repo with a chatty hook', async () => { + resetCounter(); + const discovery = await fixtureDiscovery('hooks-additional-context'); + const result = await withHermeticHome(() => + scan(resolve(FIXTURES, 'hooks-additional-context'), discovery), + ); + const lever = result.findings.find( + f => f.scanner === 'GAP' && /chatty_hooks=/.test(f.evidence || ''), + ); + assert.ok(lever, `expected filter-before lever; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(lever.severity, 'info'); + }); + + it('does NOT emit the lever for a repo with only quiet hooks', async () => { + resetCounter(); + const discovery = await fixtureDiscovery('hooks-quiet'); + const result = await withHermeticHome(() => + scan(resolve(FIXTURES, 'hooks-quiet'), discovery), + ); + const lever = result.findings.find( + f => f.scanner === 'GAP' && /chatty_hooks=/.test(f.evidence || ''), + ); + assert.equal(lever, undefined, `expected no filter-before lever; got id=${lever?.id}`); + }); +}); diff --git a/tests/scanners/hook-additional-context.test.mjs b/tests/scanners/hook-additional-context.test.mjs new file mode 100644 index 0000000..46a0e05 --- /dev/null +++ b/tests/scanners/hook-additional-context.test.mjs @@ -0,0 +1,90 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { assessHookAdditionalContext } from '../../scanners/lib/hook-additional-context.mjs'; + +// B5 (v5.10) — pure heuristic: does a hook script build +// hookSpecificOutput.additionalContext from un-grepped / verbose command output? +// Low precision by design (ships as an info advisory, never severity-bearing). + +describe('assessHookAdditionalContext — not applicable', () => { + it('empty / missing input → not flagged', () => { + assert.equal(assessHookAdditionalContext({}).flagged, false); + assert.equal(assessHookAdditionalContext({ scriptContent: '' }).flagged, false); + assert.equal(assessHookAdditionalContext().flagged, false); + }); + + it('script with no additionalContext reference → not flagged even if it cats a file', () => { + const a = assessHookAdditionalContext({ + scriptContent: 'CONTENT="$(cat /var/log/big.log)"\necho "$CONTENT"\n', + }); + assert.equal(a.buildsAdditionalContext, false); + assert.equal(a.flagged, false); + }); +}); + +describe('assessHookAdditionalContext — shell hooks', () => { + it('additionalContext built from an unfiltered cat → flagged', () => { + const a = assessHookAdditionalContext({ + scriptContent: + 'BODY="$(cat /tmp/notes.md)"\n' + + 'echo "{\\"hookSpecificOutput\\":{\\"additionalContext\\":\\"$BODY\\"}}"\n', + }); + assert.equal(a.buildsAdditionalContext, true); + assert.equal(a.hasVerboseCapture, true); + assert.equal(a.hasFilter, false); + assert.equal(a.capturesUnfiltered, true); + assert.equal(a.flagged, true); + }); + + it('additionalContext from a git log → flagged', () => { + const a = assessHookAdditionalContext({ + scriptContent: + 'LOG="$(git log --oneline)"\n' + + 'printf \'{"hookSpecificOutput":{"additionalContext":"%s"}}\' "$LOG"\n', + }); + assert.equal(a.flagged, true); + }); + + it('additionalContext from cat piped through grep → NOT flagged (filtered)', () => { + const a = assessHookAdditionalContext({ + scriptContent: + 'BODY="$(cat /tmp/test.log | grep ERROR)"\n' + + 'echo "{\\"hookSpecificOutput\\":{\\"additionalContext\\":\\"$BODY\\"}}"\n', + }); + assert.equal(a.hasFilter, true); + assert.equal(a.capturesUnfiltered, false); + assert.equal(a.flagged, false); + }); + + it('additionalContext from a cheap command only ($(date)) → NOT flagged', () => { + const a = assessHookAdditionalContext({ + scriptContent: + 'NOW="$(date)"\n' + + 'echo "{\\"hookSpecificOutput\\":{\\"additionalContext\\":\\"$NOW\\"}}"\n', + }); + assert.equal(a.hasVerboseCapture, false); + assert.equal(a.flagged, false); + }); +}); + +describe('assessHookAdditionalContext — node hooks', () => { + it('execSync(git log) into additionalContext with no slice → flagged', () => { + const a = assessHookAdditionalContext({ + scriptContent: + "const out = execSync('git log --oneline').toString();\n" + + "process.stdout.write(JSON.stringify({ hookSpecificOutput: { additionalContext: out } }));\n", + }); + assert.equal(a.flagged, true); + }); + + it('readFileSync sliced before additionalContext → NOT flagged (bounded)', () => { + const a = assessHookAdditionalContext({ + scriptContent: + "const raw = readFileSync('big.log', 'utf8');\n" + + "const out = raw.slice(0, 400);\n" + + "process.stdout.write(JSON.stringify({ hookSpecificOutput: { additionalContext: out } }));\n", + }); + assert.equal(a.hasFilter, true); + assert.equal(a.flagged, false); + }); +}); diff --git a/tests/scanners/hook-validator.test.mjs b/tests/scanners/hook-validator.test.mjs index 125cc87..8c8f73a 100644 --- a/tests/scanners/hook-validator.test.mjs +++ b/tests/scanners/hook-validator.test.mjs @@ -101,6 +101,46 @@ describe('HKV scanner — verbose hook output (v5 M5)', () => { }); }); +describe('HKV scanner — additionalContext injection advisory (v5.10 B5)', () => { + it('flags a hook that injects unfiltered command output into additionalContext (info advisory)', async () => { + resetCounter(); + const path = resolve(FIXTURES, 'hooks-additional-context'); + const discovery = await discoverConfigFiles(path); + const result = await scan(path, discovery); + const f = result.findings.find( + x => x.scanner === 'HKV' && /additional_context_unfiltered=true/.test(x.evidence || ''), + ); + assert.ok(f, `expected additionalContext advisory; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'info', `expected info (advisory), got ${f.severity}`); + // The chatty.sh script (SessionStart) is the one flagged, not filtered.sh. + assert.match(f.file || '', /chatty\.sh$/); + // Precision caveat must be disclosed in the description (low-precision advisory). + assert.match(f.description || '', /every time|enters Claude's context|advisory/i); + }); + + it('does NOT flag the filtered (grep|head) sibling hook', async () => { + resetCounter(); + const path = resolve(FIXTURES, 'hooks-additional-context'); + const discovery = await discoverConfigFiles(path); + const result = await scan(path, discovery); + const filtered = result.findings.find( + x => x.scanner === 'HKV' && /additional_context_unfiltered=true/.test(x.evidence || '') && /filtered\.sh$/.test(x.file || ''), + ); + assert.equal(filtered, undefined, `filtered.sh should not be flagged; got id=${filtered?.id}`); + }); + + it('does NOT flag a quiet hook with no additionalContext', async () => { + resetCounter(); + const path = resolve(FIXTURES, 'hooks-quiet'); + const discovery = await discoverConfigFiles(path); + const result = await scan(path, discovery); + const f = result.findings.find( + x => x.scanner === 'HKV' && /additional_context_unfiltered=true/.test(x.evidence || ''), + ); + assert.equal(f, undefined, `expected no additionalContext advisory; got id=${f?.id}`); + }); +}); + describe('HKV scanner — CC 2.1.152 MessageDisplay event (Batch 1 false-positive fix)', () => { // The pre-write path-guard blocks committing settings.json/hooks.json, so // this suite materializes a hermetic temp fixture at runtime. From fa1ddd963a246fbc7bcf416b405b89f6beec46d6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 20:45:12 +0200 Subject: [PATCH 21/53] feat(cps): scan @imported files for volatile cached-prefix content (v5.10 B6) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPS originally inspected only files discovery classifies as claude-md, but a CLAUDE.md can pull arbitrary files into the cached prefix via @import — and those targets (e.g. @shared/conventions.md) are usually not claude-md in discovery, so their inlined content was never scanned. Neither TOK Pattern A (top-30 of cascade files) nor the in-file CPS scan reaches past the importing file, so volatility inside an imported file was invisible. B6 closes the gap: for each @import whose import site sits within the cached-prefix window (imp.line <= CACHED_PREFIX_LINES), CPS resolves the path (resolveImportPath, mirroring import-resolver/token-hotspots semantics), reads the target, and runs findVolatileLines over its first 150 lines. A hit emits a distinct medium finding — "Volatile content in @imported file breaks cached prefix" — keyed on the resolved file, evidence naming the importer. Scope boundaries (deliberate): - One hop only; imports-of-imports stay with IMP (deep-chain owner). - No lines-1-30 skip for imported content — that exclusion is root-file-specific to avoid Pattern A overlap, which never reaches imported files. - No double-reporting: an import resolving to a discovered claude-md is skipped (own iteration); a reportedImports set dedupes a target imported by several CLAUDE.md files. Dropped from B6 (per plan verdict): confident behavioral cache-buster detection (opusplan/model-switch is runtime, not static config) and jq-transcript automation. "No overstated behavioral finding ships" — even the permitted opusplan info-advisory was left out; the @import extension is the whole of B6. Byte-stability: the in-file finding keeps the same condition + byte-identical evidence/description (continue-skip refactored to if-emit, behaviour-preserving); new findings fire only on a volatile import, which no frozen v5.0.0 fixture has. docs: README + scanner-internals CPS rows + full B6 note; CLAUDE.md kept lean ([skip-docs]). Suite 1254 -> 1257 green; snapshots + SC-5 untouched. Version/badges/CHANGELOG wait for the v5.10 release cut. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 2 +- docs/scanner-internals.md | 37 +++++- scanners/cache-prefix-scanner.mjs | 117 ++++++++++++++---- .../fixtures/import-volatile/clean/CLAUDE.md | 7 ++ .../import-volatile/clean/shared/stable.md | 5 + .../import-volatile/positive/CLAUDE.md | 8 ++ .../positive/shared/conventions.md | 8 ++ tests/scanners/cache-prefix.test.mjs | 25 ++++ 8 files changed, 183 insertions(+), 26 deletions(-) create mode 100644 tests/fixtures/import-volatile/clean/CLAUDE.md create mode 100644 tests/fixtures/import-volatile/clean/shared/stable.md create mode 100644 tests/fixtures/import-volatile/positive/CLAUDE.md create mode 100644 tests/fixtures/import-volatile/positive/shared/conventions.md diff --git a/README.md b/README.md index 01f3b68..e60bfd9 100644 --- a/README.md +++ b/README.md @@ -323,7 +323,7 @@ By default, `/config-audit` auto-detects scope from your git context. Override w | `conflict-detector.mjs` | CNF | Settings contradictions across scopes, permission conflicts, hook duplicates | | `feature-gap-scanner.mjs` | GAP | 25 feature checks shown as opportunities, not grades — plus a conditional `disableBundledSkills` recommendation when the active skill listing is over budget, and a conditional **filter-before-Claude-reads** lever when a hook injects unfiltered output into `additionalContext` (companion to the HKV advisory; cites the documented `filter-test-output.sh` pattern) | | `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascades, bloated skill descriptions, MCP tool-schema budget, and stale `~/.claude/plugins/cache` versions (disk-cleanup, zero live-context impact) — cache-aware ranking excludes superseded plugin versions by default (`--no-exclude-cache` to include) | -| `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of the CLAUDE.md cascade — beyond the cache-prefix window but still re-loaded every turn | +| `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of the CLAUDE.md cascade — beyond Pattern A's top-30 window but still re-loaded every turn — **plus** volatile content inside `@import`-ed files (inlined into the cached prefix, one hop, otherwise invisible to per-file scans) | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries: (1) tools in BOTH `permissions.deny` and `permissions.allow` — deny wins (incl. the `Tool(*)` deny-all glob, equivalent to a bare deny); (2) unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — valid only as `mcp____*`; (3) `Tool(param:value)` rules whose key is the tool's own canonicalizing field (`command`/`file_path`/`path`/`notebook_path`/`url`) — CC ignores these and emits a startup warning | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions; user-vs-plugin overlaps | | `skill-listing-scanner.mjs` | SKL | Skill-listing token budget: a single skill description over the ~1,536-char listing cap Claude Code truncates (`CA-SKL-001`), and the summed active-skill descriptions exceeding the ~2%-of-context listing budget (`CA-SKL-002`) | diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index a1a5893..befa00d 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -19,7 +19,7 @@ Scanner CLI: `node scanners/scan-orchestrator.mjs [--global] [--full-mach | `conflict-detector.mjs` | CNF | Settings conflicts, permission contradictions, hook duplicates | | `feature-gap-scanner.mjs` | GAP | 25 feature checks across 4 tiers — shown as opportunities, not grades | | `token-hotspots.mjs` | TOK | Cache-breaking volatile content, redundant tool permissions, deep import chains, oversized cascade, bloated SKILL.md descriptions, MCP tool-schema budget, MCP tool-schema deferral (CA-TOK-006), stale plugin-cache disk-cleanup (prompt-cache patterns) | -| `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of CLAUDE.md cascade (beyond Pattern A's top-30 window) | +| `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of CLAUDE.md cascade (beyond Pattern A's top-30 window); plus volatile content inside `@import`-ed files (v5.10 B6, one hop) | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries (low). (1) Tools in BOTH `permissions.deny` AND `permissions.allow` — deny wins; dominance is param-aware and treats the `Tool(*)` deny-all glob as equivalent to a bare deny (covers a bare allow). (2) Unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — CC accepts allow globs only after a literal glob-free `mcp____` prefix. Predicates shared with CNF live in `lib/permission-rules.mjs` | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions (low); user-vs-plugin overlaps (medium); `details.namespaces` payload | | `skill-listing-scanner.mjs` | SKL | (1) `CA-SKL-001` (medium): active skill descriptions over the verified 1,536-char listing cap (CC 2.1.105) → silently truncated in the model's skill listing. (2) `CA-SKL-002` (low): sum of active descriptions (each counted up to the cap) over the listing budget (~2% of context, CC 2.1.32), anchored on a conservative 200k window with a calibration note that the budget scales 5× on 1M-context models — leads with the measured sum, an estimate not telemetry. HOME-scoped (all user + plugin skills). Remediation surfaces `disableBundledSkills` / `skillOverrides` / trim. Distinct lens from TOK pattern F (project-local 500-char bloat heuristic) | @@ -219,6 +219,41 @@ returns ≥1 chatty hook — surfaces the documented **filter-before-Claude-read grep ERROR and return only matches instead of a 10,000-line log). No chatty hook → silent (opportunity, not noise — same contract as the cliOverMcp / bundledSkills levers). +### cache-prefix-scanner — @import extension (v5.10 B6) + +CPS originally scanned only the files discovery classifies as `claude-md`. But a CLAUDE.md can pull +arbitrary files into context with `@import` directives, and those targets are usually *not* `claude-md` +in discovery (e.g. `@shared/conventions.md`) — so their content was inlined into the cached prefix yet +never inspected. Neither TOK Pattern A (top-30 of cascade files) nor the in-file CPS scan reaches past +the importing file, so volatility in an imported file was invisible. + +B6 closes the gap: for each `@import` whose **import site** sits within the cached-prefix window +(`imp.line ≤ CACHED_PREFIX_LINES`), CPS resolves the path (`resolveImportPath`, mirroring +import-resolver/token-hotspots semantics), reads the target, and runs `findVolatileLines` over its first +150 lines. A hit emits a distinct medium finding — *"Volatile content in @imported file breaks cached +prefix"* — keyed on the resolved file (so the fix points at the right place), with evidence naming the +importer (`imported by (@ at line N)`). + +**Scope boundaries (deliberate):** +- **One hop only.** Imports-of-imports are not followed — IMP owns deep-chain analysis. The verified win + is the direct import; transitive resolution adds cycle/depth complexity for marginal coverage. +- **No lines-1–30 skip for imported content.** That exclusion exists only to avoid duplicating TOK + Pattern A's territory in the *root* file; Pattern A never reads imported files, so all of the imported + prefix counts. +- **No double-reporting.** An import resolving to a file that is itself a discovered `claude-md` is + skipped (it gets its own in-file iteration); a `reportedImports` set dedupes a target imported by + several CLAUDE.md files. + +**Byte-stability.** The in-file finding is emitted under exactly the same condition and with byte-identical +evidence/description as before (the `continue`-skip was refactored to an `if`-emit — behaviour-preserving). +New findings fire only when a discovered CLAUDE.md imports a volatile file, which no frozen v5.0.0 fixture +does — snapshots and SC-5 verified untouched by the full suite. + +**Dropped from B6 (per plan `verdict`).** Confident behavioral cache-buster detection (opusplan / +model-switch is a *runtime* behaviour, not static config a scanner can reliably flag) and jq-transcript +automation. "No overstated behavioral finding ships" — so even the permitted opusplan *info*-advisory was +left out; the verified @import extension is the whole of B6. + ### CML scanner — context-window-scaled char budget Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors diff --git a/scanners/cache-prefix-scanner.mjs b/scanners/cache-prefix-scanner.mjs index ff42208..62895ea 100644 --- a/scanners/cache-prefix-scanner.mjs +++ b/scanners/cache-prefix-scanner.mjs @@ -15,9 +15,12 @@ * Zero external dependencies. */ +import { resolve, dirname } from 'node:path'; +import { tmpdir } from 'node:os'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; +import { findImports } from './lib/yaml-parser.mjs'; const SCANNER = 'CPS'; @@ -41,6 +44,22 @@ const VOLATILE_PATTERNS = [ { rx: /\$\{[A-Z_][A-Z0-9_]*\}/, label: '${VAR} substitution' }, ]; +/** + * Resolve an @import path relative to the file that declares it. + * Mirrors import-resolver.mjs / token-hotspots.mjs path semantics. + * @param {string} importPath + * @param {string} containingFile + * @returns {string} absolute resolved path + */ +function resolveImportPath(importPath, containingFile) { + if (importPath.startsWith('~')) { + const home = process.env.HOME || process.env.USERPROFILE || tmpdir(); + return resolve(importPath.replace(/^~/, home)); + } + if (importPath.startsWith('/')) return importPath; + return resolve(dirname(containingFile), importPath); +} + /** * Scan content for volatile lines within the cached prefix window. * Returns array of {line, label, snippet}. @@ -75,40 +94,90 @@ export async function scan(targetPath, discovery) { const findings = []; let filesScanned = 0; + // Files already scanned in-file below — an @import resolving to one of these + // is reported by its own iteration, not duplicated as an import finding. + const discoveredClaudeMd = new Set( + discovery.files.filter(f => f.type === 'claude-md').map(f => f.absPath)); + // @imported files reported once, even when several CLAUDE.md files import them. + const reportedImports = new Set(); + for (const f of discovery.files) { if (f.type !== 'claude-md') continue; filesScanned++; const content = await readTextFile(f.absPath); if (!content) continue; - const volatile = findVolatileLines(content); - if (volatile.length === 0) continue; + // --- In-file volatility (unchanged behavior) --- + const volatile = findVolatileLines(content); // Skip volatility that's already covered by TOK Pattern A (lines 1–30) — // CPS' value is in the 31–150 range. Pattern A handles 1–30. const beyondTopThirty = volatile.filter(v => v.line > 30); - if (beyondTopThirty.length === 0) continue; + if (beyondTopThirty.length > 0) { + const evidence = + beyondTopThirty.slice(0, 5) + .map(v => `line ${v.line} (${v.label}): ${v.snippet}`) + .join('; '); + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.medium, + title: 'Volatile content inside cached prefix breaks reuse', + description: + `${f.relPath || f.absPath} contains ${beyondTopThirty.length} volatile ` + + `entr${beyondTopThirty.length === 1 ? 'y' : 'ies'} between lines 31 and ` + + `${CACHED_PREFIX_LINES}. The prompt cache covers the file's prefix; ` + + 'any volatility forces a fresh cache write from that line down on every turn.', + file: f.absPath, + evidence, + recommendation: + 'Move volatile sections (timestamps, !shell-exec, ${VAR} substitutions, dated logs) ' + + `below line ${CACHED_PREFIX_LINES} or extract them to an @import-ed file outside the ` + + 'cached prefix. Stable content above, volatile content below.', + category: 'token-efficiency', + })); + } - const evidence = - beyondTopThirty.slice(0, 5) - .map(v => `line ${v.line} (${v.label}): ${v.snippet}`) - .join('; '); - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.medium, - title: 'Volatile content inside cached prefix breaks reuse', - description: - `${f.relPath || f.absPath} contains ${beyondTopThirty.length} volatile ` + - `entr${beyondTopThirty.length === 1 ? 'y' : 'ies'} between lines 31 and ` + - `${CACHED_PREFIX_LINES}. The prompt cache covers the file's prefix; ` + - 'any volatility forces a fresh cache write from that line down on every turn.', - file: f.absPath, - evidence, - recommendation: - 'Move volatile sections (timestamps, !shell-exec, ${VAR} substitutions, dated logs) ' + - `below line ${CACHED_PREFIX_LINES} or extract them to an @import-ed file outside the ` + - 'cached prefix. Stable content above, volatile content below.', - category: 'token-efficiency', - })); + // --- v5.10 B6: volatility inside @imported files --- + // @import-ed content is inlined into the cached prefix at the import site. + // TOK Pattern A and the in-file scan above never look past the importing + // file, so volatility in an imported file is otherwise invisible. We scan + // direct imports only (one hop); IMP owns deep-chain analysis. The whole + // imported-file prefix counts (no lines-1–30 skip — that exclusion is + // root-file-specific to avoid Pattern A overlap, which does not reach here). + for (const imp of findImports(content)) { + if (imp.line > CACHED_PREFIX_LINES) continue; // import site outside prefix + const resolved = resolveImportPath(imp.path, f.absPath); + if (discoveredClaudeMd.has(resolved)) continue; // scanned in its own iteration + if (reportedImports.has(resolved)) continue; + reportedImports.add(resolved); + const importedContent = await readTextFile(resolved); + if (!importedContent) continue; + const importedVolatile = findVolatileLines(importedContent); + if (importedVolatile.length === 0) continue; + + const importEvidence = + `imported by ${f.relPath || f.absPath} (@${imp.path} at line ${imp.line}); ` + + importedVolatile.slice(0, 5) + .map(v => `line ${v.line} (${v.label}): ${v.snippet}`) + .join('; '); + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.medium, + title: 'Volatile content in @imported file breaks cached prefix', + description: + `@${imp.path} (imported by ${f.relPath || f.absPath} at line ${imp.line}) contains ` + + `${importedVolatile.length} volatile entr${importedVolatile.length === 1 ? 'y' : 'ies'} ` + + `within its first ${CACHED_PREFIX_LINES} lines. @import-ed content is inlined into the ` + + 'prompt-cache prefix, so volatility there forces a fresh cache write every turn — even ' + + 'when the importing CLAUDE.md is itself byte-stable.', + file: resolved, + evidence: importEvidence, + recommendation: + 'Move volatile content (timestamps, !shell-exec, ${VAR} substitutions, dated logs) out ' + + 'of the @imported file, or import it below the cached-prefix window. Keep imported config ' + + 'byte-stable so the importing file\'s cache survives.', + category: 'token-efficiency', + })); + } } return scannerResult(SCANNER, 'ok', findings, filesScanned, Date.now() - start); diff --git a/tests/fixtures/import-volatile/clean/CLAUDE.md b/tests/fixtures/import-volatile/clean/CLAUDE.md new file mode 100644 index 0000000..d9be4e5 --- /dev/null +++ b/tests/fixtures/import-volatile/clean/CLAUDE.md @@ -0,0 +1,7 @@ +# Clean Import Project + +Stable preamble. + +@shared/stable.md + +All content is stable. diff --git a/tests/fixtures/import-volatile/clean/shared/stable.md b/tests/fixtures/import-volatile/clean/shared/stable.md new file mode 100644 index 0000000..5b3621c --- /dev/null +++ b/tests/fixtures/import-volatile/clean/shared/stable.md @@ -0,0 +1,5 @@ +# Stable Conventions + +Fully stable content. +No timestamps, no shell-exec, no variable substitutions. +Just plain prose that is identical on every turn. diff --git a/tests/fixtures/import-volatile/positive/CLAUDE.md b/tests/fixtures/import-volatile/positive/CLAUDE.md new file mode 100644 index 0000000..739c4f2 --- /dev/null +++ b/tests/fixtures/import-volatile/positive/CLAUDE.md @@ -0,0 +1,8 @@ +# Imported Volatile Project + +Stable preamble. No volatile content lives in this file. + +@shared/conventions.md + +This importing file is byte-stable on its own. +Nothing below changes between turns. diff --git a/tests/fixtures/import-volatile/positive/shared/conventions.md b/tests/fixtures/import-volatile/positive/shared/conventions.md new file mode 100644 index 0000000..4a13156 --- /dev/null +++ b/tests/fixtures/import-volatile/positive/shared/conventions.md @@ -0,0 +1,8 @@ +# Conventions + +These conventions are mostly stable prose. + +Last build: ${BUILD_TIMESTAMP} +!git log -1 --format=%cd + +More conventions text that does not change. diff --git a/tests/scanners/cache-prefix.test.mjs b/tests/scanners/cache-prefix.test.mjs index 4e719fa..aecea00 100644 --- a/tests/scanners/cache-prefix.test.mjs +++ b/tests/scanners/cache-prefix.test.mjs @@ -61,6 +61,31 @@ describe('CPS scanner — does not duplicate TOK Pattern A territory', () => { }); }); +describe('CPS scanner — volatile content in @imported files (v5.10 B6)', () => { + it('flags volatile content inside an @imported file even when the root file is stable', async () => { + const result = await runScanner('import-volatile/positive'); + const f = result.findings.find(x => /volatile content in @imported file/i.test(x.title || '')); + assert.ok(f, `expected @import-volatile finding; got: ${result.findings.map(x => x.title).join(' | ')}`); + assert.equal(f.severity, 'medium', `expected medium, got ${f.severity}`); + assert.match(String(f.evidence || ''), /imported by/i); + assert.match(String(f.evidence || ''), /BUILD_TIMESTAMP|git log|VAR substitution|shell-exec/i); + assert.equal(f.category, 'token-efficiency'); + }); + + it('does NOT emit the in-file CPS finding when the importing file itself is stable', async () => { + // Root CLAUDE.md carries no volatile lines; only the @imported file does. + const result = await runScanner('import-volatile/positive'); + const inFile = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.equal(inFile, undefined, 'a byte-stable importing file must not trip the in-file finding'); + }); + + it('does NOT flag a clean @imported file', async () => { + const result = await runScanner('import-volatile/clean'); + const f = result.findings.find(x => /volatile content in @imported file/i.test(x.title || '')); + assert.equal(f, undefined, `expected no finding for a clean import; got: ${f?.title}`); + }); +}); + describe('CPS scanner — orchestrator wiring', () => { it('CPS appears in scan-orchestrator scanner list', async () => { const orch = await import('../../scanners/scan-orchestrator.mjs'); From c2e3a56a201088b3a147356b1767896c77a70c6d Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 21:02:14 +0200 Subject: [PATCH 22/53] =?UTF-8?q?release:=20v5.10.0=20=E2=80=94=20"Deferra?= =?UTF-8?q?l=20&=20injection=20hygiene"=20(B4+B5+B6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three additive hardening levers extending existing scanners toward a tighter always-loaded prefix (scanner count stays 16, agents 7, commands 21): - B4 (CA-TOK-006): MCP tool-schema deferral check + CLI-over-MCP lever - B5: hook additionalContext-injection advisory + filter-before lever - B6: CPS @import volatile-content scan Version sync: plugin.json 5.10.0, README version/tests badges (1215+ -> 1257+) + version-history row, tokens pattern-count 7 -> 8 (README + CLAUDE.md), CLAUDE.md test tally 1215/68 -> 1257/71 (36 -> 39 scanner files), CHANGELOG [5.10.0]. self-audit --check-readme green; suite 1257 pass / 0 fail. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 4 ++-- README.md | 7 ++++--- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 132f2bd..c886b5b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.9.0", + "version": "5.10.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index ac95a4c..5b428a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,48 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.10.0] - 2026-06-23 + +### Summary +"Deferral & injection hygiene" — three additive hardening levers that extend existing scanners +toward a tighter always-loaded prefix. **B4** detects config that forces full MCP tool schemas into +the always-loaded prefix (deferral defeated), with a CLI-over-MCP companion lever. **B5** adds a +hook `additionalContext`-injection advisory plus a filter-before-Claude-reads lever. **B6** extends +the cache-prefix scanner to follow `@import`s and flag volatile content in imported files. Scanner +count stays **16** (all three extend existing scanners), agents **7**, commands **21**; +`--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1257** tests. + +### Added +- **B4 — MCP tool-schema deferral (`CA-TOK-006`; tokens patterns 7 → 8).** By default Claude Code + defers MCP tool schemas (names-only, ~120 tok; full schemas load on demand via tool search). + `CA-TOK-006` detects config-file signals that force the FULL schemas into the always-loaded prefix + every turn: `settings.json` `env.ENABLE_TOOL_SEARCH="false"` (high), `"ToolSearch"` in + `permissions.deny` (high), a configured Haiku model (medium), and per-server `.mcp.json` + `alwaysLoad:true` (CC v2.1.121+, high). Severity scales with the aggregate forced-upfront tokens + (medium-confidence reasons cap at medium). New pure engine `lib/mcp-deferral.mjs` + (`assessMcpDeferral`, unit-tested, no IO) shared by TOK and GAP. A feature-gap **CLI-over-MCP** + lever fires only as a companion to `CA-TOK-006` (prefer `gh`/`aws`/`gcloud` over MCP for common + operations — CLI adds zero context tokens until used). + - **Honest scoping (Verifiseringsplikt):** triggers on config files ONLY, never `process.env` + shell vars. Vertex / custom `ANTHROPIC_BASE_URL` / a runtime `/model` switch are launch state + (would flap snapshots machine-dependently), so they are DISCLOSED in every finding, not + triggered. Mechanism verified 2026-06-23 against code.claude.com/docs (`context-window.md`, + `mcp.md#configure-tool-search` + `#exempt-a-server-from-deferral`, `costs.md`); the + prefix-cache-invalidation claim was NOT-CONFIRMED in docs and is not asserted. +- **B5 — hook `additionalContext`-injection advisory + filter-before lever.** HKV emits an info + advisory when a hook injects unfiltered command output into `additionalContext` (it enters context + every turn the hook fires). A feature-gap **filter-before-Claude-reads** companion lever cites the + documented `filter-test-output.sh` pattern (filter at the hook, not after Claude reads). +- **B6 — CPS `@import` volatile scan.** The cache-prefix scanner now follows `@import`s (one hop) and + flags volatile content in the imported file that breaks the cached prefix — a new medium finding + ("Volatile content in @imported file breaks cached prefix"), keyed on the resolved file with + evidence "imported by (@ at line N)". Scoped to one hop (the IMP scanner owns deep + chains); resolved files that are themselves discovered CLAUDE.md are skipped (own iteration). + +### Notes +- Scanner count unchanged at **16** — B4/B5/B6 all extend existing scanners (TOK / HKV + GAP / CPS). + `--json`/`--raw` output remains byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. + ## [5.9.0] - 2026-06-23 ### Summary diff --git a/CLAUDE.md b/CLAUDE.md index c86d5b3..8c8c0ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,7 @@ Analyzes and optimizes Claude Code configuration across three pillars: |---------|-------------| | `/config-audit` | Full audit with auto-scope detection (no setup needed) | | `/config-audit posture` | Quick health scorecard (A-F grades, 10 quality areas incl. Token Efficiency, Plugin Hygiene) | -| `/config-audit tokens` | prompt-cache-aware token hotspots (7 patterns: cache-breaking, redundant perms, deep imports, oversized cascade, bloated SKILL.md desc, MCP tool-schema budget, stale plugin-cache disk-cleanup), each ranked hotspot tagged with its load pattern (always / on-demand / external) — **cache-aware** (stale `~/.claude/plugins/cache` versions excluded by default; only each plugin's active version counts; `--no-exclude-cache` for the full walk), optional `--accurate-tokens` API calibration, `--with-telemetry-recipe` cache-hit recipe pointer | +| `/config-audit tokens` | prompt-cache-aware token hotspots (8 patterns: cache-breaking, redundant perms, deep imports, oversized cascade, bloated SKILL.md desc, MCP tool-schema budget, MCP tool-schema deferral, stale plugin-cache disk-cleanup), each ranked hotspot tagged with its load pattern (always / on-demand / external) — **cache-aware** (stale `~/.claude/plugins/cache` versions excluded by default; only each plugin's active version counts; `--no-exclude-cache` for the full walk), optional `--accurate-tokens` API calibration, `--with-telemetry-recipe` cache-hit recipe pointer | | `/config-audit manifest` | Ranked table of every token source (CLAUDE.md, rules, agents, skills, output styles, MCP, hooks) sorted by estimated tokens, each tagged with its load pattern (always-loaded / on-demand / external) + an always-loaded subtotal ("tokens that enter context every turn") | | `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact (incl. a conditional `disableBundledSkills` lever when the active skill listing is over budget — remediation companion to SKL `CA-SKL-002`) | | `/config-audit optimize` | Optimization lens (mechanism-fit) — config that works but fits a better mechanism: procedure→skill (CA-OPT-001, deterministic), lifecycle→hook / unscoped path→rule / "never"→permission (prose-judgment via opus `optimization-lens-agent`). Hybrid motor; every finding cites a best-practices-register rule. Agent-driven, **not byte-stable** | @@ -123,7 +123,7 @@ Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{S node --test 'tests/**/*.test.mjs' ``` -1215 tests across 68 test files (22 lib + 36 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1257 tests across 71 test files (22 lib + 39 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. diff --git a/README.md b/README.md index e60bfd9..1951ec4 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.9.0-blue) +![Version](https://img.shields.io/badge/version-5.10.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1215+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1257+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -275,7 +275,7 @@ Your team configuration changes over time. Track it: |---------|-------------| | `/config-audit` | Full audit with auto-scope detection (no setup needed) | | `/config-audit posture` | Quick health scorecard: A-F grades across 10 quality areas (incl. Token Efficiency, Plugin Hygiene) | -| `/config-audit tokens` | prompt-cache-aware token hotspots — ranked by estimated waste, each tagged with its load pattern (always / on-demand / external); 7 patterns + optional `--accurate-tokens` API calibration. **Cache-aware:** stale `~/.claude/plugins/cache` versions (superseded installs that load on zero turns) are excluded from the ranking by default — only each plugin's active version is counted; `--no-exclude-cache` restores the full walk. Stale versions surface as a separate **Dead config** disk-cleanup finding | +| `/config-audit tokens` | prompt-cache-aware token hotspots — ranked by estimated waste, each tagged with its load pattern (always / on-demand / external); 8 patterns + optional `--accurate-tokens` API calibration. **Cache-aware:** stale `~/.claude/plugins/cache` versions (superseded installs that load on zero turns) are excluded from the ranking by default — only each plugin's active version is counted; `--no-exclude-cache` restores the full walk. Stale versions surface as a separate **Dead config** disk-cleanup finding | | `/config-audit manifest` | Ranked table of every token source (CLAUDE.md, rules, agents, skills, output styles, MCP, hooks) sorted by estimated tokens — each tagged with its **load pattern** (always-loaded / on-demand / external) plus an **always-loaded subtotal** ("≈X tokens enter context every turn before you type"). Component-level: no coarse plugin roll-up (it would double-count) | | `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact | | `/config-audit optimize` | Optimization lens (mechanism-fit): config that works but fits a better mechanism — procedure→skill, lifecycle→hook, unscoped path→rule, "never"→permission. Hybrid motor (deterministic pre-filter + opus precision gate), every finding cites a best-practices-register rule | @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.10.0** | 2026-06-23 | "Deferral & injection hygiene" — three additive hardening levers that extend existing scanners toward a tighter always-loaded prefix (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B4 — MCP tool-schema deferral (`CA-TOK-006`; tokens patterns 7→8):** Claude Code defers MCP tool schemas (names-only, ~120 tok; full schemas load on demand) by default, so `CA-TOK-006` detects config-file signals that force the FULL schemas into the always-loaded prefix every turn — `env.ENABLE_TOOL_SEARCH="false"` (high), a `"ToolSearch"` deny (high), a configured Haiku model (medium), or per-server `alwaysLoad:true` (CC v2.1.121+, high); severity scales with the aggregate forced-upfront tokens. New pure engine `lib/mcp-deferral.mjs` shared by TOK + GAP, plus a feature-gap **CLI-over-MCP** companion lever (prefer `gh`/`aws`/`gcloud`). Triggers on config files ONLY — Vertex / custom `ANTHROPIC_BASE_URL` / a runtime `/model` switch are launch state and are disclosed, never triggered; the prefix-cache-invalidation claim was NOT-CONFIRMED in docs and is not asserted. **B5 — hook `additionalContext` advisory + filter-before lever:** HKV emits an info advisory when a hook injects unfiltered output into `additionalContext`, with a feature-gap **filter-before-Claude-reads** companion citing the documented `filter-test-output.sh` pattern. **B6 — CPS `@import` volatile scan:** the cache-prefix scanner now follows `@import`s (one hop) and flags volatile content in the imported file that breaks the cached prefix — a new medium finding, keyed on the resolved file. 1257 tests | | **5.9.0** | 2026-06-23 | "Machine-wide token lens" — the three highest-impact hardening gaps toward whole-machine token tuning. **B1 — agent-listing budget (new orchestrated scanner AGT, count 15→16):** the always-loaded agent listing (name+description re-sent every turn) is now measured — `CA-AGT-001` per-agent description bloat (advisory), `CA-AGT-002` aggregate listing over budget; both LOW and explicitly **inferred / upper-bound** (the mechanism is undocumented — the evidence discloses it rather than overstating). **B2 — machine-wide always-loaded token roll-up:** the campaign ledger now carries a token bill — `campaign refresh-tokens` does a live cross-repo sweep that counts the **shared global always-loaded layer once** + per-repo deltas, with a ranked "most expensive repos" table (the `whats-active` double-count, avoided by construction). **B3 — cache-aware filtering (folds in B0):** `~/.claude/plugins/cache` holds *both* active and stale plugin versions (installPaths point INTO it), so token-hotspots + CNF are now **version-aware** — `--exclude-cache` (default ON) keeps each plugin's active version and drops only stale ones (`installed_plugins.json`-driven), so stale versions stop polluting the hotspot ranking and inflating duplicate-hook conflicts; stale versions surface as a separate **Dead config** disk-cleanup finding (zero live-context impact). `--json`/`--raw` byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. 1215 tests | | **5.8.0** | 2026-06-23 | "Campaign motor" — a durable, machine-wide audit **campaign** that sits ABOVE individual sessions (one repo = one session; a fleet of repos = a campaign). **Ledger:** `~/.claude/config-audit/campaign-ledger.json` (outside the plugin dir → survives uninstall/upgrade) tracks a repo list + per-repo lifecycle (pending→audited→planned→implemented) + a machine-wide roll-up by status & severity; pure transforms with injected `now`. **`/config-audit campaign` (commands 20→21):** read-only report (`campaign-cli`) + human-approved writes (`campaign-write-cli`: init / add / set-status) — reports first, mutates only on explicit approval, never hand-edits the ledger. **Cross-repo backlog:** one severity-weighted prioritized pick-list (`buildBacklog`, `critical:1000/high:100/medium:10/low:1`). **Plan export + execution-by-reuse:** `campaign-export-cli --write` drops a planned repo's plan verbatim into its own `docs/`; execution reuses the existing `/config-audit implement` + `rollback` (no new execution machinery). All campaign code is `-cli`/lib → scanner count stays **15**, agents **7**, byte-stable. Plus pre-release cleanup: `knowledge-refresh` wired into the router + help; CLAUDE.md trimmed 540→134 lines (impl notes → `docs/scanner-internals.md`, config grade B→A). 1168 tests | | **5.7.0** | 2026-06-21 | "Optimization lens" — first detector of the «optimally shaped?» axis (vs «correct?»), plus a living knowledge layer. **Register:** `knowledge/best-practices.json`, a provenance-stamped, schema-validated best-practices register (first runtime-consumed `knowledge/` file). **OPT scanner (count 14→15):** `CA-OPT-001` (LOW) a ≥6-step CLAUDE.md procedure that would fit better as a skill, citing register entry `BP-MECH-003`. **`/config-audit optimize` + `optimization-lens-agent` (opus, agents 6→7):** prose-judgment lens for lifecycle→hook (`BP-MECH-001`), unscoped path→rule (`BP-MECH-002`), "never"→permission (`BP-MECH-004`); pre-filter recall + opus precision gate. **`/config-audit knowledge-refresh` (commands 19→20):** deterministic stale-check (injected reference date, 90-day cadence) + web re-verify/poll, human-approved writes only. Last two are agent/web-driven (not byte-stable). 1091 tests | From 27988801be07054db07e3b42d69d9cf96281ce7e Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 21:36:20 +0200 Subject: [PATCH 23/53] feat(skl): flag oversized skill bodies on demand (v5.11 B7) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New CA-SKL-003 (low): a SKILL.md body over ~5,000 tokens (~500 lines) should split reference content into supporting files / use context: fork. - measureActiveSkillListing() now returns body metrics (chars/lines/tokens); the body was already read in full, only the frontmatter was parsed before. - Honest framing: BODY_CALIBRATION_NOTE marks this as ON-DEMAND cost (loads only when the skill is invoked, NOT every turn like the always-loaded listing) and an estimate — hence low severity. - 5 new tests; full suite 1262 green; snapshots byte-stable (default branch untouched; new finding fires only on bodies >5k tok, none in fixtures). Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/lib/skill-listing-budget.mjs | 24 ++++- scanners/skill-listing-scanner.mjs | 35 +++++++ tests/scanners/skill-listing-scanner.test.mjs | 91 +++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) diff --git a/scanners/lib/skill-listing-budget.mjs b/scanners/lib/skill-listing-budget.mjs index 4afed8d..d68682e 100644 --- a/scanners/lib/skill-listing-budget.mjs +++ b/scanners/lib/skill-listing-budget.mjs @@ -48,6 +48,19 @@ export const BUDGET_CALIBRATION_NOTE = `window; at ${withCommas(LARGE_CONTEXT_WINDOW)} context the budget is ~${withCommas(LARGE_CONTEXT_BUDGET_TOKENS)} ` + 'tok and you are likely within it. this is an estimate, not measured telemetry'; +// Skill-body size guidance (CA-SKL-003). A SKILL.md body over ~5,000 tokens +// (~500 lines / ~20k chars) should split reference content into supporting files +// (Claude Code skill-authoring guidance). Unlike the listing budget above, the +// body is an ON-DEMAND cost: it loads only when the skill is invoked, not every +// turn — so this is a LOW-severity efficiency signal, not an always-loaded bill. +export const BODY_TOKEN_THRESHOLD = 5000; + +// Honest framing for the body-size finding: distinguishes on-demand from +// always-loaded cost and flags the figure as an estimate. Appended to evidence. +export const BODY_CALIBRATION_NOTE = + 'this is the skill BODY (SKILL.md below the frontmatter), which loads ON DEMAND only when the ' + + 'skill is invoked - NOT every turn like the always-loaded listing. estimate (chars/4), not measured telemetry'; + /** * @typedef {object} BudgetAssessment * @property {number} scanned - number of descriptions assessed @@ -91,6 +104,9 @@ export function assessSkillListingBudget(descLengths) { * @property {string|null} pluginName * @property {string} path * @property {number} descLength + * @property {number} bodyChars - SKILL.md body length below the frontmatter (on-demand cost) + * @property {number} bodyLines - body line count + * @property {number} bodyTokens - estimateTokens(bodyChars, 'markdown') */ /** @@ -110,14 +126,20 @@ export async function measureActiveSkillListing() { if (!skill || typeof skill.path !== 'string') continue; const content = await readTextFile(skill.path); if (!content) continue; - const fm = parseFrontmatter(content)?.frontmatter || null; + const parsed = parseFrontmatter(content); + const fm = parsed?.frontmatter || null; const desc = (fm && typeof fm.description === 'string') ? fm.description : ''; + const body = (parsed && typeof parsed.body === 'string') ? parsed.body : ''; + const bodyChars = body.length; skills.push({ name: skill.name, source: skill.source, pluginName: skill.pluginName, path: skill.path, descLength: desc.length, + bodyChars, + bodyLines: bodyChars === 0 ? 0 : body.split('\n').length, + bodyTokens: estimateTokens(bodyChars, 'markdown'), }); } diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs index 838253a..cbbf16e 100644 --- a/scanners/skill-listing-scanner.mjs +++ b/scanners/skill-listing-scanner.mjs @@ -40,6 +40,8 @@ import { DESCRIPTION_CAP, AGGREGATE_BUDGET_TOKENS, BUDGET_CALIBRATION_NOTE, + BODY_TOKEN_THRESHOLD, + BODY_CALIBRATION_NOTE, measureActiveSkillListing, } from './lib/skill-listing-budget.mjs'; @@ -116,5 +118,38 @@ export async function scan(_targetPath, _discovery) { })); } + // CA-SKL-003 (oversized body). Emitted last so the common single-issue cases + // read cleanly. Unlike the listing budget, this is an ON-DEMAND cost — the body + // loads only when the skill is invoked, not every turn — hence low severity and + // an explicit on-demand calibration note. + for (const skill of skills) { + if (skill.bodyTokens <= BODY_TOKEN_THRESHOLD) continue; + + const sourceLabel = skill.source === 'plugin' + ? `plugin:${skill.pluginName}` + : 'user'; + + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Skill body is large (loads on demand when the skill runs)', + description: + `Skill "${skill.name}" (${sourceLabel}) has a body of about ${skill.bodyTokens} tokens ` + + `(${skill.bodyLines} lines), over the ~${BODY_TOKEN_THRESHOLD}-token guidance for a skill body. ` + + 'The body is not in the always-loaded listing — it loads only when the skill is invoked — but ' + + 'once loaded a large body consumes context for the rest of that session. Claude Code skill ' + + 'guidance is to keep the body lean and move heavy reference material into supporting files.', + file: skill.path, + evidence: + `body_tokens~${skill.bodyTokens}; body_lines=${skill.bodyLines}; body_chars=${skill.bodyChars}; ` + + `threshold=${BODY_TOKEN_THRESHOLD} tok; skill="${skill.name}"; source=${sourceLabel} - ${BODY_CALIBRATION_NOTE}`, + recommendation: + 'Move reference content into supporting files the skill loads only when needed, and consider ' + + '`context: fork` in the skill frontmatter for heavy skills so the body runs in a forked context ' + + 'instead of consuming the main thread.', + category: 'token-efficiency', + })); + } + return scannerResult(SCANNER, 'ok', findings, aggregate.scanned, Date.now() - start); } diff --git a/tests/scanners/skill-listing-scanner.test.mjs b/tests/scanners/skill-listing-scanner.test.mjs index aa5f31f..92d7b97 100644 --- a/tests/scanners/skill-listing-scanner.test.mjs +++ b/tests/scanners/skill-listing-scanner.test.mjs @@ -61,6 +61,22 @@ async function homeWithNUserSkills(count, descLen, prefix = 'agg') { return home; } +/** + * Write one user skill whose SKILL.md *body* (below the frontmatter) is about + * `bodyChars` chars, spread across multiple lines (81 chars incl. newline). + * The description stays tiny so body-size tests stay isolated from the cap/aggregate checks. + */ +async function addUserSkillWithBody(home, name, bodyChars) { + const dir = join(home, '.claude', 'skills', name); + await mkdir(dir, { recursive: true }); + const line = `${'x'.repeat(80)}\n`; + const body = line.repeat(Math.ceil(bodyChars / line.length)); + await writeFile( + join(dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: short.\n---\n${body}`, + ); +} + // Aggregate listing budget: 2% (CC 2.1.32) of the conservative 200k anchor. const AGGREGATE_BUDGET_TOKENS = 4000; // 0.02 * 200_000 // Token heuristic mirrors estimateTokens('markdown'): ceil(chars / 4). @@ -303,6 +319,81 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { }); }); +describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => { + const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || '')); + + it('flags a skill whose body exceeds ~5,000 tokens', async () => { + const home = uniqueDir('bigbody'); + await addUserSkillWithBody(home, 'heavy', 20_400); // ~5,100 tok body + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f, 'expected an oversized-body finding for a >5k-token skill body'); + assert.equal(f.severity, 'low', 'body size is on-demand cost -> low severity'); + assert.match(String(f.file), /heavy[\\/]SKILL\.md$/); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('does NOT flag a skill whose body is under the threshold', async () => { + const home = uniqueDir('smallbody'); + await addUserSkillWithBody(home, 'light', 8_000); // ~2,000 tok body + try { + const result = await runScannerWithHome(home); + assert.equal(findBody(result.findings), undefined, 'a small body must not fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('evidence reports body tokens + lines and an honest on-demand calibration note', async () => { + const home = uniqueDir('bodyev'); + await addUserSkillWithBody(home, 'heavy', 20_400); + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f); + const ev = String(f.evidence); + assert.match(ev, /body_tokens~\d+/, 'evidence states estimated body tokens'); + assert.match(ev, /body_lines=\d+/, 'evidence states the body line count'); + assert.match(ev, /on demand/i, 'evidence discloses on-demand (not always-loaded) cost'); + assert.match(ev, /not every turn/i, 'evidence contrasts with the always-loaded listing'); + assert.match(ev, /estimate/i, 'evidence flags the figure as an estimate, not telemetry'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('recommends context:fork and moving reference content into supporting files', async () => { + const home = uniqueDir('bodyrec'); + await addUserSkillWithBody(home, 'heavy', 20_400); + try { + const result = await runScannerWithHome(home); + const f = findBody(result.findings); + assert.ok(f); + const rec = String(f.recommendation); + assert.match(rec, /context.?\s*fork/i, 'recommendation should mention context: fork'); + assert.match(rec, /supporting file/i, 'recommendation should mention supporting files'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('a large body alone does NOT trip the per-description cap or aggregate findings', async () => { + const home = uniqueDir('bodyonly'); + await addUserSkillWithBody(home, 'heavy', 20_400); // big body, tiny description + try { + const result = await runScannerWithHome(home); + assert.equal(findCaps(result.findings).length, 0, 'body size is independent of the 1,536-char description cap'); + assert.equal(findAggregate(result.findings), undefined, 'one tiny description stays under the aggregate budget'); + assert.ok(findBody(result.findings), 'only the body finding should fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + describe('SKL scanner — suppression compatibility', () => { it('CA-SKL-001 is NOT matched by a CA-TOK-* glob suppression', async () => { const { applySuppressions } = await import('../../scanners/lib/suppression.mjs'); From 2082b7d112c0f75db41f37db819282094cfefca7 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 21:44:52 +0200 Subject: [PATCH 24/53] feat(skl,cml): --context-window calibration, advisory when unknown (v5.11 B8) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SKL-002 (skill-listing budget) and CML char-budget now calibrate to a resolved context window instead of always anchoring at 200k: - resolveContextWindow(): --context-window calibrates; 'auto' keeps the conservative 200k anchor but marks advisory (model→window probing deferred to B8b); no flag → 200k anchor, byte-identical to pre-B8 default. - scaleForWindow(): linear off the 200k anchor (identity at the anchor). - SKL + CML each keep an untouched default branch (window===200k && !advisory) for byte-stability and a calibrated branch; advisory downgrades the budget finding from a breach (low/medium) to info. - Flag wired through scan-orchestrator + posture; runAllScanners resolves once and threads { contextWindow } to scanners (others ignore the 3rd arg). - CPS intentionally excluded: it has no window-anchored budget (fixed 150-line volatility heuristic), so there is nothing to calibrate. 15 new tests; e2e CLI verified (1M suppresses SKL-002, auto → info, default unchanged); full suite 1279 green; snapshots byte-stable. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/claude-md-linter.mjs | 59 +++++++++--- scanners/lib/context-window.mjs | 49 ++++++++++ scanners/lib/skill-listing-budget.mjs | 17 ++-- scanners/posture.mjs | 4 + scanners/scan-orchestrator.mjs | 12 ++- scanners/skill-listing-scanner.mjs | 92 ++++++++++++++----- tests/lib/context-window.test.mjs | 76 +++++++++++++++ tests/lib/skill-listing-budget.test.mjs | 14 +++ tests/scanners/claude-md-linter.test.mjs | 32 +++++++ tests/scanners/skill-listing-scanner.test.mjs | 54 +++++++++++ 10 files changed, 364 insertions(+), 45 deletions(-) create mode 100644 tests/lib/context-window.test.mjs diff --git a/scanners/claude-md-linter.mjs b/scanners/claude-md-linter.mjs index 316f702..8878416 100644 --- a/scanners/claude-md-linter.mjs +++ b/scanners/claude-md-linter.mjs @@ -9,13 +9,18 @@ import { finding, scannerResult, resetCounter } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { parseFrontmatter, extractSections, findImports } from './lib/yaml-parser.mjs'; import { lineCount, truncate } from './lib/string-utils.mjs'; -import { LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, withCommas } from './lib/context-window.mjs'; +import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, LARGE_CONTEXT_SCALE, scaleForWindow, withCommas } from './lib/context-window.mjs'; import { dirname } from 'node:path'; const SCANNER = 'CML'; const MAX_RECOMMENDED_LINES = 200; const MAX_ABSOLUTE_LINES = 500; +// Shared remediation for the char-budget finding (byte-identical across the +// default and the B8 window-calibrated branches). +const CHAR_BUDGET_RECOMMENDATION = + 'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.'; + // Claude Code's own startup warning ("Large CLAUDE.md will impact performance // (X chars > 40.0k)") fires once a CLAUDE.md passes ~40.0k chars on a // 200k-context model. CC 2.1.169 made that threshold scale with the model's @@ -39,10 +44,20 @@ const RECOMMENDED_SECTIONS = [ * @param {{ files: import('./lib/file-discovery.mjs').ConfigFile[] }} discovery * @returns {Promise} */ -export async function scan(targetPath, discovery) { +export async function scan(targetPath, discovery, opts = {}) { const start = Date.now(); const claudeFiles = discovery.files.filter(f => f.type === 'claude-md'); + // B8 — calibrate the char-budget threshold to the resolved context window. The + // default (no opts) is the conservative 200k anchor (40k chars) at full + // severity — byte-identical to the pre-B8 finding. An unknown (advisory) window + // keeps the anchor but downgrades the finding to info instead of a breach. + const cw = opts.contextWindow; + const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR; + const advisory = !!(cw && cw.advisory); + const isDefaultWindow = window === CONTEXT_WINDOW_ANCHOR && !advisory; + const charThreshold = scaleForWindow(CLAUDE_MD_CHAR_WARN_ANCHOR, window); + if (claudeFiles.length === 0) { return scannerResult(SCANNER, 'ok', [ finding({ @@ -122,17 +137,35 @@ export async function scan(targetPath, discovery) { // this budget (short lines), or short by lines yet over it (long lines), so // this is complementary to the line-count checks above. const chars = content.length; - if (chars > CLAUDE_MD_CHAR_WARN_ANCHOR) { - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.medium, - title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', - description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`, - file: file.absPath, - evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`, - recommendation: 'Split detail into @imports and .claude/rules/ files so only the relevant rules load, and keep the top of CLAUDE.md byte-stable for cache hits.', - autoFixable: false, - })); + if (chars > charThreshold) { + if (isDefaultWindow) { + // Conservative 200k anchor — byte-identical to the pre-B8 finding. + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.medium, + title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', + description: `${file.relPath} is ${withCommas(chars)} chars. Claude Code shows a startup warning ("Large CLAUDE.md will impact performance ... chars > 40.0k") once a CLAUDE.md passes ~40.0k chars on a 200k-context model — it loads in full on every turn. CC 2.1.169 scales that threshold with the context window, so on a ${withCommas(LARGE_CONTEXT_WINDOW)}-token model it relaxes to ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} chars and you are likely within it.`, + file: file.absPath, + evidence: `${withCommas(chars)} chars > 40.0k (200k-context anchor; ~${withCommas(CLAUDE_MD_CHAR_WARN_LARGE)} at ${withCommas(LARGE_CONTEXT_WINDOW)} context). This is an estimate, not measured telemetry.`, + recommendation: CHAR_BUDGET_RECOMMENDATION, + autoFixable: false, + })); + } else { + // B8 — window-calibrated. Advisory (unknown window) downgrades to info. + const winLabel = withCommas(window); + const threshLabel = withCommas(charThreshold); + findings.push(finding({ + scanner: SCANNER, + severity: advisory ? SEVERITY.info : SEVERITY.medium, + title: 'CLAUDE.md exceeds Claude Code\'s performance-warning threshold', + description: `${file.relPath} is ${withCommas(chars)} chars, over the ~${threshLabel}-char performance-warning threshold Claude Code applies at a ${winLabel}-token context window (it scales the ~40.0k-char @ 200k warning by the context window, CC 2.1.169) — it loads in full on every turn.` + + (advisory ? ' Your context window is unknown, so this anchors on the conservative 200k window — advisory.' : ''), + file: file.absPath, + evidence: `${withCommas(chars)} chars > ${threshLabel} (calibrated to a ${winLabel}-token context window). This is an estimate, not measured telemetry.`, + recommendation: CHAR_BUDGET_RECOMMENDATION, + autoFixable: false, + })); + } } // --- Empty file --- diff --git a/scanners/lib/context-window.mjs b/scanners/lib/context-window.mjs index de33374..15c914a 100644 --- a/scanners/lib/context-window.mjs +++ b/scanners/lib/context-window.mjs @@ -26,3 +26,52 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR; // Dependency-free thousands separator (repo invariant: zero external deps). export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); + +/** + * @typedef {object} ResolvedContextWindow + * @property {number} window - the context window budgets calibrate against + * @property {boolean} advisory - true when the window is unknown: keep the anchor + * but downgrade budget findings to info instead of + * firing them as a breach + * @property {'default'|'explicit'|'auto-unresolved'} source + */ + +/** + * Resolve the raw `--context-window` CLI value into a window + advisory flag. + * + * Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior — + * the conservative 200k anchor at full severity. Only an explicit value changes + * calibration. `auto` asks the tool to figure out the window; until model→window + * probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the + * result advisory so SKL/CML downgrade their budget findings to info rather than + * "crying wolf" with a breach on a window we cannot confirm. + * + * @param {string|number|null|undefined} arg + * @returns {ResolvedContextWindow} + */ +export function resolveContextWindow(arg) { + if (arg == null) { + return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; + } + if (String(arg).trim().toLowerCase() === 'auto') { + return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' }; + } + const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10); + if (Number.isFinite(n) && n > 0) { + return { window: n, advisory: false, source: 'explicit' }; + } + // Unparseable / non-positive: fall back to the conservative default (no advisory). + return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; +} + +/** + * Scale a 200k-anchored budget to a given context window. Linear in the window, + * so it is the identity at the anchor (keeps the default byte-stable). + * + * @param {number} anchorValue - the budget/threshold defined at the 200k anchor + * @param {number} window - the target context window + * @returns {number} + */ +export function scaleForWindow(anchorValue, window) { + return Math.round(anchorValue * (window / CONTEXT_WINDOW_ANCHOR)); +} diff --git a/scanners/lib/skill-listing-budget.mjs b/scanners/lib/skill-listing-budget.mjs index d68682e..ff8886b 100644 --- a/scanners/lib/skill-listing-budget.mjs +++ b/scanners/lib/skill-listing-budget.mjs @@ -77,23 +77,26 @@ export const BODY_CALIBRATION_NOTE = * flags it — so the aggregate does not double-count it). * * @param {number[]} descLengths - one entry per active skill (description char count) + * @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - the listing budget to + * measure against. Defaults to the 200k-anchored 4,000 tok; B8 passes a + * window-calibrated budget. Defaulting keeps existing callers byte-stable. * @returns {BudgetAssessment} */ -export function assessSkillListingBudget(descLengths) { +export function assessSkillListingBudget(descLengths, budgetTokens = AGGREGATE_BUDGET_TOKENS) { let aggregateChars = 0; for (const len of descLengths) { const safe = (typeof len === 'number' && Number.isFinite(len) && len > 0) ? len : 0; aggregateChars += Math.min(safe, DESCRIPTION_CAP); } const aggregateTokens = estimateTokens(aggregateChars, 'markdown'); - const overBudget = aggregateTokens > AGGREGATE_BUDGET_TOKENS; + const overBudget = aggregateTokens > budgetTokens; return { scanned: descLengths.length, aggregateChars, aggregateTokens, - budgetTokens: AGGREGATE_BUDGET_TOKENS, + budgetTokens, overBudget, - overBy: overBudget ? aggregateTokens - AGGREGATE_BUDGET_TOKENS : 0, + overBy: overBudget ? aggregateTokens - budgetTokens : 0, }; } @@ -115,9 +118,11 @@ export function assessSkillListingBudget(descLengths) { * enumerateSkills). Callers that run under test MUST override HOME (see the * hermetic-home helper / runScannerWithHome pattern). * + * @param {number} [budgetTokens=AGGREGATE_BUDGET_TOKENS] - listing budget for the + * aggregate assessment (B8 window-calibration); defaults keep callers byte-stable. * @returns {Promise<{ skills: ActiveSkillEntry[], aggregate: BudgetAssessment }>} */ -export async function measureActiveSkillListing() { +export async function measureActiveSkillListing(budgetTokens = AGGREGATE_BUDGET_TOKENS) { const plugins = await enumeratePlugins(); const allSkills = await enumerateSkills(plugins); @@ -143,7 +148,7 @@ export async function measureActiveSkillListing() { }); } - const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength)); + const aggregate = assessSkillListingBudget(skills.map((s) => s.descLength), budgetTokens); return { skills, aggregate }; } diff --git a/scanners/posture.mjs b/scanners/posture.mjs index 4a480e5..db12564 100644 --- a/scanners/posture.mjs +++ b/scanners/posture.mjs @@ -63,10 +63,13 @@ async function main() { let rawMode = false; let includeGlobal = false; let fullMachine = false; + let contextWindow = null; for (let i = 0; i < args.length; i++) { if (args[i] === '--output-file' && args[i + 1]) { outputFile = args[++i]; + } else if (args[i] === '--context-window' && args[i + 1]) { + contextWindow = args[++i]; } else if (args[i] === '--json') { jsonMode = true; } else if (args[i] === '--raw') { @@ -89,6 +92,7 @@ async function main() { fullMachine, filterFixtures, humanizedProgress, + contextWindow, }); // stdout JSON path: --json and --raw both write the v5.0.0-shape result diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 096eb12..71864ea 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -14,6 +14,7 @@ import { envelope } from './lib/output.mjs'; import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths } from './lib/file-discovery.mjs'; import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs'; import { humanizeEnvelope } from './lib/humanizer.mjs'; +import { resolveContextWindow } from './lib/context-window.mjs'; // Scanner registry — import order determines execution order import { scan as scanClaudeMd } from './claude-md-linter.mjs'; @@ -94,6 +95,11 @@ export async function runAllScanners(targetPath, opts = {}) { // and CNF duplicate-hook findings with config that loads on zero turns. (B3) const excludeCache = opts.excludeCache !== false; + // B8 — resolve the context window once and thread it to budget-aware scanners + // (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is + // byte-identical to the pre-B8 default; other scanners ignore the third arg. + const contextWindow = resolveContextWindow(opts.contextWindow); + // Shared file discovery — scanners reuse this let discovery; if (opts.fullMachine) { @@ -112,7 +118,7 @@ export async function runAllScanners(targetPath, opts = {}) { resetCounter(); const scanStart = Date.now(); try { - const result = await scanner.fn(resolvedPath, discovery); + const result = await scanner.fn(resolvedPath, discovery, { contextWindow }); results.push(result); const count = result.findings.length; const label = opts.humanizedProgress @@ -206,10 +212,13 @@ async function main() { let outputFile = null; let saveBaseline = false; let baselinePath = null; + let contextWindow = null; for (let i = 0; i < args.length; i++) { if (args[i] === '--output-file' && args[i + 1]) { outputFile = args[++i]; + } else if (args[i] === '--context-window' && args[i + 1]) { + contextWindow = args[++i]; } else if (args[i] === '--save-baseline') { saveBaseline = true; } else if (args[i] === '--baseline' && args[i + 1]) { @@ -254,6 +263,7 @@ async function main() { filterFixtures, excludeCache, humanizedProgress, + contextWindow, }); // Default mode runs the humanizer; --json and --raw bypass for v5.0.0 byte-equal output. diff --git a/scanners/skill-listing-scanner.mjs b/scanners/skill-listing-scanner.mjs index cbbf16e..30ceb08 100644 --- a/scanners/skill-listing-scanner.mjs +++ b/scanners/skill-listing-scanner.mjs @@ -44,6 +44,15 @@ import { BODY_CALIBRATION_NOTE, measureActiveSkillListing, } from './lib/skill-listing-budget.mjs'; +import { CONTEXT_WINDOW_ANCHOR, scaleForWindow, withCommas } from './lib/context-window.mjs'; + +// Shared remediation for the aggregate-budget finding (byte-identical across the +// default and the B8 window-calibrated branches). +const AGGREGATE_RECOMMENDATION = + 'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' + + 'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' + + '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' + + 'trigger phrases.'; const SCANNER = 'SKL'; @@ -53,11 +62,21 @@ const SCANNER = 'SKL'; * @param {string} _targetPath unused (skill listing is HOME-scoped) * @param {object} _discovery unused (ignores project discovery) */ -export async function scan(_targetPath, _discovery) { +export async function scan(_targetPath, _discovery, opts = {}) { const start = Date.now(); const findings = []; - const { skills, aggregate } = await measureActiveSkillListing(); + // B8 — calibrate the aggregate budget to the resolved context window. The + // default (no opts) is the conservative 200k anchor at full severity, which is + // byte-identical to the pre-B8 behavior. An unknown (advisory) window keeps the + // anchor but downgrades the finding to info instead of firing it as a breach. + const cw = opts.contextWindow; + const window = (cw && typeof cw.window === 'number') ? cw.window : CONTEXT_WINDOW_ANCHOR; + const advisory = !!(cw && cw.advisory); + const isDefault = window === CONTEXT_WINDOW_ANCHOR && !advisory; + const budgetTokens = scaleForWindow(AGGREGATE_BUDGET_TOKENS, window); + + const { skills, aggregate } = await measureActiveSkillListing(budgetTokens); for (const skill of skills) { if (skill.descLength <= DESCRIPTION_CAP) continue; @@ -93,29 +112,52 @@ export async function scan(_targetPath, _discovery) { // CA-SKL-002 (aggregate). Emitted after the per-skill findings so the common // "one oversized skill + aggregate" case reads 001=cap, 002=aggregate. if (aggregate.overBudget) { - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.low, - title: 'Aggregate skill descriptions may exceed the listing budget', - description: - `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + - `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + - `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` + - 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + - 'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' + - 'is an estimate — the budget scales with your actual context window (see evidence).', - evidence: - `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + - `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` + - `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` + - `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, - recommendation: - 'Reclaim skill-listing budget: set `disableBundledSkills: true` to drop bundled skills you ' + - 'do not use from the listing, use `skillOverrides` (`name-only` collapses a description, ' + - '`off` removes a skill) on the heaviest entries, and trim long descriptions toward their ' + - 'trigger phrases.', - category: 'token-efficiency', - })); + if (isDefault) { + // Conservative 200k anchor — byte-identical to the pre-B8 finding. + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.low, + title: 'Aggregate skill descriptions may exceed the listing budget', + description: + `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + + `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + + `${AGGREGATE_BUDGET_TOKENS}-token budget Claude Code allots the skill listing on a 200k ` + + 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + + 'Claude Code drops descriptions, so the model may stop seeing some skills entirely. This ' + + 'is an estimate — the budget scales with your actual context window (see evidence).', + evidence: + `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + + `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@200k=` + + `${AGGREGATE_BUDGET_TOKENS} tok (skill listing ~2% of context, CC 2.1.32); over_by~` + + `${aggregate.overBy} tok - ${BUDGET_CALIBRATION_NOTE}`, + recommendation: AGGREGATE_RECOMMENDATION, + category: 'token-efficiency', + })); + } else { + // B8 — window-calibrated. Advisory (unknown window) downgrades to info. + const winLabel = withCommas(window); + findings.push(finding({ + scanner: SCANNER, + severity: advisory ? SEVERITY.info : SEVERITY.low, + title: 'Aggregate skill descriptions may exceed the listing budget', + description: + `The ${aggregate.scanned} active skills carry about ${aggregate.aggregateTokens} tokens of description text ` + + `(each description counted up to the ${DESCRIPTION_CAP}-char listing cap), above the ` + + `${budgetTokens}-token budget Claude Code allots the skill listing at a ${winLabel}-token ` + + 'context window (about 2% of context, CC 2.1.32). When the listing overflows that budget ' + + 'Claude Code drops descriptions, so the model may stop seeing some skills entirely.' + + (advisory + ? ' Your context window is unknown, so this is advisory: it anchors on the conservative 200k window.' + : ''), + evidence: + `active_skills_scanned=${aggregate.scanned}; description_chars=${aggregate.aggregateChars} (each capped at ` + + `${DESCRIPTION_CAP}); description_tokens~${aggregate.aggregateTokens}; budget@${winLabel}=` + + `${budgetTokens} tok (skill listing ~2% of context, CC 2.1.32); over_by~${aggregate.overBy} tok` + + (advisory ? ` - ${BUDGET_CALIBRATION_NOTE}` : ' - this is an estimate, not measured telemetry'), + recommendation: AGGREGATE_RECOMMENDATION, + category: 'token-efficiency', + })); + } } // CA-SKL-003 (oversized body). Emitted last so the common single-issue cases diff --git a/tests/lib/context-window.test.mjs b/tests/lib/context-window.test.mjs new file mode 100644 index 0000000..4557d80 --- /dev/null +++ b/tests/lib/context-window.test.mjs @@ -0,0 +1,76 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + CONTEXT_WINDOW_ANCHOR, + LARGE_CONTEXT_WINDOW, + resolveContextWindow, + scaleForWindow, +} from '../../scanners/lib/context-window.mjs'; + +// B8 — context-window calibration. resolveContextWindow turns the raw +// --context-window CLI value into { window, advisory } that SKL/CML calibrate +// their budgets with. The DEFAULT (no flag) must be byte-identical to the +// pre-B8 behavior: the conservative 200k anchor at full severity (advisory=false). + +describe('resolveContextWindow — default (no flag) is the conservative anchor', () => { + it('undefined resolves to the 200k anchor, not advisory (byte-stable default)', () => { + const r = resolveContextWindow(undefined); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, false); + assert.equal(r.source, 'default'); + }); + + it('null resolves to the conservative default', () => { + const r = resolveContextWindow(null); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, false); + }); +}); + +describe('resolveContextWindow — explicit window', () => { + it('a numeric string calibrates to that window, not advisory', () => { + const r = resolveContextWindow('1000000'); + assert.equal(r.window, 1_000_000); + assert.equal(r.advisory, false); + assert.equal(r.source, 'explicit'); + }); + + it('a plain number is accepted', () => { + const r = resolveContextWindow(1_000_000); + assert.equal(r.window, 1_000_000); + assert.equal(r.advisory, false); + }); +}); + +describe('resolveContextWindow — auto / unknown downgrades to advisory', () => { + it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => { + const r = resolveContextWindow('auto'); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor'); + assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach'); + assert.equal(r.source, 'auto-unresolved'); + }); + + it('"AUTO" is case-insensitive', () => { + assert.equal(resolveContextWindow('AUTO').advisory, true); + }); + + it('an invalid value falls back to the conservative default (not advisory)', () => { + for (const bad of ['banana', '0', '-5', '']) { + const r = resolveContextWindow(bad); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, `"${bad}" -> anchor`); + assert.equal(r.advisory, false, `"${bad}" -> not advisory`); + } + }); +}); + +describe('scaleForWindow — linear scaling off the 200k anchor', () => { + it('is the identity at the 200k anchor (byte-stable default)', () => { + assert.equal(scaleForWindow(4000, CONTEXT_WINDOW_ANCHOR), 4000); + assert.equal(scaleForWindow(40_000, CONTEXT_WINDOW_ANCHOR), 40_000); + }); + + it('scales 5x at the 1M window', () => { + assert.equal(scaleForWindow(4000, LARGE_CONTEXT_WINDOW), 20_000); + assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000); + }); +}); diff --git a/tests/lib/skill-listing-budget.test.mjs b/tests/lib/skill-listing-budget.test.mjs index 8c4c826..a20b43a 100644 --- a/tests/lib/skill-listing-budget.test.mjs +++ b/tests/lib/skill-listing-budget.test.mjs @@ -102,6 +102,20 @@ describe('assessSkillListingBudget — aggregate math', () => { assert.equal(r.aggregateChars, 100); assert.equal(r.scanned, 3); }); + + it('accepts a calibrated budget (B8): 17×1000 chars is within a 20,000-tok 1M budget', () => { + const r = assessSkillListingBudget(Array(17).fill(1000), 20_000); + assert.equal(r.aggregateTokens, 4250); + assert.equal(r.budgetTokens, 20_000, 'reports the calibrated budget, not the 200k default'); + assert.equal(r.overBudget, false, 'within the relaxed 1M budget'); + assert.equal(r.overBy, 0); + }); + + it('the budget argument defaults to the 200k anchor (byte-stable for existing callers)', () => { + const r = assessSkillListingBudget(Array(17).fill(1000)); + assert.equal(r.budgetTokens, AGGREGATE_BUDGET_TOKENS); + assert.equal(r.overBudget, true); + }); }); describe('envFlag', () => { diff --git a/tests/scanners/claude-md-linter.test.mjs b/tests/scanners/claude-md-linter.test.mjs index f720dbc..5e4dd23 100644 --- a/tests/scanners/claude-md-linter.test.mjs +++ b/tests/scanners/claude-md-linter.test.mjs @@ -172,6 +172,38 @@ describe('CML scanner — char budget mirrors CC startup warning (CC 2.1.169)', }); }); +describe('CML scanner — context-window calibration (B8)', () => { + const FIXTURE = resolve(FIXTURES, 'large-claude-chars'); // 48,531 chars + const charFinding = (r) => + r.findings.find((f) => /performance-warning threshold/i.test(f.title || '')); + + async function scanWithCtx(contextWindow) { + resetCounter(); + const discovery = await discoverConfigFiles(FIXTURE); + return scan(FIXTURE, discovery, { contextWindow }); + } + + it('--context-window 1000000 relaxes the 40k char threshold so a 48k-char file does NOT fire', async () => { + const at1m = await scanWithCtx({ window: 1_000_000, advisory: false }); + assert.equal(charFinding(at1m), undefined, + '48,531 chars is under the ~200,000-char threshold at a 1M window'); + }); + + it('an unknown (advisory) window keeps the 40k anchor but downgrades to info', async () => { + const advisory = await scanWithCtx({ window: 200_000, advisory: true }); + const f = charFinding(advisory); + assert.ok(f, 'still surfaces the measurement at the conservative anchor'); + assert.equal(f.severity, 'info', 'advisory downgrades it from medium to info'); + }); + + it('no opts (default) is unchanged: fires medium at the 40k anchor', async () => { + resetCounter(); + const discovery = await discoverConfigFiles(FIXTURE); + const result = await scan(FIXTURE, discovery); + assert.equal(charFinding(result)?.severity, 'medium', 'default must stay byte-stable: medium'); + }); +}); + describe('CML scanner — large-by-lines but under the char budget (no false char finding)', () => { // large-cascade/CLAUDE.md is 1024 lines but only 37,393 chars (short lines): // under CC's 40.0k char threshold, so the char-budget finding must NOT fire — diff --git a/tests/scanners/skill-listing-scanner.test.mjs b/tests/scanners/skill-listing-scanner.test.mjs index 92d7b97..e50ff59 100644 --- a/tests/scanners/skill-listing-scanner.test.mjs +++ b/tests/scanners/skill-listing-scanner.test.mjs @@ -28,6 +28,18 @@ async function runScannerWithHome(home) { } } +/** Like runScannerWithHome but threads a resolved { window, advisory } (B8 calibration). */ +async function runScannerWithCtx(home, contextWindow) { + resetCounter(); + const original = process.env.HOME; + process.env.HOME = home; + try { + return await scan('/unused', { files: [] }, { contextWindow }); + } finally { + process.env.HOME = original; + } +} + /** Build a fake HOME with one user skill whose description has `len` chars. */ async function homeWithUserSkill(name, descLen) { const home = uniqueDir(name); @@ -319,6 +331,48 @@ describe('SKL scanner — aggregate listing budget (CA-SKL-002)', () => { }); }); +describe('SKL scanner — context-window calibration (B8)', () => { + it('--context-window 1000000 relaxes the aggregate budget so an over-200k listing does NOT fire', async () => { + // 50 skills * 400 chars = 20,000 chars -> 5,000 tok. Over the 4,000-tok 200k + // budget, under the 20,000-tok 1M budget — the acceptance case. + const home = await homeWithNUserSkills(50, 400, 'cw'); + try { + const at200k = await runScannerWithCtx(home, { window: 200_000, advisory: false }); + assert.ok(findAggregate(at200k.findings), 'control: fires at the 200k anchor'); + + const at1m = await runScannerWithCtx(home, { window: 1_000_000, advisory: false }); + assert.equal(findAggregate(at1m.findings), undefined, + 'at a 1M context window the listing is within budget and must not fire'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('an unknown (advisory) window keeps the anchor but downgrades the finding to info', async () => { + const home = await homeWithNUserSkills(17, 1000, 'cwadv'); + try { + const advisory = await runScannerWithCtx(home, { window: 200_000, advisory: true }); + const agg = findAggregate(advisory.findings); + assert.ok(agg, 'still surfaces the measurement (conservative anchor)'); + assert.equal(agg.severity, 'info', 'advisory downgrades it from a budget breach (low) to info'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('no opts (default) is unchanged: fires low at the 200k anchor', async () => { + const home = await homeWithNUserSkills(17, 1000, 'cwdef'); + try { + const result = await runScannerWithHome(home); + const agg = findAggregate(result.findings); + assert.ok(agg); + assert.equal(agg.severity, 'low', 'default behavior must be byte-stable: low severity'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + describe('SKL scanner — oversized skill body (B7, on-demand cost)', () => { const findBody = (findings) => findings.find((f) => /body is large/i.test(f.title || '')); From ad1eceb76a42780ef5478b5a1bd04cb5db808e72 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 21:49:32 +0200 Subject: [PATCH 25/53] =?UTF-8?q?release:=20v5.11.0=20=E2=80=94=20"Precisi?= =?UTF-8?q?on=20polish"=20(B7+B8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-sync: plugin.json 5.10.0→5.11.0; README version badge + tests badge 1257+→1279+ + new version-history row; CLAUDE.md test counts 1257/71→1279/72 (22→23 lib test files); CHANGELOG [5.11.0]; README SKL row documents CA-SKL-003 + the --context-window flag. B7 (CA-SKL-003) and B8 (--context-window calibration) shipped as feat commits 2798880 + 2082b7d. Scanner/agent/command counts unchanged (16/7/21); --json/--raw byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. Suite 1279 green. self-audit --check-readme: PASS (badges == filesystem). Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 35 +++++++++++++++++++++++++++++++++++ CLAUDE.md | 2 +- README.md | 7 ++++--- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c886b5b..adac250 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.10.0", + "version": "5.11.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b428a2..bd89d95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.11.0] - 2026-06-23 + +### Summary +"Precision polish" — the two LOW-priority calibration gaps from the hardening plan, both additive. +**B7** flags an oversized SKILL.md body (`CA-SKL-003`), honestly framed as an on-demand cost. **B8** +lets `CA-SKL-002` and the CML char-budget calibrate to a real context window via `--context-window`, +and downgrade to advisory when the window is unknown — so the 200k anchor stops crying wolf on a 1M +host. Scanner count stays **16** (both extend the existing SKL/CML scanners), agents **7**, commands +**21**; `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1279** tests. + +### Added +- **B7 — oversized skill body (`CA-SKL-003`, low).** `measureActiveSkillListing()` now measures the + SKILL.md **body** below the frontmatter (the file was already read in full; only the frontmatter was + parsed). A body over ~5,000 tokens (`BODY_TOKEN_THRESHOLD`) fires `CA-SKL-003`, recommending a + supporting-file split and `context: fork` for heavy skills. + - **Honest framing (Verifiseringsplikt):** `BODY_CALIBRATION_NOTE` marks this as an **on-demand** + cost — the body loads only when the skill is invoked, **not** every turn like the always-loaded + listing — and an estimate (chars/4), hence low severity. Distinct from the always-loaded + listing-budget findings. +- **B8 — context-window calibration (`--context-window`).** `CA-SKL-002` (skill-listing budget) and + the CML char-budget threshold now calibrate to a resolved context window instead of always + anchoring at 200k. `lib/context-window.mjs` gains `resolveContextWindow()` and `scaleForWindow()`: + - `--context-window ` calibrates the budget to `n` (e.g. `1000000` relaxes the SKL listing budget + to ~20,000 tok, so an over-200k listing is within budget and does not fire). + - `--context-window auto` keeps the conservative 200k anchor but marks the result **advisory** — + SKL/CML emit the finding at **info** rather than as a budget breach (model→window auto-probing is + deferred to a later B8b). + - No flag → the conservative 200k anchor at full severity, **byte-identical** to the pre-B8 default. + - Both SKL and CML keep an untouched default branch (`window === 200k && !advisory`) for + byte-stability plus a calibrated branch. The flag is wired through `scan-orchestrator` and + `posture`; `runAllScanners` resolves it once and threads `{ contextWindow }` to the scanners + (others ignore the third arg). + - **CPS intentionally excluded:** it has no window-anchored budget (a fixed 150-line volatility + heuristic), so there is nothing to calibrate. + ## [5.10.0] - 2026-06-23 ### Summary diff --git a/CLAUDE.md b/CLAUDE.md index 8c8c0ef..97054c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -123,7 +123,7 @@ Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{S node --test 'tests/**/*.test.mjs' ``` -1257 tests across 71 test files (22 lib + 39 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. +1279 tests across 72 test files (23 lib + 39 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. diff --git a/README.md b/README.md index 1951ec4..a930847 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.10.0-blue) +![Version](https://img.shields.io/badge/version-5.11.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1257+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1279+-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -326,7 +326,7 @@ By default, `/config-audit` auto-detects scope from your git context. Override w | `cache-prefix-scanner.mjs` | CPS | Volatile content in lines 31–150 of the CLAUDE.md cascade — beyond Pattern A's top-30 window but still re-loaded every turn — **plus** volatile content inside `@import`-ed files (inlined into the cached prefix, one hop, otherwise invisible to per-file scans) | | `disabled-in-schema-scanner.mjs` | DIS | Dead/ineffective permission entries: (1) tools in BOTH `permissions.deny` and `permissions.allow` — deny wins (incl. the `Tool(*)` deny-all glob, equivalent to a bare deny); (2) unanchored allow wildcards (`*`, `B*`, `mcp__*`) that Claude Code silently skips — valid only as `mcp____*`; (3) `Tool(param:value)` rules whose key is the tool's own canonicalizing field (`command`/`file_path`/`path`/`notebook_path`/`url`) — CC ignores these and emits a startup warning | | `collision-scanner.mjs` | COL | Cross-plugin skill name collisions; user-vs-plugin overlaps | -| `skill-listing-scanner.mjs` | SKL | Skill-listing token budget: a single skill description over the ~1,536-char listing cap Claude Code truncates (`CA-SKL-001`), and the summed active-skill descriptions exceeding the ~2%-of-context listing budget (`CA-SKL-002`) | +| `skill-listing-scanner.mjs` | SKL | Skill-listing token budget: a single skill description over the ~1,536-char listing cap Claude Code truncates (`CA-SKL-001`), the summed active-skill descriptions exceeding the ~2%-of-context listing budget (`CA-SKL-002`), and an oversized SKILL.md **body** over ~5,000 tokens (`CA-SKL-003`, low — on-demand cost: the body loads only when the skill runs, not every turn; recommends supporting-file split + `context: fork`). The `CA-SKL-002` (and CML char-budget) findings accept `--context-window ` to calibrate to your real window instead of the conservative 200k anchor (`--context-window auto` keeps the anchor but downgrades to advisory) | | `output-style-scanner.mjs` | OST | Output-style validation: a custom (user/project) style missing `keep-coding-instructions: true` that silently strips built-in software-engineering instructions (`CA-OST-001`), a plugin style with `force-for-plugin: true` overriding the user's selected `outputStyle` (`CA-OST-002`), and a settings `outputStyle` resolving to no built-in or custom style — dead config (`CA-OST-003`) | | `optimization-lens-scanner.mjs` | OPT | Optimization lens (mechanism-fit): a multi-step procedure in CLAUDE.md that would fit better as a skill (`CA-OPT-001`) — reads the machine-readable best-practices register, framed as an opportunity, not a failure. The deterministic half of the lens; prose-judgment cases (lifecycle→hook, unscoped path→rule, "never"→permission) are judged by the opus `optimization-lens-agent` via `/config-audit optimize` | | `agent-listing-scanner.mjs` | AGT | Always-loaded agent-listing budget: a per-agent description over the soft bloat cap (`CA-AGT-001`, advisory) and the summed active-agent name+description listing — re-sent every turn — exceeding the listing budget (`CA-AGT-002`). Both LOW and explicitly **inferred / upper-bound**: the agent-listing mechanism is undocumented, so the evidence discloses the estimate and heuristic budget rather than overstating certainty | @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.11.0** | 2026-06-23 | "Precision polish" — the two LOW-priority calibration gaps, both additive (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B7 — oversized skill body (`CA-SKL-003`, low):** the SKL scanner now measures the SKILL.md **body** (it already read the file in full) and flags bodies over ~5,000 tokens, recommending a supporting-file split + `context: fork`. Honestly framed as an **on-demand** cost — the body loads only when the skill is invoked, **not** every turn like the always-loaded listing — hence low severity. **B8 — context-window calibration (`--context-window`):** `CA-SKL-002` (skill-listing budget) and the CML char-budget now calibrate to a real context window via `--context-window ` (e.g. `1000000` stops the 200k anchor crying wolf on a 1M host) instead of always anchoring at 200k; `--context-window auto` keeps the conservative anchor but **downgrades budget findings to info/advisory** rather than firing a breach (model→window auto-probing deferred to a later B8b). No flag → byte-identical to the pre-B8 200k default. CPS is intentionally excluded (no window-anchored budget to calibrate). 1279 tests | | **5.10.0** | 2026-06-23 | "Deferral & injection hygiene" — three additive hardening levers that extend existing scanners toward a tighter always-loaded prefix (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B4 — MCP tool-schema deferral (`CA-TOK-006`; tokens patterns 7→8):** Claude Code defers MCP tool schemas (names-only, ~120 tok; full schemas load on demand) by default, so `CA-TOK-006` detects config-file signals that force the FULL schemas into the always-loaded prefix every turn — `env.ENABLE_TOOL_SEARCH="false"` (high), a `"ToolSearch"` deny (high), a configured Haiku model (medium), or per-server `alwaysLoad:true` (CC v2.1.121+, high); severity scales with the aggregate forced-upfront tokens. New pure engine `lib/mcp-deferral.mjs` shared by TOK + GAP, plus a feature-gap **CLI-over-MCP** companion lever (prefer `gh`/`aws`/`gcloud`). Triggers on config files ONLY — Vertex / custom `ANTHROPIC_BASE_URL` / a runtime `/model` switch are launch state and are disclosed, never triggered; the prefix-cache-invalidation claim was NOT-CONFIRMED in docs and is not asserted. **B5 — hook `additionalContext` advisory + filter-before lever:** HKV emits an info advisory when a hook injects unfiltered output into `additionalContext`, with a feature-gap **filter-before-Claude-reads** companion citing the documented `filter-test-output.sh` pattern. **B6 — CPS `@import` volatile scan:** the cache-prefix scanner now follows `@import`s (one hop) and flags volatile content in the imported file that breaks the cached prefix — a new medium finding, keyed on the resolved file. 1257 tests | | **5.9.0** | 2026-06-23 | "Machine-wide token lens" — the three highest-impact hardening gaps toward whole-machine token tuning. **B1 — agent-listing budget (new orchestrated scanner AGT, count 15→16):** the always-loaded agent listing (name+description re-sent every turn) is now measured — `CA-AGT-001` per-agent description bloat (advisory), `CA-AGT-002` aggregate listing over budget; both LOW and explicitly **inferred / upper-bound** (the mechanism is undocumented — the evidence discloses it rather than overstating). **B2 — machine-wide always-loaded token roll-up:** the campaign ledger now carries a token bill — `campaign refresh-tokens` does a live cross-repo sweep that counts the **shared global always-loaded layer once** + per-repo deltas, with a ranked "most expensive repos" table (the `whats-active` double-count, avoided by construction). **B3 — cache-aware filtering (folds in B0):** `~/.claude/plugins/cache` holds *both* active and stale plugin versions (installPaths point INTO it), so token-hotspots + CNF are now **version-aware** — `--exclude-cache` (default ON) keeps each plugin's active version and drops only stale ones (`installed_plugins.json`-driven), so stale versions stop polluting the hotspot ranking and inflating duplicate-hook conflicts; stale versions surface as a separate **Dead config** disk-cleanup finding (zero live-context impact). `--json`/`--raw` byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. 1215 tests | | **5.8.0** | 2026-06-23 | "Campaign motor" — a durable, machine-wide audit **campaign** that sits ABOVE individual sessions (one repo = one session; a fleet of repos = a campaign). **Ledger:** `~/.claude/config-audit/campaign-ledger.json` (outside the plugin dir → survives uninstall/upgrade) tracks a repo list + per-repo lifecycle (pending→audited→planned→implemented) + a machine-wide roll-up by status & severity; pure transforms with injected `now`. **`/config-audit campaign` (commands 20→21):** read-only report (`campaign-cli`) + human-approved writes (`campaign-write-cli`: init / add / set-status) — reports first, mutates only on explicit approval, never hand-edits the ledger. **Cross-repo backlog:** one severity-weighted prioritized pick-list (`buildBacklog`, `critical:1000/high:100/medium:10/low:1`). **Plan export + execution-by-reuse:** `campaign-export-cli --write` drops a planned repo's plan verbatim into its own `docs/`; execution reuses the existing `/config-audit implement` + `rollback` (no new execution machinery). All campaign code is `-cli`/lib → scanner count stays **15**, agents **7**, byte-stable. Plus pre-release cleanup: `knowledge-refresh` wired into the router + help; CLAUDE.md trimmed 540→134 lines (impl notes → `docs/scanner-internals.md`, config grade B→A). 1168 tests | From cf75249b5ecf0cca6cf049c03016744f219c0465 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 22:29:12 +0200 Subject: [PATCH 26/53] =?UTF-8?q?feat(skl,cml):=20--context-window=20auto?= =?UTF-8?q?=20model=E2=86=92window=20probe=20(v5.12=20B8b)=20[skip-docs]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the deferred B8 half. `--context-window auto` now probes the configured model and calibrates SKL/CML budgets to its real window instead of always falling back to the conservative advisory anchor. - lib/context-window.mjs: pure modelToContextWindow() maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit [1m] tier tag, dated/provider-prefixed IDs, and the opus/sonnet/fable aliases) to the 1M window; unknown/unconfirmed -> null (caller keeps the conservative anchor). resolveContextWindow() auto branch now probes opts.model: recognized -> auto-probed (not advisory); unknown or unpinned -> auto-unresolved (advisory, pre-B8b behavior). - lib/active-model.mjs (new): resolveActiveModel() reads the model the way Claude Code resolves it — shell ANTHROPIC_MODEL override, then settings cascade local > project > user. Injectable env, hermetic under test HOME. - scan-orchestrator: resolves the active model only when the flag is `auto` and threads it into resolveContextWindow; posture inherits via runAllScanners. Default (no flag) and explicit --context-window paths ignore the model and stay byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. TDD: 17 new tests (context-window mapping/probe + active-model cascade). Suite 1296 green. Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/lib/active-model.mjs | 51 ++++++++++++++++ scanners/lib/context-window.mjs | 67 +++++++++++++++++++-- scanners/scan-orchestrator.mjs | 9 ++- tests/lib/active-model.test.mjs | 96 +++++++++++++++++++++++++++++++ tests/lib/context-window.test.mjs | 93 +++++++++++++++++++++++++++++- 5 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 scanners/lib/active-model.mjs create mode 100644 tests/lib/active-model.test.mjs diff --git a/scanners/lib/active-model.mjs b/scanners/lib/active-model.mjs new file mode 100644 index 0000000..0ccbc76 --- /dev/null +++ b/scanners/lib/active-model.mjs @@ -0,0 +1,51 @@ +/** + * Active-model resolution for the `--context-window auto` probe (B8b). + * + * Reads the configured model the way Claude Code itself resolves it, so the + * window probe (context-window.mjs `modelToContextWindow`) sees the real model: + * 1. the shell `ANTHROPIC_MODEL` override (applies to the launched session); + * 2. otherwise the settings cascade `model` field — user `~/.claude`, then + * project `.claude`, then project-local `.claude` (local > project > user). + * + * Reads the cascade files directly (like isBundledSkillsDisabled) rather than via + * config-discovery classification, and takes an injectable `env` so it is + * deterministic and hermetic under the test HOME. Returns null when no model is + * pinned anywhere — the honest signal that `auto` must fall back to advisory. + * + * Zero external dependencies (repo invariant). + */ + +import { join } from 'node:path'; +import { readTextFile } from './file-discovery.mjs'; +import { parseJson } from './yaml-parser.mjs'; + +/** + * @param {string|null|undefined} projectPath - project root, to also read project + local settings + * @param {{ env?: Record }} [opts] + * @returns {Promise} the resolved model id/alias, or null if unset + */ +export async function resolveActiveModel(projectPath, { env = process.env } = {}) { + // 1. Shell ANTHROPIC_MODEL overrides settings (CC: applies to the session). + const envModel = typeof env?.ANTHROPIC_MODEL === 'string' ? env.ANTHROPIC_MODEL.trim() : ''; + if (envModel) return envModel; + + // 2. Settings cascade: user -> project -> project-local, later wins. + const home = (env && (env.HOME || env.USERPROFILE)) || ''; + const candidates = []; + if (home) candidates.push(join(home, '.claude', 'settings.json')); + if (projectPath) { + candidates.push(join(projectPath, '.claude', 'settings.json')); + candidates.push(join(projectPath, '.claude', 'settings.local.json')); + } + + let model = null; + for (const p of candidates) { + const content = await readTextFile(p); + if (!content) continue; + const parsed = parseJson(content); + if (parsed && typeof parsed.model === 'string' && parsed.model.trim()) { + model = parsed.model.trim(); + } + } + return model; +} diff --git a/scanners/lib/context-window.mjs b/scanners/lib/context-window.mjs index 15c914a..596d846 100644 --- a/scanners/lib/context-window.mjs +++ b/scanners/lib/context-window.mjs @@ -27,13 +27,58 @@ export const LARGE_CONTEXT_SCALE = LARGE_CONTEXT_WINDOW / CONTEXT_WINDOW_ANCHOR; // Dependency-free thousands separator (repo invariant: zero external deps). export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); +// Model families whose context window is the large (1M) tier. Verified June 2026 +// (platform.claude.com models overview): Fable 5, Opus 4.8/4.7/4.6 and Sonnet 4.6 +// all run a 1M context window. Matched by substring so dated IDs +// (claude-opus-4-8-20260528) and provider-prefixed IDs +// (us.anthropic.claude-opus-4-8) resolve too. Models we cannot confirm (e.g. +// Haiku, older 200k-era IDs) are deliberately left out: the caller then keeps the +// conservative anchor rather than guess a relaxed budget. +export const LARGE_CONTEXT_MODEL_IDS = [ + 'claude-fable-5', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-sonnet-4-6', +]; + +// Short aliases Claude Code accepts in the `model` setting / ANTHROPIC_MODEL that +// currently resolve to a 1M-tier model (`opusplan` plans on an Opus-tier model). +export const LARGE_CONTEXT_MODEL_ALIASES = new Set(['opus', 'sonnet', 'fable', 'opusplan']); + +/** + * Map a configured model id/alias to its context window, or null when we cannot + * confirm it. Pure: no IO. Used by the `--context-window auto` probe (B8b) so + * known 1M-tier models calibrate budgets instead of falling back to the + * conservative advisory anchor. + * + * @param {string} modelId - e.g. "claude-opus-4-8[1m]", "claude-sonnet-4-6", "opus" + * @returns {number|null} the context window, or null if unrecognized + */ +export function modelToContextWindow(modelId) { + if (typeof modelId !== 'string') return null; + const id = modelId.trim().toLowerCase(); + if (!id) return null; + // Explicit tier tag wins — the running session model surfaces as e.g. + // "claude-opus-4-8[1m]". This is the strongest, most future-proof signal. + if (id.includes('[1m]')) return LARGE_CONTEXT_WINDOW; + // Known 1M-tier families (substring → tolerant of date/provider-prefix variants). + for (const fam of LARGE_CONTEXT_MODEL_IDS) { + if (id.includes(fam)) return LARGE_CONTEXT_WINDOW; + } + // Short aliases. + if (LARGE_CONTEXT_MODEL_ALIASES.has(id)) return LARGE_CONTEXT_WINDOW; + // Unknown: cannot confirm the window — keep the conservative anchor (null). + return null; +} + /** * @typedef {object} ResolvedContextWindow * @property {number} window - the context window budgets calibrate against * @property {boolean} advisory - true when the window is unknown: keep the anchor * but downgrade budget findings to info instead of * firing them as a breach - * @property {'default'|'explicit'|'auto-unresolved'} source + * @property {'default'|'explicit'|'auto-probed'|'auto-unresolved'} source */ /** @@ -41,19 +86,29 @@ export const withCommas = (n) => String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ',') * * Design (B8): the DEFAULT (no flag) is byte-identical to the pre-B8 behavior — * the conservative 200k anchor at full severity. Only an explicit value changes - * calibration. `auto` asks the tool to figure out the window; until model→window - * probing ships (B8b) it cannot, so it keeps the conservative anchor but marks the - * result advisory so SKL/CML downgrade their budget findings to info rather than - * "crying wolf" with a breach on a window we cannot confirm. + * calibration. `auto` asks the tool to figure out the window. + * + * B8b: `auto` now probes the configured model (`opts.model`, resolved from the + * settings cascade / ANTHROPIC_MODEL by the orchestrator). A recognized 1M-tier + * model calibrates to its window (source `auto-probed`, not advisory). When the + * model is unknown or unpinned, it keeps the conservative anchor but marks the + * result advisory (source `auto-unresolved`) so SKL/CML downgrade their budget + * findings to info rather than "crying wolf" on a window we cannot confirm. * * @param {string|number|null|undefined} arg + * @param {{ model?: string|null }} [opts] - probe input for `auto` (ignored on the + * default/explicit paths, which stay byte-stable). * @returns {ResolvedContextWindow} */ -export function resolveContextWindow(arg) { +export function resolveContextWindow(arg, opts = {}) { if (arg == null) { return { window: CONTEXT_WINDOW_ANCHOR, advisory: false, source: 'default' }; } if (String(arg).trim().toLowerCase() === 'auto') { + const probed = modelToContextWindow(opts.model); + if (probed) { + return { window: probed, advisory: false, source: 'auto-probed' }; + } return { window: CONTEXT_WINDOW_ANCHOR, advisory: true, source: 'auto-unresolved' }; } const n = typeof arg === 'number' ? arg : parseInt(String(arg).trim(), 10); diff --git a/scanners/scan-orchestrator.mjs b/scanners/scan-orchestrator.mjs index 71864ea..6beed16 100644 --- a/scanners/scan-orchestrator.mjs +++ b/scanners/scan-orchestrator.mjs @@ -15,6 +15,7 @@ import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs'; import { humanizeEnvelope } from './lib/humanizer.mjs'; import { resolveContextWindow } from './lib/context-window.mjs'; +import { resolveActiveModel } from './lib/active-model.mjs'; // Scanner registry — import order determines execution order import { scan as scanClaudeMd } from './claude-md-linter.mjs'; @@ -98,7 +99,13 @@ export async function runAllScanners(targetPath, opts = {}) { // B8 — resolve the context window once and thread it to budget-aware scanners // (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is // byte-identical to the pre-B8 default; other scanners ignore the third arg. - const contextWindow = resolveContextWindow(opts.contextWindow); + // B8b — `--context-window auto` probes the configured model (settings cascade / + // ANTHROPIC_MODEL) so a 1M-tier host self-calibrates; unknown/unpinned → advisory. + let probedModel = null; + if (String(opts.contextWindow ?? '').trim().toLowerCase() === 'auto') { + probedModel = await resolveActiveModel(resolvedPath, { env: process.env }); + } + const contextWindow = resolveContextWindow(opts.contextWindow, { model: probedModel }); // Shared file discovery — scanners reuse this let discovery; diff --git a/tests/lib/active-model.test.mjs b/tests/lib/active-model.test.mjs new file mode 100644 index 0000000..1ae1dd3 --- /dev/null +++ b/tests/lib/active-model.test.mjs @@ -0,0 +1,96 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { resolveActiveModel } from '../../scanners/lib/active-model.mjs'; + +// B8b — resolveActiveModel reads the configured model so `--context-window auto` +// can probe it. Source precedence mirrors Claude Code's own resolution: the shell +// ANTHROPIC_MODEL override wins, otherwise the settings cascade (local > project > +// user). Fully file/env-injectable so it is hermetic under the test HOME. + +async function withTempHome(fn) { + const home = await mkdtemp(join(tmpdir(), 'config-audit-model-home-')); + try { + return await fn(home); + } finally { + await rm(home, { recursive: true, force: true }); + } +} + +async function withTempProject(fn) { + const project = await mkdtemp(join(tmpdir(), 'config-audit-model-proj-')); + try { + return await fn(project); + } finally { + await rm(project, { recursive: true, force: true }); + } +} + +async function writeSettings(dir, file, obj) { + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, file), JSON.stringify(obj)); +} + +describe('resolveActiveModel — env override (ANTHROPIC_MODEL)', () => { + it('returns the shell ANTHROPIC_MODEL when set, ignoring settings', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + const env = { HOME: home, ANTHROPIC_MODEL: 'claude-opus-4-8[1m]' }; + assert.equal(await resolveActiveModel(null, { env }), 'claude-opus-4-8[1m]'); + }); + }); + + it('an empty ANTHROPIC_MODEL is treated as unset (falls through to the cascade)', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + const env = { HOME: home, ANTHROPIC_MODEL: ' ' }; + assert.equal(await resolveActiveModel(null, { env }), 'claude-sonnet-4-6'); + }); + }); +}); + +describe('resolveActiveModel — settings cascade (local > project > user)', () => { + it('reads the model from user ~/.claude/settings.json', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-opus-4-8' }); + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), 'claude-opus-4-8'); + }); + }); + + it('project settings.json overrides user settings', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 'claude-opus-4-8' }); + await withTempProject(async (project) => { + await writeSettings(join(project, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + assert.equal(await resolveActiveModel(project, { env: { HOME: home } }), 'claude-sonnet-4-6'); + }); + }); + }); + + it('settings.local.json overrides project settings (local wins)', async () => { + await withTempHome(async (home) => { + await withTempProject(async (project) => { + await writeSettings(join(project, '.claude'), 'settings.json', { model: 'claude-sonnet-4-6' }); + await writeSettings(join(project, '.claude'), 'settings.local.json', { model: 'claude-opus-4-8[1m]' }); + assert.equal(await resolveActiveModel(project, { env: { HOME: home } }), 'claude-opus-4-8[1m]'); + }); + }); + }); +}); + +describe('resolveActiveModel — nothing configured', () => { + it('returns null on a clean HOME with no model anywhere (honest advisory fallback)', async () => { + await withTempHome(async (home) => { + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), null); + }); + }); + + it('ignores a non-string model field', async () => { + await withTempHome(async (home) => { + await writeSettings(join(home, '.claude'), 'settings.json', { model: 123 }); + assert.equal(await resolveActiveModel(null, { env: { HOME: home } }), null); + }); + }); +}); diff --git a/tests/lib/context-window.test.mjs b/tests/lib/context-window.test.mjs index 4557d80..0c003e4 100644 --- a/tests/lib/context-window.test.mjs +++ b/tests/lib/context-window.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { CONTEXT_WINDOW_ANCHOR, LARGE_CONTEXT_WINDOW, + modelToContextWindow, resolveContextWindow, scaleForWindow, } from '../../scanners/lib/context-window.mjs'; @@ -43,13 +44,20 @@ describe('resolveContextWindow — explicit window', () => { }); describe('resolveContextWindow — auto / unknown downgrades to advisory', () => { - it('"auto" keeps the conservative anchor but flags advisory (no model probe yet)', () => { + it('"auto" with no model keeps the conservative anchor but flags advisory', () => { const r = resolveContextWindow('auto'); assert.equal(r.window, CONTEXT_WINDOW_ANCHOR, 'window stays the conservative anchor'); assert.equal(r.advisory, true, 'unknown window -> advisory, not a budget breach'); assert.equal(r.source, 'auto-unresolved'); }); + it('"auto" with an unrecognized model also stays advisory', () => { + const r = resolveContextWindow('auto', { model: 'totally-made-up-model' }); + assert.equal(r.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(r.advisory, true); + assert.equal(r.source, 'auto-unresolved'); + }); + it('"AUTO" is case-insensitive', () => { assert.equal(resolveContextWindow('AUTO').advisory, true); }); @@ -74,3 +82,86 @@ describe('scaleForWindow — linear scaling off the 200k anchor', () => { assert.equal(scaleForWindow(40_000, LARGE_CONTEXT_WINDOW), 200_000); }); }); + +// B8b — model -> context-window mapping. `--context-window auto` probes the +// configured model and maps known 1M-tier model IDs to the large window so SKL/CML +// self-calibrate instead of falling back to the conservative advisory anchor. +// Verified June 2026 (platform.claude.com models overview): Fable 5, Opus +// 4.8/4.7/4.6 and Sonnet 4.6 run a 1M context window. + +describe('modelToContextWindow — known 1M-tier models map to the large window', () => { + it('the explicit [1m] tier tag wins (the running session model surfaces this way)', () => { + assert.equal(modelToContextWindow('claude-opus-4-8[1m]'), LARGE_CONTEXT_WINDOW); + // Suffix wins even for a base ID we do not otherwise enumerate. + assert.equal(modelToContextWindow('claude-future-9[1m]'), LARGE_CONTEXT_WINDOW); + }); + + it('known 1M base IDs map to 1M', () => { + for (const id of [ + 'claude-fable-5', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-sonnet-4-6', + ]) { + assert.equal(modelToContextWindow(id), LARGE_CONTEXT_WINDOW, id); + } + }); + + it('tolerates dated suffixes and provider prefixes (substring match)', () => { + assert.equal(modelToContextWindow('claude-opus-4-8-20260528'), LARGE_CONTEXT_WINDOW); + assert.equal(modelToContextWindow('us.anthropic.claude-sonnet-4-6'), LARGE_CONTEXT_WINDOW); + assert.equal(modelToContextWindow('CLAUDE-OPUS-4-8'), LARGE_CONTEXT_WINDOW); // case-insensitive + }); + + it('short aliases that resolve to a 1M-tier model map to 1M', () => { + for (const alias of ['opus', 'sonnet', 'fable', 'opusplan']) { + assert.equal(modelToContextWindow(alias), LARGE_CONTEXT_WINDOW, alias); + } + }); +}); + +describe('modelToContextWindow — unknown / unconfirmed models return null (conservative)', () => { + it('returns null for models whose window we do not confirm', () => { + // Haiku is intentionally NOT enumerated: keep the conservative anchor rather + // than guess. null -> caller keeps the advisory 200k anchor. + assert.equal(modelToContextWindow('claude-haiku-4-5-20251001'), null); + assert.equal(modelToContextWindow('haiku'), null); + assert.equal(modelToContextWindow('some-unknown-model'), null); + assert.equal(modelToContextWindow('default'), null); + }); + + it('returns null for non-string / empty input', () => { + assert.equal(modelToContextWindow(undefined), null); + assert.equal(modelToContextWindow(null), null); + assert.equal(modelToContextWindow(''), null); + assert.equal(modelToContextWindow(' '), null); + assert.equal(modelToContextWindow(42), null); + }); +}); + +describe('resolveContextWindow — auto probes the model (B8b)', () => { + it('"auto" + a recognized 1M model calibrates to 1M, not advisory', () => { + const r = resolveContextWindow('auto', { model: 'claude-opus-4-8[1m]' }); + assert.equal(r.window, LARGE_CONTEXT_WINDOW); + assert.equal(r.advisory, false, 'a confirmed window is not advisory'); + assert.equal(r.source, 'auto-probed'); + }); + + it('"auto" + a recognized base ID (no tag) also calibrates', () => { + const r = resolveContextWindow('auto', { model: 'claude-sonnet-4-6' }); + assert.equal(r.window, LARGE_CONTEXT_WINDOW); + assert.equal(r.source, 'auto-probed'); + }); + + it('the model is ignored on the explicit and default paths (byte-stable)', () => { + const explicit = resolveContextWindow('1000000', { model: 'claude-opus-4-8' }); + assert.equal(explicit.source, 'explicit'); + assert.equal(explicit.window, 1_000_000); + + const def = resolveContextWindow(undefined, { model: 'claude-opus-4-8[1m]' }); + assert.equal(def.source, 'default'); + assert.equal(def.window, CONTEXT_WINDOW_ANCHOR); + assert.equal(def.advisory, false); + }); +}); From 6bb08cc84d8db36943750cab7320bd77e50e48ad Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 23 Jun 2026 22:29:21 +0200 Subject: [PATCH 27/53] =?UTF-8?q?release:=20v5.12.0=20=E2=80=94=20"Auto-ca?= =?UTF-8?q?libration"=20(B8b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-sync for the B8b model→window auto-probe shipped in cf75249: - plugin.json 5.11.0 -> 5.12.0 - README version badge + version-history row (1296 tests) - CHANGELOG [5.12.0] entry Scanner count stays 16, agents 7, commands 21. No new finding ID or scanner. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ README.md | 3 ++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index adac250..b788502 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.11.0", + "version": "5.12.0", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index bd89d95..76fa697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.0] - 2026-06-23 + +### Summary +"Auto-calibration" — completes the deferred half of B8. `--context-window auto` now **probes the +configured model** and calibrates SKL/CML budgets to its real context window, instead of always +falling back to the conservative advisory anchor. A 1M-tier host self-calibrates without the manual +`--context-window 1000000`. No new finding ID or scanner (count stays **16**, agents **7**, commands +**21**); the default and explicit `--context-window` paths are unchanged, so `--json`/`--raw` stay +byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1296** tests. + +### Added +- **B8b — model→window auto-probe.** `lib/context-window.mjs` gains a pure `modelToContextWindow()` + that maps a configured model id/alias to its context window: + - the explicit `[1m]` tier tag wins (the running session model surfaces as e.g. + `claude-opus-4-8[1m]`); + - known 1M-tier families `LARGE_CONTEXT_MODEL_IDS` (`claude-fable-5`, `claude-opus-4-8`, + `claude-opus-4-7`, `claude-opus-4-6`, `claude-sonnet-4-6` — verified June 2026 against the + platform.claude.com models overview) match by substring, so dated (`-20260528`) and + provider-prefixed (`us.anthropic.…`) IDs resolve too; + - the short aliases `opus` / `sonnet` / `fable` / `opusplan` resolve to 1M. + - Models we cannot confirm (Haiku, older 200k-era IDs, unknown) return `null` — the caller then + keeps the conservative anchor rather than guess a relaxed budget. +- **New IO helper `lib/active-model.mjs` `resolveActiveModel()`** reads the configured model the way + Claude Code resolves it: the shell `ANTHROPIC_MODEL` override first, otherwise the settings cascade + `model` field (user `~/.claude` → project `.claude` → project-local, local wins). Reads the cascade + files directly (like `isBundledSkillsDisabled`) and takes an injectable `env`, so it stays + deterministic and hermetic under the test HOME. Returns `null` when no model is pinned anywhere. + +### Changed +- **`resolveContextWindow(arg, opts)` — the `auto` branch now probes.** It maps `opts.model` via + `modelToContextWindow()`: a recognized 1M-tier model calibrates to its window (source `auto-probed`, + **not** advisory); an unknown or unpinned model keeps the conservative 200k anchor and stays advisory + (source `auto-unresolved`, the pre-B8b `auto` behavior). `scan-orchestrator` resolves the active + model (only when the flag is `auto`) and threads it in; `posture` inherits this via `runAllScanners`. + The default (no flag) and explicit `--context-window ` paths ignore `opts.model` and are unchanged. + ## [5.11.0] - 2026-06-23 ### Summary diff --git a/README.md b/README.md index a930847..4db637a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.11.0-blue) +![Version](https://img.shields.io/badge/version-5.12.0-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.0** | 2026-06-23 | "Auto-calibration" — completes the deferred B8 half (**B8b**): `--context-window auto` now **probes the configured model** instead of always falling back to advisory. New pure `modelToContextWindow()` maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit `[1m]` tier tag, dated/provider-prefixed IDs, and the `opus`/`sonnet`/`fable` aliases) to the 1M window; new IO helper `lib/active-model.mjs` `resolveActiveModel()` reads the model the way Claude Code resolves it (shell `ANTHROPIC_MODEL` override, then the settings cascade local > project > user). When `auto` resolves a recognized model the budget calibrates to its window (`auto-probed`, not advisory); when no model is pinned or it is unrecognized it keeps the conservative anchor and stays advisory (`auto-unresolved`) — the honest fallback. No new finding ID or scanner (count stays **16**, agents **7**, commands **21**); the default and explicit `--context-window` paths are unchanged, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1296** tests. | | **5.11.0** | 2026-06-23 | "Precision polish" — the two LOW-priority calibration gaps, both additive (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B7 — oversized skill body (`CA-SKL-003`, low):** the SKL scanner now measures the SKILL.md **body** (it already read the file in full) and flags bodies over ~5,000 tokens, recommending a supporting-file split + `context: fork`. Honestly framed as an **on-demand** cost — the body loads only when the skill is invoked, **not** every turn like the always-loaded listing — hence low severity. **B8 — context-window calibration (`--context-window`):** `CA-SKL-002` (skill-listing budget) and the CML char-budget now calibrate to a real context window via `--context-window ` (e.g. `1000000` stops the 200k anchor crying wolf on a 1M host) instead of always anchoring at 200k; `--context-window auto` keeps the conservative anchor but **downgrades budget findings to info/advisory** rather than firing a breach (model→window auto-probing deferred to a later B8b). No flag → byte-identical to the pre-B8 200k default. CPS is intentionally excluded (no window-anchored budget to calibrate). 1279 tests | | **5.10.0** | 2026-06-23 | "Deferral & injection hygiene" — three additive hardening levers that extend existing scanners toward a tighter always-loaded prefix (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B4 — MCP tool-schema deferral (`CA-TOK-006`; tokens patterns 7→8):** Claude Code defers MCP tool schemas (names-only, ~120 tok; full schemas load on demand) by default, so `CA-TOK-006` detects config-file signals that force the FULL schemas into the always-loaded prefix every turn — `env.ENABLE_TOOL_SEARCH="false"` (high), a `"ToolSearch"` deny (high), a configured Haiku model (medium), or per-server `alwaysLoad:true` (CC v2.1.121+, high); severity scales with the aggregate forced-upfront tokens. New pure engine `lib/mcp-deferral.mjs` shared by TOK + GAP, plus a feature-gap **CLI-over-MCP** companion lever (prefer `gh`/`aws`/`gcloud`). Triggers on config files ONLY — Vertex / custom `ANTHROPIC_BASE_URL` / a runtime `/model` switch are launch state and are disclosed, never triggered; the prefix-cache-invalidation claim was NOT-CONFIRMED in docs and is not asserted. **B5 — hook `additionalContext` advisory + filter-before lever:** HKV emits an info advisory when a hook injects unfiltered output into `additionalContext`, with a feature-gap **filter-before-Claude-reads** companion citing the documented `filter-test-output.sh` pattern. **B6 — CPS `@import` volatile scan:** the cache-prefix scanner now follows `@import`s (one hop) and flags volatile content in the imported file that breaks the cached prefix — a new medium finding, keyed on the resolved file. 1257 tests | | **5.9.0** | 2026-06-23 | "Machine-wide token lens" — the three highest-impact hardening gaps toward whole-machine token tuning. **B1 — agent-listing budget (new orchestrated scanner AGT, count 15→16):** the always-loaded agent listing (name+description re-sent every turn) is now measured — `CA-AGT-001` per-agent description bloat (advisory), `CA-AGT-002` aggregate listing over budget; both LOW and explicitly **inferred / upper-bound** (the mechanism is undocumented — the evidence discloses it rather than overstating). **B2 — machine-wide always-loaded token roll-up:** the campaign ledger now carries a token bill — `campaign refresh-tokens` does a live cross-repo sweep that counts the **shared global always-loaded layer once** + per-repo deltas, with a ranked "most expensive repos" table (the `whats-active` double-count, avoided by construction). **B3 — cache-aware filtering (folds in B0):** `~/.claude/plugins/cache` holds *both* active and stale plugin versions (installPaths point INTO it), so token-hotspots + CNF are now **version-aware** — `--exclude-cache` (default ON) keeps each plugin's active version and drops only stale ones (`installed_plugins.json`-driven), so stale versions stop polluting the hotspot ranking and inflating duplicate-hook conflicts; stale versions surface as a separate **Dead config** disk-cleanup finding (zero live-context impact). `--json`/`--raw` byte-stable; frozen v5.0.0 + SC-5 snapshots untouched. 1215 tests | From 45efed3dbfd8c36c23ef7297cc7319dae45876ad Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 10:04:47 +0200 Subject: [PATCH 28/53] fix(tok): live-session caveat in Pattern H stale-cache recommendation Pattern H ("Stale plugin-cache versions") recommended deleting stale version dirs under ~/.claude/plugins/cache with no warning that a currently-running session may still hold one of those versions for its whole lifetime. "Stale" is judged against installed_plugins.json (what NEW sessions load), so the recommendation could reproduce the exact footgun that broke a live session during C4: deleting the dir pulls the files out from under the running session, which then breaks and must /exit + restart. Extend the recommendation text with the live-session caveat. No new finding ID/scanner, no token counts changed (recommendation string only) -> --json/--raw stay byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. +1 test (1297). Co-Authored-By: Claude Opus 4.8 (1M context) --- scanners/token-hotspots.mjs | 9 +++++++-- tests/scanners/token-hotspots.test.mjs | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index 1dc9eae..87efcbe 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -579,8 +579,13 @@ export async function scan(targetPath, discovery) { '(installed_plugins.json points at newer versions)', recommendation: 'Delete the listed stale version directories under ~/.claude/plugins/cache to reclaim ' + - 'disk (reinstall/prune via the plugin manager, or remove the dirs directly). Re-run ' + - 'with --no-exclude-cache to include cached versions in the token/conflict scan.', + 'disk (reinstall/prune via the plugin manager, or remove the dirs directly). ' + + 'Caution: do NOT delete a version a running session is still using — "stale" is judged ' + + 'against installed_plugins.json (what NEW sessions load), but an already-running session ' + + 'can hold an older version for its whole lifetime. Removing it mid-session pulls the files ' + + 'out from under that session, which then breaks and must /exit + restart to pick up the ' + + 'active version. Run with --no-exclude-cache to include cached versions in the ' + + 'token/conflict scan.', category: 'plugin-cache-hygiene', })); } diff --git a/tests/scanners/token-hotspots.test.mjs b/tests/scanners/token-hotspots.test.mjs index 039103b..bb42ed9 100644 --- a/tests/scanners/token-hotspots.test.mjs +++ b/tests/scanners/token-hotspots.test.mjs @@ -383,6 +383,16 @@ describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => { assert.match(f.evidence, /mkt\/config-audit\/5\.6\.0/); }); + it('warns against deleting a version a running session still uses (/exit caveat)', async () => { + resetCounter(); + const result = await withHermeticHome(() => scan('/tmp/x', staleDiscovery)); + const f = result.findings.find(x => /stale plugin-cache/i.test(x.title || '')); + assert.match(f.recommendation, /running session|active session/i, + 'recommendation must caution about sessions still using a version'); + assert.match(f.recommendation, /\/exit/, + 'recommendation must tell affected sessions to /exit + restart'); + }); + it('does NOT fire when there are no stale versions', async () => { resetCounter(); const result = await withHermeticHome(() => scan('/tmp/x', { files: [], staleCacheVersions: [] })); From 0f9e319c8572c3f1ad3e46fda733e855fe8ffb81 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 10:15:54 +0200 Subject: [PATCH 29/53] =?UTF-8?q?release:=20v5.12.1=20=E2=80=94=20"Footgun?= =?UTF-8?q?=20guard"=20(Pattern=20H=20live-session=20caveat)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-sync for the Pattern H recommendation fix shipped in 45efed3: - plugin.json 5.12.0 -> 5.12.1 - README version badge + version-history row (1297 tests) - CHANGELOG [5.12.1] entry Recommendation string only — no new finding ID or scanner (count stays 16, agents 7, commands 21), no token figures changed, so --json/--raw stay byte-stable and frozen v5.0.0 + SC-5 + default-output snapshots are untouched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 18 ++++++++++++++++++ README.md | 3 ++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index b788502..cd197a2 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.12.0", + "version": "5.12.1", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 76fa697..99e88cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.1] - 2026-06-24 + +### Summary +"Footgun guard" — Pattern H (stale plugin-cache versions, `token-hotspots`) recommended deleting +stale version directories without warning that a currently-running session may still hold one of +those versions for its whole lifetime. "Stale" is judged against `installed_plugins.json` (what NEW +sessions load), so the recommendation could reproduce the exact failure that broke a live session: +deleting the directory pulls the files out from under the running session, which then breaks and +must `/exit` + restart. Recommendation text only — no new finding ID or scanner (count stays **16**, +agents **7**, commands **21**), no token figures changed, so `--json`/`--raw` stay byte-stable and +the frozen v5.0.0 + SC-5 + default-output snapshots are untouched. **1297** tests. + +### Fixed +- **Pattern H live-session caveat (`token-hotspots`, `plugin-cache-hygiene`).** The stale-cache + cleanup recommendation now cautions against deleting a version a running session still uses, and + tells affected sessions to `/exit` + restart to pick up the active version — closing the footgun + that broke a live session during the C4 cache cleanup. + ## [5.12.0] - 2026-06-23 ### Summary diff --git a/README.md b/README.md index 4db637a..0e45a67 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.12.0-blue) +![Version](https://img.shields.io/badge/version-5.12.1-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.1** | 2026-06-24 | "Footgun guard" — Pattern H (stale plugin-cache versions, `token-hotspots`) recommended deleting stale version dirs with **no warning** that a currently-running session may still hold one of those versions for its whole lifetime. "Stale" is judged against `installed_plugins.json` (what NEW sessions load), so the recommendation could reproduce the exact failure that breaks a live session: deleting the dir pulls the files out from under the running session, which then must `/exit` + restart. The `plugin-cache-hygiene` recommendation now carries the live-session caveat. **Recommendation string only** — no new finding ID or scanner (count stays **16**, agents **7**, commands **21**), no token figures changed, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 + default-output snapshots are untouched. **1297** tests. | | **5.12.0** | 2026-06-23 | "Auto-calibration" — completes the deferred B8 half (**B8b**): `--context-window auto` now **probes the configured model** instead of always falling back to advisory. New pure `modelToContextWindow()` maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit `[1m]` tier tag, dated/provider-prefixed IDs, and the `opus`/`sonnet`/`fable` aliases) to the 1M window; new IO helper `lib/active-model.mjs` `resolveActiveModel()` reads the model the way Claude Code resolves it (shell `ANTHROPIC_MODEL` override, then the settings cascade local > project > user). When `auto` resolves a recognized model the budget calibrates to its window (`auto-probed`, not advisory); when no model is pinned or it is unrecognized it keeps the conservative anchor and stays advisory (`auto-unresolved`) — the honest fallback. No new finding ID or scanner (count stays **16**, agents **7**, commands **21**); the default and explicit `--context-window` paths are unchanged, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1296** tests. | | **5.11.0** | 2026-06-23 | "Precision polish" — the two LOW-priority calibration gaps, both additive (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B7 — oversized skill body (`CA-SKL-003`, low):** the SKL scanner now measures the SKILL.md **body** (it already read the file in full) and flags bodies over ~5,000 tokens, recommending a supporting-file split + `context: fork`. Honestly framed as an **on-demand** cost — the body loads only when the skill is invoked, **not** every turn like the always-loaded listing — hence low severity. **B8 — context-window calibration (`--context-window`):** `CA-SKL-002` (skill-listing budget) and the CML char-budget now calibrate to a real context window via `--context-window ` (e.g. `1000000` stops the 200k anchor crying wolf on a 1M host) instead of always anchoring at 200k; `--context-window auto` keeps the conservative anchor but **downgrades budget findings to info/advisory** rather than firing a breach (model→window auto-probing deferred to a later B8b). No flag → byte-identical to the pre-B8 200k default. CPS is intentionally excluded (no window-anchored budget to calibrate). 1279 tests | | **5.10.0** | 2026-06-23 | "Deferral & injection hygiene" — three additive hardening levers that extend existing scanners toward a tighter always-loaded prefix (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B4 — MCP tool-schema deferral (`CA-TOK-006`; tokens patterns 7→8):** Claude Code defers MCP tool schemas (names-only, ~120 tok; full schemas load on demand) by default, so `CA-TOK-006` detects config-file signals that force the FULL schemas into the always-loaded prefix every turn — `env.ENABLE_TOOL_SEARCH="false"` (high), a `"ToolSearch"` deny (high), a configured Haiku model (medium), or per-server `alwaysLoad:true` (CC v2.1.121+, high); severity scales with the aggregate forced-upfront tokens. New pure engine `lib/mcp-deferral.mjs` shared by TOK + GAP, plus a feature-gap **CLI-over-MCP** companion lever (prefer `gh`/`aws`/`gcloud`). Triggers on config files ONLY — Vertex / custom `ANTHROPIC_BASE_URL` / a runtime `/model` switch are launch state and are disclosed, never triggered; the prefix-cache-invalidation claim was NOT-CONFIRMED in docs and is not asserted. **B5 — hook `additionalContext` advisory + filter-before lever:** HKV emits an info advisory when a hook injects unfiltered output into `additionalContext`, with a feature-gap **filter-before-Claude-reads** companion citing the documented `filter-test-output.sh` pattern. **B6 — CPS `@import` volatile scan:** the cache-prefix scanner now follows `@import`s (one hop) and flags volatile content in the imported file that breaks the cached prefix — a new medium finding, keyed on the resolved file. 1257 tests | From be1056aac069278795bccadc6b20fa2a0d05647a Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 10:50:28 +0200 Subject: [PATCH 30/53] fix(acr): enumeratePlugins honors enabledPlugins + polyrepo cache installPaths (M-BUG-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit active-config-reader walked ~/.claude/plugins/marketplaces and ignored both the enabledPlugins toggle and the polyrepo cache layout. On a polyrepo machine it counted disabled/uninstalled marketplaces plugins as "active" while MISSING the actually-enabled plugins installed under plugins/cache. This corrupted the agent listing and every pluginList consumer (manifest, AGT, whats-active, hooks, rules). Now: when installed_plugins.json is present, inject only plugins that are in the manifest AND enabledPlugins[key]===true, each resolved to its active installPath (incl. cache/). When the manifest is absent (fixtures/pre-v2 installs), fall back to the historic marketplaces walk rather than silently dropping config — mirrors file-discovery.mjs's "trust installed_plugins.json" contract. Verified on real machine: agent listing 114->104, ghost plugins (newsletter, content-machine, harness, kiur, ...) gone, voyage/linkedin/ms-ai/okr now correctly counted. Full suite 1301/0; byte-stable snapshots untouched (hermetic empty HOME). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CrTb8ktf1XZWEVwgz5MTTo --- scanners/lib/active-config-reader.mjs | 115 +++++++++++++++++++++-- tests/lib/active-config-reader.test.mjs | 119 ++++++++++++++++++++++++ 2 files changed, 227 insertions(+), 7 deletions(-) diff --git a/scanners/lib/active-config-reader.mjs b/scanners/lib/active-config-reader.mjs index 5fa017e..a5a46e1 100644 --- a/scanners/lib/active-config-reader.mjs +++ b/scanners/lib/active-config-reader.mjs @@ -327,19 +327,120 @@ export async function readClaudeJsonProjectSlice(repoPath) { // ───────────────────────────────────────────────────────────────────────── /** - * Enumerate all plugins installed under ~/.claude/plugins/marketplaces. - * For each plugin: counts commands, agents, skills, hooks, rules; reads version from plugin.json. + * Enumerate the plugins Claude Code actually injects for a repo. * + * Authoritative source is `~/.claude/plugins/installed_plugins.json` (the install + * manifest) gated by the `enabledPlugins` toggle map. Only plugins that are both + * installed AND `enabledPlugins[key] === true` are injected, so only those are + * counted — each resolved to its ACTIVE `installPath`, which for polyrepo plugins + * lives under `plugins/cache` (never under `plugins/marketplaces`, so the historic + * marketplaces walk missed them entirely while also counting disabled/uninstalled + * marketplaces plugins). Mirrors file-discovery.mjs's "trust installed_plugins.json" + * contract: when the manifest is absent (test fixtures, pre-v2 installs) we cannot + * tell enabled from installed, so we fall back to discovering everything under + * `plugins/marketplaces` rather than silently dropping config. (M-BUG-1) + * + * @param {string} [repoPath] - when given, project/local-scoped installs and + * project-level `enabledPlugins` overrides are resolved relative to it; omit for + * HOME/global scope (only user-scope installs + user `enabledPlugins`). * @returns {Promise>} */ -export async function enumeratePlugins() { +export async function enumeratePlugins(repoPath) { const home = process.env.HOME || process.env.USERPROFILE || ''; if (!home) return []; - const marketplacesRoot = join(home, '.claude', 'plugins', 'marketplaces'); - const pluginRoots = await discoverAllPluginsUnder(marketplacesRoot); + const installed = await readInstalledPluginsManifest(home); - // Dedupe via realpath (symlinks are common) + let pluginRoots; + if (installed) { + // Manifest present → inject only ENABLED plugins, from their active installPath. + const enabled = await readEnabledPluginsMap(home, repoPath); + pluginRoots = []; + for (const [key, recs] of Object.entries(installed)) { + if (enabled[key] !== true) continue; // not explicitly enabled → not injected + const rec = pickActivePluginRecord(recs, repoPath); + if (!rec || !rec.installPath) continue; + try { + await stat(rec.installPath); // skip enabled-but-missing installPaths + pluginRoots.push(rec.installPath); + } catch { /* installPath gone → not loadable */ } + } + } else { + // No manifest → cannot tell enabled from installed → discover all on disk. + pluginRoots = await discoverAllPluginsUnder(join(home, '.claude', 'plugins', 'marketplaces')); + } + + return buildPluginRecords(pluginRoots); +} + +/** + * Read the install manifest's `plugins` map ({ "name@marketplace": [record, …] }). + * Returns null when absent/unparseable so callers fall back to disk discovery. + */ +async function readInstalledPluginsManifest(home) { + const p = join(home, '.claude', 'plugins', 'installed_plugins.json'); + let raw; + try { raw = await readFile(p, 'utf-8'); } catch { return null; } + const parsed = parseJson(raw); + if (!parsed || !parsed.plugins || typeof parsed.plugins !== 'object') return null; + return parsed.plugins; +} + +/** + * Merge the `enabledPlugins` toggle map across the scopes Claude Code reads: + * user settings.json, then (when repoPath given) project settings + local + the + * ~/.claude.json project slice. Later scopes override earlier ones. + */ +async function readEnabledPluginsMap(home, repoPath) { + const merged = {}; + const sources = [join(home, '.claude', 'settings.json')]; + if (repoPath) { + sources.push(join(repoPath, '.claude', 'settings.json')); + sources.push(join(repoPath, '.claude', 'settings.local.json')); + } + for (const s of sources) { + try { + const parsed = parseJson(await readFile(s, 'utf-8')); + if (parsed && parsed.enabledPlugins && typeof parsed.enabledPlugins === 'object') { + Object.assign(merged, parsed.enabledPlugins); + } + } catch { /* missing/unreadable scope */ } + } + if (repoPath) { + try { + const slice = await readClaudeJsonProjectSlice(repoPath); + if (slice && slice.enabledPlugins && typeof slice.enabledPlugins === 'object') { + Object.assign(merged, slice.enabledPlugins); + } + } catch { /* ignore */ } + } + return merged; +} + +/** + * Pick the applicable install record for a plugin. User-scope records apply + * everywhere; project/local-scope records only when repoPath is within their + * projectPath (so a project-scoped plugin never leaks into HOME/global scope). + */ +function pickActivePluginRecord(recs, repoPath) { + if (!Array.isArray(recs) || recs.length === 0) return null; + const applicable = recs.filter((r) => { + if (!r || !r.installPath) return false; + const scope = r.scope || 'user'; + if (scope === 'user') return true; + if (!repoPath || !r.projectPath) return false; + const target = normalizePath(resolve(repoPath)); + const pp = normalizePath(resolve(r.projectPath)); + return target === pp || target.startsWith(pp + sep); + }); + return applicable.find((r) => (r.scope || 'user') === 'user') || applicable[0] || null; +} + +/** + * Build plugin records from a list of plugin root paths: dedupe via realpath, + * count items, read plugin.json name/version. + */ +async function buildPluginRecords(pluginRoots) { const seen = new Set(); const results = []; for (const root of pluginRoots) { @@ -1017,7 +1118,7 @@ export async function readActiveConfig(repoPath, opts = {}) { detectGitRoot(absRepoPath), walkClaudeMdCascade(absRepoPath), readClaudeJsonProjectSlice(absRepoPath), - enumeratePlugins(), + enumeratePlugins(absRepoPath), readSettingsCascade(absRepoPath), ]); diff --git a/tests/lib/active-config-reader.test.mjs b/tests/lib/active-config-reader.test.mjs index 61fd5ec..0576727 100644 --- a/tests/lib/active-config-reader.test.mjs +++ b/tests/lib/active-config-reader.test.mjs @@ -415,6 +415,125 @@ describe('enumeratePlugins', () => { }); }); +// ───────────────────────────────────────────────────────────────────────── +// enumeratePlugins — enabledPlugins + installed_plugins.json (M-BUG-1) +// ───────────────────────────────────────────────────────────────────────── + +/** + * Build a fake HOME that mirrors a real machine: an installed_plugins.json + * manifest (polyrepo plugins live under plugins/cache, monorepo ones under + * plugins/marketplaces) + an enabledPlugins toggle map in settings.json. + * + * The enumerator must inject ONLY enabled plugins, resolving each to its ACTIVE + * installPath — including cache/ paths the marketplaces walk never sees — and + * must NOT count disabled plugins, plugins absent from enabledPlugins, or + * marketplaces plugins absent from the manifest entirely (the real M-BUG-1). + */ +async function buildManifestHome(home) { + const cacheRoot = join(home, '.claude', 'plugins', 'cache', 'mp'); + const mpRoot = join(home, '.claude', 'plugins', 'marketplaces', 'mp', 'plugins'); + + async function makePlugin(root, name, version) { + await mkdir(join(root, '.claude-plugin'), { recursive: true }); + await writeFile( + join(root, '.claude-plugin', 'plugin.json'), + JSON.stringify({ name, description: name, version }, null, 2), + ); + await mkdir(join(root, 'agents'), { recursive: true }); + await writeFile( + join(root, 'agents', `${name}-agent.md`), + `---\nname: ${name}-agent\ndescription: ${name} agent\n---\nbody\n`, + ); + } + + const enabledCachePath = join(cacheRoot, 'enabled-cache', '1.0.0'); + const disabledCachePath = join(cacheRoot, 'disabled-cache', '1.0.0'); + const enabledMpPath = join(mpRoot, 'enabled-mp'); + const notToggledMpPath = join(mpRoot, 'not-toggled-mp'); + const ghostMpPath = join(mpRoot, 'ghost-mp'); // on disk, absent from manifest + + await makePlugin(enabledCachePath, 'enabled-cache', '1.0.0'); + await makePlugin(disabledCachePath, 'disabled-cache', '1.0.0'); + await makePlugin(enabledMpPath, 'enabled-mp', '0.1.0'); + await makePlugin(notToggledMpPath, 'not-toggled-mp', '0.1.0'); + await makePlugin(ghostMpPath, 'ghost-mp', '0.1.0'); + + await mkdir(join(home, '.claude', 'plugins'), { recursive: true }); + await writeFile( + join(home, '.claude', 'plugins', 'installed_plugins.json'), + JSON.stringify({ + version: 2, + plugins: { + 'enabled-cache@mp': [{ scope: 'user', installPath: enabledCachePath, version: '1.0.0' }], + 'disabled-cache@mp': [{ scope: 'user', installPath: disabledCachePath, version: '1.0.0' }], + 'enabled-mp@mp': [{ scope: 'user', installPath: enabledMpPath, version: '0.1.0' }], + 'not-toggled-mp@mp': [{ scope: 'user', installPath: notToggledMpPath, version: '0.1.0' }], + // ghost-mp deliberately absent from the manifest + }, + }, null, 2), + ); + + await writeFile( + join(home, '.claude', 'settings.json'), + JSON.stringify({ + enabledPlugins: { + 'enabled-cache@mp': true, + 'disabled-cache@mp': false, + 'enabled-mp@mp': true, + // not-toggled-mp@mp deliberately absent (treated as not enabled) + }, + }, null, 2), + ); +} + +describe('enumeratePlugins — enabledPlugins + installed_plugins.json (M-BUG-1)', () => { + let home, originalHome; + beforeEach(async () => { + home = uniqueDir('manifest-home'); + await mkdir(home, { recursive: true }); + originalHome = process.env.HOME; + process.env.HOME = home; + }); + afterEach(async () => { + process.env.HOME = originalHome; + await rm(home, { recursive: true, force: true }); + }); + + it('injects only ENABLED plugins (honors enabledPlugins toggle)', async () => { + await buildManifestHome(home); + const names = (await enumeratePlugins()).map(p => p.name).sort(); + assert.deepEqual(names, ['enabled-cache', 'enabled-mp']); + }); + + it('excludes disabled, not-toggled, and manifest-absent (ghost) plugins', async () => { + await buildManifestHome(home); + const names = (await enumeratePlugins()).map(p => p.name); + assert.ok(!names.includes('disabled-cache'), 'disabled plugin must not be injected'); + assert.ok(!names.includes('not-toggled-mp'), 'plugin absent from enabledPlugins must not be injected'); + assert.ok(!names.includes('ghost-mp'), 'marketplaces plugin absent from manifest must not be injected'); + }); + + it('resolves an enabled plugin from plugins/cache (polyrepo installPath)', async () => { + await buildManifestHome(home); + const cached = (await enumeratePlugins()).find(p => p.name === 'enabled-cache'); + assert.ok(cached, 'cache-installed enabled plugin must be discovered'); + assert.ok(cached.path.includes(join('plugins', 'cache')), `expected a cache/ path, got ${cached.path}`); + assert.equal(cached.agents, 1); + }); + + it('falls back to the marketplaces walk when installed_plugins.json is absent', async () => { + // No manifest → cannot tell enabled from installed → discover all on disk. + const legacyRoot = join(home, '.claude', 'plugins', 'marketplaces', 'mp', 'plugins', 'legacy'); + await mkdir(join(legacyRoot, '.claude-plugin'), { recursive: true }); + await writeFile( + join(legacyRoot, '.claude-plugin', 'plugin.json'), + JSON.stringify({ name: 'legacy', version: '9.9.9' }, null, 2), + ); + const plugins = await enumeratePlugins(); + assert.ok(plugins.find(p => p.name === 'legacy'), 'legacy marketplaces plugin discovered via fallback'); + }); +}); + // ───────────────────────────────────────────────────────────────────────── // enumerateSkills // ───────────────────────────────────────────────────────────────────────── From a1e786ba4f91275b9cbb9ed6677777e695e79976 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 14:59:43 +0200 Subject: [PATCH 31/53] =?UTF-8?q?release:=20v5.12.2=20=E2=80=94=20"Honest?= =?UTF-8?q?=20census"=20(M-BUG-1:=20honest=20plugin=20enumeration)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit enumeratePlugins now honors enabledPlugins (disabled plugins no longer contribute phantom agents/skills/commands) and enumerates polyrepo plugins from their active installPath in installed_plugins.json, not only marketplaces//plugins/. Fixes manifest/whats-active/AGT/token-hotspots for any user with disabled plugins or a polyrepo marketplace. No count change (scanners 16, agents 7, commands 21); --json/--raw byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. 1301 tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XYFipiVaRtbimkDjDnnKvY --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 23 +++++++++++++++++++++++ README.md | 5 +++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cd197a2..afa83f2 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.12.1", + "version": "5.12.2", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 99e88cc..21ee440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.2] - 2026-06-24 + +### Summary +"Honest census" — fixes a plugin-enumeration bug (`M-BUG-1`, dogfooding find) that made the +always-loaded inventory untrustworthy on two common setups: machines with **disabled plugins** and +**polyrepo marketplaces**. `enumeratePlugins` walked `~/.claude/plugins/marketplaces//plugins/` +and ignored both enable-state and the polyrepo cache layout, so it **over-counted phantom agents** +from disabled/unenabled plugins while **missing the entire enabled polyrepo set** (whose plugins +live under `cache/`, not `marketplaces//plugins/`). It now gates on `installed_plugins.json` + +`enabledPlugins` and enumerates each plugin from its active `installPath`, with the marketplaces +walk as fallback. Affects `manifest`, `whats-active`, the agent-listing (AGT) and `token-hotspots` +for every such user. No new finding ID or scanner (count stays **16**, agents **7**, commands +**21**); `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 + default-output snapshots +are untouched. **1301** tests (+4). + +### Fixed +- **`active-config-reader` plugin enumeration (`M-BUG-1`).** `enumeratePlugins(repoPath)` now honors + `enabledPlugins` (disabled plugins no longer contribute phantom agents/skills/commands) and + enumerates polyrepo plugins from their active `installPath` in `installed_plugins.json` (not only + `marketplaces//plugins/`). Real-machine verify: the always-loaded agent listing dropped from + 114 to 104 with the phantom ghosts gone and the true enabled set present. TDD: 4 failing tests → + fix → full suite 1301/0, snapshots untouched. + ## [5.12.1] - 2026-06-24 ### Summary diff --git a/README.md b/README.md index 0e45a67..eabf02d 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.12.1-blue) +![Version](https://img.shields.io/badge/version-5.12.2-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1279+-brightgreen) +![Tests](https://img.shields.io/badge/tests-1301-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.2** | 2026-06-24 | "Honest census" — fixes `M-BUG-1` (dogfooding find): `enumeratePlugins` walked `~/.claude/plugins/marketplaces//plugins/` and ignored both enable-state and the polyrepo cache layout, so it **over-counted phantom agents** from disabled plugins while **missing the entire enabled polyrepo set** (whose plugins live under `cache/`). It now gates on `installed_plugins.json` + `enabledPlugins` and enumerates each plugin from its active `installPath`, with the marketplaces walk as fallback. Fixes `manifest`/`whats-active`/AGT/`token-hotspots` for any user with disabled plugins or a polyrepo marketplace. No count change (scanners **16**, agents **7**, commands **21**); `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. Real-machine verify: agent listing 114→104, ghosts gone. **1301** tests. | | **5.12.1** | 2026-06-24 | "Footgun guard" — Pattern H (stale plugin-cache versions, `token-hotspots`) recommended deleting stale version dirs with **no warning** that a currently-running session may still hold one of those versions for its whole lifetime. "Stale" is judged against `installed_plugins.json` (what NEW sessions load), so the recommendation could reproduce the exact failure that breaks a live session: deleting the dir pulls the files out from under the running session, which then must `/exit` + restart. The `plugin-cache-hygiene` recommendation now carries the live-session caveat. **Recommendation string only** — no new finding ID or scanner (count stays **16**, agents **7**, commands **21**), no token figures changed, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 + default-output snapshots are untouched. **1297** tests. | | **5.12.0** | 2026-06-23 | "Auto-calibration" — completes the deferred B8 half (**B8b**): `--context-window auto` now **probes the configured model** instead of always falling back to advisory. New pure `modelToContextWindow()` maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit `[1m]` tier tag, dated/provider-prefixed IDs, and the `opus`/`sonnet`/`fable` aliases) to the 1M window; new IO helper `lib/active-model.mjs` `resolveActiveModel()` reads the model the way Claude Code resolves it (shell `ANTHROPIC_MODEL` override, then the settings cascade local > project > user). When `auto` resolves a recognized model the budget calibrates to its window (`auto-probed`, not advisory); when no model is pinned or it is unrecognized it keeps the conservative anchor and stays advisory (`auto-unresolved`) — the honest fallback. No new finding ID or scanner (count stays **16**, agents **7**, commands **21**); the default and explicit `--context-window` paths are unchanged, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1296** tests. | | **5.11.0** | 2026-06-23 | "Precision polish" — the two LOW-priority calibration gaps, both additive (scanner count stays **16**, agents **7**, commands **21**; `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 untouched). **B7 — oversized skill body (`CA-SKL-003`, low):** the SKL scanner now measures the SKILL.md **body** (it already read the file in full) and flags bodies over ~5,000 tokens, recommending a supporting-file split + `context: fork`. Honestly framed as an **on-demand** cost — the body loads only when the skill is invoked, **not** every turn like the always-loaded listing — hence low severity. **B8 — context-window calibration (`--context-window`):** `CA-SKL-002` (skill-listing budget) and the CML char-budget now calibrate to a real context window via `--context-window ` (e.g. `1000000` stops the 200k anchor crying wolf on a 1M host) instead of always anchoring at 200k; `--context-window auto` keeps the conservative anchor but **downgrades budget findings to info/advisory** rather than firing a breach (model→window auto-probing deferred to a later B8b). No flag → byte-identical to the pre-B8 200k default. CPS is intentionally excluded (no window-anchored budget to calibrate). 1279 tests | From 7f097d524f5e48239015d1975b7be4398cb8d8fb Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 00:19:27 +0200 Subject: [PATCH 32/53] =?UTF-8?q?fix(acr):=20enumerateAgents=20counts=20on?= =?UTF-8?q?ly=20CC-registered=20agents=20=E2=80=94=20recurse=20+=20frontma?= =?UTF-8?q?tter=20filter=20+=20HOME=20dedup=20(M-BUG-3/4/5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit enumerateAgents previously counted every .md in an agents dir as an always-loaded agent. Per the official CC subagents docs, CC registers a subagent only when its frontmatter declares both name and description, scans agents dirs recursively, and (at a HOME self-scan) must not count ~/.claude/agents twice. - M-BUG-5: require valid name+description frontmatter; frontmatter-less files are registration no-ops costing 0 always-loaded tokens. Fixes the user-agent over-count (this machine: 13 -> 0). - M-BUG-3: listMarkdownFiles gains opt-in recursion; enumerateAgents recurses so agents in subfolders (agents/review/x.md) are counted, matching CC. - M-BUG-4: configDirs dedupes project==user paths, so a `manifest --global` self-scan (repoPath===$HOME) counts ~/.claude once (user scope), killing the spurious "project 13" double-count. Benefits rules/agents/output-styles. TDD: 4 failing tests -> green. Full suite 1305/0; --json/--raw byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched (no snapshot records agent enumeration rows). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- scanners/lib/active-config-reader.mjs | 38 +++++++++++++-- tests/lib/active-config-reader.test.mjs | 64 +++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/scanners/lib/active-config-reader.mjs b/scanners/lib/active-config-reader.mjs index a5a46e1..fc31ee5 100644 --- a/scanners/lib/active-config-reader.mjs +++ b/scanners/lib/active-config-reader.mjs @@ -572,14 +572,20 @@ async function countPluginItems(pluginRoot) { return counts; } -async function listMarkdownFiles(dir) { +async function listMarkdownFiles(dir, recursive = false) { const out = []; let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return out; } for (const e of entries) { + const full = join(dir, e.name); + if (e.isDirectory()) { + // Opt-in recursion (M-BUG-3): CC scans agents dirs recursively, so agents + // organized into subfolders must be enumerated too. Other callers stay flat. + if (recursive) out.push(...await listMarkdownFiles(full, true)); + continue; + } if (!e.isFile()) continue; if (!e.name.endsWith('.md')) continue; - const full = join(dir, e.name); try { const s = await stat(full); out.push({ path: full, size: s.size }); @@ -662,6 +668,11 @@ export async function enumerateSkills(pluginList = []) { // Rules, agents, output styles (v5.6 Foundation enumeration) // ───────────────────────────────────────────────────────────────────────── +/** True when `v` is a non-empty, non-whitespace string (a usable frontmatter field). */ +function hasText(v) { + return typeof v === 'string' && v.trim().length > 0; +} + /** * Build the project/user/plugin directory list for a per-kind enumerator. * Project + user dirs live under `.claude/`; plugins under each of the @@ -669,8 +680,16 @@ export async function enumerateSkills(pluginList = []) { */ function configDirs(repoPath, pluginList, subdir, pluginSubdirs = [subdir]) { const home = process.env.HOME || process.env.USERPROFILE || ''; - const dirs = [{ dir: join(repoPath, '.claude', subdir), source: 'project', pluginName: null }]; - if (home) dirs.push({ dir: join(home, '.claude', subdir), source: 'user', pluginName: null }); + const projectDir = join(repoPath, '.claude', subdir); + const userDir = home ? join(home, '.claude', subdir) : null; + const dirs = []; + // M-BUG-4: when repoPath === $HOME (the `manifest --global` self-scan), the + // project dir resolves to the same path as the user dir. Count it once, as + // user scope, instead of enumerating the same directory twice. + if (!(userDir && userDir === projectDir)) { + dirs.push({ dir: projectDir, source: 'project', pluginName: null }); + } + if (userDir) dirs.push({ dir: userDir, source: 'user', pluginName: null }); for (const p of pluginList) { for (const sub of pluginSubdirs) { dirs.push({ dir: join(p.path, sub), source: 'plugin', pluginName: p.name }); @@ -730,8 +749,17 @@ export async function enumerateAgents(repoPath, pluginList = []) { const lp = deriveLoadPattern('agent'); const dirs = configDirs(repoPath, pluginList, 'agents'); for (const { dir, source, pluginName } of dirs) { - const files = await listMarkdownFiles(dir); + const files = await listMarkdownFiles(dir, true); // M-BUG-3: CC scans agents dirs recursively for (const f of files) { + // M-BUG-5: CC registers a subagent only when its frontmatter declares both + // `name` and `description` (docs: identity comes only from `name`; both are + // required). Frontmatter-less / incomplete files are registration no-ops + // that cost zero always-loaded tokens — don't count them as agents. + let frontmatter; + try { + ({ frontmatter } = parseFrontmatter(await readFile(f.path, 'utf-8'))); + } catch { continue; } + if (!hasText(frontmatter && frontmatter.name) || !hasText(frontmatter && frontmatter.description)) continue; out.push({ name: basename(f.path).replace(/\.md$/, ''), source, diff --git a/tests/lib/active-config-reader.test.mjs b/tests/lib/active-config-reader.test.mjs index 0576727..10c1874 100644 --- a/tests/lib/active-config-reader.test.mjs +++ b/tests/lib/active-config-reader.test.mjs @@ -952,6 +952,70 @@ describe('enumerateAgents (v5.6)', () => { await mkdir(root, { recursive: true }); assert.deepEqual(await enumerateAgents(root, []), []); }); + + // M-BUG-5: CC registers a subagent only when its frontmatter declares both + // `name` and `description` (docs: identity comes only from `name`; both are + // required). Frontmatter-less / incomplete files are registration no-ops that + // cost zero always-loaded tokens — they must NOT be counted as agents. + it('M-BUG-5: skips files with no frontmatter at all', async () => { + const dir = join(root, '.claude', 'agents'); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'reviewer.md'), '---\nname: reviewer\ndescription: reviews code\n---\nbody\n'); + await writeFile(join(dir, 'not-an-agent.md'), '# Not An Agent\n\nJust a markdown doc, no frontmatter.\n'); + const agents = await enumerateAgents(root, []); + assert.deepEqual(agents.map(a => a.name).sort(), ['reviewer']); + }); + + it('M-BUG-5: skips files missing name or description', async () => { + const dir = join(root, '.claude', 'agents'); + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'reviewer.md'), '---\nname: reviewer\ndescription: reviews code\n---\nbody\n'); + await writeFile(join(dir, 'name-only.md'), '---\nname: name-only\n---\nbody\n'); + await writeFile(join(dir, 'desc-only.md'), '---\ndescription: has no name\n---\nbody\n'); + await writeFile(join(dir, 'empty-name.md'), '---\nname:\ndescription: blank name\n---\nbody\n'); + const agents = await enumerateAgents(root, []); + assert.deepEqual(agents.map(a => a.name).sort(), ['reviewer']); + }); + + // M-BUG-3: CC scans agents dirs recursively, so a valid agent in a subfolder + // (e.g. agents/review/security.md) is registered and must be counted. + it('M-BUG-3: recurses into agent subdirectories', async () => { + const sub = join(root, '.claude', 'agents', 'review'); + await mkdir(sub, { recursive: true }); + await writeFile(join(sub, 'security.md'), '---\nname: security\ndescription: security review\n---\nbody\n'); + const agents = await enumerateAgents(root, []); + const a = agents.find(x => x.name === 'security'); + assert.ok(a, 'agent in subdir should be counted'); + assert.equal(a.source, 'project'); + }); +}); + +// M-BUG-4: when repoPath === $HOME (the `manifest --global` self-scan), the +// project agents dir resolves to the SAME path as the user agents dir. The +// shared configDirs helper must count it once (as user scope), not twice. +describe('enumerateAgents — HOME self-scan dedup (M-BUG-4)', () => { + let home, originalHome; + beforeEach(async () => { + home = uniqueDir('agents-home-selfscan'); + await mkdir(join(home, '.claude', 'agents'), { recursive: true }); + originalHome = process.env.HOME; + process.env.HOME = home; + }); + afterEach(async () => { + process.env.HOME = originalHome; + await rm(home, { recursive: true, force: true }); + }); + + it('does not double-count user agents when repoPath === HOME', async () => { + await writeFile( + join(home, '.claude', 'agents', 'solo.md'), + '---\nname: solo\ndescription: only one of me\n---\nbody\n', + ); + const agents = await enumerateAgents(home, []); + const solos = agents.filter(a => a.name === 'solo'); + assert.equal(solos.length, 1, 'agent at HOME must be counted once, not twice'); + assert.equal(solos[0].source, 'user', 'HOME self-scan agents are user scope'); + }); }); describe('enumerateOutputStyles (v5.6)', () => { From 4ad1875b31852155ffe55a8db7b0920597753118 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 00:37:40 +0200 Subject: [PATCH 33/53] =?UTF-8?q?release:=20v5.12.3=20=E2=80=94=20"Phantom?= =?UTF-8?q?=20agents"=20(M-BUG-3/4/5:=20enumerateAgents=20counts=20only=20?= =?UTF-8?q?CC-registered=20agents)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releases commit 7f097d5. enumerateAgents now counts only the agents Claude Code actually registers: recurse into agent subdirs (M-BUG-3), dedupe project==user path when the scope root is $HOME (M-BUG-4, root cause — also fixed rules and output-styles), and require valid name+description frontmatter before counting a file (M-BUG-5). Real-machine verify: user-agent count 13->0 (all 12 user agents plus REMEMBER.md are frontmatter-less, so CC registers none), HOME project-dup 13->0; corrected always-loaded baseline is ~53, not 66. No count change (scanners 16, agents 7, commands 21); agent enumeration is machine-dependent and absent from the frozen snapshots, so v5.0.0 + SC-5 + default-output stay byte-stable. 1305 tests. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ README.md | 5 +++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index afa83f2..cb42360 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.12.2", + "version": "5.12.3", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ee440..8b70746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.3] - 2026-06-26 + +### Summary +"Phantom agents" — fixes `M-BUG-3/4/5` (dogfooding finds) in `scanners/lib/active-config-reader.mjs` +so that `enumerateAgents` counts only the agents Claude Code actually **registers**. Per the official +subagents documentation, an agent file must carry valid `name`+`description` frontmatter, and CC +scans the agents directory **recursively** while silently skipping frontmatter-less files. The reader +violated all three rules: it counted every `.md` regardless of frontmatter (`M-BUG-5`), never recursed +into agent subdirectories (`M-BUG-3`), and double-counted entries when the project directory equals the +user directory — the case when the scope root is `$HOME` (`M-BUG-4`, the root cause, which also affected +rules and output-styles). Real-machine verify: the user-agent count dropped **13→0** (all 12 user +agents plus `REMEMBER.md` are frontmatter-less, so CC registers none of them) and the HOME `project` +duplicate dropped **13→0**; the corrected always-loaded baseline is ≈ **53**, not 66. Agent enumeration +is machine-dependent and therefore absent from the frozen snapshots, so the v5.0.0 + SC-5 + +default-output snapshots stay byte-stable; no count change (scanners **16**, agents **7**, commands +**21**). **1305** tests (+4). + +### Fixed +- **`active-config-reader` agent enumeration (`M-BUG-3/4/5`).** Three surgical fixes: + - `listMarkdownFiles` gains an opt-in `recursive` flag so agent enumeration descends into + subdirectories the way Claude Code does (`M-BUG-3`). + - `configDirs` now de-duplicates the project and user paths when they resolve to the same directory + (the case when the scope root is `$HOME`), the root cause that also double-counted rules and + output-styles (`M-BUG-4`). + - `enumerateAgents` requires a valid `name`+`description` frontmatter block before counting a file, + matching CC's actual registration rule — frontmatter-less files are silently skipped (`M-BUG-5`). + - Added a `hasText` frontmatter helper. TDD: 4 failing tests (one per bug) → fix → full suite + 1305/0, frozen v5.0.0 + SC-5 + default-output snapshots untouched (agent enumeration is + machine-dependent and never seeded into a snapshot). + ## [5.12.2] - 2026-06-24 ### Summary diff --git a/README.md b/README.md index eabf02d..275281d 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.12.2-blue) +![Version](https://img.shields.io/badge/version-5.12.3-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1301-brightgreen) +![Tests](https://img.shields.io/badge/tests-1305-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.3** | 2026-06-26 | "Phantom agents" — fixes `M-BUG-3/4/5` (dogfooding finds) in `scanners/lib/active-config-reader.mjs`: `enumerateAgents` now counts only the agents Claude Code actually **registers**. Per the official subagents doc, an agent needs valid `name`+`description` frontmatter, and CC scans recursively and silently skips frontmatter-less files. The reader previously (`M-BUG-5`) counted every `.md` regardless of frontmatter, (`M-BUG-3`) never recursed into agent subdirs, and (`M-BUG-4`) double-counted when the project dir equals the user dir (scanning `$HOME` — root cause, also affecting rules/output-styles). Real-machine verify: user-agent count **13→0** (all 12 user agents + `REMEMBER.md` are frontmatter-less → CC registers none), HOME `project`-dup **13→0**; corrected always-loaded baseline ≈ **53** (was 66). Agent enumeration is machine-dependent and absent from the frozen snapshots, so the v5.0.0 + SC-5 + default-output snapshots stay byte-stable; no count change (scanners **16**, agents **7**, commands **21**). **1305** tests. | | **5.12.2** | 2026-06-24 | "Honest census" — fixes `M-BUG-1` (dogfooding find): `enumeratePlugins` walked `~/.claude/plugins/marketplaces//plugins/` and ignored both enable-state and the polyrepo cache layout, so it **over-counted phantom agents** from disabled plugins while **missing the entire enabled polyrepo set** (whose plugins live under `cache/`). It now gates on `installed_plugins.json` + `enabledPlugins` and enumerates each plugin from its active `installPath`, with the marketplaces walk as fallback. Fixes `manifest`/`whats-active`/AGT/`token-hotspots` for any user with disabled plugins or a polyrepo marketplace. No count change (scanners **16**, agents **7**, commands **21**); `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. Real-machine verify: agent listing 114→104, ghosts gone. **1301** tests. | | **5.12.1** | 2026-06-24 | "Footgun guard" — Pattern H (stale plugin-cache versions, `token-hotspots`) recommended deleting stale version dirs with **no warning** that a currently-running session may still hold one of those versions for its whole lifetime. "Stale" is judged against `installed_plugins.json` (what NEW sessions load), so the recommendation could reproduce the exact failure that breaks a live session: deleting the dir pulls the files out from under the running session, which then must `/exit` + restart. The `plugin-cache-hygiene` recommendation now carries the live-session caveat. **Recommendation string only** — no new finding ID or scanner (count stays **16**, agents **7**, commands **21**), no token figures changed, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 + default-output snapshots are untouched. **1297** tests. | | **5.12.0** | 2026-06-23 | "Auto-calibration" — completes the deferred B8 half (**B8b**): `--context-window auto` now **probes the configured model** instead of always falling back to advisory. New pure `modelToContextWindow()` maps known 1M-tier model IDs (Fable 5, Opus 4.8/4.7/4.6, Sonnet 4.6 — verified June 2026 — plus the explicit `[1m]` tier tag, dated/provider-prefixed IDs, and the `opus`/`sonnet`/`fable` aliases) to the 1M window; new IO helper `lib/active-model.mjs` `resolveActiveModel()` reads the model the way Claude Code resolves it (shell `ANTHROPIC_MODEL` override, then the settings cascade local > project > user). When `auto` resolves a recognized model the budget calibrates to its window (`auto-probed`, not advisory); when no model is pinned or it is unrecognized it keeps the conservative anchor and stays advisory (`auto-unresolved`) — the honest fallback. No new finding ID or scanner (count stays **16**, agents **7**, commands **21**); the default and explicit `--context-window` paths are unchanged, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 snapshots are untouched. **1296** tests. | From 18af5a24e92554a4505326cf3dc79a23d59dba6e Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 10:56:39 +0200 Subject: [PATCH 34/53] fix(acr): RUL resolves rule glob against the rule's own project root, not the scan root (M-BUG-9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rule's paths:/globs: pattern scopes relative to the directory containing the rule's .claude/, not the outer scan target. countGlobMatches globbed against the scan root and collectProjectFiles' depth>4 cutoff never reached deep matching files, so a live rule in a nested repo (e.g. a marketplace checkout under ~/.claude) was wrongly flagged "matches no files / never activates" (high) — a false F-grade for any user with rules in a nested repo. Same scope-conflation family as M-BUG-1/2/8. - deriveProjectRoot(ruleAbsPath): parent of the rule's .claude segment. - collect + glob per project root (cached), relative to that root — so a nested repo's rule resolves against its own tree, where its files live. - user-global rules (root === HOME) skip the no-match check: they scope against whatever project is active at runtime, not a fixed tree, so "matches 0 files here" is not a dead-rule signal (and avoids a HOME walk). TDD: 2 failing tests (nested-repo false-positive + HOME guard) -> green. Full suite 1307/0; frozen v5.0.0 + default-output snapshots unchanged (RUL appears in none; the fix is a no-op when projectRoot === targetPath, i.e. the common single-repo scan). Dogfooding C6: clears the 2 ktg-privat false positives on the real machine and surfaces a previously-hidden genuine dead rule (false negative) in the bundled optimal-setup example. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- scanners/rules-validator.mjs | 66 +++++++++++++----- tests/scanners/rules-validator.test.mjs | 91 +++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 18 deletions(-) diff --git a/scanners/rules-validator.mjs b/scanners/rules-validator.mjs index 3bd358f..e277132 100644 --- a/scanners/rules-validator.mjs +++ b/scanners/rules-validator.mjs @@ -10,7 +10,7 @@ import { SEVERITY } from './lib/severity.mjs'; import { parseFrontmatter } from './lib/yaml-parser.mjs'; import { lineCount, truncate } from './lib/string-utils.mjs'; import { readdir, stat } from 'node:fs/promises'; -import { join, resolve, relative } from 'node:path'; +import { join, resolve, relative, sep } from 'node:path'; const SCANNER = 'RUL'; @@ -30,8 +30,16 @@ export async function scan(targetPath, discovery) { return scannerResult(SCANNER, 'skipped', [], 0, Date.now() - start); } - // Collect all real files in the project for glob matching - const projectFiles = await collectProjectFiles(targetPath); + // Rule path patterns scope relative to the rule's OWN project root (the dir + // containing its .claude/), not the outer scan root. Resolve + cache per root. + const home = process.env.HOME || process.env.USERPROFILE || ''; + const projectFilesByRoot = new Map(); + async function projectFilesFor(root) { + if (!projectFilesByRoot.has(root)) { + projectFilesByRoot.set(root, await collectProjectFiles(root)); + } + return projectFilesByRoot.get(root); + } for (const file of ruleFiles) { const content = await readTextFile(file.absPath); @@ -74,22 +82,32 @@ export async function scan(targetPath, discovery) { if (paths) { const patterns = Array.isArray(paths) ? paths : [paths]; - for (const pattern of patterns) { - if (typeof pattern !== 'string') continue; + // A rule scopes relative to its own project root (parent of its .claude/), + // not the scan root. User-global rules (root === HOME) match against the + // active project at runtime, so "matches no files here" is not meaningful. + const projectRoot = deriveProjectRoot(file.absPath) || targetPath; + const isUserGlobal = home && projectRoot === home; - // Check if pattern matches any real files - const matchCount = countGlobMatches(pattern, projectFiles, targetPath); - if (matchCount === 0) { - findings.push(finding({ - scanner: SCANNER, - severity: SEVERITY.high, - title: 'Rule path pattern matches no files', - description: `${file.relPath}: pattern "${pattern}" matches 0 files. This rule will never activate.`, - file: file.absPath, - evidence: `paths: "${pattern}"`, - recommendation: 'Check the glob pattern. Common issues: wrong directory name, missing **, incorrect extension.', - autoFixable: false, - })); + if (!isUserGlobal) { + const projectFiles = await projectFilesFor(projectRoot); + + for (const pattern of patterns) { + if (typeof pattern !== 'string') continue; + + // Check if pattern matches any real files (relative to the rule's root) + const matchCount = countGlobMatches(pattern, projectFiles, projectRoot); + if (matchCount === 0) { + findings.push(finding({ + scanner: SCANNER, + severity: SEVERITY.high, + title: 'Rule path pattern matches no files', + description: `${file.relPath}: pattern "${pattern}" matches 0 files. This rule will never activate.`, + file: file.absPath, + evidence: `paths: "${pattern}"`, + recommendation: 'Check the glob pattern. Common issues: wrong directory name, missing **, incorrect extension.', + autoFixable: false, + })); + } } } } @@ -195,6 +213,18 @@ async function collectProjectFiles(targetPath, depth = 0) { * @param {string} basePath * @returns {number} */ +/** + * Resolve the project root a rule scopes against: the directory containing the + * `.claude/` dir the rule lives under. `/a/b/.claude/rules/x.md` → `/a/b`. + * Returns null if the path has no `.claude` segment. + */ +function deriveProjectRoot(ruleAbsPath) { + const parts = ruleAbsPath.split(sep); + const idx = parts.lastIndexOf('.claude'); + if (idx <= 0) return null; + return parts.slice(0, idx).join(sep); +} + function countGlobMatches(pattern, files, basePath) { try { const regex = globToRegex(pattern); diff --git a/tests/scanners/rules-validator.test.mjs b/tests/scanners/rules-validator.test.mjs index c3fedab..4cf9390 100644 --- a/tests/scanners/rules-validator.test.mjs +++ b/tests/scanners/rules-validator.test.mjs @@ -200,3 +200,94 @@ describe('RUL — block-sequence-scoped rule is correctly scoped (parser regress assert.equal(f.severity, 'low'); }); }); + +describe('RUL — nested-repo rule glob resolves to its own project root (M-BUG-9)', () => { + // A rule living in a NESTED repo's .claude/rules/ scopes its paths: pattern + // relative to that nested repo's root — NOT relative to the outer scan root. + // Before the fix, countGlobMatches globbed against the scan root and + // collectProjectFiles' depth>4 cutoff never reached the deep matching files, + // so a live rule was wrongly flagged "matches no files / never activates". + // (Real-machine surface: ~/.claude/plugins/marketplaces/ktg-privat/.claude/rules/.) + let tmpRoot; + let result; + + async function writeNestedRepo(scanRoot) { + // Nested repo sits 3 dirs below the scan root → its matching files land + // past the old depth>4 cutoff when walked from the scan root. + const nested = join(scanRoot, 'level1', 'level2', 'nested-repo'); + await mkdir(join(nested, '.claude', 'rules'), { recursive: true }); + await mkdir(join(nested, 'plugins', 'app'), { recursive: true }); + await mkdir(join(nested, 'plugins', 'app', 'hooks'), { recursive: true }); + // Files that the rule patterns match — relative to the NESTED repo root. + await writeFile(join(nested, 'plugins', 'app', 'CLAUDE.md'), '# App\n', 'utf8'); + await writeFile(join(nested, 'plugins', 'app', 'hooks', 'hooks.json'), '{}\n', 'utf8'); + // Two rules mirroring the real ktg-privat ones. + await writeFile( + join(nested, '.claude', 'rules', 'plugin-convention.md'), + '---\npaths: "plugins/*/CLAUDE.md"\n---\n\n# Convention\nbody\n', + 'utf8', + ); + await writeFile( + join(nested, '.claude', 'rules', 'hook-format.md'), + '---\npaths: "**/hooks/hooks.json"\n---\n\n# Hook format\nbody\n', + 'utf8', + ); + } + + beforeEach(async () => { + resetCounter(); + tmpRoot = await mkdtemp(join(tmpdir(), 'ca-rul-nested-')); + await writeNestedRepo(tmpRoot); + const discovery = await discoverConfigFiles(tmpRoot); + result = await scan(tmpRoot, discovery); + }); + + afterEach(async () => { + if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true }); + }); + + it('does NOT flag the nested rules as "matches no files"', () => { + const dead = result.findings.filter(f => f.title.includes('matches no files')); + assert.equal( + dead.length, + 0, + `nested-repo rules matched real files but were flagged dead: ${dead.map(f => f.evidence).join(' | ')}`, + ); + }); +}); + +describe('RUL — user-global rule is not flagged "matches no files" (M-BUG-9 guard)', () => { + // A user-global ~/.claude/rules/ rule scopes against whatever project is + // active at runtime, not a fixed tree — so "matches 0 files here" is not a + // meaningful dead-rule signal and must not be flagged (and must not trigger a + // HOME-wide file walk). + let tmpHome; + let savedHome; + let result; + + beforeEach(async () => { + resetCounter(); + tmpHome = await mkdtemp(join(tmpdir(), 'ca-rul-home-')); + savedHome = process.env.HOME; + process.env.HOME = tmpHome; + await mkdir(join(tmpHome, '.claude', 'rules'), { recursive: true }); + await writeFile( + join(tmpHome, '.claude', 'rules', 'global.md'), + '---\npaths: "src/**/*.rs"\n---\n\n# Global rust rule\nbody\n', + 'utf8', + ); + // Scan from HOME so the rule is discovered AND its derived project root === HOME. + const discovery = await discoverConfigFiles(tmpHome); + result = await scan(tmpHome, discovery); + }); + + afterEach(async () => { + process.env.HOME = savedHome; + if (tmpHome) await rm(tmpHome, { recursive: true, force: true }); + }); + + it('does NOT flag a user-global rule as "matches no files"', () => { + const dead = result.findings.filter(f => f.title.includes('matches no files')); + assert.equal(dead.length, 0, `user-global rule wrongly flagged dead: ${dead.map(f => f.evidence).join(' | ')}`); + }); +}); From 346dfac6fad023d3e30271eff9c95a87f50ce9ef Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 11:04:30 +0200 Subject: [PATCH 35/53] =?UTF-8?q?release:=20v5.12.4=20=E2=80=94=20"Rooted?= =?UTF-8?q?=20rules"=20(M-BUG-9:=20RUL=20resolves=20rule=20glob=20against?= =?UTF-8?q?=20the=20rule's=20own=20project=20root)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-sync for the M-BUG-9 fix (code already in 18af5a2): - plugin.json 5.12.3 -> 5.12.4 - README version badge -> 5.12.4, tests badge 1305 -> 1307, new version-history row - CHANGELOG [5.12.4] section checkReadmeBadges: passed:true (tests 1307, scanners 16, commands 21, agents 7, hooks 4 — all match filesystem). Full suite 1307/0. Frozen v5.0.0 + default-output snapshots byte-stable (the fix is a no-op when projectRoot === targetPath; RUL appears in no snapshot). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ README.md | 5 +++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cb42360..589062b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.12.3", + "version": "5.12.4", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b70746..28dc95d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.4] - 2026-06-26 + +### Summary +"Rooted rules" — fixes `M-BUG-9` (dogfooding find) in `scanners/rules-validator.mjs`. The RUL +"Rule path pattern matches no files" check resolved a rule's `paths:`/`globs:` glob against the +outer **scan root** instead of the rule's **own project root** (the directory containing its +`.claude/`), and `collectProjectFiles` carried a `depth>4` cutoff that never reached deep matching +files. As a result, a live rule in a **nested repo** — e.g. a marketplace checkout under +`~/.claude/plugins/marketplaces//.claude/rules/` — was wrongly flagged "never activates" (high +severity), a false F-grade for any user with rules in a nested repo. Same scope-conflation family as +`M-BUG-1/2` (the scanner treats a nested repo's config as scoped to the outer scan root). The fix is a +no-op for the common single-repo scan (`projectRoot === targetPath`), so the frozen v5.0.0 + +default-output snapshots stay byte-stable; no count change (scanners **16**, agents **7**, commands +**21**). **1307** tests (+2). + +### Fixed +- **`rules-validator` glob base (`M-BUG-9`).** The dead-rule check now resolves each rule against its + own project root: + - `deriveProjectRoot(ruleAbsPath)` returns the parent of the rule's `.claude` segment. + - Project files are collected and globbed **per project root** (cached), relative to that root, so a + nested repo's rule matches against its own tree where its files live. This also sidesteps the old + `depth>4` cutoff, because the walk now starts at the nearby project root. + - User-global rules (`projectRoot === $HOME`, i.e. `~/.claude/rules/`) skip the no-match check: they + scope against whatever project is active at runtime, not a fixed tree, so "matches 0 files here" is + not a dead-rule signal (and this avoids a `$HOME`-wide file walk). + - TDD: 2 failing tests (nested-repo false-positive + HOME guard) → fix → full suite 1307/0, frozen + v5.0.0 + default-output snapshots untouched (RUL findings appear in none). Real-machine verify: the + two `ktg-privat` false positives clear and a previously-hidden genuine dead rule (a false negative) + surfaces in the bundled `optimal-setup` example; zero new false positives. + ## [5.12.3] - 2026-06-26 ### Summary diff --git a/README.md b/README.md index 275281d..e6e98e9 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.12.3-blue) +![Version](https://img.shields.io/badge/version-5.12.4-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1305-brightgreen) +![Tests](https://img.shields.io/badge/tests-1307-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.4** | 2026-06-26 | "Rooted rules" — fixes `M-BUG-9` (dogfooding find) in `scanners/rules-validator.mjs`: the RUL "Rule path pattern matches no files" check now resolves a rule's `paths:`/`globs:` pattern against the rule's **own project root** (the dir containing its `.claude/`), not the outer scan root. Previously `countGlobMatches` globbed against the scan target and `collectProjectFiles`' `depth>4` cutoff never reached deep matching files, so a live rule in a **nested repo** (e.g. a marketplace checkout under `~/.claude`) was wrongly flagged "never activates" (high) — a false F-grade for anyone with rules in a nested repo. The fix derives each rule's project root, collects+globs per root (cached), and skips the check for user-global rules (`root === $HOME`), which scope against the active project at runtime. Same scope-conflation family as `M-BUG-1/2`. No count change (scanners **16**, agents **7**, commands **21**); the fix is a no-op when `projectRoot === targetPath` (the common single-repo scan), so frozen v5.0.0 + default-output snapshots stay byte-stable. **1307** tests (+2 TDD: nested-repo false-positive + HOME guard). | | **5.12.3** | 2026-06-26 | "Phantom agents" — fixes `M-BUG-3/4/5` (dogfooding finds) in `scanners/lib/active-config-reader.mjs`: `enumerateAgents` now counts only the agents Claude Code actually **registers**. Per the official subagents doc, an agent needs valid `name`+`description` frontmatter, and CC scans recursively and silently skips frontmatter-less files. The reader previously (`M-BUG-5`) counted every `.md` regardless of frontmatter, (`M-BUG-3`) never recursed into agent subdirs, and (`M-BUG-4`) double-counted when the project dir equals the user dir (scanning `$HOME` — root cause, also affecting rules/output-styles). Real-machine verify: user-agent count **13→0** (all 12 user agents + `REMEMBER.md` are frontmatter-less → CC registers none), HOME `project`-dup **13→0**; corrected always-loaded baseline ≈ **53** (was 66). Agent enumeration is machine-dependent and absent from the frozen snapshots, so the v5.0.0 + SC-5 + default-output snapshots stay byte-stable; no count change (scanners **16**, agents **7**, commands **21**). **1305** tests. | | **5.12.2** | 2026-06-24 | "Honest census" — fixes `M-BUG-1` (dogfooding find): `enumeratePlugins` walked `~/.claude/plugins/marketplaces//plugins/` and ignored both enable-state and the polyrepo cache layout, so it **over-counted phantom agents** from disabled plugins while **missing the entire enabled polyrepo set** (whose plugins live under `cache/`). It now gates on `installed_plugins.json` + `enabledPlugins` and enumerates each plugin from its active `installPath`, with the marketplaces walk as fallback. Fixes `manifest`/`whats-active`/AGT/`token-hotspots` for any user with disabled plugins or a polyrepo marketplace. No count change (scanners **16**, agents **7**, commands **21**); `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. Real-machine verify: agent listing 114→104, ghosts gone. **1301** tests. | | **5.12.1** | 2026-06-24 | "Footgun guard" — Pattern H (stale plugin-cache versions, `token-hotspots`) recommended deleting stale version dirs with **no warning** that a currently-running session may still hold one of those versions for its whole lifetime. "Stale" is judged against `installed_plugins.json` (what NEW sessions load), so the recommendation could reproduce the exact failure that breaks a live session: deleting the dir pulls the files out from under the running session, which then must `/exit` + restart. The `plugin-cache-hygiene` recommendation now carries the live-session caveat. **Recommendation string only** — no new finding ID or scanner (count stays **16**, agents **7**, commands **21**), no token figures changed, so `--json`/`--raw` stay byte-stable and the frozen v5.0.0 + SC-5 + default-output snapshots are untouched. **1297** tests. | From bfd577aeee88ada7e6955b0851a722fa676af74a Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 12:23:43 +0200 Subject: [PATCH 36/53] =?UTF-8?q?fix(acr):=20file-discovery=20skips=20back?= =?UTF-8?q?ups/=20dirs=20=E2=80=94=20never=20live=20config=20(M-BUG-8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A directory named `backups` holds backup COPIES, not live config, so walking it during a config audit produces stale findings. config-audit's own session backups (~/.claude/config-audit/backups//files/.../CLAUDE.md) were the canonical case: a ~/.claude-scope audit walked 36 frozen config copies as if live, polluting CPS (C3) and HKV/RUL (C6) results. Same non-live-noise family as M-BUG-2. - Add `backups` to SKIP_DIRS (broad, name-based — consistent with vendor/dist/ .cache): the rule is general, a backups/ dir is never live config. TDD: 2 failing tests (no config under backups/ discovered + backups/ counted as skipped) -> green; a third asserts live config beside the backups tree is still discovered. Full suite 1310/0 (+3). Byte-stable: no fixture is named `backups` and `backups` appears in zero frozen snapshots, so v5.0.0 + SC-5 + default-output outputs are unchanged. Dogfooding ~/.claude: files under /backups/ drop from 36 -> 0; 717 live config files retained. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- scanners/lib/file-discovery.mjs | 5 ++++ tests/lib/file-discovery.test.mjs | 48 ++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/scanners/lib/file-discovery.mjs b/scanners/lib/file-discovery.mjs index 7f32bad..eb6e832 100644 --- a/scanners/lib/file-discovery.mjs +++ b/scanners/lib/file-discovery.mjs @@ -11,6 +11,11 @@ const SKIP_DIRS = new Set([ 'node_modules', '.git', 'dist', 'build', 'coverage', '__pycache__', '.next', '.nuxt', '.output', '.cache', '.turbo', '.parcel-cache', 'vendor', 'venv', '.venv', '.tox', + // A `backups` dir holds backup COPIES, not live config — auditing it as if + // live produces stale findings. config-audit's own session backups + // (~/.claude/config-audit/backups//files/.../CLAUDE.md) are the canonical + // case (M-BUG-8), but the rule is general: backups are never live config. + 'backups', ]); // Path marker for the plugin install cache (~/.claude/plugins/cache). diff --git a/tests/lib/file-discovery.test.mjs b/tests/lib/file-discovery.test.mjs index 036fe4f..3a4a09f 100644 --- a/tests/lib/file-discovery.test.mjs +++ b/tests/lib/file-discovery.test.mjs @@ -1,6 +1,6 @@ import { describe, it, before, after } from 'node:test'; import assert from 'node:assert/strict'; -import { join } from 'node:path'; +import { join, sep } from 'node:path'; import { mkdir, writeFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { @@ -509,3 +509,49 @@ describe('discoverConfigFiles — cache-aware filtering', () => { assert.ok(!keys.has(STALE1) && !keys.has(STALE2), 'stale dropped via Multi'); }); }); + +// ─────────────────────────────────────────────────────────────── +// Group 8: backups/ skip (M-BUG-8) — a directory named `backups` +// holds backup COPIES, not live config. config-audit's own session +// backups (~/.claude/config-audit/backups//files/plugins/*/CLAUDE.md) +// were walked as if live, producing stale findings on a ~/.claude audit. +// ─────────────────────────────────────────────────────────────── + +describe('discoverConfigFiles — skips backups/ (M-BUG-8)', () => { + let dir; + + before(async () => { + dir = tempDir('backups'); + // A live CLAUDE.md at the root (must still be discovered). + await mkdir(dir, { recursive: true }); + await writeFile(join(dir, 'CLAUDE.md'), '# Live config'); + // config-audit's own backup tree — frozen copies, NOT live config. + const backupClaude = join(dir, 'config-audit', 'backups', '20260518_103233', 'files', 'plugins', 'voyage'); + await mkdir(backupClaude, { recursive: true }); + await writeFile(join(backupClaude, 'CLAUDE.md'), '# Frozen backup copy'); + const backupSettings = join(dir, 'config-audit', 'backups', '20260518_103233', 'files', '.claude'); + await mkdir(backupSettings, { recursive: true }); + await writeFile(join(backupSettings, 'settings.json'), '{}'); + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('discovers the live CLAUDE.md', async () => { + const { files } = await discoverConfigFiles(dir); + const live = files.find(f => f.relPath === 'CLAUDE.md'); + assert.ok(live, 'live CLAUDE.md at root should be discovered'); + }); + + it('does NOT discover config files under a backups/ directory', async () => { + const { files } = await discoverConfigFiles(dir); + const inBackups = files.filter(f => f.absPath.includes(`backups${sep}`)); + assert.equal(inBackups.length, 0, 'no files under backups/ should be discovered'); + }); + + it('counts backups/ as a skipped directory', async () => { + const { skipped } = await discoverConfigFiles(dir); + assert.ok(skipped >= 1, 'backups/ should be counted as skipped'); + }); +}); From dd9db60fc9886c11d6a9b4192c3d05a3c277a132 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 12:45:14 +0200 Subject: [PATCH 37/53] fix(acr): CPS ignores fenced/inline code + CC-stable path vars (M-BUG-7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CPS flagged ${CLAUDE_PLUGIN_ROOT}/${CLAUDE_PROJECT_DIR} (CC-provided stable paths) and {date}/timestamp tokens shown in documentation as cache-busters. Fix: skip fenced code blocks, strip inline-code spans, and whitelist CC-stable vars before pattern-matching. Suppress-only — frozen v5.0.0 snapshots untouched (CPS yields findings:[] there), no re-seed. Suite 1316/0 (+6). Dogfood ~/.claude 5->2 (3 doc false-positives suppressed; 2 remaining = own volatile test fixtures). --- scanners/cache-prefix-scanner.mjs | 60 +++++++++++++++---- .../cps-fenced/inside-fence/CLAUDE.md | 46 ++++++++++++++ tests/fixtures/cps-fenced/mixed/CLAUDE.md | 55 +++++++++++++++++ .../fixtures/cps-inline-code/mixed/CLAUDE.md | 55 +++++++++++++++++ tests/fixtures/cps-inline-code/pure/CLAUDE.md | 46 ++++++++++++++ .../cps-stable-vars/non-whitelisted/CLAUDE.md | 46 ++++++++++++++ .../cps-stable-vars/whitelisted/CLAUDE.md | 46 ++++++++++++++ tests/scanners/cache-prefix.test.mjs | 60 +++++++++++++++++++ 8 files changed, 403 insertions(+), 11 deletions(-) create mode 100644 tests/fixtures/cps-fenced/inside-fence/CLAUDE.md create mode 100644 tests/fixtures/cps-fenced/mixed/CLAUDE.md create mode 100644 tests/fixtures/cps-inline-code/mixed/CLAUDE.md create mode 100644 tests/fixtures/cps-inline-code/pure/CLAUDE.md create mode 100644 tests/fixtures/cps-stable-vars/non-whitelisted/CLAUDE.md create mode 100644 tests/fixtures/cps-stable-vars/whitelisted/CLAUDE.md diff --git a/scanners/cache-prefix-scanner.mjs b/scanners/cache-prefix-scanner.mjs index 62895ea..0e41822 100644 --- a/scanners/cache-prefix-scanner.mjs +++ b/scanners/cache-prefix-scanner.mjs @@ -30,7 +30,28 @@ const SCANNER = 'CPS'; // hits per turn, not to chase every inline date in a long backlog file. const CACHED_PREFIX_LINES = 150; -// Volatile-pattern set (extends token-hotspots.mjs Pattern A). +// CC-provided substitution variables that resolve to a stable per-install or +// per-project path (e.g. "${CLAUDE_PLUGIN_ROOT}/hooks/x.mjs"). CC expands them +// to the same value every turn, so they never break the prompt cache — unlike a +// runtime ${TIMESTAMP}. Excluded from the ${VAR} volatile flag (M-BUG-7). +const STABLE_CC_VARS = new Set(['CLAUDE_PLUGIN_ROOT', 'CLAUDE_PROJECT_DIR']); + +// Matches every ${VAR} occurrence on a line so a line carrying only stable CC +// vars is not mistaken for a runtime cache-buster. +const VAR_RX = /\$\{([A-Z_][A-Z0-9_]*)\}/g; + +/** True when a line contains at least one non-CC-stable ${VAR} substitution. */ +function hasVolatileVar(line) { + VAR_RX.lastIndex = 0; + let m; + while ((m = VAR_RX.exec(line)) !== null) { + if (!STABLE_CC_VARS.has(m[1])) return true; + } + return false; +} + +// Volatile-pattern set (extends token-hotspots.mjs Pattern A). The ${VAR} entry +// is `varAware` — flagged via hasVolatileVar() so CC-stable vars are excluded. const VOLATILE_PATTERNS = [ { rx: /\{timestamp\}/i, label: '{timestamp} placeholder' }, { rx: /\{uuid\}/i, label: '{uuid} placeholder' }, @@ -41,7 +62,7 @@ const VOLATILE_PATTERNS = [ { rx: /^\s*\[\d{4}-\d{2}-\d{2}/, label: 'dated log line [YYYY-MM-DD ...]' }, // v5 N3 extensions: { rx: /^\s*!/, label: 'shell-exec line (! prefix)' }, - { rx: /\$\{[A-Z_][A-Z0-9_]*\}/, label: '${VAR} substitution' }, + { rx: /\$\{[A-Z_][A-Z0-9_]*\}/, label: '${VAR} substitution', varAware: true }, ]; /** @@ -68,16 +89,33 @@ function findVolatileLines(content) { const out = []; if (!content) return out; const lines = content.split('\n').slice(0, CACHED_PREFIX_LINES); + let inFence = false; for (let i = 0; i < lines.length; i++) { - for (const { rx, label } of VOLATILE_PATTERNS) { - if (rx.test(lines[i])) { - out.push({ - line: i + 1, - label, - snippet: lines[i].length > 120 ? lines[i].slice(0, 117) + '...' : lines[i], - }); - break; - } + const line = lines[i]; + // Fenced code blocks (``` or ~~~) hold illustrative, byte-stable literal + // text — a ${VAR} or timestamp shown inside one is documentation, not a + // runtime cache-buster — so the fence delimiters and their content are + // skipped (M-BUG-7). + if (/^\s*(```|~~~)/.test(line)) { + inFence = !inFence; + continue; + } + if (inFence) continue; + // Strip `inline code` spans before pattern-testing: a {date} or ${VAR} + // shown inside backticks is literal documentation text, byte-stable, not a + // runtime cache-buster (M-BUG-7). The original line is still reported as the + // snippet so context is preserved. + const probe = line.replace(/`[^`]*`/g, ''); + for (const { rx, label, varAware } of VOLATILE_PATTERNS) { + // The ${VAR} pattern flags only non-CC-stable substitutions; every other + // pattern keeps its plain line test. + if (varAware ? !hasVolatileVar(probe) : !rx.test(probe)) continue; + out.push({ + line: i + 1, + label, + snippet: line.length > 120 ? line.slice(0, 117) + '...' : line, + }); + break; } } return out; diff --git a/tests/fixtures/cps-fenced/inside-fence/CLAUDE.md b/tests/fixtures/cps-fenced/inside-fence/CLAUDE.md new file mode 100644 index 0000000..5ccb2b6 --- /dev/null +++ b/tests/fixtures/cps-fenced/inside-fence/CLAUDE.md @@ -0,0 +1,46 @@ +# Fenced Documentation Project + +Stable preamble. All volatility below is illustrative, inside a code fence. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +```bash +export STAMP=${TIMESTAMP} +!deploy.sh --at 2026-06-26T10:00:00 +echo {date} > /tmp/log +[2026-06-26 12:00] starting +``` +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. diff --git a/tests/fixtures/cps-fenced/mixed/CLAUDE.md b/tests/fixtures/cps-fenced/mixed/CLAUDE.md new file mode 100644 index 0000000..cd55d8a --- /dev/null +++ b/tests/fixtures/cps-fenced/mixed/CLAUDE.md @@ -0,0 +1,55 @@ +# Fenced Plus Prose Project + +Stable preamble. A fence below, plus one genuine volatile prose line. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +```bash +export STAMP=${TIMESTAMP} # documented, not live +``` +Body line 38. +Body line 39. +Body line 40. +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. +Body line 47. +Body line 48. +Body line 49. +!git log -1 # genuine shell-exec in live prose at line 50 +Body line 51. +Body line 52. +Body line 53. +Body line 54. +Body line 55. diff --git a/tests/fixtures/cps-inline-code/mixed/CLAUDE.md b/tests/fixtures/cps-inline-code/mixed/CLAUDE.md new file mode 100644 index 0000000..c39f00e --- /dev/null +++ b/tests/fixtures/cps-inline-code/mixed/CLAUDE.md @@ -0,0 +1,55 @@ +# Inline Code Mixed Project + +Stable preamble. Backticked template plus a live var in prose. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +Body line 35. +Body line 36. +Body line 37. +Body line 38. +Body line 39. +Output template is `run-{date}.md` (documented). +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. +Body line 47. +Body line 48. +Body line 49. +Deployed at ${RELEASE_STAMP} on every push. +Body line 51. +Body line 52. +Body line 53. +Body line 54. +Body line 55. diff --git a/tests/fixtures/cps-inline-code/pure/CLAUDE.md b/tests/fixtures/cps-inline-code/pure/CLAUDE.md new file mode 100644 index 0000000..241c303 --- /dev/null +++ b/tests/fixtures/cps-inline-code/pure/CLAUDE.md @@ -0,0 +1,46 @@ +# Inline Code Doc Project + +Stable preamble. A filename template is documented in backticks below. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +Body line 35. +Body line 36. +Body line 37. +Body line 38. +Body line 39. +| `--brief ` | writes to `.claude/plans/run-{date}-{slug}.md` | +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. diff --git a/tests/fixtures/cps-stable-vars/non-whitelisted/CLAUDE.md b/tests/fixtures/cps-stable-vars/non-whitelisted/CLAUDE.md new file mode 100644 index 0000000..dd1ffe9 --- /dev/null +++ b/tests/fixtures/cps-stable-vars/non-whitelisted/CLAUDE.md @@ -0,0 +1,46 @@ +# Non-Whitelisted Var Project + +Stable preamble with a genuine runtime substitution below. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +Body line 35. +Body line 36. +Body line 37. +Body line 38. +Body line 39. +Deployed build tag: ${DEPLOY_TIMESTAMP} (changes every release). +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. diff --git a/tests/fixtures/cps-stable-vars/whitelisted/CLAUDE.md b/tests/fixtures/cps-stable-vars/whitelisted/CLAUDE.md new file mode 100644 index 0000000..3fb0961 --- /dev/null +++ b/tests/fixtures/cps-stable-vars/whitelisted/CLAUDE.md @@ -0,0 +1,46 @@ +# Stable CC Vars Project + +Stable preamble — only CC-provided path vars appear below. +Body line 4. +Body line 5. +Body line 6. +Body line 7. +Body line 8. +Body line 9. +Body line 10. +Body line 11. +Body line 12. +Body line 13. +Body line 14. +Body line 15. +Body line 16. +Body line 17. +Body line 18. +Body line 19. +Body line 20. +Body line 21. +Body line 22. +Body line 23. +Body line 24. +Body line 25. +Body line 26. +Body line 27. +Body line 28. +Body line 29. +Body line 30. +Body line 31. +Body line 32. +Body line 33. +Body line 34. +Body line 35. +Body line 36. +Body line 37. +Body line 38. +Body line 39. +Hooks resolve under ${CLAUDE_PLUGIN_ROOT}/hooks and project root is ${CLAUDE_PROJECT_DIR}. +Body line 41. +Body line 42. +Body line 43. +Body line 44. +Body line 45. +Body line 46. diff --git a/tests/scanners/cache-prefix.test.mjs b/tests/scanners/cache-prefix.test.mjs index aecea00..e4f3d00 100644 --- a/tests/scanners/cache-prefix.test.mjs +++ b/tests/scanners/cache-prefix.test.mjs @@ -86,6 +86,66 @@ describe('CPS scanner — volatile content in @imported files (v5.10 B6)', () => }); }); +describe('CPS scanner — CC-stable vars are not cache-busters (M-BUG-7)', () => { + it('does NOT flag ${CLAUDE_PLUGIN_ROOT} / ${CLAUDE_PROJECT_DIR} in prose', async () => { + // These are CC-provided path substitutions that resolve to a stable value + // every turn — referencing them in CLAUDE.md prose is not a cache-buster. + const result = await runScanner('cps-stable-vars/whitelisted'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.equal(f, undefined, + `CC-stable path vars must not trip CPS; got: ${f?.evidence}`); + }); + + it('STILL flags a genuine non-CC ${VAR} substitution (whitelist is selective)', async () => { + const result = await runScanner('cps-stable-vars/non-whitelisted'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.ok(f, `expected a finding for a genuine runtime var; got none`); + assert.match(String(f.evidence || ''), /line 40/); + assert.match(String(f.evidence || ''), /\$\{VAR\} substitution/i); + }); +}); + +describe('CPS scanner — volatility inside fenced code blocks is documentation (M-BUG-7)', () => { + it('does NOT flag volatile-looking lines inside a ```fence```', async () => { + // Content inside a fenced code block is illustrative, byte-stable literal + // text — not runtime volatility — so it must not break the cached prefix. + const result = await runScanner('cps-fenced/inside-fence'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.equal(f, undefined, + `fenced code content must not trip CPS; got: ${f?.evidence}`); + }); + + it('STILL flags genuine volatility in live prose outside the fence', async () => { + const result = await runScanner('cps-fenced/mixed'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.ok(f, `expected a finding for the out-of-fence volatile line; got none`); + assert.match(String(f.evidence || ''), /line 50/); + assert.match(String(f.evidence || ''), /shell-exec/i); + assert.doesNotMatch(String(f.evidence || ''), /line 3[567]/, + 'fenced lines 35–37 must not appear in the evidence'); + }); +}); + +describe('CPS scanner — volatility inside inline code is documentation (M-BUG-7)', () => { + it('does NOT flag a {date} placeholder shown inside `inline code`', async () => { + // A filename template like `.claude/plans/run-{date}.md` in backticks is + // literal documentation text — byte-stable, not a runtime substitution. + const result = await runScanner('cps-inline-code/pure'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.equal(f, undefined, + `inline-code documentation must not trip CPS; got: ${f?.evidence}`); + }); + + it('STILL flags a live ${VAR} that sits outside backticks', async () => { + const result = await runScanner('cps-inline-code/mixed'); + const f = result.findings.find(x => /volatile content inside cached prefix/i.test(x.title || '')); + assert.ok(f, `expected a finding for the out-of-backtick volatile line; got none`); + assert.match(String(f.evidence || ''), /line 50/); + assert.doesNotMatch(String(f.evidence || ''), /line 40/, + 'the backticked {date} on line 40 must not appear in the evidence'); + }); +}); + describe('CPS scanner — orchestrator wiring', () => { it('CPS appears in scan-orchestrator scanner list', async () => { const orch = await import('../../scanners/scan-orchestrator.mjs'); From 7e949105665958f508e144db4834506207c395f6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 14:29:24 +0200 Subject: [PATCH 38/53] fix(acr): token estimator discounts block-level HTML comments (M-BUG-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md token estimates counted block-level HTML comments toward always-loaded tokens, but CC strips them before injection (preserved only inside code fences, per code.claude.com/docs/en/memory). Fix: new stripInjectedHtmlComments + effectiveMemoryBytes in active-config-reader; the CML cascade (walkClaudeMdCascade) and token-hotspots now size CLAUDE.md from effective (stripped) bytes, while raw byte figures stay honest. Block-level only — inline comments retained (conservative, verified scope). Suite 1329/0 (+13). Frozen v5.0.0 snapshots untouched (no fixture has `) that lie OUTSIDE fenced code + * blocks. Claude Code strips these before injecting a CLAUDE.md / memory file + * into context (code.claude.com/docs/en/memory: "block-level HTML comments are + * stripped before the content is injected"), preserving them only inside fenced + * code blocks (``` / ~~~). A byte-accurate token estimate must therefore discount + * them. (M-BUG-6) + * + * Conservative scope — only *block-level* comments are removed (a comment that + * occupies its own line(s)); inline comments sharing a line with other text are + * retained, since the verified CC behavior covers block-level stripping only. + * + * @param {string} content + * @returns {string} content with out-of-fence block comments removed + */ +export function stripInjectedHtmlComments(content) { + if (typeof content !== 'string' || content === '') return ''; + const lines = content.split('\n'); + const out = []; + let inFence = false; + let inComment = false; + for (const line of lines) { + if (inComment) { + // Inside a multi-line block comment: drop lines until the closing `-->`, + // keeping any real content that trails the close on the same line. + const end = line.indexOf('-->'); + if (end !== -1) { + inComment = false; + const rest = line.slice(end + 3); + if (rest.trim() !== '') out.push(rest); + } + continue; + } + // Fence delimiters (``` / ~~~) toggle a preserve-verbatim region. + if (/^\s*(```|~~~)/.test(line)) { + inFence = !inFence; + out.push(line); + continue; + } + if (inFence) { + out.push(line); + continue; + } + // Whole line is a single self-contained block comment → CC strips it. + if (/^\s*\s*$/.test(line)) continue; + // Block comment opening with nothing but whitespace before it and no close + // on this line → runs onto following lines. + const openIdx = line.indexOf('', openIdx) === -1 && line.slice(0, openIdx).trim() === '') { + inComment = true; + continue; + } + out.push(line); + } + return out.join('\n'); +} + +/** + * Effective injected byte length of a CLAUDE.md / memory source: raw UTF-8 bytes + * minus the block-level HTML comments CC strips before injection. Used wherever a + * CLAUDE.md token estimate must reflect what actually enters context. (M-BUG-6) + * + * @param {string} content + * @returns {number} + */ +export function effectiveMemoryBytes(content) { + if (typeof content !== 'string') return 0; + return Buffer.byteLength(stripInjectedHtmlComments(content), 'utf8'); +} + // ───────────────────────────────────────────────────────────────────────── // Load-pattern model (v5.6 Foundation) // ───────────────────────────────────────────────────────────────────────── @@ -202,7 +272,11 @@ export async function walkClaudeMdCascade(repoPath) { const totalBytes = files.reduce((sum, f) => sum + f.bytes, 0); const totalLines = files.reduce((sum, f) => sum + f.lines, 0); - const estimatedTokens = estimateTokens(totalBytes, 'markdown'); + // Token estimate is computed from the *effective* (injected) byte count — CC + // strips block-level HTML comments before injection — while totalBytes stays + // the honest on-disk figure. (M-BUG-6) + const effectiveBytes = files.reduce((sum, f) => sum + (f.effectiveBytes ?? f.bytes), 0); + const estimatedTokens = estimateTokens(effectiveBytes, 'markdown'); return { files, totalBytes, totalLines, estimatedTokens }; } @@ -217,6 +291,7 @@ async function tryAddClaudeMd(absPath, scope, parent, files, seen) { path: absPath, scope, bytes: s.size, + effectiveBytes: effectiveMemoryBytes(content), lines: lineCount(content), parent, }; diff --git a/scanners/token-hotspots.mjs b/scanners/token-hotspots.mjs index 87efcbe..f0f3e6c 100644 --- a/scanners/token-hotspots.mjs +++ b/scanners/token-hotspots.mjs @@ -22,12 +22,12 @@ */ import { resolve, dirname, isAbsolute } from 'node:path'; -import { stat } from 'node:fs/promises'; +import { stat, readFile } from 'node:fs/promises'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { findImports, parseJson, parseFrontmatter } from './lib/yaml-parser.mjs'; -import { estimateTokens, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; +import { estimateTokens, effectiveMemoryBytes, readActiveConfig, deriveLoadPattern } from './lib/active-config-reader.mjs'; import { assessMcpDeferralForRepo, severityForForcedSchemas, @@ -261,6 +261,26 @@ function detectRedundantPermissions(settings) { return issues; } +/** + * Byte count to feed the token estimator for a discovered file. CLAUDE.md / + * memory files are sized from their *effective* (injected) content — CC strips + * block-level HTML comments before injection — so a raw byte read over-counts + * them. Every other source uses the raw on-disk size. (M-BUG-6) + * + * @param {{type:string, absPath?:string, size:number}} f + * @returns {Promise} + */ +async function tokenBytesFor(f) { + if (f.type === 'claude-md' && f.absPath) { + try { + return effectiveMemoryBytes(await readFile(f.absPath, 'utf-8')); + } catch { + return f.size; + } + } + return f.size; +} + /** * Build the ranked hotspots array. * @@ -272,7 +292,7 @@ async function buildHotspots(discovery, targetPath, activeConfig) { const ranked = []; for (const f of discovery.files) { const kind = tokenKind(f.type); - const tokens = estimateTokens(f.size, kind); + const tokens = estimateTokens(await tokenBytesFor(f), kind); if (tokens <= 0) continue; ranked.push({ absPath: f.absPath, @@ -651,7 +671,7 @@ export async function scan(targetPath, discovery) { // ── Total estimated tokens (sum of every discovered source + activeConfig MCP) ── let totalTokens = 0; for (const f of discovery.files) { - totalTokens += estimateTokens(f.size, tokenKind(f.type)); + totalTokens += estimateTokens(await tokenBytesFor(f), tokenKind(f.type)); } if (activeConfig && Array.isArray(activeConfig.mcpServers)) { for (const m of activeConfig.mcpServers) { diff --git a/tests/lib/active-config-reader.test.mjs b/tests/lib/active-config-reader.test.mjs index 10c1874..7753022 100644 --- a/tests/lib/active-config-reader.test.mjs +++ b/tests/lib/active-config-reader.test.mjs @@ -5,6 +5,8 @@ import { mkdir, writeFile, rm, readFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { estimateTokens, + stripInjectedHtmlComments, + effectiveMemoryBytes, detectGitRoot, walkClaudeMdCascade, readClaudeJsonProjectSlice, @@ -196,6 +198,85 @@ describe('estimateTokens', () => { }); }); +// ───────────────────────────────────────────────────────────────────────── +// stripInjectedHtmlComments / effectiveMemoryBytes (M-BUG-6) +// Claude Code strips block-level HTML comments from a CLAUDE.md/memory file +// before injecting it into context (code.claude.com/docs/en/memory), preserving +// them only inside fenced code blocks. A byte-accurate token estimate must +// discount them. Inline comments (text on the same line) are conservatively +// retained — only block-level stripping is verified behavior. +// ───────────────────────────────────────────────────────────────────────── + +describe('stripInjectedHtmlComments (M-BUG-6)', () => { + it('strips a single-line block-level comment outside code fences', () => { + const src = '# Title\n\n\n\nBody text.\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(!out.includes('maintainer note'), 'comment text should be removed'); + assert.ok(out.includes('# Title') && out.includes('Body text.'), 'surrounding content preserved'); + }); + + it('strips a multi-line block comment outside fences', () => { + const src = 'A\n\nB\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(!out.includes('line one') && !out.includes('line two'), 'all comment lines removed'); + assert.ok(out.includes('A') && out.includes('B'), 'surrounding content preserved'); + }); + + it('preserves an HTML comment inside a ``` fenced code block', () => { + const src = '# Title\n\n```html\n\n```\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(out.includes('kept: this is example code'), 'fenced comment must be preserved (CC keeps it)'); + }); + + it('preserves an HTML comment inside a ~~~ fenced code block', () => { + const src = '~~~\n\n~~~\n'; + const out = stripInjectedHtmlComments(src); + assert.ok(out.includes('kept tilde'), 'tilde-fenced comment must be preserved'); + }); + + it('keeps inline comments (only block-level stripping is verified)', () => { + // Text on the same line as the comment → conservatively retained; the + // verified CC behavior covers block-level comments only (Verifiseringsplikt). + const src = 'Visible tail\n'; + assert.equal(stripInjectedHtmlComments(src), src); + }); + + it('returns content unchanged when there are no comments', () => { + const src = '# Plain\n\nNo comments here.\n'; + assert.equal(stripInjectedHtmlComments(src), src); + }); + + it('handles empty and non-string input', () => { + assert.equal(stripInjectedHtmlComments(''), ''); + assert.equal(stripInjectedHtmlComments(undefined), ''); + assert.equal(stripInjectedHtmlComments(null), ''); + }); +}); + +describe('effectiveMemoryBytes (M-BUG-6)', () => { + it('discounts out-of-fence block comments from the byte count', () => { + const src = '# Title\n\n\n\nBody.\n'; + const raw = Buffer.byteLength(src, 'utf8'); + const eff = effectiveMemoryBytes(src); + assert.ok(eff < raw, `effective (${eff}) should be below raw (${raw})`); + }); + + it('counts comments inside fences (CC keeps them)', () => { + const src = '```\n\n```\n'; + assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8')); + }); + + it('equals raw bytes when no comments are present', () => { + const src = '# Plain markdown\n\nbody\n'; + assert.equal(effectiveMemoryBytes(src), Buffer.byteLength(src, 'utf8')); + }); + + it('returns 0 for non-string input', () => { + assert.equal(effectiveMemoryBytes(undefined), 0); + assert.equal(effectiveMemoryBytes(null), 0); + }); +}); + // ───────────────────────────────────────────────────────────────────────── // detectGitRoot // ───────────────────────────────────────────────────────────────────────── @@ -291,6 +372,23 @@ describe('walkClaudeMdCascade', () => { assert.equal(result.estimatedTokens, Math.ceil(result.totalBytes / 4)); }); + it('discounts block-level HTML comments from estimatedTokens (M-BUG-6)', async () => { + // CC strips block-level HTML comments before injection, so a CLAUDE.md + // padded with a maintainer-note comment must estimate FEWER tokens than its + // raw byte size would imply — totalBytes stays the honest on-disk figure. + const comment = ``; + await writeFile( + join(fixture.root, 'CLAUDE.md'), + `# Project Instructions\n\n${comment}\n\nBuild with care.\n`, + ); + const result = await walkClaudeMdCascade(fixture.root); + assert.ok( + result.estimatedTokens < Math.ceil(result.totalBytes / 4), + `expected discounted tokens (${result.estimatedTokens}) below raw heuristic ` + + `(${Math.ceil(result.totalBytes / 4)})`, + ); + }); + it('handles missing user CLAUDE.md gracefully', async () => { // Remove user CLAUDE.md await rm(join(fixture.fakeHome, '.claude', 'CLAUDE.md')); diff --git a/tests/scanners/token-hotspots.test.mjs b/tests/scanners/token-hotspots.test.mjs index bb42ed9..7ade2c6 100644 --- a/tests/scanners/token-hotspots.test.mjs +++ b/tests/scanners/token-hotspots.test.mjs @@ -1,7 +1,9 @@ import { describe, it, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve } from 'node:path'; +import { resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { scan } from '../../scanners/token-hotspots.mjs'; import { discoverConfigFiles } from '../../scanners/lib/file-discovery.mjs'; @@ -405,3 +407,36 @@ describe('TOK scanner — H stale plugin-cache versions (v5.9 B3)', () => { assert.ok(!result.findings.some(x => /stale plugin-cache/i.test(x.title || ''))); }); }); + +describe('TOK scanner — CLAUDE.md HTML-comment token discount (M-BUG-6)', () => { + it('estimates a comment-padded CLAUDE.md below its raw byte heuristic', async () => { + const dir = join( + tmpdir(), + `config-audit-tok-mbug6-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ); + await mkdir(dir, { recursive: true }); + try { + // Big block-level comment CC strips before injection — the hotspot's + // estimated_tokens must reflect the stripped size, not the raw on-disk byte + // count. (Hotspot output exposes `path` + `estimated_tokens`, not `size`.) + const comment = ``; + const content = `# Root\n\n${comment}\n\nReal instruction body.\n`; + const rawBytes = Buffer.byteLength(content, 'utf8'); + await writeFile(join(dir, 'CLAUDE.md'), content); + resetCounter(); + const discovery = await discoverConfigFiles(dir); + const result = await withHermeticHome(() => scan(dir, discovery)); + const hs = result.hotspots.find( + h => typeof h.path === 'string' && h.path.endsWith('CLAUDE.md'), + ); + assert.ok(hs, 'expected a CLAUDE.md hotspot for the fixture'); + assert.ok( + hs.estimated_tokens < Math.ceil(rawBytes / 4), + `expected discounted tokens (${hs.estimated_tokens}) below raw heuristic ` + + `(${Math.ceil(rawBytes / 4)}) for ${rawBytes} raw bytes`, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); From 3cf5c714a212ac220262a7858aab7d7f92d296f0 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 15:15:14 +0200 Subject: [PATCH 39/53] fix(acr): SET typo-gates unknown-key false positives (M-BUG-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CC settings schema is passthrough (verified against the 2.1.193 binary): it forwards unrecognized keys unchanged rather than rejecting them, so an arbitrary unknown key is valid/forward-compatible, not an error — the finding's "silently ignored" claim was factually wrong. The only real risk is a TYPO of a real key (the intended setting then silently has no effect). Fix: flag an unknown key only when it closely matches a known key (new levenshtein helper; edit distance <= 2, both keys >= 4 chars); severity medium -> low; honest passthrough framing in the scanner + humanizer. Also refreshed KNOWN_KEYS with 6 binary-verified keys (agentPushNotifEnabled, remoteControlAtStartup, skipAutoPermissionPrompt, skipDangerousModePermissionPrompt, skipWorkflowUsageWarning, tui). Suite 1341/0 (+12). Frozen v5.0.0 snapshots untouched (0 CA-SET findings there), no re-seed. Dogfood ~/.claude/settings.json 6->0 (all 6 keys above were false unknown-key findings; 0 typo flags introduced across 167 walked files). --- scanners/lib/humanizer-data.mjs | 8 +-- scanners/lib/string-utils.mjs | 35 ++++++++++++ scanners/settings-validator.mjs | 44 ++++++++++++--- tests/lib/string-utils.test.mjs | 38 ++++++++++++- tests/scanners/settings-validator.test.mjs | 65 +++++++++++++++++++++- 5 files changed, 174 insertions(+), 16 deletions(-) diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index 2356547..adf6802 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -96,10 +96,10 @@ export const TRANSLATIONS = { // ───────────────────────────────────────────────────────────── SET: { static: { - 'Unknown settings key': { - title: 'A settings key isn\'t recognized', - description: 'A key in your settings file isn\'t one Claude Code understands. It will be ignored.', - recommendation: 'Check the key name for typos, or remove the key if it\'s no longer in use.', + 'Possible typo in settings key': { + title: 'A settings key looks like a typo', + description: 'A key in your settings file isn\'t recognized, but it\'s very close to a real one — likely a typo. Claude Code forwards unrecognized keys unchanged rather than rejecting them, so a misspelled key silently has no effect.', + recommendation: 'Check the suggested key name. Fix the spelling, or keep the key if it\'s intentional (e.g. a newer key this audit doesn\'t know yet).', }, 'Deprecated settings key': { title: 'A settings key is no longer supported', diff --git a/scanners/lib/string-utils.mjs b/scanners/lib/string-utils.mjs index e0e14cf..0f27bf1 100644 --- a/scanners/lib/string-utils.mjs +++ b/scanners/lib/string-utils.mjs @@ -43,6 +43,41 @@ export function isSimilar(a, b, threshold = 0.8) { return similarity >= threshold; } +/** + * Levenshtein edit distance between two strings (insertions, deletions, + * substitutions; a transposition counts as 2). Used for typo detection on + * settings keys. Zero external dependencies, O(a*b) with two rolling rows. + * @param {string} a + * @param {string} b + * @returns {number} + */ +export function levenshtein(a, b) { + if (a === b) return 0; + const al = a.length; + const bl = b.length; + if (al === 0) return bl; + if (bl === 0) return al; + let prev = new Array(bl + 1); + let curr = new Array(bl + 1); + for (let j = 0; j <= bl; j++) prev[j] = j; + for (let i = 1; i <= al; i++) { + curr[0] = i; + const ac = a.charCodeAt(i - 1); + for (let j = 1; j <= bl; j++) { + const cost = ac === b.charCodeAt(j - 1) ? 0 : 1; + curr[j] = Math.min( + prev[j] + 1, // deletion + curr[j - 1] + 1, // insertion + prev[j - 1] + cost, // substitution + ); + } + const tmp = prev; + prev = curr; + curr = tmp; + } + return prev[bl]; +} + /** * Extract all key-like patterns from a settings.json or similar config. * @param {object} obj diff --git a/scanners/settings-validator.mjs b/scanners/settings-validator.mjs index 03a52ec..bf84de8 100644 --- a/scanners/settings-validator.mjs +++ b/scanners/settings-validator.mjs @@ -8,11 +8,11 @@ import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; import { parseJson } from './lib/yaml-parser.mjs'; -import { extractKeys } from './lib/string-utils.mjs'; +import { extractKeys, levenshtein } from './lib/string-utils.mjs'; const SCANNER = 'SET'; -/** Known top-level settings.json keys (as of CC 2.1.181 / June 2026) */ +/** Known top-level settings.json keys (as of CC 2.1.193 / June 2026) */ const KNOWN_KEYS = new Set([ 'additionalDirectories', 'agent', 'allowAllClaudeAiMcps', 'allowedChannelPlugins', 'allowedHttpHookUrls', @@ -37,6 +37,9 @@ const KNOWN_KEYS = new Set([ 'spinnerTipsOverride', 'spinnerVerbs', 'statusLine', 'strictKnownMarketplaces', 'useAutoModeDuringPlan', 'voiceEnabled', 'wheelScrollAccelerationEnabled', 'worktree', '$schema', + // CC 2.1.193 binary-verified (M-BUG-10): present as quoted string literals in the binary + 'agentPushNotifEnabled', 'remoteControlAtStartup', 'skipAutoPermissionPrompt', + 'skipDangerousModePermissionPrompt', 'skipWorkflowUsageWarning', 'tui', ]); /** Deprecated keys with migration info */ @@ -75,6 +78,16 @@ const VALID_EFFORT_LEVELS = new Set(['low', 'medium', 'high', 'xhigh', 'max']); * a project root to walks/discovery, inflating per-turn cost and confusing scope. */ const ADDITIONAL_DIRS_THRESHOLD = 2; +/** M-BUG-10: the CC settings schema is passthrough — it forwards unrecognized + * keys unchanged rather than rejecting them, so an arbitrary unknown key is + * valid/forward-compatible, not an error. The only real risk is a TYPO of a + * real key (the intended setting silently does nothing), so an unknown key is + * flagged ONLY when it closely matches a known key: edit distance within + * TYPO_MAX_DISTANCE and both keys at least TYPO_MIN_LEN chars (short keys are + * too noisy for reliable edit-distance matching). */ +const TYPO_MAX_DISTANCE = 2; +const TYPO_MIN_LEN = 4; + /** The only valid sub-keys of `autoMode`, each a prose-rule string array * (the literal "$defaults" is a valid entry). Verified against * code.claude.com/docs/en/auto-mode-config. */ @@ -115,17 +128,32 @@ export async function scan(targetPath, discovery) { continue; } - // Check for unknown keys + // Check for unknown keys — typo gate (M-BUG-10). The CC settings schema is + // passthrough, so an unrecognized key is NOT an error; only a typo of a real + // key is (the intended setting silently does nothing). Flag a key only when + // it closely matches a known key; an unknown key far from every known key is + // treated as valid/forward-compatible and emitted nothing. for (const key of Object.keys(parsed)) { - if (!KNOWN_KEYS.has(key)) { + if (KNOWN_KEYS.has(key)) continue; + let nearest = null; + let best = Infinity; + for (const known of KNOWN_KEYS) { + if (Math.min(key.length, known.length) < TYPO_MIN_LEN) continue; + const d = levenshtein(key, known); + if (d <= TYPO_MAX_DISTANCE && d < best) { + best = d; + nearest = known; + } + } + if (nearest) { findings.push(finding({ scanner: SCANNER, - severity: SEVERITY.medium, - title: 'Unknown settings key', - description: `${file.relPath}: "${key}" is not a recognized settings.json key. It will be silently ignored.`, + severity: SEVERITY.low, + title: 'Possible typo in settings key', + description: `${file.relPath}: "${key}" is not a recognized settings.json key, but it closely matches "${nearest}". Claude Code forwards unrecognized keys unchanged (it does not reject them), so if "${key}" is a typo of "${nearest}" the intended setting silently has no effect.`, file: file.absPath, evidence: key, - recommendation: 'Check spelling. See https://json.schemastore.org/claude-code-settings.json for valid keys.', + recommendation: `Did you mean "${nearest}"? Fix the spelling, or keep "${key}" if it is intentional (e.g. a newer settings key this audit does not recognize yet).`, autoFixable: false, })); } diff --git a/tests/lib/string-utils.test.mjs b/tests/lib/string-utils.test.mjs index 9855b2d..2629bff 100644 --- a/tests/lib/string-utils.test.mjs +++ b/tests/lib/string-utils.test.mjs @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { lineCount, truncate, isSimilar, extractKeys, normalizePath } from '../../scanners/lib/string-utils.mjs'; +import { lineCount, truncate, isSimilar, extractKeys, normalizePath, levenshtein } from '../../scanners/lib/string-utils.mjs'; describe('lineCount', () => { it('counts lines correctly', () => { @@ -90,6 +90,42 @@ describe('extractKeys', () => { }); }); +describe('levenshtein', () => { + it('returns 0 for identical strings', () => { + assert.strictEqual(levenshtein('permissions', 'permissions'), 0); + }); + + it('counts a single substitution as 1', () => { + assert.strictEqual(levenshtein('model', 'modet'), 1); + }); + + it('counts a single deletion as 1 (permisions → permissions)', () => { + assert.strictEqual(levenshtein('permisions', 'permissions'), 1); + }); + + it('counts a single insertion as 1', () => { + assert.strictEqual(levenshtein('hooks', 'hooks2'), 1); + }); + + it('counts a transposition as 2 (standard Levenshtein)', () => { + assert.strictEqual(levenshtein('import', 'improt'), 2); + }); + + it('returns the other length when one string is empty', () => { + assert.strictEqual(levenshtein('', 'abc'), 3); + assert.strictEqual(levenshtein('abc', ''), 3); + assert.strictEqual(levenshtein('', ''), 0); + }); + + it('is symmetric', () => { + assert.strictEqual(levenshtein('effortLevel', 'efortLevel'), levenshtein('efortLevel', 'effortLevel')); + }); + + it('gives a large distance for unrelated keys', () => { + assert.ok(levenshtein('unknownKey123', 'permissions') > 2); + }); +}); + describe('normalizePath', () => { it('expands ~ to HOME', () => { const home = process.env.HOME; diff --git a/tests/scanners/settings-validator.test.mjs b/tests/scanners/settings-validator.test.mjs index 5aac01a..2612e10 100644 --- a/tests/scanners/settings-validator.test.mjs +++ b/tests/scanners/settings-validator.test.mjs @@ -47,10 +47,12 @@ describe('SET scanner — broken project', () => { result = await scan(resolve(FIXTURES, 'broken-project'), discovery); }); - it('detects unknown settings key', () => { - // CA-SET-001 in broken-project, evidence='unknownKey123'. + it('does NOT flag an arbitrary unknown key (CC schema is passthrough)', () => { + // M-BUG-10: "unknownKey123" is not close to any known key, so it is a + // valid forward-compatible / passthrough key — NOT a typo. Claude Code + // forwards unrecognized keys unchanged, so flagging it is a false positive. const found = result.findings.some(f => f.scanner === 'SET' && /unknownKey123/.test(f.evidence || '')); - assert.ok(found, 'Should detect unknownKey123'); + assert.ok(!found, 'unknownKey123 is far from every known key → must not be flagged'); }); it('detects deprecated key (includeCoAuthoredBy)', () => { @@ -192,6 +194,63 @@ describe('SET scanner — CC 2.1.114→181 valid keys (Batch 1 false-positive fi }); }); +describe('SET scanner — unknown-key typo gate (M-BUG-10)', () => { + // The CC settings schema is passthrough: it forwards unrecognized keys + // unchanged rather than rejecting them, so an arbitrary unknown key is + // valid/forward-compatible and must NOT be flagged. The only real risk is a + // TYPO of a real key (the intended setting silently has no effect), so the + // scanner flags ONLY unknown keys that closely match a known key. + // Path-guard blocks committing settings.json fixtures → materialize a temp one. + let tmpRoot; + let result; + const TYPO_TITLE = 'Possible typo in settings key'; + + beforeEach(async () => { + resetCounter(); + tmpRoot = await mkdtemp(join(tmpdir(), 'ca-set-typo-')); + await mkdir(join(tmpRoot, '.claude'), { recursive: true }); + const settings = { + $schema: 'https://json.schemastore.org/claude-code-settings.json', + permisions: { deny: ['Read(./.env)'] }, // TYPO of "permissions" (edit distance 1) + someNewKey2027: true, // forward-compat / passthrough — far from every known key + permissions: { deny: ['Read(./.env)'], allow: ['Bash(npm run *)'] }, + }; + await writeFile( + join(tmpRoot, '.claude', 'settings.json'), + JSON.stringify(settings, null, 2) + '\n', + 'utf8', + ); + const discovery = await discoverConfigFiles(tmpRoot); + result = await scan(tmpRoot, discovery); + }); + + afterEach(async () => { + if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true }); + }); + + it('flags a likely typo ("permisions") and suggests the nearest known key', () => { + const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions'); + assert.ok(typo, `expected a typo finding for "permisions"; got: ${result.findings.map(f => `${f.title}:${f.evidence || ''}`).join(' | ')}`); + assert.equal(typo.title, TYPO_TITLE); + assert.match(typo.recommendation, /permissions/, 'recommendation suggests the nearest known key'); + }); + + it('marks the typo finding as low severity (passthrough → not a hard error)', () => { + const typo = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'permisions'); + assert.equal(typo.severity, 'low'); + }); + + it('does NOT flag a forward-compatible key far from any known key', () => { + const fwd = result.findings.find(f => f.scanner === 'SET' && f.evidence === 'someNewKey2027'); + assert.equal(fwd, undefined, 'someNewKey2027 is far from every known key → passthrough, not flagged'); + }); + + it('emits exactly one typo finding for this file', () => { + const typos = result.findings.filter(f => f.scanner === 'SET' && f.title === TYPO_TITLE); + assert.equal(typos.length, 1, `expected exactly one typo finding; got: ${typos.map(f => f.evidence).join(', ')}`); + }); +}); + describe('SET scanner — empty project', () => { let result; beforeEach(async () => { From e8afb148d305e1c1c61791f6869397d79a2219dc Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 17:24:08 +0200 Subject: [PATCH 40/53] fix(acr): conflict-detector segregates plugin-bundled configs (M-BUG-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CNF compared every discovered settings.json/hooks.json pairwise regardless of origin, so it treated installed plugins' bundled configs — each plugin's own settings.json/hooks.json plus its shipped test fixtures and examples under ~/.claude/plugins/ — as if they were the user's authored cascade. A "conflict" between two plugins' bundled test fixtures is not something a user can resolve, yet these dominated the count: 339 CNF findings on this machine (315 high-sev permission allow/deny "conflicts", 18 duplicate-hook, 6 settings-key), almost all sourced from plugin-internal fixtures. The Conflicts grade was F on pure noise. Fix: CNF excludes any file whose path is under `.claude/plugins/` from conflict analysis (new isPluginBundled predicate; absPath marker). Kept CNF-local rather than a discovery-level skip on purpose: an active plugin's contributed hooks.json/.mcp.json legitimately lives in plugins/cache and other scanners need it — only conflict analysis must ignore plugin-bundled files. Same class as M-BUG-8 (non-live config trees treated as live). Suite 1344/0 (+3: plugin-bundled exclusion, discovery-side sanity, over-exclusion guard). Frozen v5.0.0 + SC-5 snapshots untouched (marketplace-medium has no plugins/ paths), no re-seed. Dogfood ~/.claude CNF 339->0 (F-grade was 100% plugin-bundled noise; the ~3 genuine user-scope local settings have no actual conflicting keys, matching the plan C5 "real surface ~3 files" prediction). Follow-up (not in this fix): classifyScope tags plugin-bundled files by checking basePath instead of the file's own path, so scope:'plugin' is effectively dead for a ~/.claude-rooted scan. Fixing it would let every scanner trust the scope field, but that is a discovery-layer change beyond this bug's scope. --- scanners/conflict-detector.mjs | 25 +++++- tests/scanners/conflict-detector.test.mjs | 97 ++++++++++++++++++++++- 2 files changed, 117 insertions(+), 5 deletions(-) diff --git a/scanners/conflict-detector.mjs b/scanners/conflict-detector.mjs index 44038b2..fbbcb23 100644 --- a/scanners/conflict-detector.mjs +++ b/scanners/conflict-detector.mjs @@ -5,6 +5,7 @@ * Finding IDs: CA-CNF-NNN */ +import { sep } from 'node:path'; import { readTextFile } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; @@ -17,6 +18,22 @@ const SCANNER = 'CNF'; // Keys checked separately or not meaningful to compare const SKIP_KEYS = new Set(['$schema', 'hooks', 'permissions']); +// Files under `.claude/plugins/` are shipped by installed plugins — the plugin's +// own settings.json/hooks.json plus bundled test fixtures and examples. They are +// not the user's authored cascade and a "conflict" between them is not something +// the user can resolve, so they must be excluded from cross-scope conflict +// analysis. (Other scanners still need active plugin config, so this exclusion is +// CNF-local, not a discovery-level skip. M-BUG-2.) +const PLUGIN_TREE_MARKER = `.claude${sep}plugins${sep}`; + +/** + * @param {import('./lib/file-discovery.mjs').ConfigFile} file + * @returns {boolean} true if the file is shipped by an installed plugin + */ +function isPluginBundled(file) { + return file.absPath.includes(PLUGIN_TREE_MARKER); +} + /** * Flatten an object's top-level keys into a simple key→value map. * Only first level — we compare top-level settings, not nested. @@ -63,10 +80,10 @@ export async function scan(targetPath, discovery) { const start = Date.now(); const findings = []; - // Collect settings files - const settingsFiles = discovery.files.filter(f => f.type === 'settings-json'); - // Collect hooks files - const hooksFiles = discovery.files.filter(f => f.type === 'hooks-json'); + // Collect settings files (excluding plugin-bundled — see PLUGIN_TREE_MARKER) + const settingsFiles = discovery.files.filter(f => f.type === 'settings-json' && !isPluginBundled(f)); + // Collect hooks files (excluding plugin-bundled) + const hooksFiles = discovery.files.filter(f => f.type === 'hooks-json' && !isPluginBundled(f)); const totalFiles = settingsFiles.length + hooksFiles.length; diff --git a/tests/scanners/conflict-detector.test.mjs b/tests/scanners/conflict-detector.test.mjs index f0c3226..cc06cc2 100644 --- a/tests/scanners/conflict-detector.test.mjs +++ b/tests/scanners/conflict-detector.test.mjs @@ -1,6 +1,6 @@ import { describe, it, beforeEach, before, after } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve, join } from 'node:path'; +import { resolve, join, sep } from 'node:path'; import { fileURLToPath } from 'node:url'; import { mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -208,3 +208,98 @@ describe('CNF scanner — cache exclusion drops duplicate-hook count (B3)', () = `cache exclusion must drop CNF count: include=${includeCount} exclude=${excludeCount}`); }); }); + +// M-BUG-2: CNF must segregate plugin-bundled configs from the user's authored +// cascade. Installed plugins ship their OWN settings.json / hooks.json — plus +// bundled test fixtures and examples — under ~/.claude/plugins/. None of these +// are user-authored config the user can edit, and a "conflict" between two +// plugins' bundled files is not something the user can resolve. Yet CNF compared +// every settings-json/hooks-json pairwise regardless of origin, inflating the +// Conflicts grade to F with hundreds of bogus findings (339 on this machine, +// ~315 of them high-severity permission "conflicts" between plugin test +// fixtures). CNF must exclude any file whose path is under `.claude/plugins/`. +// The exclusion belongs in CNF, NOT discovery: an active plugin's contributed +// hooks.json/.mcp.json legitimately lives in plugins/cache and other scanners +// need it — only conflict analysis must ignore it. +describe('CNF scanner — excludes plugin-bundled configs (M-BUG-2)', () => { + let dir, result, discovery; + + before(async () => { + dir = join(tmpdir(), `config-audit-cnf-mbug2-${Date.now()}`); + // Live, user-authored cascade (project-scope settings). + const liveClaude = join(dir, '.claude'); + await mkdir(liveClaude, { recursive: true }); + await writeFile(join(liveClaude, 'settings.json'), JSON.stringify({ + model: 'opus', + permissions: { deny: ['Bash(curl:*)'] }, + hooks: { PreToolUse: [{ matcher: 'Edit' }] }, + })); + // An installed plugin ships its OWN settings.json + hooks.json (active version). + const p1 = join(liveClaude, 'plugins', 'cache', 'mkt', 'p1', '1.0.0'); + await mkdir(join(p1, '.claude'), { recursive: true }); + await writeFile(join(p1, '.claude', 'settings.json'), JSON.stringify({ + model: 'sonnet', + permissions: { allow: ['Bash(curl:*)'] }, + hooks: { PreToolUse: [{ matcher: 'Edit' }] }, + })); + await mkdir(join(p1, 'hooks'), { recursive: true }); + await writeFile(join(p1, 'hooks', 'hooks.json'), JSON.stringify({ + hooks: { PreToolUse: [{ matcher: 'Edit' }] }, + })); + // Another plugin ships a TEST FIXTURE settings.json — pure noise, never live. + const p2fix = join(liveClaude, 'plugins', 'cache', 'mkt', 'p2', '2.0.0', + 'tests', 'fixtures', 'proj', '.claude'); + await mkdir(p2fix, { recursive: true }); + await writeFile(join(p2fix, 'settings.json'), JSON.stringify({ + model: 'haiku', + permissions: { allow: ['Bash(rm:*)'] }, + })); + + resetCounter(); + discovery = await discoverConfigFiles(dir); + result = await scan(dir, discovery); + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('discovery surfaces the plugin-bundled settings (exclusion must be CNF-side, not discovery-side)', () => { + const pluginSettings = discovery.files.filter( + f => f.type === 'settings-json' && f.absPath.includes(`.claude${sep}plugins${sep}`)); + assert.ok(pluginSettings.length >= 2, + `expected ≥2 plugin-bundled settings discovered; got ${pluginSettings.length}`); + }); + + it('does not flag any conflict sourced from plugin-bundled configs', () => { + assert.equal(result.findings.length, 0, + `expected 0 CNF findings (only noise is plugin-bundled); got ${result.findings.length}: ${result.findings.map(f => f.title).join(', ')}`); + }); +}); + +// Guard against over-exclusion: the fix must NOT silence genuine conflicts +// between the user's own authored files (user/project/local), which are not +// under `.claude/plugins/`. +describe('CNF scanner — genuine live conflict still flagged (M-BUG-2 guard)', () => { + let dir, result; + + before(async () => { + dir = join(tmpdir(), `config-audit-cnf-mbug2-guard-${Date.now()}`); + const claude = join(dir, '.claude'); + await mkdir(claude, { recursive: true }); + await writeFile(join(claude, 'settings.json'), JSON.stringify({ model: 'opus' })); + await writeFile(join(claude, 'settings.local.json'), JSON.stringify({ model: 'haiku' })); + resetCounter(); + const discovery = await discoverConfigFiles(dir); + result = await scan(dir, discovery); + }); + + after(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('still flags the genuine cross-scope key conflict between user-authored files', () => { + assert.ok(result.findings.some(f => f.title.includes('model')), + `expected a genuine "model" conflict; got: ${result.findings.map(f => f.title).join(', ') || '(none)'}`); + }); +}); From 1bdaefc2686f767412b66df120e823e2556d2252 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 26 Jun 2026 18:04:20 +0200 Subject: [PATCH 41/53] =?UTF-8?q?release:=20v5.12.5=20=E2=80=94=20"Dogfood?= =?UTF-8?q?=20denoise"=20(M-BUG-2/6/7/8/10=20scanner=20false-positive=20ba?= =?UTF-8?q?tch)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-sync for the Fase-3 scanner false-positive batch (code already shipped in bfd577a / dd9db60 / 7e94910 / 3cf5c71 / e8afb14): - plugin.json 5.12.4 -> 5.12.5 - README version badge -> 5.12.5, tests badge 1307 -> 1344, new version-history row - CHANGELOG [5.12.5] section (per-bug Fixed entries) Batch theme: five scanners stop counting non-user / non-live config as the user's authored cascade (plugin-bundled config, frozen backups, doc examples, forward-compatible settings keys). checkReadmeBadges: passed:true (tests 1344, scanners 16, commands 21, agents 7, hooks 4 — all match filesystem). Full suite 1344/0. Frozen v5.0.0 + SC-5 + default-output snapshots byte-stable; no re-seed across all five fixes (each affected fixture's findings are genuinely unchanged). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EnUvKEqyEa1m9gy6Aqhdqq --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++ README.md | 5 ++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 589062b..4654ec0 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "config-audit", "description": "Multi-agent workflow for analyzing, reporting, and optimizing Claude Code configuration across your entire machine", - "version": "5.12.4", + "version": "5.12.5", "author": { "name": "Kjell Tore Guttormsen" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index 28dc95d..dfd4e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,63 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.12.5] - 2026-06-26 + +### Summary +"Dogfood denoise" — a samle-release of the Fase-3 scanner false-positive batch: `M-BUG-2/6/7/8/10`, +all dogfooding finds from running config-audit on the maintainer's real `~/.claude`. The shared theme +is **non-user / non-live config wrongly counted as the user's authored cascade**: installed plugins' +bundled config, frozen backup copies, doc examples, and forward-compatible settings keys all produced +findings the user could neither act on nor was responsible for. No new scanner, command, agent, or hook +(counts stay scanners **16**, agents **7**, commands **21**, hooks **4**); all five fixes are byte-stable +— the frozen v5.0.0 + SC-5 + default-output snapshots are untouched and **no fixture was re-seeded** +(verified per bug: each affected fixture's findings are genuinely unchanged because the snapshot fixtures +contain none of the triggering paths/tokens). **1344** tests (+37). + +### Fixed +- **`conflict-detector` segregates plugin-bundled config (`M-BUG-2`).** CNF compared every discovered + `settings.json`/`hooks.json` pairwise regardless of origin, so it treated installed plugins' bundled + configs — each plugin's own settings/hooks plus its shipped fixtures and examples under + `~/.claude/plugins/` — as the user's cascade. A "conflict" between two plugins' bundled test fixtures + is not user-resolvable, yet these dominated the count (dogfood **339** findings: 315 high-sev + allow/deny, 18 duplicate-hook, 6 settings-key — Conflicts grade F on ~100% plugin noise). Fix: a new + `isPluginBundled` predicate excludes any file whose absolute path is under `.claude/plugins/` from + conflict analysis. Kept **CNF-local, not a discovery-level skip** on purpose — an active plugin's + contributed `hooks.json`/`.mcp.json` legitimately lives in `plugins/cache` and other scanners need it; + only conflict analysis must ignore plugin-bundled files. Same class as `M-BUG-8`. Dogfood **339→0** + (the ~3 genuine user-scope local settings have no actually-conflicting keys). +3 tests (plugin-bundled + exclusion, discovery-side sanity, over-exclusion guard). +- **`file-discovery` skips `backups/` (`M-BUG-8`).** A directory named `backups` holds backup COPIES, not + live config, so walking it during an audit produces stale findings. config-audit's own session backups + (`~/.claude/config-audit/backups//files/.../CLAUDE.md`) were the canonical case: a `~/.claude`-scope + audit walked 36 frozen copies as if live, polluting CPS and HKV/RUL. Fix: add `backups` to `SKIP_DIRS` + (broad, name-based — consistent with `vendor`/`dist`/`.cache`). Dogfood files-under-`/backups/` + **36→0**, 717 live config files retained. +3 tests. +- **token estimator discounts block-level HTML comments (`M-BUG-6`).** CLAUDE.md token estimates counted + block-level `` comments toward always-loaded tokens, but CC strips them before injection + (preserved only inside code fences, per `code.claude.com/docs/en/memory`). Fix: new + `stripInjectedHtmlComments` + `effectiveMemoryBytes` in `active-config-reader`; the CML cascade and + `token-hotspots` now size CLAUDE.md from effective (stripped) bytes while raw byte figures stay honest. + Block-level only — inline comments retained (conservative, verified scope). Dogfood `~/.claude` CLAUDE.md + ~3386→3301 tok (~85 tok). +13 tests. +- **`cache-prefix-stability` ignores code + CC-stable path vars (`M-BUG-7`).** CPS flagged + `${CLAUDE_PLUGIN_ROOT}`/`${CLAUDE_PROJECT_DIR}` (CC-provided stable paths) and `{date}`/timestamp tokens + shown in documentation as cache-busters. Fix: skip fenced code blocks, strip inline-code spans, and + whitelist CC-stable vars before pattern-matching. Suppress-only — frozen v5.0.0 snapshots untouched + (CPS yields `findings:[]` there). Dogfood **5→2** (3 doc false-positives suppressed; 2 remaining are + own volatile test fixtures). +6 tests. +- **`settings-validator` typo-gates unknown keys (`M-BUG-10`).** The CC settings schema is passthrough + (verified against the 2.1.193 binary): it forwards unrecognized keys unchanged rather than rejecting + them, so an arbitrary unknown key is forward-compatible, not an error — the finding's "silently ignored" + claim was factually wrong. The only real risk is a TYPO of a real key (the intended setting then + silently has no effect). Fix: flag an unknown key only when it closely matches a known key (new + `levenshtein` helper; edit distance ≤2, both keys ≥4 chars); severity medium→low; honest passthrough + framing in scanner + humanizer. Also refreshed `KNOWN_KEYS` with 6 binary-verified keys + (`agentPushNotifEnabled`, `remoteControlAtStartup`, `skipAutoPermissionPrompt`, + `skipDangerousModePermissionPrompt`, `skipWorkflowUsageWarning`, `tui`). Dogfood + `~/.claude/settings.json` **6→0** (all 6 were false unknown-key findings; 0 typo flags introduced across + 167 walked files). +12 tests. + ## [5.12.4] - 2026-06-26 ### Summary diff --git a/README.md b/README.md index e6e98e9..fd6db59 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ *AI-generated: all code produced by Claude Code through dialog-driven development. [Full disclosure →](../../README.md#ai-generated-code-disclosure)* -![Version](https://img.shields.io/badge/version-5.12.4-blue) +![Version](https://img.shields.io/badge/version-5.12.5-blue) ![Platform](https://img.shields.io/badge/platform-Claude_Code_Plugin-purple) ![Scanners](https://img.shields.io/badge/scanners-16-cyan) ![Commands](https://img.shields.io/badge/commands-21-green) ![Agents](https://img.shields.io/badge/agents-7-orange) ![Hooks](https://img.shields.io/badge/hooks-4-red) -![Tests](https://img.shields.io/badge/tests-1307-brightgreen) +![Tests](https://img.shields.io/badge/tests-1344-brightgreen) ![License](https://img.shields.io/badge/license-MIT-lightgrey) A Claude Code plugin that checks configuration health, suggests context-aware improvements, and auto-fixes issues — `CLAUDE.md`, `settings.json`, hooks, rules, MCP servers, `@imports`, and plugins. 16 deterministic scanners across 10 quality areas, context-aware feature recommendations, auto-fix with backup/rollback, a prompt-cache-aware Token Hotspots scanner with optional API-calibrated `--accurate-tokens` mode, plus cache-prefix stability, dead-tool, cross-plugin collision, output-style, and always-loaded agent-listing-budget detection. Zero external dependencies. @@ -662,6 +662,7 @@ This plugin is cautious by design — configuration files are important, and a b | Version | Date | Highlights | |---------|------|-----------| +| **5.12.5** | 2026-06-26 | "Dogfood denoise" — a samle-release of the Fase-3 scanner false-positive batch (`M-BUG-2/6/7/8/10`, all dogfooding finds on the maintainer's real machine). Five scanners stop counting non-user / non-live config as the user's: **CNF** (`M-BUG-2`) excludes files under `.claude/plugins/` from conflict analysis (`isPluginBundled`) — installed plugins' bundled settings/hooks/fixtures are not a user-resolvable cascade (dogfood **339→0**, Conflicts was an F on pure plugin noise). **file-discovery** (`M-BUG-8`) adds `backups` to `SKIP_DIRS` — a `backups/` tree holds frozen copies, never live config (dogfood files-under-`/backups/` **36→0**, 717 live retained). **token estimator** (`M-BUG-6`) strips block-level `` HTML comments from CLAUDE.md sizing — CC strips them before injection, so they were never always-loaded tokens (dogfood ~3386→3301, ~85 tok). **CPS** (`M-BUG-7`) skips fenced/inline code and whitelists CC-stable path vars (`${CLAUDE_PLUGIN_ROOT}`/`${CLAUDE_PROJECT_DIR}`) before cache-buster matching (dogfood **5→2**). **SET** (`M-BUG-10`) typo-gates the unknown-settings-key finding — the CC schema is passthrough (verified against the 2.1.193 binary), so an unknown key is forward-compatible, not an error; it now flags only a near-miss of a known key (levenshtein ≤2), severity medium→low, +6 binary-verified `KNOWN_KEYS` (dogfood **6→0**). No count change (scanners **16**, agents **7**, commands **21**, hooks **4**); all five are byte-stable — frozen v5.0.0 + SC-5 + default-output snapshots untouched, no re-seed (each fixture's findings are genuinely unchanged). **1344** tests (+37). | | **5.12.4** | 2026-06-26 | "Rooted rules" — fixes `M-BUG-9` (dogfooding find) in `scanners/rules-validator.mjs`: the RUL "Rule path pattern matches no files" check now resolves a rule's `paths:`/`globs:` pattern against the rule's **own project root** (the dir containing its `.claude/`), not the outer scan root. Previously `countGlobMatches` globbed against the scan target and `collectProjectFiles`' `depth>4` cutoff never reached deep matching files, so a live rule in a **nested repo** (e.g. a marketplace checkout under `~/.claude`) was wrongly flagged "never activates" (high) — a false F-grade for anyone with rules in a nested repo. The fix derives each rule's project root, collects+globs per root (cached), and skips the check for user-global rules (`root === $HOME`), which scope against the active project at runtime. Same scope-conflation family as `M-BUG-1/2`. No count change (scanners **16**, agents **7**, commands **21**); the fix is a no-op when `projectRoot === targetPath` (the common single-repo scan), so frozen v5.0.0 + default-output snapshots stay byte-stable. **1307** tests (+2 TDD: nested-repo false-positive + HOME guard). | | **5.12.3** | 2026-06-26 | "Phantom agents" — fixes `M-BUG-3/4/5` (dogfooding finds) in `scanners/lib/active-config-reader.mjs`: `enumerateAgents` now counts only the agents Claude Code actually **registers**. Per the official subagents doc, an agent needs valid `name`+`description` frontmatter, and CC scans recursively and silently skips frontmatter-less files. The reader previously (`M-BUG-5`) counted every `.md` regardless of frontmatter, (`M-BUG-3`) never recursed into agent subdirs, and (`M-BUG-4`) double-counted when the project dir equals the user dir (scanning `$HOME` — root cause, also affecting rules/output-styles). Real-machine verify: user-agent count **13→0** (all 12 user agents + `REMEMBER.md` are frontmatter-less → CC registers none), HOME `project`-dup **13→0**; corrected always-loaded baseline ≈ **53** (was 66). Agent enumeration is machine-dependent and absent from the frozen snapshots, so the v5.0.0 + SC-5 + default-output snapshots stay byte-stable; no count change (scanners **16**, agents **7**, commands **21**). **1305** tests. | | **5.12.2** | 2026-06-24 | "Honest census" — fixes `M-BUG-1` (dogfooding find): `enumeratePlugins` walked `~/.claude/plugins/marketplaces//plugins/` and ignored both enable-state and the polyrepo cache layout, so it **over-counted phantom agents** from disabled plugins while **missing the entire enabled polyrepo set** (whose plugins live under `cache/`). It now gates on `installed_plugins.json` + `enabledPlugins` and enumerates each plugin from its active `installPath`, with the marketplaces walk as fallback. Fixes `manifest`/`whats-active`/AGT/`token-hotspots` for any user with disabled plugins or a polyrepo marketplace. No count change (scanners **16**, agents **7**, commands **21**); `--json`/`--raw` byte-stable, frozen v5.0.0 + SC-5 + default-output snapshots untouched. Real-machine verify: agent listing 114→104, ghosts gone. **1301** tests. | From 96e32df87bb033bc8dbde60e263d8e8b43c50c11 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Mon, 29 Jun 2026 08:32:44 +0200 Subject: [PATCH 42/53] =?UTF-8?q?docs(claude-md):=20trim=20project=20CLAUD?= =?UTF-8?q?E.md=20to=20invariants=20(=E2=88=92662=20always-tok)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin's own CLAUDE.md is loaded every turn while working in this repo (measured 2,178 always-loaded tokens via `manifest`, the largest slice of the 2,745-tok project delta). Much of it was reference-grade prose that duplicates README.md, `/config-audit help`, and docs/ — verbose per-command feature lists, the full plain-language-output spec, the session-dir ASCII tree — none of which is invariant for working on the plugin. Trimmed to what is invariant: command names + one-line purpose, the agent / hook tables (model/color/tools, script/event), finding-ID format, enforced .claude/rules conventions, coding style, test command, gotchas. Detail now points to README / docs/. Also dropped two stale badge-duplicate figures that had already rotted (the README badge owns them): "18 commands" (now 21) and "1279 tests / 72 files" (now 1344) — removed rather than re-pinned so they can't go stale again. Measured: CLAUDE.md 134->102 lines, 8844->6170 B, 2,178->1,516 tok (-30%); config-audit project always-delta 2,745->2,083 tok (-24%). Full suite 1344/0; frozen v5.0.0 + SC-5 + default-output snapshots byte-stable (no scanner/snapshot reads this repo's CLAUDE.md). Docs-only — no version bump, no catalog ref change. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- CLAUDE.md | 92 ++++++++++++++++++------------------------------------- 1 file changed, 30 insertions(+), 62 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 97054c1..a367750 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,13 +1,8 @@ # Config-Audit Plugin -Claude Code Configuration Intelligence — know if your configuration is correct, find what could improve it, fix it automatically. +Claude Code Configuration Intelligence — know if your config is correct, find what could improve it, fix it automatically. Three pillars: **Health** (deterministic scanners), **Opportunities** (context-aware recommendations), **Action** (auto-fix with backup/rollback). -## What this plugin does - -Analyzes and optimizes Claude Code configuration across three pillars: -- **Health** — Deterministic scanners verify correctness, consistency, and completeness -- **Opportunities** — Context-aware recommendations for features that could benefit your project -- **Action** — Auto-fix with backup/rollback +Per-command flags, patterns, and feature lists live in `README.md` and `/config-audit help`. This file carries what's invariant for working on the plugin. ## Commands @@ -15,15 +10,15 @@ Analyzes and optimizes Claude Code configuration across three pillars: | Command | Description | |---------|-------------| -| `/config-audit` | Full audit with auto-scope detection (no setup needed) | -| `/config-audit posture` | Quick health scorecard (A-F grades, 10 quality areas incl. Token Efficiency, Plugin Hygiene) | -| `/config-audit tokens` | prompt-cache-aware token hotspots (8 patterns: cache-breaking, redundant perms, deep imports, oversized cascade, bloated SKILL.md desc, MCP tool-schema budget, MCP tool-schema deferral, stale plugin-cache disk-cleanup), each ranked hotspot tagged with its load pattern (always / on-demand / external) — **cache-aware** (stale `~/.claude/plugins/cache` versions excluded by default; only each plugin's active version counts; `--no-exclude-cache` for the full walk), optional `--accurate-tokens` API calibration, `--with-telemetry-recipe` cache-hit recipe pointer | -| `/config-audit manifest` | Ranked table of every token source (CLAUDE.md, rules, agents, skills, output styles, MCP, hooks) sorted by estimated tokens, each tagged with its load pattern (always-loaded / on-demand / external) + an always-loaded subtotal ("tokens that enter context every turn") | -| `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact (incl. a conditional `disableBundledSkills` lever when the active skill listing is over budget — remediation companion to SKL `CA-SKL-002`) | -| `/config-audit optimize` | Optimization lens (mechanism-fit) — config that works but fits a better mechanism: procedure→skill (CA-OPT-001, deterministic), lifecycle→hook / unscoped path→rule / "never"→permission (prose-judgment via opus `optimization-lens-agent`). Hybrid motor; every finding cites a best-practices-register rule. Agent-driven, **not byte-stable** | +| `/config-audit` | Full audit with auto-scope detection | +| `/config-audit posture` | A-F health scorecard (10 quality areas) | +| `/config-audit tokens` | Prompt-cache-aware token hotspots, each tagged with its load pattern; cache-aware | +| `/config-audit manifest` | Ranked table of every token source + always-loaded subtotal | +| `/config-audit feature-gap` | Context-aware feature recommendations grouped by impact | +| `/config-audit optimize` | Mechanism-fit lens (procedure→skill, lifecycle→hook, path→rule, never→permission). Agent-driven, **not byte-stable** | | `/config-audit fix` | Auto-fix deterministic issues with backup + verification | | `/config-audit rollback` | Restore configuration from backup | -| `/config-audit plan` | Create action plan from audit findings | +| `/config-audit plan` | Create action plan from findings | | `/config-audit implement` | Execute plan with backups + auto-verify | | `/config-audit help` | Show all commands | @@ -33,9 +28,9 @@ Analyzes and optimizes Claude Code configuration across three pillars: |---------|-------------| | `/config-audit drift` | Compare current config against saved baseline | | `/config-audit plugin-health` | Audit plugin structure, frontmatter, cross-plugin coherence | -| `/config-audit whats-active` | Read-only inventory of plugins, skills, MCP, hooks, CLAUDE.md active for a repo (with token estimates) | -| `/config-audit knowledge-refresh` | Keep the best-practices register fresh — deterministic stale check (sources older than ~90d) + web candidate poll (CC changelog + Anthropic blog); **human-approved writes only** (Verifiseringsplikt). The "living" half of the knowledge base. Web/judgment-driven, **not byte-stable** | -| `/config-audit campaign` | Machine-wide audit campaign — durable ledger ABOVE sessions: per-repo lifecycle (pending→audited→planned→implemented) + machine-wide roll-up by severity + **machine-wide always-loaded token bill** (`refresh-tokens` live cross-repo sweep — shared global layer counted once + per-repo deltas) + cross-repo prioritized backlog + plan **export** (drop a planned repo's plan into its own `docs/`), resumable across sessions. Read-only report (campaign-cli) + **human-approved** writes via deterministic write/export CLIs. THIN: tracks+routes state, reuses existing implement/rollback for execution. Judgment-driven, **not byte-stable** | +| `/config-audit whats-active` | Read-only inventory of active plugins/skills/MCP/hooks/CLAUDE.md (with token estimates) | +| `/config-audit knowledge-refresh` | Refresh the best-practices register (stale check + web poll). Human-approved writes; **not byte-stable** | +| `/config-audit campaign` | Machine-wide audit ledger + token bill across repos. Human-approved writes; **not byte-stable** | | `/config-audit discover` | Run discovery phase only | | `/config-audit analyze` | Run analysis phase only | | `/config-audit interview` | Gather user preferences (opt-in) | @@ -51,68 +46,43 @@ Analyzes and optimizes Claude Code configuration across three pillars: | planner-agent | Create action plan | opus | yellow | Read, Glob, Write | | implementer-agent | Execute changes | sonnet | magenta | Read, Write, Edit, Bash, Glob | | verifier-agent | Verify results | sonnet | purple | Read, Glob, Grep | -| feature-gap-agent | Context-aware feature recommendations | opus | green | Read, Glob, Grep, Write | -| optimization-lens-agent | Mechanism-fit precision gate (prose-judgment lens cases) | opus | orange | Read, Glob, Grep, Write | +| feature-gap-agent | Feature recommendations | opus | green | Read, Glob, Grep, Write | +| optimization-lens-agent | Mechanism-fit precision gate | opus | orange | Read, Glob, Grep, Write | ## Hooks | Event | Script | Purpose | |-------|--------|---------| -| PreToolUse | `auto-backup-config.mjs` | Auto-backup config files before Edit/Write | -| PostToolUse | `post-edit-verify.mjs` | Verify config files after Edit/Write, block on new critical/high | -| SessionStart | `session-start.mjs` | Checks for active (unfinished) sessions | -| Stop | `stop-session-reminder.mjs` | Reminds about current session phase | +| PreToolUse | `auto-backup-config.mjs` | Backup config files before Edit/Write | +| PostToolUse | `post-edit-verify.mjs` | Verify after Edit/Write, block on new critical/high | +| SessionStart | `session-start.mjs` | Check for active (unfinished) sessions | +| Stop | `stop-session-reminder.mjs` | Remind about current session phase | ## Reference docs (read on demand) -- **Scanner inventory, lib modules, action engines, knowledge base, per-scanner/per-block implementation notes:** `docs/scanner-internals.md` -- **Plain-language output (v5.1.0), humanizer vocabularies, output modes:** `docs/humanizer.md` +- `docs/scanner-internals.md` — scanner inventory, lib modules, action engines, knowledge base, per-scanner/per-block implementation notes (design rationale, primary-source verification, byte-stability lessons) +- `docs/humanizer.md` — plain-language output (v5.1.0), humanizer vocabularies, output modes -## Plain-Language Output (v5.1.0) — summary +## Plain-Language Output (v5.1.0) -Default output of all 18 commands routes through `humanizeEnvelope` from `lib/humanizer.mjs`. Findings get three decorated fields: - -- `userImpactCategory` — Configuration mistake / Conflict / Wasted tokens / Dead config / Missed opportunity -- `userActionLanguage` — Fix this now / Fix soon / Fix when convenient / Optional cleanup / FYI (derived from severity) -- `relevanceContext` — `affects-everyone` (default) / `affects-this-machine-only` (`*.local.*` files) / `test-fixture-no-impact` - -`--raw` bypasses the humanizer for byte-stable v5.0.0 output. `--json` is also byte-stable. Full detail and Wave 5 lessons: `docs/humanizer.md`. +Default output of all commands routes through `humanizeEnvelope` (`lib/humanizer.mjs`), decorating each finding with `userImpactCategory`, `userActionLanguage`, and `relevanceContext`. `--raw` and `--json` bypass the humanizer for byte-stable v5.0.0 output. Full detail: `docs/humanizer.md`. ## Suppressions -Create `.config-audit-ignore` at project root to suppress known findings: -``` -CA-SET-003 # Exact ID -CA-GAP-* # Glob pattern (all GAP findings) -``` -Suppressed findings tracked in envelope's `suppressed_findings` for audit trail. Disable with `--no-suppress`. +Create `.config-audit-ignore` at project root — one exact ID or glob per line (`CA-SET-003`, `CA-GAP-*`). Suppressed findings are tracked in the envelope's `suppressed_findings` for audit trail. Disable with `--no-suppress`. ## Architecture -### Workflow -``` -/config-audit → discover + analyze (auto) → plan → implement → verify -``` -Default: auto-detects scope from git context. Override with `/config-audit full|repo|home|current`. Delta mode: `--delta` (incremental). +Workflow: `/config-audit → discover + analyze (auto) → plan → implement → verify`. Auto-detects scope from git context; override with `full|repo|home|current`; `--delta` for incremental. Session state lives under `~/.claude/config-audit/sessions/{id}/` (scope.yaml, discovery.json, state.yaml, findings/, analysis-report.md, action-plan.md, backups/, implementation-log.md). -### Session Directory -``` -~/.claude/config-audit/sessions/{session-id}/ -├── scope.yaml, discovery.json, state.yaml -├── findings/, analysis-report.md, action-plan.md -├── backups/, implementation-log.md -└── interview.md (if interview run) -``` - -### Finding ID Format -`CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-DIS-001`, `CA-COL-001`, `CA-SKL-001`, `CA-OST-001`, `CA-OPT-001`, `CA-AGT-001` +Finding ID format: `CA-{SCANNER}-{NNN}` — e.g. `CA-CML-001`, `CA-SET-003`, `CA-HKV-002`, `CA-RUL-005`, `CA-TOK-005`, `CA-CPS-001`, `CA-SKL-001`, `CA-OST-001`, `CA-OPT-001`, `CA-AGT-001`. ## Conventions -Enforced project conventions live in `.claude/rules/` (auto-loaded as project instructions): - -- `ux-rules.md` — output/narration/formatting rules for all commands (never dump raw JSON, narrate before each step, space-separated command suggestions) +Enforced conventions live in `.claude/rules/` (auto-loaded as project instructions): +- `ux-rules.md` — output/narration/formatting for all commands (never dump raw JSON, narrate before each step, space-separated command suggestions) - `command-development.md` — required command frontmatter + `plugin:action` naming +- `agent-development.md` — agent frontmatter + "when to use" conventions - `state-management.md` — update `state.yaml` after every workflow phase Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{SCANNER}-{NNN}` ID format; byte-stable CLIs are verified against frozen `tests/snapshots/v5.0.0/` baselines. @@ -123,12 +93,10 @@ Coding style: scanners are zero-dependency Node ESM; new findings use the `CA-{S node --test 'tests/**/*.test.mjs' ``` -1279 tests across 72 test files (23 lib + 39 scanner + 1 hook + 1 agent + 3 commands + 1 knowledge + 4 top-level). Test fixtures in `tests/fixtures/`. Top-level humanizer tests: `json-backcompat.test.mjs`, `raw-backcompat.test.mjs`, `scenario-read-test.test.mjs`, `snapshot-default-output.test.mjs`. - -Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. +Test fixtures in `tests/fixtures/`. Per-scanner and per-build-block implementation notes (design rationale, primary-source verification, byte-stability lessons) live in `docs/scanner-internals.md` → **Implementation notes**. ## Gotchas - Session directories accumulate — use `/config-audit cleanup` to manage -- Scanners run on Node.js >= 18 (uses node:test, node:fs/promises) +- Scanners run on Node.js ≥ 18 (uses node:test, node:fs/promises) - Plugin CLAUDE.md files in node_modules should be excluded via scope From 1d634926177002d5fcaf11cc3023164acda00ce1 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 06:44:36 +0200 Subject: [PATCH 43/53] fix(acr): optimize lens scopes out plugin-bundled CLAUDE.md + unique candidate paths (M-BUG-11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The optimize lens CLI fed its precision-gate agent every CLAUDE.md that discovery returned, including the 256 files under ~/.claude/plugins/ — vendored plugin CLAUDE.md across every cached version (7 config-audit, 6 ms-ai-architect, 5 okr, ...) plus their bundled tests/fixtures and examples. Running `optimize --global` on this machine produced 454 candidates across 92 "files", ~250 of them sourced from plugin-internal files a user cannot act on (the plugin overwrites them on update). Same class as M-BUG-2: plugin-bundled config is not the user's cascade. Second defect: candidates were keyed by `relPath || absPath`, but relPath collides across scopes — a repo-root `CLAUDE.md` and the user-global `~/.claude/CLAUDE.md` both relPath to `CLAUDE.md`. The two files that actually matter were merged into one indistinguishable bucket (21 candidates), the agent's Read(file) would resolve the wrong one, and cache-file relPaths were not readable relative to cwd at all. Fix (lens-CLI-local, surgical): - Filter isPluginBundled (absPath under `.claude/plugins/`) from discovery for BOTH halves of the motor (candidate loop + the OPT scanner, which reads discovery.files directly). Drops vendored files regardless of active/stale version, so excludeCache is unnecessary here. - Key each candidate by absPath: unique + readable. No change to file-discovery.mjs or the OPT scanner, so their byte-stable snapshots are untouched. Suite 1348/0 (+4: candidate scoping, real-config survives, deterministic scoping, absolute-path identity). Frozen v5.0.0 + SC-5 snapshots untouched (the lens CLI has no snapshot; the command is agent-driven, not byte-stable). Dogfood ~/.claude `optimize --global`: candidates 454->45, deterministic 2->0 (both were stale plugin-cache copies), distinct files 92->11, repo vs user-global now distinct (12 + 9 = 21). Residual 45 includes config-audit's own tests/fixtures CLAUDE.md (repo-specific dogfooding artifact, not a general bug — left alone). --- scanners/optimize-lens-cli.mjs | 25 +++++- tests/scanners/optimize-lens-cli.test.mjs | 102 ++++++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 tests/scanners/optimize-lens-cli.test.mjs diff --git a/scanners/optimize-lens-cli.mjs b/scanners/optimize-lens-cli.mjs index 21d0bc5..aa3b112 100644 --- a/scanners/optimize-lens-cli.mjs +++ b/scanners/optimize-lens-cli.mjs @@ -24,7 +24,7 @@ * Exit codes: 0=ok, 3=unrecoverable error. Zero external dependencies. */ -import { resolve } from 'node:path'; +import { resolve, sep } from 'node:path'; import { writeFile, readFile, stat } from 'node:fs/promises'; import { discoverConfigFiles } from './lib/file-discovery.mjs'; import { resetCounter } from './lib/output.mjs'; @@ -33,6 +33,15 @@ import { loadRegister, getEntry } from './lib/best-practices-register.mjs'; import { prefilterClaudeMd, LENS_DETECTORS } from './lib/lens-prefilter.mjs'; import { scan as optScan } from './optimization-lens-scanner.mjs'; +// Files under `.claude/plugins/` are shipped by an installed plugin — vendored +// CLAUDE.md plus its bundled tests/fixtures and examples. They are not the user's +// authored config, so a mechanism-fit suggestion against them is not actionable +// (the user can't edit a file the plugin overwrites on update). Excluded from the +// lens regardless of active/stale version. (M-BUG-11; mirrors the M-BUG-2 rule +// that keeps plugin-bundled config out of the conflict detector.) +const PLUGIN_TREE_MARKER = `.claude${sep}plugins${sep}`; +const isPluginBundled = (file) => (file.absPath || '').includes(PLUGIN_TREE_MARKER); + /** Confirmed register entry for `id`, or null. */ function confirmedEntry(register, id) { const e = getEntry(register, id); @@ -72,7 +81,13 @@ async function main() { } resetCounter(); - const discovery = await discoverConfigFiles(absPath, { includeGlobal }); + const rawDiscovery = await discoverConfigFiles(absPath, { includeGlobal }); + // Scope the lens to the user's authored config: drop plugin-bundled files for + // BOTH halves of the motor (the OPT scanner reads discovery.files directly). + const discovery = { + ...rawDiscovery, + files: (rawDiscovery.files || []).filter((f) => !isPluginBundled(f)), + }; // ── Deterministic half: the OPT scanner (CA-OPT-001) ── const opt = await optScan(absPath, discovery); @@ -94,7 +109,11 @@ async function main() { const entry = register ? confirmedEntry(register, cand.registerId) : null; if (!entry) continue; // never surface an unverifiable recommendation candidates.push({ - file: file.relPath || file.absPath, + // Absolute path: unique + readable. relPath collides across scopes + // (a repo-root `CLAUDE.md` and the user-global `~/.claude/CLAUDE.md` + // both relPath to `CLAUDE.md`), which would send the agent's Read() to + // the wrong file. (M-BUG-11) + file: file.absPath, line: bodyStartLine - 1 + cand.line, lensCheck: cand.lensCheck, mechanism: cand.mechanism, diff --git a/tests/scanners/optimize-lens-cli.test.mjs b/tests/scanners/optimize-lens-cli.test.mjs new file mode 100644 index 0000000..bb45fed --- /dev/null +++ b/tests/scanners/optimize-lens-cli.test.mjs @@ -0,0 +1,102 @@ +/** + * optimize-lens-cli tests — payload scoping + candidate identity (M-BUG-11). + * + * The lens CLI feeds the precision-gate agent. Two correctness invariants it must + * hold, both established elsewhere in the codebase but originally missing here: + * + * 1. SCOPING — plugin-bundled CLAUDE.md (anything under `.claude/plugins/`: + * vendored plugin config + its bundled tests/fixtures + examples, active or + * stale) is NOT the user's authored config. The user can't act on a + * mechanism-fit suggestion against a file a plugin ships. So the lens must + * drop them — the M-BUG-2 `isPluginBundled` rule, applied to the lens. + * + * 2. IDENTITY — a candidate's `file` must uniquely name a readable file. The + * user-global `~/.claude/CLAUDE.md` and a repo-root `CLAUDE.md` both have + * relPath `CLAUDE.md`; labelling candidates by relPath collides them, and + * the agent's `Read(file)` then resolves the wrong one. Candidates carry an + * absolute path. + * + * Hermetic: a temp target with a real CLAUDE.md and a nested plugin-bundled one. + * `.claude/plugins/` is not in SKIP_DIRS, so the normal walk discovers it — + * no ~/.claude / --global needed. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join, isAbsolute } from 'node:path'; +import { mkdtemp, mkdir, writeFile, rm, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +const execFileP = promisify(execFile); +const CLI = fileURLToPath(new URL('../../scanners/optimize-lens-cli.mjs', import.meta.url)); + +/** Build a temp target, run the lens CLI on it, return the parsed payload. */ +async function runLens(files) { + const root = await mkdtemp(join(tmpdir(), 'ca-lens-cli-')); + try { + for (const [rel, content] of Object.entries(files)) { + const abs = join(root, rel); + await mkdir(join(abs, '..'), { recursive: true }); + await writeFile(abs, content, 'utf-8'); + } + const out = join(root, 'payload.json'); + await execFileP('node', [CLI, root, '--output-file', out]); + return JSON.parse(await readFile(out, 'utf-8')); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +const REAL = '# Project\n\nNever commit secrets to the repo.\n'; +const PLUGIN = '# Vendored plugin\n\nNever delete the plugin cache directory.\n'; + +describe('optimize-lens-cli — scoping (M-BUG-11)', () => { + it('excludes plugin-bundled CLAUDE.md from candidates', async () => { + const payload = await runLens({ + 'CLAUDE.md': REAL, + '.claude/plugins/cache/mkt/plug/1.0.0/CLAUDE.md': PLUGIN, + }); + const bundled = payload.candidates.filter( + (c) => c.file.includes(`.claude/plugins/`) || c.file.includes(`/plugins/`), + ); + assert.deepEqual( + bundled.map((c) => c.file), + [], + 'no candidate should come from a file under .claude/plugins/', + ); + }); + + it('still surfaces the user’s real CLAUDE.md', async () => { + const payload = await runLens({ + 'CLAUDE.md': REAL, + '.claude/plugins/cache/mkt/plug/1.0.0/CLAUDE.md': PLUGIN, + }); + const real = payload.candidates.filter((c) => c.file.endsWith(`${join('', 'CLAUDE.md')}`)); + assert.ok(real.length >= 1, 'the real CLAUDE.md never-instruction should survive scoping'); + }); + + it('excludes plugin-bundled CLAUDE.md from deterministic findings too', async () => { + const procedure = + '# Plugin release\n\n' + + Array.from({ length: 7 }, (_, i) => `${i + 1}. Do release step ${i + 1} and verify.`).join('\n') + + '\n'; + const payload = await runLens({ + 'CLAUDE.md': REAL, + '.claude/plugins/cache/mkt/plug/1.0.0/CLAUDE.md': procedure, + }); + const bundled = (payload.deterministic || []).filter((f) => String(f.file).includes('plugins/')); + assert.deepEqual(bundled, [], 'deterministic CA-OPT-001 must not fire on vendored plugin CLAUDE.md'); + }); +}); + +describe('optimize-lens-cli — candidate identity (M-BUG-11)', () => { + it('labels every candidate with an absolute, unique path', async () => { + const payload = await runLens({ 'CLAUDE.md': REAL }); + assert.ok(payload.candidates.length >= 1, 'expected at least one candidate'); + for (const c of payload.candidates) { + assert.ok(isAbsolute(c.file), `candidate file must be absolute, got: ${c.file}`); + } + }); +}); From b58393099a29edddb141a4e42dd2c4a990831434 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 09:10:07 +0200 Subject: [PATCH 44/53] fix(acr): posture --output-file humanizes findings in default mode (M-BUG-12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feature-gap.md (Step 3-4) and posture.md (Step 3-4) read findings from `posture.mjs --output-file` and group on the humanizer fields (userActionLanguage / userImpactCategory / relevanceContext). But posture.mjs only humanized the stderr scorecard — its --output-file JSON wrote the raw v5.0.0-shape `result`, so every finding's humanizer fields were `undefined`. Both commands silently degraded to the raw tier-fallback: v5.1.0 plain-language output was dead for feature-gap and for posture's finding-level grouping. Re-derived on tests/fixtures/marketplace-medium: 17 GAP findings, all three humanizer fields undefined in the default --output-file JSON. Fix (posture-CLI-local, surgical): humanize the output-file payload in default mode, mirroring scan-orchestrator.mjs:277 — but posture nests the scanner envelope under `result.scannerEnvelope` (its `result` has no top-level `scanners` array), so humanizeEnvelope is applied to `result.scannerEnvelope`, not `result` (the latter would no-op). --json / --raw stay raw, so the explicit-v5.0.0-shape contract and snapshot byte-compat are preserved. TDD: red-first test in posture-humanizer.test.mjs default-mode block asserts GAP findings in the output file carry userActionLanguage/userImpactCategory; a --raw --output-file guard asserts the raw shape is unchanged. Suite 1350/0 (+2). Frozen v5.0.0 + SC-5/6/7 + default-output snapshots byte-stable: --json/--raw bypass the humanizer (their snapshot tests use those flags), and the default --output-file JSON is not snapshot-pinned. Committed, not released — batches with M-BUG-11 in a later hardening release. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- scanners/posture.mjs | 10 +++++- tests/scanners/posture-humanizer.test.mjs | 43 ++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/scanners/posture.mjs b/scanners/posture.mjs index db12564..16a89f0 100644 --- a/scanners/posture.mjs +++ b/scanners/posture.mjs @@ -10,6 +10,7 @@ import { resolve } from 'node:path'; import { writeFile } from 'node:fs/promises'; import { runAllScanners } from './scan-orchestrator.mjs'; +import { humanizeEnvelope } from './lib/humanizer.mjs'; import { calculateUtilization, determineMaturityLevel, @@ -114,7 +115,14 @@ async function main() { } if (outputFile) { - const json = JSON.stringify(result, null, 2); + // Consumers (feature-gap.md, posture.md) read scannerEnvelope.scanners[].findings + // and group on humanizer fields. posture's result nests the envelope under + // `scannerEnvelope`, so humanize THAT (not `result`, which has no top-level + // `scanners` array — humanizeEnvelope would no-op). --json/--raw stay raw. + const fileEnv = (jsonMode || rawMode) + ? result + : { ...result, scannerEnvelope: humanizeEnvelope(result.scannerEnvelope) }; + const json = JSON.stringify(fileEnv, null, 2); await writeFile(outputFile, json, 'utf-8'); process.stderr.write(`\nResults written to ${outputFile}\n`); } diff --git a/tests/scanners/posture-humanizer.test.mjs b/tests/scanners/posture-humanizer.test.mjs index fc297f9..22c99d4 100644 --- a/tests/scanners/posture-humanizer.test.mjs +++ b/tests/scanners/posture-humanizer.test.mjs @@ -1,6 +1,7 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve, dirname } from 'node:path'; +import { resolve, dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; @@ -141,4 +142,44 @@ describe('posture humanizer wiring (Step 6)', () => { 'humanized stderr must differ from v5.0.0 verbatim stderr'); }); }); + + // M-BUG-12: feature-gap.md/posture.md read findings from posture.mjs --output-file + // and group on humanizer fields (userActionLanguage etc.). Default-mode output-file + // must therefore humanize findings inside the nested scannerEnvelope; --raw stays raw. + describe('default mode --output-file (M-BUG-12: humanized findings)', () => { + it('writes humanized GAP findings (userActionLanguage defined) to the output file', async () => { + const tmp = join(tmpdir(), `ca-posture-outfile-${process.pid}.json`); + try { + await runPosture(['--output-file', tmp]); + const env = JSON.parse(await readFile(tmp, 'utf-8')); + const gap = env.scannerEnvelope.scanners.find(s => s.scanner === 'GAP'); + assert.ok(gap, 'GAP scanner must be present in the output file'); + assert.ok(gap.findings.length > 0, 'fixture must yield at least one GAP finding'); + for (const f of gap.findings) { + assert.notEqual(f.userActionLanguage, undefined, + `${f.id}: default-mode output-file findings must carry userActionLanguage`); + assert.notEqual(f.userImpactCategory, undefined, + `${f.id}: default-mode output-file findings must carry userImpactCategory`); + } + } finally { + await unlink(tmp).catch(() => {}); + } + }); + + it('--raw --output-file keeps v5.0.0 raw finding shape (no humanizer fields)', async () => { + const tmp = join(tmpdir(), `ca-posture-outfile-raw-${process.pid}.json`); + try { + await runPosture(['--raw', '--output-file', tmp]); + const env = JSON.parse(await readFile(tmp, 'utf-8')); + for (const s of env.scannerEnvelope.scanners) { + for (const f of s.findings) { + assert.equal(f.userActionLanguage, undefined, + `${f.id}: --raw output-file must not carry userActionLanguage`); + } + } + } finally { + await unlink(tmp).catch(() => {}); + } + }); + }); }); From f4bf3ae2cb77745ecae8631ce09b03c28f207345 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 10:09:10 +0200 Subject: [PATCH 45/53] fix(acr): feature-gap scopes presence checks to authored config + reads settings cascade (M-BUG-13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GAP scanner's 25 presence checks ran over the full includeGlobal discovery, so this plugin's own examples/optimal-setup (vendored across plugin-cache versions) satisfied every tier-3 check — masking real feature gaps to GAP=0 on ANY target. And the real ~/.claude/settings.json is invisible to the settings-key checks (includeGlobal gotcha + maxFiles cap), which would flip statusLine/autoMode to false positives once the maskers were removed. - isAuthoredConfig: exclude plugin-bundled (~/.claude/plugins/) + nested examples/ and tests/fixtures/ (relPath-relative, so a fixture scanned AS the target keeps its own files) from ctx.files + parsedSettings. - readSettingsCascade: read the user->project->local settings cascade directly and merge into parsedSettings — immune to the discovery cap/gotcha. Empty target: ~0 (masked) -> 18 humanized opportunities; no statusLine/autoMode false positives. Frozen v5.0.0 snapshots + SC-5/6/7 byte-stable (marketplace-medium has no nested demo trees; hermetic-HOME cascade adds nothing). Suite 1350->1355/0. Found by dogfooding feature-gap against the machine. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- docs/scanner-internals.md | 33 +++++++ scanners/feature-gap-scanner.mjs | 82 ++++++++++++++++- tests/scanners/feature-gap-scanner.test.mjs | 97 ++++++++++++++++++++- 3 files changed, 207 insertions(+), 5 deletions(-) diff --git a/docs/scanner-internals.md b/docs/scanner-internals.md index befa00d..a59e1f6 100644 --- a/docs/scanner-internals.md +++ b/docs/scanner-internals.md @@ -254,6 +254,39 @@ model-switch is a *runtime* behaviour, not static config a scanner can reliably automation. "No overstated behavioral finding ships" — so even the permitted opusplan *info*-advisory was left out; the verified @import extension is the whole of B6. +### GAP scanner — authored-config scoping + direct cascade read (M-BUG-13) + +The 25 presence checks ask "does the user's effective config have feature X?" and GAP **always** +runs `includeGlobal: true`. Two failure modes made the answer wrong on a real machine, both surfaced +by dogfooding `feature-gap`/`posture --global`: + +1. **Demo/vendored config masks real gaps.** This plugin's own `examples/optimal-setup/` is a complete + config (sets `outputStyle`/`statusLine`/`worktree`/`model`/`keybindings.json`/`.lsp.json`), and its + copies vendored under `~/.claude/plugins/cache/.../config-audit//examples/` are pulled into the + includeGlobal discovery. Because `anySettingsHas`/`files.some(...)` accept ANY discovered file, that + one demo file drove every tier-3 check to "present" → **GAP=0 on any target** (false negative). + Fix: `isAuthoredConfig` filters `ctx.files`/`parsedSettings` to the user's authored cascade — + excludes `~/.claude/plugins/` (absPath marker, mirrors CNF's M-BUG-2 exclusion) and any file whose + path **relative to the scan target** sits under `examples/` or `tests/fixtures/`. relPath (not + absPath) is deliberate: a fixture scanned AS the target keeps its own files, so the frozen v5.0.0 + snapshots (scanned from `tests/fixtures/marketplace-medium`, which has no such nested trees) are + byte-stable. + +2. **The real `~/.claude/settings.json` is invisible to the settings-key checks.** Discovery misses it + (its relPath carries no `.claude` segment when the walk root IS `~/.claude` — the gotcha) AND, when + vendored plugins flood the walk, the `maxFiles=2000` cap drops it. After (1) removed the demo + maskers, `statusLine`/`autoMode`/`permissions` (which the user HAS) would flip to false **positives**. + Fix: `readSettingsCascade` reads the four canonical cascade paths (user `settings.json`/`.local`, + project `settings.json`/`.local`) directly and merges them INTO `parsedSettings` — immune to the + cap and the gotcha. Merge (not replace) keeps non-canonical project settings and leaves the snapshot + (hermetic empty HOME → cascade adds nothing new) byte-stable. + +Net: an empty target now surfaces ~18 humanized opportunities (was masked to ~0); config-audit's own +repo still shows 0 in output via its intentional `.config-audit-ignore` `CA-GAP-*` self-suppression +(a plugin repo legitimately lacks user-project features) — suppression is an envelope-layer concern, +orthogonal to this scanner fix. Scoped GAP-local; the includeGlobal discovery gotcha itself is left +to other consumers (see auto-memory `discovery-includeglobal-user-settings-gotcha`). + ### CML scanner — context-window-scaled char budget Beyond the line-count checks (200/500 lines, both MEDIUM), the CML scanner mirrors diff --git a/scanners/feature-gap-scanner.mjs b/scanners/feature-gap-scanner.mjs index 00a5c23..0a68c6f 100644 --- a/scanners/feature-gap-scanner.mjs +++ b/scanners/feature-gap-scanner.mjs @@ -7,7 +7,7 @@ * Finding IDs: CA-GAP-NNN */ -import { resolve } from 'node:path'; +import { resolve, join, sep } from 'node:path'; import { readTextFile, discoverConfigFiles } from './lib/file-discovery.mjs'; import { finding, scannerResult } from './lib/output.mjs'; import { SEVERITY } from './lib/severity.mjs'; @@ -46,6 +46,68 @@ function isTargetLocal(ctx, f) { return f.absPath.startsWith(ctx.targetPath); } +// Files that are test/demo/vendored config — NOT part of the user's authored +// cascade — must not satisfy "is feature X present?" checks, or they mask real +// gaps. The canonical case: this plugin's own examples/optimal-setup sets +// outputStyle/statusLine/worktree/model/keybindings/.lsp.json, and (because GAP +// always runs includeGlobal) its copies vendored under ~/.claude/plugins/cache +// drive every tier-3 presence check to "present" — hiding the user's real gaps +// on ANY target. Two classes to exclude: +// - plugin-bundled: anything under ~/.claude/plugins/ (absPath marker, mirrors +// the CNF conflict-detector exclusion from M-BUG-2). +// - nested demo/test data: a file whose path RELATIVE TO THE SCAN TARGET sits +// under an examples/ or tests/fixtures/ subtree. relPath (not absPath) is +// deliberate: a fixture scanned AS the target keeps its own files, so the +// frozen v5.0.0 byte-snapshots (scanned from tests/fixtures/marketplace-medium) +// are untouched. (M-BUG-13) +const PLUGIN_TREE_MARKER = `.claude${sep}plugins${sep}`; + +/** + * @param {import('./lib/file-discovery.mjs').ConfigFile} file + * @returns {boolean} true if the file is part of the user's authored config + */ +function isAuthoredConfig(file) { + if (file.absPath.includes(PLUGIN_TREE_MARKER)) return false; + const segs = (file.relPath || '').split(sep); + if (segs.includes('examples')) return false; + const ti = segs.indexOf('tests'); + if (ti !== -1 && segs[ti + 1] === 'fixtures') return false; + return true; +} + +/** + * Read the user→project→local settings cascade directly from the filesystem. + * The settings-key gap checks ask "does the USER's resolved config set X?" — a + * question the includeGlobal discovery answers unreliably on a real machine: the + * top-level ~/.claude/settings.json is missed (its relPath carries no `.claude` + * segment when the walk root IS ~/.claude) and, when many vendored plugins flood + * the walk, dropped by the discovery file cap. Reading the canonical cascade + * paths directly is immune to both. Merged INTO (not replacing) the discovery + * settings so any non-canonical project settings still count and the frozen + * snapshots stay byte-stable. (M-BUG-13) + * @param {string} targetPath + * @returns {Promise>} + */ +async function readSettingsCascade(targetPath) { + const home = process.env.HOME || process.env.USERPROFILE || ''; + const paths = []; + if (home) { + paths.push(['user', join(home, '.claude', 'settings.json')]); + paths.push(['user-local', join(home, '.claude', 'settings.local.json')]); + } + paths.push(['project', join(targetPath, '.claude', 'settings.json')]); + paths.push(['local', join(targetPath, '.claude', 'settings.local.json')]); + + const out = []; + for (const [scope, p] of paths) { + const content = await readTextFile(p); + if (!content) continue; + const parsed = parseJson(content); + if (parsed && typeof parsed === 'object') out.push({ key: `cascade:${scope}:${p}`, parsed }); + } + return out; +} + const TIER_SEVERITY = { t1: SEVERITY.medium, t2: SEVERITY.low, @@ -479,18 +541,30 @@ export async function scan(targetPath, sharedDiscovery) { ? sharedDiscovery : await discoverConfigFiles(resolve(targetPath), { includeGlobal: true }); - // Parse all settings files upfront + // Presence checks ("does the user have feature X?") must see only the user's + // authored cascade — not bundled/vendored/demo config, which masks real gaps + // (M-BUG-13, see isAuthoredConfig). + const authoredFiles = discovery.files.filter(isAuthoredConfig); + + // Parse all settings files upfront (authored discovery files) ... const parsedSettings = new Map(); - for (const file of discovery.files.filter(f => f.type === 'settings-json')) { + for (const file of authoredFiles.filter(f => f.type === 'settings-json')) { const content = await readTextFile(file.absPath); if (content) { const parsed = parseJson(content); parsedSettings.set(`${file.scope}:${file.relPath}`, parsed); } } + // ... plus the real user→project→local cascade read directly, so settings-key + // checks see the true resolved config regardless of the discovery cap/gotcha + // (M-BUG-13). Merged, not replacing — keeps non-canonical project settings and + // the frozen byte-snapshots unchanged. + for (const { key, parsed } of await readSettingsCascade(resolve(targetPath))) { + parsedSettings.set(key, parsed); + } const ctx = { - files: discovery.files, + files: authoredFiles, targetPath: resolve(targetPath), parsedSettings, fileContents: new Map(), diff --git a/tests/scanners/feature-gap-scanner.test.mjs b/tests/scanners/feature-gap-scanner.test.mjs index ed9cf45..c31730a 100644 --- a/tests/scanners/feature-gap-scanner.test.mjs +++ b/tests/scanners/feature-gap-scanner.test.mjs @@ -1,6 +1,6 @@ import { describe, it, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; -import { resolve, join } from 'node:path'; +import { resolve, join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -411,3 +411,98 @@ describe('GAP scanner — filter-before lever wiring (chatty hook fixture)', () assert.equal(lever, undefined, `expected no filter-before lever; got id=${lever?.id}`); }); }); + +describe('GAP scanner — test/demo data must not mask real gaps (M-BUG-13)', () => { + // Build a throwaway project (+ optional hermetic HOME settings) and run GAP + // exactly as posture --global does: includeGlobal discovery + scan(). + async function runGap({ projectFiles = {}, homeSettings = null } = {}) { + const project = await mkdtemp(join(tmpdir(), 'config-audit-gap-mask-proj-')); + const home = await mkdtemp(join(tmpdir(), 'config-audit-gap-mask-home-')); + for (const [rel, content] of Object.entries(projectFiles)) { + const abs = join(project, rel); + await mkdir(dirname(abs), { recursive: true }); + await writeFile(abs, content); + } + if (homeSettings) { + await mkdir(join(home, '.claude'), { recursive: true }); + await writeFile(join(home, '.claude', 'settings.json'), JSON.stringify(homeSettings)); + } + const original = process.env.HOME; + process.env.HOME = home; + try { + resetCounter(); + const discovery = await discoverConfigFiles(project, { includeGlobal: true }); + const result = await scan(project, discovery); + return result; + } finally { + process.env.HOME = original; + await rm(project, { recursive: true, force: true }); + await rm(home, { recursive: true, force: true }); + } + } + + const hasGap = (result, titleRe) => + result.findings.some(f => f.scanner === 'GAP' && titleRe.test(f.title || '')); + + it('a nested examples/ settings.json does NOT satisfy the outputStyle check (settings-key)', async () => { + const result = await runGap({ + projectFiles: { + '.claude/CLAUDE.md': '# proj', + 'examples/optimal-setup/.claude/settings.json': JSON.stringify({ outputStyle: 'Explanatory' }), + }, + }); + assert.ok( + hasGap(result, /output style/i), + `example settings.json must not mask the outputStyle gap; got: ${result.findings.map(f => f.title).join(' | ')}`, + ); + }); + + it('a nested examples/ keybindings.json does NOT satisfy the keybindings check (file-type)', async () => { + const result = await runGap({ + projectFiles: { + '.claude/CLAUDE.md': '# proj', + 'examples/optimal-setup/.claude/keybindings.json': JSON.stringify({}), + }, + }); + assert.ok( + hasGap(result, /keybinding/i), + `example keybindings.json must not mask the keybindings gap; got: ${result.findings.map(f => f.title).join(' | ')}`, + ); + }); + + it('a nested tests/fixtures/ settings.json does NOT satisfy the model check', async () => { + const result = await runGap({ + projectFiles: { + '.claude/CLAUDE.md': '# proj', + 'tests/fixtures/demo/.claude/settings.json': JSON.stringify({ model: 'opus' }), + }, + }); + assert.ok( + hasGap(result, /model config/i), + `fixture settings.json must not mask the model gap; got: ${result.findings.map(f => f.title).join(' | ')}`, + ); + }); + + it("the project's OWN real .claude/settings.json IS counted (not over-excluded)", async () => { + const result = await runGap({ + projectFiles: { + '.claude/CLAUDE.md': '# proj', + '.claude/settings.json': JSON.stringify({ outputStyle: 'Explanatory' }), + }, + }); + assert.ok( + !hasGap(result, /output style/i), + `real project settings.json must satisfy outputStyle; got: ${result.findings.map(f => f.title).join(' | ')}`, + ); + }); + + it('the real ~/.claude/settings.json IS seen for settings-key checks (gotcha fix end-to-end)', async () => { + const result = await runGap({ + homeSettings: { statusLine: { type: 'command', command: 'x' } }, + }); + assert.ok( + !hasGap(result, /status line/i), + `~/.claude/settings.json statusLine must be discovered; got: ${result.findings.map(f => f.title).join(' | ')}`, + ); + }); +}); From eb0b3fd29d6abd43f6bda80d4de5fb385dacd6fa Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 11:02:45 +0200 Subject: [PATCH 46/53] fix(acr): size-neutral copy for "CLAUDE.md not modular" GAP (M-BUG-14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The t2_2 feature-gap check is a pure presence check (rules file OR @import present) with no length gate, consistent with its t2_3/t2_4/t2_5 siblings. But the humanized copy claimed the file is "one big block" and that splitting makes it "easier on the loading time" — an unconditional size/load-cost overclaim. For a ~625-token non-modular CLAUDE.md the load saving is trivial, so the description lied. Found by dogfooding feature-gap against linkedin-posts. Decided fix = copy-softening, NOT a length gate in t2_2 (that would make it the only size-gated check among the presence-check siblings + risks byte-stability if a short non-modular CLAUDE.md fixture exists in snapshots). - humanizer-data.mjs: title "one big block" -> "all live in one file"; description drops "long"/"loading time", keeps the honest structural "split into linked files with @import or .claude/rules/" framing. recommendation unchanged. - RED-first unit test pins the size-neutral copy (no "big"/"long"/"loading time" in title/desc, structural split framing kept). Frozen v5.0.0 snapshots untouched (RAW envelope bypasses humanizer); SC-5 default-output byte-stable (no non-modular GAP finding in those fixtures). Suite 1355->1356/0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- scanners/lib/humanizer-data.mjs | 4 ++-- tests/lib/humanizer-data.test.mjs | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index adf6802..a0e1dcf 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -435,8 +435,8 @@ export const TRANSLATIONS = { recommendation: 'Consider moving team-wide settings to project scope and keeping personal ones at user or local scope.', }, 'CLAUDE.md not modular': { - title: 'Your instructions file is one big block', - description: 'Splitting long instructions into smaller linked files makes them easier to maintain and easier on the loading time.', + title: 'Your instructions all live in one file', + description: 'Splitting your instructions into smaller linked files with `@import` or `.claude/rules/` keeps each part focused and easier to maintain.', recommendation: 'Break out long sections into separate files and link them with `@import`.', }, 'No path-scoped rules': { diff --git a/tests/lib/humanizer-data.test.mjs b/tests/lib/humanizer-data.test.mjs index e0df242..fc4683c 100644 --- a/tests/lib/humanizer-data.test.mjs +++ b/tests/lib/humanizer-data.test.mjs @@ -162,6 +162,23 @@ test('CML, SET, HKV, RUL, MCP, IMP, GAP, TOK, PLH have non-empty static maps', ( } }); +test('GAP "CLAUDE.md not modular" copy is size-neutral (M-BUG-14)', () => { + // The t2_2 check is a pure presence check (no length gate), so a small + // non-modular CLAUDE.md must not be told it is "big"/"long" or that it + // costs "loading time". Title + description must stay size/cost-neutral + // and keep only the honest structural "split into linked files" framing. + const t = TRANSLATIONS.GAP.static['CLAUDE.md not modular']; + assert.ok(t, 'GAP static map missing "CLAUDE.md not modular"'); + const title = t.title.toLowerCase(); + const desc = t.description.toLowerCase(); + assert.ok(!/\bbig\b/.test(title), `title overclaims size: "${t.title}"`); + assert.ok(!/\blong\b/.test(title), `title overclaims size: "${t.title}"`); + assert.ok(!/\bbig\b/.test(desc), `description overclaims size: "${t.description}"`); + assert.ok(!/\blong\b/.test(desc), `description overclaims size: "${t.description}"`); + assert.ok(!/loading time/.test(desc), `description overclaims load cost: "${t.description}"`); + assert.ok(/split/.test(desc), `description should keep structural split framing: "${t.description}"`); +}); + test('CNF, COL, PLH have at least one pattern entry (template-literal titles)', () => { // These scanners use template-literal titles for some findings. for (const prefix of ['CNF', 'COL', 'PLH']) { From 2975b0563f2510974395da2b8ff0800b813946d8 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 12:57:40 +0200 Subject: [PATCH 47/53] fix(acr): honest absence-state copy for two GAP enhancement findings (M-BUG-15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The t2_3 ("No path-scoped rules") and t3_6 ("No subagent isolation") feature-gap checks iterate a collection (rule files / agent files) and return false for an EMPTY one — so they fire even when the user has zero rules / zero subagents, the same state their presence-gap siblings flag. But the humanized titles presupposed the feature already exists: - "Your rules all load on every conversation" (with zero rules) - "Your subagents share Claude's main work folder" (with zero subagents) The second directly contradicts GAP-005 "You haven't set up any specialized helper agents yet" in the same report — a user cannot simultaneously have no subagents and have subagents that lack isolation. Found by dogfooding the analyze pipeline against linkedin-posts (premise-verifying each finding before trusting the analyzer-agent's report). Decided fix = align the two titles with the house "You haven't set up X yet" absence framing (state-neutral: honest for both the zero-state and the has-but-unconfigured state), NOT a base-feature presence gate in the scanner — that would change the frozen v5.0.0 marketplace-medium baseline (zero rules/agents, freezes these gaps firing) and break the RAW byte contract. Mirrors M-BUG-14's humanizer-layer, copy-only approach. - humanizer-data.mjs: title "Your rules all load on every conversation" -> "You haven't set up path-scoped rules yet"; title "Your subagents share Claude's main work folder" -> "You haven't set up subagent isolation yet". description + recommendation unchanged. t3_5/t3_7 ("Your skills don't ...") left as-is: their possessive is correct in the common case where skills exist; the zero-skills edge is latent, not manifest here. - RED-first unit test pins both titles existence-neutral (forbids "your rules all load" / "your subagents"; feature still named). Frozen v5.0.0 snapshots untouched (RAW bypasses humanizer); default-output snapshot regenerated (2 titles only, no collateral). Verified on linkedin-posts: GAP-004/005/011 now all consistently absence-framed. Suite 1356->1357/0. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- scanners/lib/humanizer-data.mjs | 4 ++-- tests/lib/humanizer-data.test.mjs | 24 +++++++++++++++++++ .../default-output/scan-orchestrator.json | 4 ++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index a0e1dcf..d4817eb 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -440,7 +440,7 @@ export const TRANSLATIONS = { recommendation: 'Break out long sections into separate files and link them with `@import`.', }, 'No path-scoped rules': { - title: 'Your rules all load on every conversation', + title: 'You haven\'t set up path-scoped rules yet', description: 'Path-scoped rules only load when you\'re working with files that match — keeps each conversation focused.', recommendation: 'Add scoping to your rules so they only load for the files they apply to.', }, @@ -490,7 +490,7 @@ export const TRANSLATIONS = { recommendation: 'Add fields like `model`, `tools`, or `description` to your skill files where useful.', }, 'No subagent isolation': { - title: 'Your subagents share Claude\'s main work folder', + title: 'You haven\'t set up subagent isolation yet', description: 'Isolated subagents run in their own copy of the repo so they can\'t accidentally disturb your main work.', recommendation: 'Add `isolation: worktree` to subagents that do destructive or experimental work.', }, diff --git a/tests/lib/humanizer-data.test.mjs b/tests/lib/humanizer-data.test.mjs index fc4683c..696461d 100644 --- a/tests/lib/humanizer-data.test.mjs +++ b/tests/lib/humanizer-data.test.mjs @@ -179,6 +179,30 @@ test('GAP "CLAUDE.md not modular" copy is size-neutral (M-BUG-14)', () => { assert.ok(/split/.test(desc), `description should keep structural split framing: "${t.description}"`); }); +test('GAP enhancement-gap titles do not presuppose the feature exists (M-BUG-15)', () => { + // t2_3 ("No path-scoped rules") and t3_6 ("No subagent isolation") iterate a + // collection and return false for an EMPTY one — so they fire even when the + // user has zero rules / zero subagents. The humanized title must therefore + // not assert the feature exists, or it contradicts the sibling presence gap + // ("No custom subagents" → "You haven't set up any specialized helper agents + // yet"). Mirror the house "You haven't set up X yet" framing — honest for + // both the zero-state and the has-but-unconfigured state. + const rules = TRANSLATIONS.GAP.static['No path-scoped rules']; + const iso = TRANSLATIONS.GAP.static['No subagent isolation']; + assert.ok(rules, 'GAP static map missing "No path-scoped rules"'); + assert.ok(iso, 'GAP static map missing "No subagent isolation"'); + // Title must not presuppose existing rules / subagents. + assert.ok(!/your rules all load/i.test(rules.title), + `t2_3 title presupposes existing rules: "${rules.title}"`); + assert.ok(!/your subagents/i.test(iso.title), + `t3_6 title presupposes existing subagents: "${iso.title}"`); + // Title must still name the feature it points at. + assert.ok(/path-scoped rules/i.test(rules.title), + `t2_3 title should still name path-scoped rules: "${rules.title}"`); + assert.ok(/isolation/i.test(iso.title), + `t3_6 title should still name subagent isolation: "${iso.title}"`); +}); + test('CNF, COL, PLH have at least one pattern entry (template-literal titles)', () => { // These scanners use template-literal titles for some findings. for (const prefix of ['CNF', 'COL', 'PLH']) { diff --git a/tests/snapshots/default-output/scan-orchestrator.json b/tests/snapshots/default-output/scan-orchestrator.json index 6b23edc..2573415 100644 --- a/tests/snapshots/default-output/scan-orchestrator.json +++ b/tests/snapshots/default-output/scan-orchestrator.json @@ -165,7 +165,7 @@ "id": "CA-GAP-003", "scanner": "GAP", "severity": "low", - "title": "Your rules all load on every conversation", + "title": "You haven't set up path-scoped rules yet", "description": "Path-scoped rules only load when you're working with files that match — keeps each conversation focused.", "file": null, "line": null, @@ -309,7 +309,7 @@ "id": "CA-GAP-012", "scanner": "GAP", "severity": "info", - "title": "Your subagents share Claude's main work folder", + "title": "You haven't set up subagent isolation yet", "description": "Isolated subagents run in their own copy of the repo so they can't accidentally disturb your main work.", "file": null, "line": null, From 239e88cecbfc785dd5ef2216dc3909896cd3d633 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:30:22 +0200 Subject: [PATCH 48/53] fix(acr): on-demand copy for oversized skill-body finding (M-BUG-16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skill-listing check emits a third finding for an oversized skill BODY (v5.11 B7, RAW title "Skill body is large (loads on demand when the skill runs)"). The body is an ON-DEMAND cost — it loads only when the skill is invoked, not the always-loaded listing Claude reads every turn. The scanner is careful to distinguish the two (RAW title + comment + evidence note). But the humanizer-data SKL.static map had no entry for this title, so it fell through to SKL._default ("A skill is using more of the listing budget than it should"). The humanized title therefore claimed a listing-budget cost and directly contradicted the finding's own humanized evidence ("loads ON DEMAND only ... NOT every turn like the always-loaded listing") — the same internal contradiction class as M-BUG-15/M-BUG-14, and the same "new finding type added without a matching humanizer entry" gap the scanner checklist warns about. Found by finding-granularity premise-verification of the linkedin-posts scan before feeding it to the analyze pipeline (the prior session's pass focused on the GAP findings and did not catch the SKL fall-through). - humanizer-data.mjs: add SKL.static entry for "Skill body is large (loads on demand when the skill runs)" with on-demand-correct title ("A skill's body is large (it loads only when that skill runs)"), description, and recommendation. No listing-budget language; tier1/tier3 forbidden-word checks pass. - RED-first tests at both layers: humanizer.test.mjs (humanizeFinding path: title is not the listing-budget _default, conveys on-demand body) and humanizer-data.test.mjs (static entry exists, on-demand-correct). RAW envelope unaffected (humanizer bypassed for --raw/--json), frozen v5.0.0 snapshots untouched, default-output fixtures contain no oversized-body skill so no snapshot regen. Suite 1357->1359/0 (+2). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- scanners/lib/humanizer-data.mjs | 5 +++++ tests/lib/humanizer-data.test.mjs | 16 ++++++++++++++++ tests/lib/humanizer.test.mjs | 21 +++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/scanners/lib/humanizer-data.mjs b/scanners/lib/humanizer-data.mjs index d4817eb..0575ddd 100644 --- a/scanners/lib/humanizer-data.mjs +++ b/scanners/lib/humanizer-data.mjs @@ -817,6 +817,11 @@ export const TRANSLATIONS = { description: 'Claude Code keeps every active skill\'s description in one shared listing it reads to choose which skill to use, and that listing has a limited size. Added up, your skills\' descriptions run past that size on a smaller setup, so Claude Code may drop some of them — and stop seeing those skills. This is an estimate; a larger setup has more room.', recommendation: 'Free up room: turn off bundled skills you do not use, collapse the heaviest ones so only their names show, or shorten the longest descriptions. The details show the measured total and the room available.', }, + 'Skill body is large (loads on demand when the skill runs)': { + title: 'A skill\'s body is large (it loads only when that skill runs)', + description: 'This skill\'s instructions run longer than the rough guidance for a skill body. The body is not part of the always-loaded listing Claude reads every turn — it loads only when you invoke the skill, so it costs nothing until then. Once it loads, though, it stays in context for the rest of that session.', + recommendation: 'Move reference material into supporting files the skill opens only when needed, so the body stays lean. For a heavy skill you can also run its body in a separate context with `context: fork` in the skill\'s settings.', + }, }, patterns: [], _default: { diff --git a/tests/lib/humanizer-data.test.mjs b/tests/lib/humanizer-data.test.mjs index 696461d..a79b065 100644 --- a/tests/lib/humanizer-data.test.mjs +++ b/tests/lib/humanizer-data.test.mjs @@ -203,6 +203,22 @@ test('GAP enhancement-gap titles do not presuppose the feature exists (M-BUG-15) `t3_6 title should still name subagent isolation: "${iso.title}"`); }); +test('SKL oversized-body copy is on-demand, not listing-budget (M-BUG-16)', () => { + // The skill-listing check emits a third finding for an oversized skill BODY, + // which loads ON DEMAND only when the skill runs — not the always-loaded + // listing. Before the fix this title had no static entry and fell through to + // the SKL _default ("using more of the listing budget"), contradicting the + // finding's own evidence ("NOT every turn like the always-loaded listing"). + const t = TRANSLATIONS.SKL.static['Skill body is large (loads on demand when the skill runs)']; + assert.ok(t, 'SKL static map missing "Skill body is large (loads on demand when the skill runs)"'); + assert.ok(!/listing budget/i.test(t.title), `body title must not claim listing budget: "${t.title}"`); + assert.ok(!/listing budget/i.test(t.description), `body description must not claim listing budget: "${t.description}"`); + // Must convey the on-demand body cost. + assert.ok(/body/i.test(t.title), `body title should name the body: "${t.title}"`); + assert.ok(/loads only|on demand|when (it|that|the) skill runs|when you invoke/i.test(t.title + ' ' + t.description), + `copy should convey the on-demand load: "${t.title}" / "${t.description}"`); +}); + test('CNF, COL, PLH have at least one pattern entry (template-literal titles)', () => { // These scanners use template-literal titles for some findings. for (const prefix of ['CNF', 'COL', 'PLH']) { diff --git a/tests/lib/humanizer.test.mjs b/tests/lib/humanizer.test.mjs index c049a70..c426c93 100644 --- a/tests/lib/humanizer.test.mjs +++ b/tests/lib/humanizer.test.mjs @@ -116,6 +116,27 @@ test('humanizeFinding falls back to _default when title unknown', () => { assert.ok(/instructions file/i.test(out.title), `expected CML _default title, got: ${out.title}`); }); +test('humanizeFinding maps SKL oversized-body finding to an on-demand title, not the listing-budget _default', () => { + // RAW title emitted by skill-listing-scanner for body > threshold (v5.11 B7). + // It is an ON-DEMAND body cost, NOT the always-loaded listing budget — the + // scanner is careful to distinguish them, so the humanized title must not + // regress into "listing budget" language via the SKL _default (M-BUG-16). + const input = makeFinding({ + scanner: 'SKL', + severity: 'low', + title: 'Skill body is large (loads on demand when the skill runs)', + description: 'Skill "repo-init" (user) has a body of about 6223 tokens (712 lines), over the ~5000-token guidance for a skill body.', + evidence: 'body_tokens~6223; threshold=5000 tok; skill="repo-init"; source=user - this is the skill BODY which loads ON DEMAND only when the skill is invoked - NOT every turn like the always-loaded listing.', + }); + const out = humanizeFinding(input); + assert.doesNotMatch(out.title, /listing budget/i, + `body finding must not be framed as a listing-budget cost, got: ${out.title}`); + assert.notEqual(out.title, 'A skill is using more of the listing budget than it should', + 'body finding must have its own mapping, not the SKL _default'); + assert.match(out.title, /body|on demand|when (it|the skill) runs/i, + `humanized title must convey the on-demand body cost, got: ${out.title}`); +}); + test('humanizeFinding passes through original strings when scanner prefix unknown', () => { const input = makeFinding({ scanner: 'XXX', title: 'whatever' }); const out = humanizeFinding(input); From 97867dbf370271588d771b3fb41c40badc47d5b5 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 30 Jun 2026 13:51:55 +0200 Subject: [PATCH 49/53] fix(acr): AGT findings humanize to "Wasted tokens" not "Other" (M-BUG-17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-listing scanner (AGT) emits an always-loaded per-turn token cost ("Agent description is long, re-sent every turn in the always-loaded listing"; scanner category 'token-efficiency', "the dominant single always-loaded source"). But SCANNER_TO_CATEGORY in humanizer.mjs had no AGT entry, so its findings fell through to the 'Other' fallback (humanizer.mjs:140) — a bucket that isn't even in the analyzer-agent's category list. Neither the scanner prefix nor the per-finding category ('token-efficiency' is not in CATEGORY_TO_IMPACT) resolved AGT to its true impact. Same class as M-BUG-16/15: a finding type without its matching humanizer mapping landing on a default that mismatches its own evidence. The analogous SKL body finding correctly buckets "Wasted tokens"; AGT (the same always-loaded token-waste mechanism) silently landed under the meaningless "Other". Found during analyze-prep premise-verification of the linkedin-posts scan: the 3 AGT findings bucketed "Other" while the analogous SKL findings bucketed "Wasted tokens". Fix: add AGT: 'Wasted tokens' to SCANNER_TO_CATEGORY, alongside TOK/CPS/SKL. RED-first (extended the Wasted-tokens category test to include AGT; the 'Other' fallback test still uses a synthetic 'XXX' scanner, unaffected). Frozen v5.0.0 untouched (AGT post-dates it; humanizer bypassed for --raw/--json); no default-output snapshot contains AGT -> 0 regen. Suite 1359/0. Verified end-to-end on linkedin-posts: 3 AGT findings now "Wasted tokens", 0 "Other" remaining, 28 findings unchanged (category-only). All 16 orchestrator scanner prefixes now covered by the category map (class closed). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01683eAqVecv9VZfQzL8CQ9h --- scanners/lib/humanizer.mjs | 1 + tests/lib/humanizer.test.mjs | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scanners/lib/humanizer.mjs b/scanners/lib/humanizer.mjs index be4f788..ddb0da7 100644 --- a/scanners/lib/humanizer.mjs +++ b/scanners/lib/humanizer.mjs @@ -38,6 +38,7 @@ const SCANNER_TO_CATEGORY = { TOK: 'Wasted tokens', CPS: 'Wasted tokens', SKL: 'Wasted tokens', + AGT: 'Wasted tokens', DIS: 'Dead config', GAP: 'Missed opportunity', PLH: 'Configuration mistake', diff --git a/tests/lib/humanizer.test.mjs b/tests/lib/humanizer.test.mjs index c426c93..6b8f64c 100644 --- a/tests/lib/humanizer.test.mjs +++ b/tests/lib/humanizer.test.mjs @@ -205,10 +205,13 @@ test('humanizeFinding sets category Conflict for CNF/COL', () => { } }); -test('humanizeFinding sets category Wasted tokens for TOK/CPS/SKL', () => { - for (const s of ['TOK', 'CPS', 'SKL']) { +test('humanizeFinding sets category Wasted tokens for TOK/CPS/SKL/AGT', () => { + // AGT (agent-listing budget) is an always-loaded per-turn token cost — the + // agent name+description is re-sent every turn in the listing — so it belongs + // in "Wasted tokens" alongside TOK/CPS/SKL, not the 'Other' fallback (M-BUG-17). + for (const s of ['TOK', 'CPS', 'SKL', 'AGT']) { const out = humanizeFinding(makeFinding({ scanner: s })); - assert.equal(out.userImpactCategory, 'Wasted tokens'); + assert.equal(out.userImpactCategory, 'Wasted tokens', `${s} should map to Wasted tokens`); } }); From 69a4654dd7533774ac6171343d1366d5b72bb917 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Tue, 14 Jul 2026 10:45:57 +0200 Subject: [PATCH 50/53] =?UTF-8?q?docs(plan):=20v5.13=20plan=20=E2=80=94=20?= =?UTF-8?q?model=20routing,=20effort=20awareness,=20dead=20references?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Video-derived audit ('The Model Isn't the Moat') cross-checked against primary sources. Verified: orchestrator+cheap-worker pattern and 5-level per-agent effort tuning (official docs); rejected: the 'Fable low ≈ Opus high' chart claim (contradicted by Anthropic's own pages). Five chunks: register entries BP-MODEL-001/002, fix-engine xhigh hygiene, CA-CML dead prose references, feature-gap model/effort opportunity, planner-agent adversarial gate. Sequenced AFTER DEL B pipeline dogfood + batch release. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CTontYwY5JGS4nL2AuiASy --- ...v5.13-model-routing-effort-deadref-plan.md | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 docs/v5.13-model-routing-effort-deadref-plan.md diff --git a/docs/v5.13-model-routing-effort-deadref-plan.md b/docs/v5.13-model-routing-effort-deadref-plan.md new file mode 100644 index 0000000..844c73e --- /dev/null +++ b/docs/v5.13-model-routing-effort-deadref-plan.md @@ -0,0 +1,157 @@ +# v5.13 Plan — Model Routing, Effort Awareness, Dead References + +Derived from an external video analysis ("The Model Isn't the Moat", 2026-07) cross-checked +against primary sources and against what config-audit already encodes. Every claim acted on +here was verified against Anthropic's own docs; video-only claims are explicitly rejected below. + +## Source verification (done 2026-07-14) + +| Claim from video | Verdict | Source | +|---|---|---| +| Orchestrator + cheaper worker models is a supported, recommended pattern | VERIFIED | code.claude.com/docs/en/sub-agents ("Control costs by routing tasks to faster, cheaper models like Haiku"), code.claude.com/docs/en/workflows | +| Reasoning effort is tunable per settings / session / launch / **per-agent frontmatter** / SDK; levels `low, medium, high, xhigh, max` | VERIFIED | code.claude.com/docs/en/model-config#adjust-effort-level, sub-agents doc | +| Leaked Fable 5 system prompt principles ("partial recognition ≠ current knowledge"; "a prompt implying a file is present doesn't mean one is"; answer-first-then-one-question; tool-call scaling 1 / 3–5 / 5–10) | VERIFIED near-verbatim, **provenance unconfirmed** (third-party leak repo, not Anthropic-confirmed) | github.com/asgeirtj/system_prompts_leaks `Anthropic/claude-fable-5.md` | +| "Fable 5 on low ≈ Opus 4.8 on high, slightly higher cost/quality" score-vs-cost chart | **CONTRADICTED** — no such chart/statement on Anthropic's pages; GPT-5.5 appears only in a testimonial | anthropic.com/news/claude-fable-5-mythos-5 | + +## Already covered — no action + +| Video idea | Existing coverage | +|---|---| +| "Process is the moat" (config/harness > raw model) | The plugin's entire thesis | +| Extract repeated procedure into a skill | BP-MECH-003 + CA-OPT-001 (`optimization-lens-scanner.mjs:121`) | +| CLAUDE.md size/ownership discipline | BP-SIZE-001 + CA-CML line/size checks (`claude-md-linter.mjs:109/:120/:140`) | +| Check that referenced imports exist | CA-IMP broken `@import` (`lib/import-resolver.mjs:88`) — but **only** `@import`, see Chunk 3 | +| Plugin's own agents are model-routed | Agents table already pins sonnet for mechanical, opus for judgment | + +## Gaps → chunks + +Verified gap summary (register-mapper sweep, 2026-07-14): no scanner audits per-agent +`model:`/`effort:` frontmatter; effort has validity-check only (`settings-validator.mjs:195`), +no recommendation; no dead-reference check for prose file mentions in CLAUDE.md; no +adversarial/failure-mode requirement in planner-agent; `fix-engine.mjs:26` effort list +omits `xhigh` (settings-validator has all five). + +### Chunk 1 — Register entries: model routing + effort (dogfoods `knowledge-refresh`) + +Add to `knowledge/best-practices.json` via the knowledge-refresh flow (human-approved write): + +- **BP-MODEL-001** (`category: model-fit`): subagents doing mechanical/read-only work can pin a + cheaper model via `model:` frontmatter; orchestrator keeps the strong model. Source: + code.claude.com/docs/en/sub-agents → `confidence: confirmed`. +- **BP-MODEL-002** (`category: model-fit`): reasoning effort is tunable at five levels in five + places (settings `effortLevel`, `/effort`, `--effort`, per-agent `effort` frontmatter, SDK); + default `high`; higher effort is not universally better for simple tasks. Source: + code.claude.com/docs/en/model-config → `confidence: confirmed`. + +Schema per `scanners/lib/best-practices-register.mjs:42-102` (id/claim/confidence/source.url/ +source.verified required). This chunk doubles as the DEL B dogfood of `/config-audit +knowledge-refresh` (each chunk is also a plugin test). + +### Chunk 2 — fix-engine effort hygiene (tiny, TDD) + +`fix-engine.mjs:26` `VALID_EFFORT_LEVELS = ['low','medium','high','max']` — missing `xhigh`. +Consequence: nearest-match "fix" for a typo like `xhig` corrects to `high`, not `xhigh`. +Red test first: `findNearestEffortLevel('xhig') === 'xhigh'`. Align list with +`settings-validator.mjs:75`. + +### Chunk 3 — CA-CML dead prose references (new deterministic check) + +The strongest video-derived principle ("a prompt implying a file is present doesn't mean one +is") applied to CLAUDE.md quality: flag file paths mentioned in CLAUDE.md **prose** that do not +exist on disk. Today only `@import` targets are existence-checked; stale pointers like +`docs/foo.md` or `scripts/bar.sh` rot silently and burn always-loaded tokens on misdirection. + +Conservative v1 to control false positives: +- Only backtick-quoted tokens that look like relative file paths (contain `/` or a known + extension), resolved against the CLAUDE.md's own directory. +- Skip URLs, globs (`*`), placeholders (`{...}`, `<...>`, `$VAR`, `${...}`), absolute and + `~/` paths (machine-specific), and paths under `.gitignore`d dirs if cheap to determine. +- Severity: low. New CA-CML-NNN (verify next free NNN at implementation — IDs are dynamic). + +Byte-stability: follow [[adding-scanner-byte-stability]] steps for a new finding type in an +EXISTING scanner — frozen `tests/snapshots/v5.0.0/` must stay untouched; default-output +snapshots regenerate (`UPDATE_SNAPSHOT=1`) only if a fixture actually carries the new type; +humanizer step 7 (M-16/M-17 lessons): `TRANSLATIONS`-static entry for the new RAW title +(CML category mapping already exists). + +### Chunk 4 — feature-gap + inventory: model/effort awareness + +- New T3 opportunity check in `feature-gap-scanner.mjs`: authored agents + (`isAuthoredConfig`, M-BUG-13 lesson) where **no** agent sets `model:` or `effort:` → + "all agents inherit the session model/effort — mechanical agents can be routed cheaper / + effort-calibrated" citing BP-MODEL-001/002. Fires only when authored agents exist + (M-BUG-15 lesson: no enhancement-check on empty collections). Opportunity framing, never + failure — deliberate max-model setups are a valid choice; finding is suppressable + (`.config-audit-ignore`). +- `whats-active` / `manifest`: surface `model`/`effort` per agent in the inventory tables. +- Humanizer wiring step 7 for the new GAP finding; verify via direct `scan()` output, not the + self-suppressed default output ([[agent-commands-need-scanner-scoping]]). +- feat commit → docs-gate: README + CLAUDE.md diffs required. + +### Chunk 5 — planner-agent adversarial gate (do AFTER DEL B pipeline dogfood) + +Add a required "Failure modes" section to `agents/planner-agent.md`'s action-plan contract: +before an action plan is emitted, list what could go wrong per change + rollback trigger. +Mirrors the video's scoping-vs-devil's-advocate distinction; currently absent (zero +adversarial requirements in agents/). **Sequencing constraint:** DEL B step 3.2 judges +planner-agent against a fasit — change the agent only after that dogfood pass, or the +fasit target moves mid-evaluation. + +## Explicitly rejected (do not revisit without new evidence) + +1. **"Fable low ≈ Opus high" cost/score framing** — contradicted by Anthropic's own pages. + Never encode in register, copy, or recommendations. +2. **Tool-call-count effort scaling (1 / 3–5 / 5–10) as a register entry** — source is an + unconfirmed third-party leak → would be `confidence: inferred`, never surfaced. Not worth + carrying. +3. **Cost/intelligence/"taste" model-routing table generator** — subjective scores don't fit + the deterministic, provenance-gated design. BP-MODEL-001 covers the actionable core. +4. **"Fable mode" skill** — a user-level skill, not configuration auditing. Out of plugin scope. +5. **CLAUDE.md prose contradiction detection** — real gap (CA-CNF only covers + settings/permissions/hooks) but not video-driven; keep this plan surgical. + +## Known tension (named, not resolved here) + +The operator's own global policy is Opus/max-effort for ALL subagents, never Haiku — the +opposite of Chunk 4's recommendation. Both are legitimate: the docs-backed routing advice +optimizes cost at equal quality for the general user; the operator deliberately buys maximum +quality. Chunk 4's copy must respect that (opportunity framing + suppressability), and on this +machine the finding will simply be suppressed or ignored. The plugin serves general users; +the operator's setup is not the target of the check. + +## Verification + +Global, after every chunk: +- `node --test 'tests/**/*.test.mjs'` → green (baseline 1359/0; count grows with new tests) +- `git status --porcelain tests/snapshots/v5.0.0/` → empty (frozen untouched) +- TDD: red test exists and fails BEFORE each production change + +Per chunk: +- **C1:** `node --test tests/lib/best-practices-register.test.mjs` green; + `node scanners/knowledge-refresh-cli.mjs` classifies BP-MODEL-001/002 as fresh +- **C2:** `findNearestEffortLevel('xhig')` → `xhigh` (red first); `grep xhigh scanners/fix-engine.mjs` non-empty +- **C3:** fixture CLAUDE.md referencing `docs/missing.md` → finding fires; existing file / + URL / glob / placeholder / `~/` path → silent; humanized output has non-contradictory copy +- **C4:** authored-agent fixture without model/effort → opportunity fires; with either set → + silent; zero authored agents → silent; `userImpactCategory` ≠ `Other` end-to-end via direct scan() +- **C5:** dogfood plan run produces a Failure-modes section; DEL B 3.2 fasit judged BEFORE the change + +## Key assumptions (test before/at implementation) + +1. **Per-agent `effort` frontmatter is official** — verified 2026-07-14 against + code.claude.com/docs/en/sub-agents + /model-config; re-fetch both pages at implementation + (docs move). +2. **New finding type in existing scanner leaves frozen snapshots untouched** — M-17 precedent + says yes when no v5.0.0 fixture carries the type; verify by running the suite and inspecting + which snapshots differ before committing. +3. **Next free CA-CML/CA-GAP finding numbers** — IDs are built dynamically; grep tests + + snapshots for the highest used NNN before assigning. + +## Sequencing vs DEL B (one plan, no relitigation) + +This plan does NOT preempt the active DEL B sequence. Recommended order: +1. DEL B step 3 pipeline dogfood (`analyze → plan → implement → rollback`) — unchanged, next. +2. Batch patch release M-11→M-17 — unchanged. +3. v5.13 chunks 1→5 (chunk 1 doubles as the `knowledge-refresh` dogfood already queued in + DEL B "Resten"; chunk 5 explicitly waits for step 3.2). Release as minor v5.13.0 via + `release-plugin.mjs` when all chunks land. From 4b7b2d9c48d9ef270fde905f8cb7035e534c9b6d Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Thu, 16 Jul 2026 20:24:12 +0200 Subject: [PATCH 51/53] fix(acr): analyze persists agent-returned report (M-BUG-18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Claude Code subagent harness instructs spawned agents NOT to write report/summary/findings/analysis .md files — the parent reads the final text message. Verified live: analyzer-agent skipped Write entirely and returned the report inline, so analysis-report.md never landed on disk and the plan/interview/status phases would find nothing to read. New contract (orchestrator-writes pattern): analyzer-agent returns the complete report as its final message; the analyze command saves it verbatim to the session directory before presenting the summary. Same class exists in plan/feature-gap/optimize/scanner agent pairs — deliberately left for their own dogfood chunks (plan is judged as-is first per the pipeline sequence). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CTontYwY5JGS4nL2AuiASy --- agents/analyzer-agent.md | 11 +++- commands/analyze.md | 17 +++-- .../analyze-report-persistence.test.mjs | 62 +++++++++++++++++++ 3 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 tests/commands/analyze-report-persistence.test.mjs diff --git a/agents/analyzer-agent.md b/agents/analyzer-agent.md index 7018314..e406563 100644 --- a/agents/analyzer-agent.md +++ b/agents/analyzer-agent.md @@ -51,11 +51,16 @@ In `--raw` mode, fall back to v5.0.0 severity prefiks and verbatim scanner title 5. **Identify optimizations**: Rules to globalize, missing configs, orphaned files 6. **Security scan**: Aggregate secret warnings, check for insecure patterns 7. **CLAUDE.md quality assessment**: Score each file against rubric, assign letter grades -8. **Generate report**: Write comprehensive markdown report — group findings by `userImpactCategory`, lead with `userActionLanguage` +8. **Generate report**: Compose the comprehensive markdown report — group findings by `userImpactCategory`, lead with `userActionLanguage` ## Output -Write to: `~/.claude/config-audit/sessions/{session-id}/analysis-report.md` +Return the complete report as your final message — do not write it to a file +yourself. The Claude Code subagent harness instructs agents not to write +report/analysis files; your text output IS the deliverable. The orchestrating +command saves your returned report verbatim to +`~/.claude/config-audit/sessions/{session-id}/analysis-report.md` for the +downstream plan/interview/status phases. **Output MUST NOT exceed 300 lines.** Prioritize findings by severity. Use tables, not prose. @@ -183,4 +188,4 @@ Verify report: all findings referenced, recommendations actionable, severity lev - Process findings in memory (typically < 1MB total) - Generate report in single pass -- No file modifications (read-only except report output) +- No file modifications (read-only; the report is returned as your final message) diff --git a/commands/analyze.md b/commands/analyze.md index 438e6f2..6387afd 100644 --- a/commands/analyze.md +++ b/commands/analyze.md @@ -60,12 +60,21 @@ Agent(subagent_type: "config-audit:analyzer-agent") raw severity. The humanizer already replaced jargon-heavy title/description/recommendation strings with plain-language equivalents — render them verbatim, do not paraphrase. - Output to: ~/.claude/config-audit/sessions/{session-id}/analysis-report.md + Return the complete report as your final message. Do not write it + to a file — the orchestrating command saves it to the session directory. ``` -### Step 4: Present summary +### Step 4: Save the report -After the agent completes, read the generated report and show a brief summary: +The agent returns the complete report as its final message — the Claude Code +subagent harness instructs agents not to write report/analysis files themselves, +so the command must persist it. Write the returned report verbatim (no edits, +no truncation) to `~/.claude/config-audit/sessions/{session-id}/analysis-report.md` +using the Write tool. Downstream phases (`plan`, `interview`, `status`) read this file. + +### Step 5: Present summary + +After saving the report, show a brief summary: ```markdown ### Analysis Complete @@ -84,6 +93,6 @@ Full report: `~/.claude/config-audit/sessions/{session-id}/analysis-report.md` - **`/config-audit fix`** — Auto-fix deterministic issues right away ``` -### Step 5: Update state +### Step 6: Update state Update `state.yaml` with `current_phase: "analyze"`, `next_phase: "plan"`. diff --git a/tests/commands/analyze-report-persistence.test.mjs b/tests/commands/analyze-report-persistence.test.mjs new file mode 100644 index 0000000..0d167f2 --- /dev/null +++ b/tests/commands/analyze-report-persistence.test.mjs @@ -0,0 +1,62 @@ +/** + * M-BUG-18 — analysis-report.md persistence contract. + * + * The Claude Code subagent harness instructs spawned agents NOT to write + * report/summary/findings/analysis .md files — the parent reads the agent's + * final text message, not files it creates. The analyzer-agent therefore + * cannot be the one that persists analysis-report.md (verified live: the + * agent skipped Write and returned the report inline). + * + * New contract (orchestrator-writes pattern): + * - analyzer-agent returns the complete report as its final message + * - the analyze command saves that returned report verbatim to + * ~/.claude/config-audit/sessions/{session-id}/analysis-report.md, + * which downstream phases (plan, interview, status) read. + */ + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFile } from 'node:fs/promises'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const COMMANDS_DIR = resolve(__dirname, '..', '..', 'commands'); +const AGENTS_DIR = resolve(__dirname, '..', '..', 'agents'); + +test('analyze.md: agent prompt does not tell the agent to write the report file', async () => { + const content = await readFile(resolve(COMMANDS_DIR, 'analyze.md'), 'utf-8'); + assert.doesNotMatch( + content, + /Output to:.*analysis-report\.md/, + 'the spawn prompt must not instruct the subagent to write analysis-report.md — the harness blocks agent-written report files' + ); +}); + +test('analyze.md: command saves the returned report to analysis-report.md', async () => { + const content = await readFile(resolve(COMMANDS_DIR, 'analyze.md'), 'utf-8'); + assert.match( + content, + /return[s]? the complete report as (its|your) final message/i, + 'analyze.md must state that the agent returns the report inline' + ); + assert.match( + content, + /Write tool[\s\S]{0,200}analysis-report\.md|analysis-report\.md[\s\S]{0,200}Write tool/, + 'analyze.md must instruct the command to persist the returned report to analysis-report.md with the Write tool' + ); +}); + +test('analyzer-agent.md: output contract is return-inline, not self-write', async () => { + const content = await readFile(resolve(AGENTS_DIR, 'analyzer-agent.md'), 'utf-8'); + assert.match( + content, + /return the complete report as your final message/i, + 'analyzer-agent must be told its final message IS the report' + ); + assert.doesNotMatch( + content, + /^Write to: .*analysis-report\.md/m, + 'analyzer-agent must not carry the old self-write output contract' + ); +}); From 0cd87e059758f7173deea9ab398b9df0860c1503 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 17 Jul 2026 03:53:06 +0200 Subject: [PATCH 52/53] fix(rul): globToRegex corrupts mid-pattern /**/ globs (M-BUG-19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ? -> [^/] replacement ran AFTER the {{GLOBSTAR_SLASH}} placeholder was restored to '(?:/.+/|/)', corrupting the group opener '(?:' into '([^/]:' — every rule pattern containing a mid-pattern '/**/' silently matched only the zero-dir branch and live rules were flagged 'matches no files' (CA-RUL). Found by dogfooding /config-audit implement on a throwaway repo copy: the implementer agent's correct 'posts/**/post.md' rule was flagged dead. Fix: run the ? replacement before placeholder restoration. Fixture outcomes byte-identical; frozen v5.0.0 baselines untouched. Co-Authored-By: Claude Fable 5 --- scanners/rules-validator.mjs | 4 +-- tests/scanners/rules-validator.test.mjs | 38 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/scanners/rules-validator.mjs b/scanners/rules-validator.mjs index e277132..4a1b61d 100644 --- a/scanners/rules-validator.mjs +++ b/scanners/rules-validator.mjs @@ -251,9 +251,9 @@ function globToRegex(pattern) { .replace(/\/\*\*\//g, '{{GLOBSTAR_SLASH}}') .replace(/\*\*/g, '{{GLOBSTAR}}') .replace(/\*/g, '[^/]*') + .replace(/\?/g, '[^/]') // must run BEFORE placeholder restore — '(?:' would corrupt .replace(/\{\{GLOBSTAR_SLASH\}\}/g, '(?:/.+/|/)') // **/ matches 0+ intermediate dirs - .replace(/\{\{GLOBSTAR\}\}/g, '.*') - .replace(/\?/g, '[^/]'); + .replace(/\{\{GLOBSTAR\}\}/g, '.*'); // Handle leading patterns if (!regex.startsWith('.*') && !regex.startsWith('/')) { diff --git a/tests/scanners/rules-validator.test.mjs b/tests/scanners/rules-validator.test.mjs index 4cf9390..cfa0d0d 100644 --- a/tests/scanners/rules-validator.test.mjs +++ b/tests/scanners/rules-validator.test.mjs @@ -291,3 +291,41 @@ describe('RUL — user-global rule is not flagged "matches no files" (M-BUG-9 gu assert.equal(dead.length, 0, `user-global rule wrongly flagged dead: ${dead.map(f => f.evidence).join(' | ')}`); }); }); + +describe('RUL — mid-pattern /**/ glob matches intermediate dirs (M-BUG-19)', () => { + // globToRegex restored the {{GLOBSTAR_SLASH}} placeholder to "(?:/.+/|/)" + // BEFORE the ? → [^/] replacement ran, corrupting the group opener "(?:" + // into "([^/]:" — so every pattern containing a mid-pattern "/**/" silently + // matched nothing but the zero-dir "|/" branch, and live rules like + // "posts/**/post.md" were flagged "matches no files". + let tmpRoot; + let result; + + beforeEach(async () => { + resetCounter(); + tmpRoot = await mkdtemp(join(tmpdir(), 'ca-rul-globstar-')); + await mkdir(join(tmpRoot, '.claude', 'rules'), { recursive: true }); + await mkdir(join(tmpRoot, 'posts', '2026-01-23-slug'), { recursive: true }); + await writeFile(join(tmpRoot, 'posts', '2026-01-23-slug', 'post.md'), '# Post\n', 'utf8'); + await writeFile( + join(tmpRoot, '.claude', 'rules', 'post-scope.md'), + '---\npaths: "posts/**/post.md"\n---\n\n# Post rule\nbody\n', + 'utf8', + ); + const discovery = await discoverConfigFiles(tmpRoot); + result = await scan(tmpRoot, discovery); + }); + + afterEach(async () => { + if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true }); + }); + + it('does NOT flag a live posts/**/post.md rule as "matches no files"', () => { + const dead = result.findings.filter(f => f.title.includes('matches no files')); + assert.equal( + dead.length, + 0, + `/**/ rule matched a real file but was flagged dead: ${dead.map(f => f.evidence).join(' | ')}`, + ); + }); +}); From b4d819b72d7fdb276d5d3911edcc91cd6e455ca5 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 17 Jul 2026 03:53:06 +0200 Subject: [PATCH 53/53] fix(acr): pin Bash >> append discipline on shared implementation log (M-BUG-20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit implement.md spawns implementer agents in parallel batches, all appending to the same implementation-log.md. Dogfooding showed agents satisfying 'Append result to:' with a full-file Write — the last writer clobbered 4 of 6 entries. Pin the mechanism in both contracts: append with Bash >> heredoc, never the Write/Edit tool on the shared log. Shape tests pin the instruction in both files (empirically verified: agents given the >> instruction appended safely). Co-Authored-By: Claude Fable 5 --- agents/implementer-agent.md | 5 +++ commands/implement.md | 2 + tests/commands/implement-log-append.test.mjs | 41 ++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/commands/implement-log-append.test.mjs diff --git a/agents/implementer-agent.md b/agents/implementer-agent.md index ecb441e..a1b066a 100644 --- a/agents/implementer-agent.md +++ b/agents/implementer-agent.md @@ -146,6 +146,11 @@ Move content from one file to another. Append to: `~/.claude/config-audit/sessions/{session-id}/implementation-log.md` +**Append discipline (shared log):** other implementer agents may be writing this +log concurrently. ALWAYS append your entry with a Bash `>>` heredoc; +NEVER use the Write or Edit tool on the log file — a full-file Write silently +clobbers entries other agents appended after you read the file. + ### Success ```markdown diff --git a/commands/implement.md b/commands/implement.md index 5f13d51..f2ff97a 100644 --- a/commands/implement.md +++ b/commands/implement.md @@ -78,6 +78,8 @@ Agent(subagent_type: "config-audit:implementer-agent") fields from the action plan (the planner already rendered them) — do not re-derive severity prose. Append result to: ~/.claude/config-audit/sessions/{session-id}/implementation-log.md + Append with Bash `>>` (heredoc) — NEVER the Write tool on this log; + parallel agents share it and a full-file Write clobbers their entries. ``` Show progress between groups using the humanized titles already present in the action plan: diff --git a/tests/commands/implement-log-append.test.mjs b/tests/commands/implement-log-append.test.mjs new file mode 100644 index 0000000..09f0d6c --- /dev/null +++ b/tests/commands/implement-log-append.test.mjs @@ -0,0 +1,41 @@ +/** + * M-BUG-20 — shared implementation-log clobbering under parallel agents. + * + * implement.md step 4 spawns implementer agents in parallel batches, and every + * agent appends its result to the SAME implementation-log.md. Dogfooding + * (2026-07-17, throwaway linkedin-posts copy) showed agents satisfying + * "Append result to:" with a full-file Write: each agent read the log, added + * its entry, and wrote the whole file back — the last writer silently + * clobbered 4 of 6 entries. + * + * Contract: both the command template and the agent prompt must pin the append + * mechanism — Bash `>>`, never the Write/Edit tool — on the shared log. + */ + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { readFile } from 'node:fs/promises'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, '..', '..'); + +const APPEND_MECHANISM_REGEX = />>/; +const FORBID_WRITE_TOOL_REGEX = /never[^.\n]*\bwrite\b[^.\n]*tool|\bwrite\b[^.\n]*tool[^.\n]*never/i; + +test('implement.md: agent-spawn template pins Bash >> append on the shared log', async () => { + const content = await readFile(resolve(ROOT, 'commands', 'implement.md'), 'utf-8'); + assert.match(content, APPEND_MECHANISM_REGEX, + 'implement.md must instruct appending to implementation-log.md with Bash >>'); + assert.match(content, FORBID_WRITE_TOOL_REGEX, + 'implement.md must forbid the Write tool on the shared implementation log'); +}); + +test('implementer-agent.md: output section pins Bash >> append and forbids Write tool on the log', async () => { + const content = await readFile(resolve(ROOT, 'agents', 'implementer-agent.md'), 'utf-8'); + assert.match(content, APPEND_MECHANISM_REGEX, + 'implementer-agent.md must instruct appending to the log with Bash >>'); + assert.match(content, FORBID_WRITE_TOOL_REGEX, + 'implementer-agent.md must forbid the Write tool on the shared implementation log'); +});