Dogfooding `campaign` + `knowledge-refresh` against a throwaway ledger. Seven
defects, all found by running the commands as written and measuring, not by
reading them.
The headline pair only existed together. `knowledge-refresh` built
`STALE_AFTER="--stale-after 30"` and expanded it unquoted, trusting the shell to
split it in two. bash does; zsh — the macOS default, and what the Bash tool runs
here — does not. The CLI got one argv entry, matched no flag, and because it had
no unknown-flag branch, silently kept the 90-day default and reported "✓ All 14
register entries were re-verified within the last 90 days": a true-sounding
sentence about a threshold the user had just overridden. Fixing either half alone
leaves a silent wrong answer or a loud one; both are fixed, and a guard now
rejects any template that packs a flag and its value into one variable.
`knowledge-refresh` also read one register and wrote another: step 6 named an
unanchored `knowledge/best-practices.json` while the CLI reads
`${CLAUDE_PLUGIN_ROOT}/…`, which for an installed plugin is the cache. The
validation gate then ran the cached test against the cached register — green no
matter what was written. The two copies were byte-identical that day, which is
exactly why it was invisible.
`campaign` vouched for repos it could not read. `add /finnes/ikke` returned
`added` + exit 0; `refresh-tokens` then put the phantom in `swept[]` with a
0-token delta and left `skipped[]` empty, so the machine-wide bill claimed
coverage of three repos on a machine with two. Paths stay tracked — an unmounted
volume is a legitimate absence — but are reported as `addedUnverified`, and the
command names them.
Two class sweeps, both measured rather than assumed. `posture` was the single
scanner (1 of 14) whose fatal catch exited 1, which ux-rules defines as a normal
WARNING grade — a crash indistinguishable from a result. And all 13 payload
writers failed on a `--output-file` whose parent did not exist, which on a fresh
machine turned `campaign`'s first run into "the ledger may be corrupt"; they now
share `scanners/lib/write-output.mjs`.
Predicted breadth was too wide for the first time in five sessions: 6 of 8 CLIs
predicted to lack unknown-flag rejection, 4 measured. `drift` and `fix` already
reject them, via a construct the grep did not recognise — a grep matches an
implementation, the invariant is a behaviour. The sweep was rewritten to run each
CLI with a bogus flag and read the exit code.
Suite 1453 → 1469/0. Frozen snapshots untouched. `optimize-lens-cli` and
`token-hotspots-cli` share the unknown-flag defect and are deferred to the v5.14
argument-handling chunk with their positional-swallow arm; the count is recorded
in the guard rather than rounded down to zero.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012NHWjN8EnoxSqRvMTLK2NE
318 lines
13 KiB
JavaScript
318 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* Config-Audit Scan Orchestrator
|
|
* Runs all registered scanners sequentially, collects findings, outputs JSON envelope.
|
|
* Usage: node scan-orchestrator.mjs <target-path> [--output-file path] [--save-baseline] [--baseline path]
|
|
* Zero external dependencies.
|
|
*/
|
|
|
|
import { resolve, sep } from 'node:path';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { writeOutputFile } from './lib/write-output.mjs';
|
|
import { resetCounter } from './lib/output.mjs';
|
|
import { envelope } from './lib/output.mjs';
|
|
import { discoverConfigFiles, discoverConfigFilesMulti, discoverFullMachinePaths } from './lib/file-discovery.mjs';
|
|
import { loadSuppressions, applySuppressions, formatSuppressionSummary } from './lib/suppression.mjs';
|
|
import { humanizeEnvelope } from './lib/humanizer.mjs';
|
|
import { resolveContextWindow } from './lib/context-window.mjs';
|
|
import { resolveActiveModel } from './lib/active-model.mjs';
|
|
|
|
// Scanner registry — import order determines execution order
|
|
import { scan as scanClaudeMd } from './claude-md-linter.mjs';
|
|
import { scan as scanSettings } from './settings-validator.mjs';
|
|
import { scan as scanHooks } from './hook-validator.mjs';
|
|
import { scan as scanRules } from './rules-validator.mjs';
|
|
import { scan as scanMcp } from './mcp-config-validator.mjs';
|
|
import { scan as scanImports } from './import-resolver.mjs';
|
|
import { scan as scanConflicts } from './conflict-detector.mjs';
|
|
import { scan as scanGap } from './feature-gap-scanner.mjs';
|
|
import { scan as scanTokenHotspots } from './token-hotspots.mjs';
|
|
import { scan as scanCachePrefix } from './cache-prefix-scanner.mjs';
|
|
import { scan as scanDisabledInSchema } from './disabled-in-schema-scanner.mjs';
|
|
import { scan as scanCollision } from './collision-scanner.mjs';
|
|
import { scan as scanSkillListing } from './skill-listing-scanner.mjs';
|
|
import { scan as scanAgentListing } from './agent-listing-scanner.mjs';
|
|
import { scan as scanOutputStyle } from './output-style-scanner.mjs';
|
|
import { scan as scanOptimizationLens } from './optimization-lens-scanner.mjs';
|
|
|
|
// Directory names that identify test fixture / example directories
|
|
const FIXTURE_DIR_NAMES = ['tests', 'examples', '__tests__', 'test-fixtures'];
|
|
|
|
/**
|
|
* Check if a finding originates from a test fixture or example directory
|
|
* relative to the scan target. Only filters when the finding's path extends
|
|
* beyond the target into a fixture subdirectory — if the target itself is
|
|
* a fixture directory, findings are NOT filtered.
|
|
* @param {object} f - Finding object
|
|
* @param {string} targetPath - Resolved scan target path
|
|
* @returns {boolean}
|
|
*/
|
|
function isFixturePath(f, targetPath) {
|
|
const p = f.file || f.path || f.location || '';
|
|
if (!p || !p.startsWith(targetPath)) return false;
|
|
// Get the path relative to target, then check if it passes through a fixture dir
|
|
const rel = p.slice(targetPath.length);
|
|
return FIXTURE_DIR_NAMES.some(dir => rel.includes(sep + dir + sep));
|
|
}
|
|
|
|
const SCANNERS = [
|
|
{ name: 'CML', fn: scanClaudeMd, label: 'CLAUDE.md Linter' },
|
|
{ name: 'SET', fn: scanSettings, label: 'Settings Validator' },
|
|
{ name: 'HKV', fn: scanHooks, label: 'Hook Validator' },
|
|
{ name: 'RUL', fn: scanRules, label: 'Rules Validator' },
|
|
{ name: 'MCP', fn: scanMcp, label: 'MCP Config Validator' },
|
|
{ name: 'IMP', fn: scanImports, label: 'Import Resolver' },
|
|
{ name: 'CNF', fn: scanConflicts, label: 'Conflict Detector' },
|
|
{ name: 'GAP', fn: scanGap, label: 'Feature Gap Scanner' },
|
|
{ name: 'TOK', fn: scanTokenHotspots, label: 'Token Hotspots' },
|
|
{ name: 'CPS', fn: scanCachePrefix, label: 'Cache-Prefix Stability' },
|
|
{ name: 'DIS', fn: scanDisabledInSchema, label: 'Disabled-In-Schema' },
|
|
{ name: 'COL', fn: scanCollision, label: 'Plugin Skill Collision' },
|
|
{ name: 'SKL', fn: scanSkillListing, label: 'Skill-Listing Budget' },
|
|
{ name: 'AGT', fn: scanAgentListing, label: 'Agent-Listing Budget' },
|
|
{ name: 'OST', fn: scanOutputStyle, label: 'Output-Style Validation' },
|
|
{ name: 'OPT', fn: scanOptimizationLens, label: 'Optimization Lens' },
|
|
];
|
|
|
|
/**
|
|
* Run all scanners against target path.
|
|
* @param {string} targetPath
|
|
* @param {object} [opts]
|
|
* @param {boolean} [opts.includeGlobal=false]
|
|
* @param {boolean} [opts.fullMachine=false] - Scan all known locations across the machine
|
|
* @param {boolean} [opts.suppress=true] - Apply suppressions from .config-audit-ignore
|
|
* @param {boolean} [opts.filterFixtures=true] - Exclude findings from test/example paths
|
|
* @param {boolean} [opts.excludeCache=true] - Drop stale ~/.claude/plugins/cache versions so findings reflect live config (B3)
|
|
* @returns {Promise<object>} Full envelope with all results
|
|
*/
|
|
// Exported for testing
|
|
export { isFixturePath, FIXTURE_DIR_NAMES };
|
|
|
|
export async function runAllScanners(targetPath, opts = {}) {
|
|
const start = Date.now();
|
|
const resolvedPath = resolve(targetPath);
|
|
|
|
// Default ON: stale cached plugin versions otherwise inflate token hotspots
|
|
// and CNF duplicate-hook findings with config that loads on zero turns. (B3)
|
|
const excludeCache = opts.excludeCache !== false;
|
|
|
|
// B8 — resolve the context window once and thread it to budget-aware scanners
|
|
// (SKL, CML). Undefined opts.contextWindow → conservative 200k anchor, which is
|
|
// byte-identical to the pre-B8 default; other scanners ignore the third arg.
|
|
// B8b — `--context-window auto` probes the configured model (settings cascade /
|
|
// ANTHROPIC_MODEL) so a 1M-tier host self-calibrates; unknown/unpinned → advisory.
|
|
let probedModel = null;
|
|
if (String(opts.contextWindow ?? '').trim().toLowerCase() === 'auto') {
|
|
probedModel = await resolveActiveModel(resolvedPath, { env: process.env });
|
|
}
|
|
const contextWindow = resolveContextWindow(opts.contextWindow, { model: probedModel });
|
|
|
|
// Shared file discovery — scanners reuse this
|
|
let discovery;
|
|
if (opts.fullMachine) {
|
|
const roots = await discoverFullMachinePaths();
|
|
discovery = await discoverConfigFilesMulti(roots, { excludeCache });
|
|
} else {
|
|
discovery = await discoverConfigFiles(resolvedPath, {
|
|
includeGlobal: opts.includeGlobal || false,
|
|
excludeCache,
|
|
});
|
|
}
|
|
|
|
const results = [];
|
|
|
|
for (const scanner of SCANNERS) {
|
|
resetCounter();
|
|
const scanStart = Date.now();
|
|
try {
|
|
const result = await scanner.fn(resolvedPath, discovery, { contextWindow });
|
|
results.push(result);
|
|
const count = result.findings.length;
|
|
const label = opts.humanizedProgress
|
|
? `\`[${scanner.name}] ${scanner.label}\``
|
|
: `[${scanner.name}] ${scanner.label}`;
|
|
process.stderr.write(` ${label}: ${count} finding(s) (${Date.now() - scanStart}ms)\n`);
|
|
} catch (err) {
|
|
results.push({
|
|
scanner: scanner.name,
|
|
status: 'error',
|
|
files_scanned: 0,
|
|
duration_ms: Date.now() - scanStart,
|
|
findings: [],
|
|
counts: { critical: 0, high: 0, medium: 0, low: 0, info: 0 },
|
|
error: err.message,
|
|
});
|
|
const label = opts.humanizedProgress
|
|
? `\`[${scanner.name}] ${scanner.label}\``
|
|
: `[${scanner.name}] ${scanner.label}`;
|
|
process.stderr.write(` ${label}: ERROR — ${err.message}\n`);
|
|
}
|
|
}
|
|
|
|
// Filter findings from test fixtures / examples (unless disabled)
|
|
const shouldFilterFixtures = opts.filterFixtures !== false;
|
|
let fixtureFindings = [];
|
|
|
|
if (shouldFilterFixtures) {
|
|
for (const result of results) {
|
|
const active = [];
|
|
const fixture = [];
|
|
for (const f of result.findings) {
|
|
if (isFixturePath(f, resolvedPath)) {
|
|
fixture.push(f);
|
|
} else {
|
|
active.push(f);
|
|
}
|
|
}
|
|
if (fixture.length > 0) {
|
|
fixtureFindings.push(...fixture);
|
|
result.findings = active;
|
|
result.counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
|
|
for (const f of active) {
|
|
if (result.counts[f.severity] !== undefined) result.counts[f.severity]++;
|
|
}
|
|
}
|
|
}
|
|
if (fixtureFindings.length > 0) {
|
|
process.stderr.write(` ${fixtureFindings.length} finding(s) from test fixtures excluded\n`);
|
|
}
|
|
}
|
|
|
|
// Apply suppressions (unless disabled)
|
|
const shouldSuppress = opts.suppress !== false;
|
|
let suppressedFindings = [];
|
|
|
|
if (shouldSuppress) {
|
|
const { suppressions } = await loadSuppressions(resolvedPath);
|
|
if (suppressions.length > 0) {
|
|
for (const result of results) {
|
|
const { active, suppressed } = applySuppressions(result.findings, suppressions);
|
|
suppressedFindings.push(...suppressed);
|
|
result.findings = active;
|
|
// Recalculate counts
|
|
result.counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
|
|
for (const f of active) {
|
|
if (result.counts[f.severity] !== undefined) result.counts[f.severity]++;
|
|
}
|
|
}
|
|
if (suppressedFindings.length > 0) {
|
|
process.stderr.write(` ${formatSuppressionSummary(suppressedFindings)}\n`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const totalMs = Date.now() - start;
|
|
const env = envelope(resolvedPath, results, totalMs);
|
|
if (fixtureFindings.length > 0) {
|
|
env.fixture_findings = fixtureFindings;
|
|
}
|
|
if (suppressedFindings.length > 0) {
|
|
env.suppressed_findings = suppressedFindings;
|
|
}
|
|
return env;
|
|
}
|
|
|
|
// --- CLI entry point ---
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
let targetPath = '.';
|
|
let outputFile = null;
|
|
let saveBaseline = false;
|
|
let baselinePath = null;
|
|
let contextWindow = null;
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
if (args[i] === '--output-file' && args[i + 1]) {
|
|
outputFile = args[++i];
|
|
} else if (args[i] === '--context-window' && args[i + 1]) {
|
|
contextWindow = args[++i];
|
|
} else if (args[i] === '--save-baseline') {
|
|
saveBaseline = true;
|
|
} else if (args[i] === '--baseline' && args[i + 1]) {
|
|
baselinePath = args[++i];
|
|
} else if (args[i] === '--global') {
|
|
// handled below
|
|
} else if (args[i] === '--full-machine') {
|
|
// handled below
|
|
} else if (args[i] === '--no-suppress') {
|
|
// handled below
|
|
} else if (args[i] === '--include-fixtures') {
|
|
// handled below
|
|
} else if (args[i] === '--exclude-cache' || args[i] === '--no-exclude-cache') {
|
|
// handled below
|
|
} else if (args[i] === '--json') {
|
|
// handled below — explicit machine-readable mode (bypass humanizer)
|
|
} else if (args[i] === '--raw') {
|
|
// handled below — v5.0.0 verbatim mode (bypass humanizer)
|
|
} else if (!args[i].startsWith('-')) {
|
|
targetPath = args[i];
|
|
}
|
|
}
|
|
|
|
const includeGlobal = args.includes('--global');
|
|
const fullMachine = args.includes('--full-machine');
|
|
const suppress = !args.includes('--no-suppress');
|
|
const filterFixtures = !args.includes('--include-fixtures');
|
|
const excludeCache = !args.includes('--no-exclude-cache');
|
|
const jsonMode = args.includes('--json');
|
|
const rawMode = args.includes('--raw');
|
|
|
|
const humanizedProgress = !jsonMode && !rawMode;
|
|
process.stderr.write(humanizedProgress ? `Config-Audit v2.2.0\n` : `Config-Audit Scanner v2.2.0\n`);
|
|
process.stderr.write(`Target: ${resolve(targetPath)}\n`);
|
|
process.stderr.write(`Scope: ${fullMachine ? 'full-machine' : includeGlobal ? 'global' : 'project'}\n`);
|
|
process.stderr.write(`Fixtures: ${filterFixtures ? 'excluded' : 'included'}\n\n`);
|
|
|
|
const result = await runAllScanners(targetPath, {
|
|
includeGlobal,
|
|
fullMachine,
|
|
suppress,
|
|
filterFixtures,
|
|
excludeCache,
|
|
humanizedProgress,
|
|
contextWindow,
|
|
});
|
|
|
|
// Default mode runs the humanizer; --json and --raw bypass for v5.0.0 byte-equal output.
|
|
const output = (jsonMode || rawMode) ? result : humanizeEnvelope(result);
|
|
const json = JSON.stringify(output, null, 2);
|
|
|
|
if (outputFile) {
|
|
await writeOutputFile(outputFile, json, 'utf-8');
|
|
process.stderr.write(`\nResults written to ${outputFile}\n`);
|
|
} else {
|
|
process.stdout.write(json + '\n');
|
|
}
|
|
|
|
if (saveBaseline) {
|
|
const bPath = baselinePath || resolve(targetPath, '.config-audit-baseline.json');
|
|
// Always save baselines as raw v5.0.0-shape envelope so future humanizer
|
|
// changes don't trigger false-positive drift findings.
|
|
await writeFile(bPath, JSON.stringify(result, null, 2), 'utf-8');
|
|
process.stderr.write(`Baseline saved to ${bPath}\n`);
|
|
}
|
|
|
|
// Summary
|
|
const agg = result.aggregate;
|
|
process.stderr.write(`\n--- Summary ---\n`);
|
|
process.stderr.write(`Findings: ${agg.total_findings} (C:${agg.counts.critical} H:${agg.counts.high} M:${agg.counts.medium} L:${agg.counts.low} I:${agg.counts.info})\n`);
|
|
process.stderr.write(`Risk: ${agg.risk_score}/100 (${agg.risk_band})\n`);
|
|
process.stderr.write(`Verdict: ${agg.verdict}\n`);
|
|
|
|
// Exit code. Set, never process.exit(): stdout is written ASYNCHRONOUSLY when
|
|
// it is a pipe, and process.exit() discards whatever is still buffered. Piping
|
|
// this envelope used to yield truncated, unparseable JSON (246 854 bytes to a
|
|
// file vs 65 536 to a pipe) — a corruption that looks like a bad file, not a
|
|
// cut-off. Letting Node exit naturally drains stdout first.
|
|
process.exitCode = agg.verdict === 'FAIL' ? 2 : agg.verdict === 'WARNING' ? 1 : 0;
|
|
}
|
|
|
|
// Only run CLI if invoked directly
|
|
const isDirectRun = process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
|
|
if (isDirectRun) {
|
|
main().catch(err => {
|
|
process.stderr.write(`Fatal: ${err.message}\n`);
|
|
process.exitCode = 3;
|
|
});
|
|
}
|