From 3c79f95e9acc429f32f67d7cfd970d3a25f2ddb5 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 07:09:26 +0200 Subject: [PATCH] feat(config-audit): self-audit --check-readme flag (v5 F6) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filesystem counts are the source of truth; README badges parsed via line-anchored substring (badge/--...). Emits readmeCheck object with counts/badges/mismatches. CLI: node scanners/self-audit.mjs --check-readme [--json] API: runSelfAudit({ checkReadme: true }) → result.readmeCheck Helper: checkReadmeBadges(pluginDir) for per-fixture testing New fixture: readme-desynced/ (commands/foo + bar, README claims 1). Note: alpha phase does NOT require result.readmeCheck.passed === true. Self-test of real plugin currently fails (scanners 10 vs 9, tests 31 vs 543); will be reconciled in Session 5 Step 28 (README sync). 582 → 586 tests, all green. --- plugins/config-audit/scanners/self-audit.mjs | 139 +++++++++++++++++- .../tests/fixtures/readme-desynced/README.md | 7 + .../fixtures/readme-desynced/commands/bar.md | 6 + .../fixtures/readme-desynced/commands/foo.md | 6 + .../tests/scanners/self-audit.test.mjs | 46 +++++- 5 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 plugins/config-audit/tests/fixtures/readme-desynced/README.md create mode 100644 plugins/config-audit/tests/fixtures/readme-desynced/commands/bar.md create mode 100644 plugins/config-audit/tests/fixtures/readme-desynced/commands/foo.md diff --git a/plugins/config-audit/scanners/self-audit.mjs b/plugins/config-audit/scanners/self-audit.mjs index a43556e..5231724 100644 --- a/plugins/config-audit/scanners/self-audit.mjs +++ b/plugins/config-audit/scanners/self-audit.mjs @@ -8,21 +8,145 @@ * Zero external dependencies. */ -import { resolve, dirname } from 'node:path'; +import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { readdir, readFile, stat } from 'node:fs/promises'; import { runAllScanners } from './scan-orchestrator.mjs'; import { scan as scanPluginHealth } from './plugin-health-scanner.mjs'; import { scoreByArea } from './lib/scoring.mjs'; import { gradeFromPassRate } from './lib/severity.mjs'; import { loadSuppressions, applySuppressions } from './lib/suppression.mjs'; +import { parseJson } from './lib/yaml-parser.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const PLUGIN_ROOT = resolve(__dirname, '..'); +// Scanner-shape detection: files in scanners/ that export `scan` and are not +// support modules. Matches the detection rule from v5 plan Step 16. +const SCANNER_EXCLUDES = new Set([ + 'scan-orchestrator.mjs', + 'self-audit.mjs', + 'whats-active.mjs', +]); + +function isScannerShape(name, content) { + if (!name.endsWith('.mjs')) return false; + if (SCANNER_EXCLUDES.has(name)) return false; + if (/-cli\.mjs$/.test(name)) return false; + if (/-engine\.mjs$/.test(name)) return false; + return /export\s+async\s+function\s+scan\b/.test(content); +} + +async function safeListDir(path) { + try { return await readdir(path, { withFileTypes: true }); } catch { return []; } +} + +async function countScannerShape(scannersDir) { + let count = 0; + for (const e of await safeListDir(scannersDir)) { + if (!e.isFile()) continue; + if (!e.name.endsWith('.mjs')) continue; + let content = ''; + try { content = await readFile(join(scannersDir, e.name), 'utf-8'); } catch { continue; } + if (isScannerShape(e.name, content)) count++; + } + return count; +} + +async function countMdFiles(dir) { + let count = 0; + for (const e of await safeListDir(dir)) { + if (e.isFile() && e.name.endsWith('.md')) count++; + } + return count; +} + +async function countTestFiles(testsRoot) { + let count = 0; + async function walk(dir) { + for (const e of await safeListDir(dir)) { + const full = join(dir, e.name); + if (e.isDirectory()) await walk(full); + else if (e.isFile() && e.name.endsWith('.test.mjs')) count++; + } + } + await walk(testsRoot); + return count; +} + +async function countHookEntries(hooksJsonPath) { + let content; + try { content = await readFile(hooksJsonPath, 'utf-8'); } catch { return 0; } + const parsed = parseJson(content); + const hooks = parsed?.hooks || parsed; + if (!hooks || typeof hooks !== 'object' || Array.isArray(hooks)) return 0; + let n = 0; + for (const handlers of Object.values(hooks)) { + if (!Array.isArray(handlers)) continue; + for (const group of handlers) { + if (!Array.isArray(group?.hooks)) continue; + n += group.hooks.length; + } + } + return n; +} + +/** + * Parse a numeric badge value from a README badge URL via line-anchored + * substring detection. Returns null if no badge for `kind` is found. + * Pattern: `badge/-(+)?-` — case-insensitive. + */ +function parseBadgeNumber(readme, kind) { + const lines = readme.split('\n'); + const rx = new RegExp(`badge\\/${kind}-([0-9]+)\\+?-`, 'i'); + for (const line of lines) { + const m = line.match(rx); + if (m) return Number(m[1]); + } + return null; +} + +/** + * Compare README badge counts against filesystem-measured counts (v5 F6). + * Filesystem counts are the source of truth. + * + * @param {string} pluginDir + * @returns {Promise<{passed: boolean, mismatches: Array<{kind:string, expected:number, foundInReadme:number}>, counts: object, badges: object}>} + */ +export async function checkReadmeBadges(pluginDir) { + const counts = { + scanners: await countScannerShape(join(pluginDir, 'scanners')), + commands: await countMdFiles(join(pluginDir, 'commands')), + agents: await countMdFiles(join(pluginDir, 'agents')), + hooks: await countHookEntries(join(pluginDir, 'hooks', 'hooks.json')), + tests: await countTestFiles(join(pluginDir, 'tests')), + knowledge: await countMdFiles(join(pluginDir, 'knowledge')), + }; + let readme = ''; + try { readme = await readFile(join(pluginDir, 'README.md'), 'utf-8'); } catch { /* missing */ } + const badges = { + scanners: parseBadgeNumber(readme, 'scanners'), + commands: parseBadgeNumber(readme, 'commands'), + agents: parseBadgeNumber(readme, 'agents'), + hooks: parseBadgeNumber(readme, 'hooks'), + tests: parseBadgeNumber(readme, 'tests'), + knowledge: parseBadgeNumber(readme, 'knowledge'), + }; + const mismatches = []; + for (const kind of Object.keys(counts)) { + if (badges[kind] === null) continue; // no badge for this kind — silent + if (counts[kind] !== badges[kind]) { + mismatches.push({ kind, expected: counts[kind], foundInReadme: badges[kind] }); + } + } + return { passed: mismatches.length === 0, mismatches, counts, badges }; +} + /** * Run self-audit on this plugin. * @param {object} [opts] * @param {boolean} [opts.fix=false] - Run fix-engine on auto-fixable findings + * @param {boolean} [opts.checkReadme=false] - Verify README badge counts (v5 F6) * @returns {Promise} Combined result */ export async function runSelfAudit(opts = {}) { @@ -80,7 +204,13 @@ export async function runSelfAudit(opts = {}) { } } - return { + // 7. Optional README badge check (v5 F6) + let readmeCheck; + if (opts.checkReadme) { + readmeCheck = await checkReadmeBadges(pluginDir); + } + + const out = { pluginDir, configGrade, configScore: avgScore, @@ -93,6 +223,8 @@ export async function runSelfAudit(opts = {}) { verdict, fixResult, }; + if (readmeCheck) out.readmeCheck = readmeCheck; + return out; } /** @@ -156,8 +288,9 @@ async function main() { const args = process.argv.slice(2); const jsonMode = args.includes('--json'); const fixMode = args.includes('--fix'); + const checkReadmeMode = args.includes('--check-readme'); - const result = await runSelfAudit({ fix: fixMode }); + const result = await runSelfAudit({ fix: fixMode, checkReadme: checkReadmeMode }); if (jsonMode) { const json = JSON.stringify(result, null, 2) + '\n'; diff --git a/plugins/config-audit/tests/fixtures/readme-desynced/README.md b/plugins/config-audit/tests/fixtures/readme-desynced/README.md new file mode 100644 index 0000000..9eb70f1 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/readme-desynced/README.md @@ -0,0 +1,7 @@ +# readme-desynced fixture + +Fixture for v5 F6 self-audit --check-readme. The badge below claims 1 command, +but `commands/` actually contains 2 (foo, bar). The check should flag this as +a low-severity mismatch. + +![Commands](https://img.shields.io/badge/commands-1-green) diff --git a/plugins/config-audit/tests/fixtures/readme-desynced/commands/bar.md b/plugins/config-audit/tests/fixtures/readme-desynced/commands/bar.md new file mode 100644 index 0000000..4ab3156 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/readme-desynced/commands/bar.md @@ -0,0 +1,6 @@ +--- +name: bar +description: Bar command for the readme-desynced fixture +--- + +# Bar command body diff --git a/plugins/config-audit/tests/fixtures/readme-desynced/commands/foo.md b/plugins/config-audit/tests/fixtures/readme-desynced/commands/foo.md new file mode 100644 index 0000000..eb8e1d1 --- /dev/null +++ b/plugins/config-audit/tests/fixtures/readme-desynced/commands/foo.md @@ -0,0 +1,6 @@ +--- +name: foo +description: Foo command for the readme-desynced fixture +--- + +# Foo command body diff --git a/plugins/config-audit/tests/scanners/self-audit.test.mjs b/plugins/config-audit/tests/scanners/self-audit.test.mjs index b929e16..c0c31e2 100644 --- a/plugins/config-audit/tests/scanners/self-audit.test.mjs +++ b/plugins/config-audit/tests/scanners/self-audit.test.mjs @@ -1,6 +1,11 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { runSelfAudit, formatSelfAudit } from '../../scanners/self-audit.mjs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { runSelfAudit, formatSelfAudit, checkReadmeBadges } from '../../scanners/self-audit.mjs'; + +const __dirname = fileURLToPath(new URL('.', import.meta.url)); +const FIXTURES = resolve(__dirname, '../fixtures'); // ======================================== // runSelfAudit @@ -69,6 +74,45 @@ describe('runSelfAudit — fixture filtering', () => { }); }); +// ======================================== +// --check-readme (v5 F6) +// ======================================== +describe('checkReadmeBadges (v5 F6)', () => { + it('detects mismatch in readme-desynced fixture', async () => { + const path = resolve(FIXTURES, 'readme-desynced'); + const result = await checkReadmeBadges(path); + assert.equal(typeof result.passed, 'boolean'); + assert.equal(result.passed, false, 'expected mismatch'); + const cmd = result.mismatches.find(m => m.kind === 'commands'); + assert.ok(cmd, `expected commands mismatch; got: ${JSON.stringify(result.mismatches)}`); + assert.equal(cmd.expected, 2, `filesystem count should be 2`); + assert.equal(cmd.foundInReadme, 1, `README badge claims 1`); + }); + + it('returns counts and badges objects', async () => { + const path = resolve(FIXTURES, 'readme-desynced'); + const result = await checkReadmeBadges(path); + assert.equal(typeof result.counts, 'object'); + assert.equal(typeof result.badges, 'object'); + assert.equal(result.counts.commands, 2); + assert.equal(result.badges.commands, 1); + }); +}); + +describe('runSelfAudit({ checkReadme: true }) (v5 F6)', () => { + it('attaches readmeCheck object to the result', async () => { + const result = await runSelfAudit({ checkReadme: true }); + assert.ok(result.readmeCheck, 'expected result.readmeCheck'); + assert.equal(typeof result.readmeCheck.passed, 'boolean'); + // Do NOT assert passed === true during alpha/beta phases — see plan Step 16. + }); + + it('omits readmeCheck when flag not set', async () => { + const result = await runSelfAudit(); + assert.equal(result.readmeCheck, undefined); + }); +}); + // ======================================== // formatSelfAudit // ========================================