fix(plugin-health): make the command able to read what the scanner found
Dogfooding `/config-audit plugin-health` against a fasit registered before the
run: 11 of 12 predictions confirmed, 1 refuted with evidence, 0 deviations.
The command's default path could not produce the report it documents.
M-BUG-21 (third arm): the argument loop ended in
`else if (!args[i].startsWith('-')) targetPath = args[i]` with no unknown-flag
branch, so `--output-file /tmp/x.json` was dropped and its value became the scan
target. Worse than in drift-cli: a non-existent path discovers no plugins, so the
scanner answered "No plugins found" (info) with exit 0 — a reassuring answer, not
an error. Unknown options and a value-less `--output-file` now exit 3.
M-BUG-33: the scanner had no `--output-file` and its default-mode report goes to
stderr, which `commands/plugin-health.md` discards with `2>/dev/null` before
telling the agent to read stdout. Zero bytes captured.
M-BUG-34: per-plugin rows and the grade formula never left `scan()` — the only
grade code, `formatPluginHealthReport`, had no caller — and cross-plugin findings
were flattened behind a `category` they share with per-plugin findings. The
mandated table and Cross-Plugin section were unbuildable, so the command had to
fabricate them. `scanDetailed()` now returns them; `scan()`'s frozen v5.0.0
envelope is unchanged by construction.
M-BUG-35: `.claude-plugin/marketplace.json` was flagged as an unknown file. It is
the documented catalog location, and `"source": "./"` makes the repo root its own
plugin, so one `.claude-plugin/` legitimately holds both.
Also: `commands/posture.md` ran both optional scanners in default mode under
`2>/dev/null` and read stdout — the same class as feature-gap.md:133 in the fix
chunk. A CLI-side flag fix does not close its callers.
Tests 1420 -> 1432, red first. Frozen v5.0.0 snapshots untouched.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XhhZ8zg1amR7YCAPqiZWdt
This commit is contained in:
parent
05f1e954d0
commit
001090261e
8 changed files with 398 additions and 33 deletions
|
|
@ -132,3 +132,29 @@ test('status.md: preserves current_phase machine field and adds humanized phase
|
|||
`status.md must include at least 3 humanized phase labels; found ${present.length}: ${present.join(', ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Økt #46 — ux-rules rule 2 for the plugin-health scanner.
|
||||
//
|
||||
// plugin-health.md passed the humanized-field assertion above while the data it
|
||||
// names was unreachable: the scanner had no --output-file, and its default-mode
|
||||
// report went to stderr, which the command discards with `2>/dev/null`. A .md
|
||||
// contract test that only greps for prose cannot catch that — these assert the
|
||||
// plumbing that makes the prose true.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('plugin-health.md invokes the scanner with --output-file (ux-rules rule 2)', async () => {
|
||||
const content = await readCommand('plugin-health.md');
|
||||
const call = content.split('\n').find(l => l.includes('plugin-health-scanner.mjs'));
|
||||
assert.ok(call, 'plugin-health.md must invoke plugin-health-scanner.mjs');
|
||||
assert.match(call, /--output-file/, 'scanner call must write to a file, not stdout/stderr');
|
||||
});
|
||||
|
||||
test('posture.md invokes the plugin-health and drift scanners with --output-file', async () => {
|
||||
const content = await readCommand('posture.md');
|
||||
for (const scanner of ['plugin-health-scanner.mjs', 'drift-cli.mjs']) {
|
||||
const call = content.split('\n').find(l => l.includes(`scanners/${scanner}`));
|
||||
assert.ok(call, `posture.md must invoke ${scanner}`);
|
||||
assert.match(call, /--output-file/, `${scanner} call in posture.md discards its output`);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@ import { describe, it, beforeEach, afterEach } from 'node:test';
|
|||
import assert from 'node:assert/strict';
|
||||
import { resolve, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
|
||||
import { mkdtemp, mkdir, writeFile, rm, readFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { scan, discoverPlugins } from '../../scanners/plugin-health-scanner.mjs';
|
||||
|
||||
|
|
@ -423,3 +424,174 @@ describe('PLH — plugin agent declares fields Claude Code ignores (E)', () => {
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Økt #46 — `plugin-health` dogfood. Four defects, all found by running the
|
||||
// command as written and comparing against a fasit registered BEFORE the run.
|
||||
//
|
||||
// F1 (M-BUG-21, third arm): the arg loop ended in
|
||||
// `else if (!args[i].startsWith('-')) targetPath = args[i]` with no
|
||||
// unknown-flag branch, so `--output-file /tmp/x.json` was dropped silently and
|
||||
// `/tmp/x.json` became the scan target. Worse than in drift-cli: scanning a
|
||||
// non-existent path yields "No plugins found" (info) and exit 0 — an
|
||||
// apparently GREEN answer, not an error.
|
||||
//
|
||||
// F3 (stderr-only class): default mode wrote the report to STDERR only, so
|
||||
// commands/plugin-health.md ("... 2>/dev/null" + "Read stdout output (JSON)")
|
||||
// captured zero bytes. There was no --output-file at all (ux-rules rule 2).
|
||||
//
|
||||
// F5/F7: per-plugin data (name/commandCount/agentCount) and the grade formula
|
||||
// never left scan(); cross-plugin findings were flattened into `findings` with
|
||||
// no marker. commands/plugin-health.md mandates a
|
||||
// `| Plugin | Grade | Commands | Agents |` table plus a separate Cross-Plugin
|
||||
// section — both unbuildable, so the command had to fabricate them.
|
||||
//
|
||||
// F9: `.claude-plugin/marketplace.json` was reported as "Unknown file". It is
|
||||
// the documented location for a marketplace catalog
|
||||
// (code.claude.com/docs/en/plugin-marketplaces), and with `"source": "./"` one
|
||||
// repo is legitimately both plugin and marketplace.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const PLH_CLI = resolve(__dirname, '../../scanners/plugin-health-scanner.mjs');
|
||||
const PLH_RUN = { encoding: 'utf-8', timeout: 20000 };
|
||||
|
||||
function plhSpawn(args) {
|
||||
const res = spawnSync('node', [PLH_CLI, ...args], PLH_RUN);
|
||||
return { status: res.status, stdout: String(res.stdout || ''), stderr: String(res.stderr || '') };
|
||||
}
|
||||
|
||||
function plhExpectFailure(args) {
|
||||
const { status, stderr } = plhSpawn(args);
|
||||
return { status, stderr };
|
||||
}
|
||||
|
||||
describe('plugin-health-scanner argument validation (F1 / M-BUG-21 third arm)', () => {
|
||||
it('rejects an unknown flag instead of swallowing its value as the scan target', () => {
|
||||
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--bogus', 'some-value', '--json']);
|
||||
assert.equal(status, 3, 'unknown flag must fail loudly, not scan "some-value"');
|
||||
assert.match(stderr, /unknown option/i);
|
||||
assert.match(stderr, /--bogus/);
|
||||
});
|
||||
|
||||
it('rejects --output-file without a value', () => {
|
||||
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--output-file']);
|
||||
assert.equal(status, 3);
|
||||
assert.match(stderr, /--output-file/);
|
||||
assert.match(stderr, /requires a value/i);
|
||||
});
|
||||
|
||||
it('still accepts a bare path as the scan target', () => {
|
||||
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--json']);
|
||||
assert.equal(status, 0);
|
||||
assert.equal(JSON.parse(stdout).files_scanned, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('plugin-health-scanner --output-file (F3 / ux-rules rule 2)', () => {
|
||||
let dir;
|
||||
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-out-')); });
|
||||
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
||||
|
||||
it('writes the payload to the file and keeps default-mode stdout empty', async () => {
|
||||
const out = join(dir, 'plh.json');
|
||||
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
||||
assert.equal(status, 0);
|
||||
assert.equal(stdout, '', 'default mode must not print to stdout (ux-rules rule 1)');
|
||||
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
||||
assert.equal(payload.scanner, 'PLH');
|
||||
assert.equal(payload.files_scanned, 1);
|
||||
});
|
||||
|
||||
it('carries humanizer fields the command renders (F4)', async () => {
|
||||
const out = join(dir, 'plh.json');
|
||||
plhSpawn([BROKEN_PLUGIN, '--output-file', out]);
|
||||
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
||||
assert.ok(payload.findings.length > 0, 'broken-plugin must produce findings');
|
||||
for (const f of payload.findings) {
|
||||
assert.ok(f.userImpactCategory, `finding ${f.id} missing userImpactCategory`);
|
||||
assert.ok(f.userActionLanguage, `finding ${f.id} missing userActionLanguage`);
|
||||
}
|
||||
});
|
||||
|
||||
it('exposes per-plugin rows with grade, score and component counts (F5)', async () => {
|
||||
const out = join(dir, 'plh.json');
|
||||
plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
||||
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
||||
assert.ok(Array.isArray(payload.plugins), 'payload must carry a plugins array');
|
||||
assert.equal(payload.plugins.length, 1);
|
||||
const p = payload.plugins[0];
|
||||
assert.equal(p.name, 'test-plugin');
|
||||
assert.ok(typeof p.commandCount === 'number');
|
||||
assert.ok(typeof p.agentCount === 'number');
|
||||
assert.ok(typeof p.score === 'number');
|
||||
assert.match(p.grade, /^[ABCDF]$/);
|
||||
});
|
||||
|
||||
it('separates cross-plugin findings from per-plugin findings (F7)', async () => {
|
||||
const out = join(dir, 'plh.json');
|
||||
plhSpawn([DUP_NAME, '--output-file', out]);
|
||||
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
||||
assert.ok(Array.isArray(payload.cross_plugin_findings), 'payload must carry cross_plugin_findings');
|
||||
assert.ok(payload.cross_plugin_findings.length > 0, 'duplicate-plugin-name must yield a namespace collision');
|
||||
for (const f of payload.cross_plugin_findings) {
|
||||
assert.equal(f.crossPlugin, true, 'cross-plugin findings must be marked');
|
||||
}
|
||||
const perPlugin = payload.findings.filter(f => f.crossPlugin !== true);
|
||||
assert.ok(
|
||||
perPlugin.length + payload.cross_plugin_findings.length === payload.findings.length,
|
||||
'cross_plugin_findings must be a subset of findings, not a parallel universe'
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves --raw and --json byte-stable (no new keys on the frozen envelope)', () => {
|
||||
const raw = JSON.parse(plhSpawn([TEST_PLUGIN, '--raw']).stdout);
|
||||
const json = JSON.parse(plhSpawn([TEST_PLUGIN, '--json']).stdout);
|
||||
for (const env of [raw, json]) {
|
||||
assert.deepEqual(
|
||||
Object.keys(env).sort(),
|
||||
['counts', 'duration_ms', 'files_scanned', 'findings', 'scanner', 'status'],
|
||||
'frozen v5.0.0 envelope must not gain keys'
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('.claude-plugin/marketplace.json is not an unknown file (F9)', () => {
|
||||
let dir;
|
||||
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-mp-')); });
|
||||
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
||||
|
||||
async function writeMarketplacePlugin(root, extraFiles = {}) {
|
||||
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
||||
await writeFile(
|
||||
join(root, '.claude-plugin', 'plugin.json'),
|
||||
JSON.stringify({ name: 'mp-plugin', description: 'd', version: '1.0.0' })
|
||||
);
|
||||
await writeFile(
|
||||
join(root, '.claude-plugin', 'marketplace.json'),
|
||||
JSON.stringify({ name: 'cat', owner: { name: 'x' }, plugins: [] })
|
||||
);
|
||||
await writeFile(join(root, 'CLAUDE.md'), '# mp-plugin\n');
|
||||
for (const [name, body] of Object.entries(extraFiles)) {
|
||||
await writeFile(join(root, '.claude-plugin', name), body);
|
||||
}
|
||||
}
|
||||
|
||||
it('does not flag marketplace.json (documented catalog location)', async () => {
|
||||
resetCounter();
|
||||
await writeMarketplacePlugin(dir);
|
||||
const result = await scan(dir);
|
||||
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
||||
assert.equal(unknown.length, 0,
|
||||
`marketplace.json is documented; got: ${unknown.map(f => f.file).join(' | ')}`);
|
||||
});
|
||||
|
||||
it('still flags a genuinely unexpected file', async () => {
|
||||
resetCounter();
|
||||
await writeMarketplacePlugin(dir, { 'notes.txt': 'scratch' });
|
||||
const result = await scan(dir);
|
||||
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
||||
assert.equal(unknown.length, 1);
|
||||
assert.match(unknown[0].file, /notes\.txt$/);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue