Dogfooding `/config-audit plugin-health` against a fasit registered before the
run: 11 of 12 predictions confirmed, 1 refuted with evidence, 0 deviations.
The command's default path could not produce the report it documents.
M-BUG-21 (third arm): the argument loop ended in
`else if (!args[i].startsWith('-')) targetPath = args[i]` with no unknown-flag
branch, so `--output-file /tmp/x.json` was dropped and its value became the scan
target. Worse than in drift-cli: a non-existent path discovers no plugins, so the
scanner answered "No plugins found" (info) with exit 0 — a reassuring answer, not
an error. Unknown options and a value-less `--output-file` now exit 3.
M-BUG-33: the scanner had no `--output-file` and its default-mode report goes to
stderr, which `commands/plugin-health.md` discards with `2>/dev/null` before
telling the agent to read stdout. Zero bytes captured.
M-BUG-34: per-plugin rows and the grade formula never left `scan()` — the only
grade code, `formatPluginHealthReport`, had no caller — and cross-plugin findings
were flattened behind a `category` they share with per-plugin findings. The
mandated table and Cross-Plugin section were unbuildable, so the command had to
fabricate them. `scanDetailed()` now returns them; `scan()`'s frozen v5.0.0
envelope is unchanged by construction.
M-BUG-35: `.claude-plugin/marketplace.json` was flagged as an unknown file. It is
the documented catalog location, and `"source": "./"` makes the repo root its own
plugin, so one `.claude-plugin/` legitimately holds both.
Also: `commands/posture.md` ran both optional scanners in default mode under
`2>/dev/null` and read stdout — the same class as feature-gap.md:133 in the fix
chunk. A CLI-side flag fix does not close its callers.
Tests 1420 -> 1432, red first. Frozen v5.0.0 snapshots untouched.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XhhZ8zg1amR7YCAPqiZWdt
597 lines
27 KiB
JavaScript
597 lines
27 KiB
JavaScript
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdtemp, mkdir, writeFile, rm, readFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { resetCounter } from '../../scanners/lib/output.mjs';
|
|
import { scan, discoverPlugins } from '../../scanners/plugin-health-scanner.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const FIXTURES = resolve(__dirname, '../fixtures');
|
|
const TEST_PLUGIN = resolve(FIXTURES, 'test-plugin');
|
|
const BROKEN_PLUGIN = resolve(FIXTURES, 'broken-plugin');
|
|
const DUP_NAME = resolve(FIXTURES, 'duplicate-plugin-name');
|
|
const DUP_CMD = resolve(FIXTURES, 'duplicate-command-name');
|
|
const SHADOW = resolve(FIXTURES, 'plugin-shadow-folder');
|
|
const SKILLS_ARR = resolve(FIXTURES, 'plugin-skills-array');
|
|
const SECTION_COV = resolve(FIXTURES, 'plugin-section-coverage');
|
|
|
|
describe('discoverPlugins', () => {
|
|
it('discovers a single plugin when pointed at plugin dir', async () => {
|
|
const plugins = await discoverPlugins(TEST_PLUGIN);
|
|
assert.equal(plugins.length, 1);
|
|
assert.ok(plugins[0].endsWith('test-plugin'));
|
|
});
|
|
|
|
it('discovers multiple plugins in parent dir', async () => {
|
|
const plugins = await discoverPlugins(FIXTURES);
|
|
// Should find test-plugin and broken-plugin (both have .claude-plugin/plugin.json)
|
|
assert.ok(plugins.length >= 2, `Expected >=2, got ${plugins.length}`);
|
|
});
|
|
|
|
it('returns empty array for dir with no plugins', async () => {
|
|
const plugins = await discoverPlugins(resolve(FIXTURES, 'empty-project'));
|
|
assert.equal(plugins.length, 0);
|
|
});
|
|
});
|
|
|
|
describe('scan on valid test-plugin', () => {
|
|
it('returns ok status', async () => {
|
|
resetCounter();
|
|
const result = await scan(TEST_PLUGIN);
|
|
assert.equal(result.scanner, 'PLH');
|
|
assert.equal(result.status, 'ok');
|
|
});
|
|
|
|
it('finds commands and agents', async () => {
|
|
resetCounter();
|
|
const result = await scan(TEST_PLUGIN);
|
|
assert.ok(result.files_scanned >= 1, 'Should scan at least 1 plugin');
|
|
// Valid plugin should have few or no findings
|
|
const criticals = result.findings.filter(f => f.severity === 'critical');
|
|
assert.equal(criticals.length, 0, 'Valid plugin should have no critical findings');
|
|
});
|
|
|
|
it('no findings for missing plugin.json fields', async () => {
|
|
resetCounter();
|
|
const result = await scan(TEST_PLUGIN);
|
|
// Anchor on PLH + a title-substring stable across humanizer rewrites.
|
|
// Raw: "Missing required field in plugin.json: <field>". Humanized: "A plugin's manifest is missing a required field".
|
|
const missingFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /(missing.{0,40}(field|manifest))|(manifest.{0,40}missing)/i.test(f.title || '')
|
|
);
|
|
assert.equal(missingFields.length, 0, 'All required fields present in test-plugin');
|
|
});
|
|
|
|
it('no findings for missing CLAUDE.md sections', async () => {
|
|
resetCounter();
|
|
const result = await scan(TEST_PLUGIN);
|
|
// Raw: "CLAUDE.md missing '<name>' section". Humanized: "A plugin's instructions file is missing a recommended section".
|
|
const missingSections = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '')
|
|
);
|
|
assert.equal(missingSections.length, 0, 'All sections present in test-plugin CLAUDE.md');
|
|
});
|
|
});
|
|
|
|
describe('CLAUDE.md section findings track present components', () => {
|
|
it('flags a missing section only for components the plugin actually ships', async () => {
|
|
resetCounter();
|
|
const result = await scan(SECTION_COV);
|
|
// section-coverage ships commands/ but no agents/ or hooks, and its CLAUDE.md omits the
|
|
// Commands section. Per the component-aware rule: flag the present component's missing
|
|
// section, never require docs for absent components.
|
|
const sectionFindings = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '')
|
|
);
|
|
assert.ok(sectionFindings.some(f => /command/i.test(f.title)),
|
|
'missing Commands section must be flagged when commands/ exists');
|
|
assert.ok(!sectionFindings.some(f => /agent/i.test(f.title)),
|
|
'agents section must not be flagged (no agents/)');
|
|
assert.ok(!sectionFindings.some(f => /hook/i.test(f.title)),
|
|
'hooks section must not be flagged (no hooks)');
|
|
});
|
|
});
|
|
|
|
describe('scan on broken-plugin', () => {
|
|
it('detects missing plugin.json fields', async () => {
|
|
resetCounter();
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// CA-PLH-001 (description) and CA-PLH-002 (version) in broken-plugin.
|
|
const missingFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && (f.id === 'CA-PLH-001' || f.id === 'CA-PLH-002')
|
|
);
|
|
assert.ok(missingFields.length >= 2, 'Should detect missing description and version');
|
|
});
|
|
|
|
it('detects missing CLAUDE.md', async () => {
|
|
resetCounter();
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// CA-PLH-003 in broken-plugin = Missing CLAUDE.md.
|
|
const missingMd = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-003');
|
|
assert.equal(missingMd.length, 1, 'Should detect missing CLAUDE.md');
|
|
});
|
|
|
|
it('detects command without frontmatter', async () => {
|
|
resetCounter();
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// CA-PLH-004 in broken-plugin = Command missing frontmatter.
|
|
const noFrontmatter = result.findings.filter(f => f.scanner === 'PLH' && f.id === 'CA-PLH-004');
|
|
assert.equal(noFrontmatter.length, 1, 'Should detect command without frontmatter');
|
|
});
|
|
|
|
it('flags missing required agent field (description) but not optional model/tools', async () => {
|
|
resetCounter();
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// Per CC sub-agents docs: only `name` and `description` are required; `model` and `tools`
|
|
// are optional (inherit / all-tools by default). bad-agent.md has `name` only.
|
|
const agentFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /Agent missing frontmatter field/.test(f.title || '')
|
|
);
|
|
assert.ok(agentFields.some(f => /description/.test(f.title)), 'missing required `description` must be flagged');
|
|
assert.ok(!agentFields.some(f => /\b(model|tools)\b/.test(f.title)),
|
|
`optional model/tools must not be flagged, got: ${agentFields.map(f => f.title).join('; ')}`);
|
|
});
|
|
});
|
|
|
|
describe('scan with no plugins', () => {
|
|
it('returns info finding for empty directory', async () => {
|
|
resetCounter();
|
|
const result = await scan(resolve(FIXTURES, 'empty-project'));
|
|
assert.equal(result.findings.length, 1);
|
|
// CA-PLH-001 in empty-project = No plugins found.
|
|
assert.equal(result.findings[0].id, 'CA-PLH-001');
|
|
assert.equal(result.findings[0].scanner, 'PLH');
|
|
assert.equal(result.findings[0].severity, 'info');
|
|
});
|
|
});
|
|
|
|
describe('cross-plugin command conflict detection', () => {
|
|
it('scans fixtures dir and reports findings for all plugins', async () => {
|
|
resetCounter();
|
|
const result = await scan(FIXTURES);
|
|
assert.equal(result.scanner, 'PLH');
|
|
assert.ok(result.files_scanned >= 2, 'Should scan multiple plugins');
|
|
});
|
|
});
|
|
|
|
describe('plugin namespace collision detection', () => {
|
|
const COLLISION_RE = /namespace collision/i;
|
|
|
|
it('flags two plugins that declare the same name', async () => {
|
|
resetCounter();
|
|
const result = await scan(DUP_NAME);
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 1, `Expected exactly one namespace collision, got ${collisions.length}`);
|
|
const f = collisions[0];
|
|
assert.equal(f.severity, 'medium', 'Namespace collision is medium severity');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('dup'), `Title should name the colliding namespace: ${f.title}`);
|
|
assert.ok(f.details && Array.isArray(f.details.namespaces), 'Should carry details.namespaces');
|
|
assert.equal(f.details.namespaces.length, 2, 'Two plugins collide on "dup"');
|
|
for (const ns of f.details.namespaces) {
|
|
assert.equal(ns.name, 'dup');
|
|
assert.ok(ns.source.startsWith('plugin:'), `source should be plugin-scoped: ${ns.source}`);
|
|
assert.ok(ns.path, 'each namespace entry carries a path');
|
|
}
|
|
});
|
|
|
|
it('excludes name-less plugins from the collision map', async () => {
|
|
resetCounter();
|
|
const result = await scan(DUP_NAME);
|
|
// gamma + delta declare no name; they must NOT form an undefined/empty collision.
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 1, 'Only the real "dup" collision; name-less plugins are ignored');
|
|
assert.ok(
|
|
!collisions.some(f => /undefined|""|''|null/.test(f.title)),
|
|
'No collision finding for an empty/undefined name'
|
|
);
|
|
});
|
|
|
|
it('does not flag a single plugin as a collision', async () => {
|
|
resetCounter();
|
|
const result = await scan(TEST_PLUGIN);
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 0, 'A single plugin must not collide with itself');
|
|
});
|
|
});
|
|
|
|
describe('cross-plugin command name ambiguity (COL-level)', () => {
|
|
const CMD_RE = /used by multiple plugins/i;
|
|
|
|
it('flags a command name shared across different plugin namespaces as low', async () => {
|
|
resetCounter();
|
|
const result = await scan(DUP_CMD);
|
|
const amb = result.findings.filter(f => f.scanner === 'PLH' && CMD_RE.test(f.title || ''));
|
|
assert.equal(amb.length, 1, `Expected one command-ambiguity finding, got ${amb.length}`);
|
|
const f = amb[0];
|
|
assert.equal(f.severity, 'low', 'Namespaced commands are ambiguity (low), not a hard conflict (high)');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('shared-cmd'), `Title should name the command: ${f.title}`);
|
|
assert.ok(f.details && Array.isArray(f.details.namespaces), 'Should carry details.namespaces');
|
|
assert.equal(f.details.namespaces.length, 2);
|
|
});
|
|
|
|
it('labels plugins by declared name, not folder basename', async () => {
|
|
resetCounter();
|
|
const result = await scan(DUP_CMD);
|
|
const f = result.findings.find(x => x.scanner === 'PLH' && CMD_RE.test(x.title || ''));
|
|
const sources = f.details.namespaces.map(n => n.source).sort();
|
|
// Folders are one/two; declared names are plugin-one/plugin-two.
|
|
assert.deepEqual(sources, ['plugin:plugin-one', 'plugin:plugin-two']);
|
|
});
|
|
|
|
it('does not emit a high-severity command conflict (legacy behavior removed)', async () => {
|
|
resetCounter();
|
|
const result = await scan(DUP_CMD);
|
|
const high = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /command name conflict/i.test(f.title || '') && f.severity === 'high'
|
|
);
|
|
assert.equal(high.length, 0, 'The old high-severity command-conflict finding must be gone');
|
|
});
|
|
|
|
it('does not flag a shared command WITHIN a colliding namespace (namespace-collision covers it)', async () => {
|
|
resetCounter();
|
|
// alpha + beta both declare name "dup" and both ship a "hello" command.
|
|
const result = await scan(DUP_NAME);
|
|
const amb = result.findings.filter(f => f.scanner === 'PLH' && CMD_RE.test(f.title || ''));
|
|
assert.equal(amb.length, 0, 'Same namespace = one /dup:hello; the namespace collision is the right signal');
|
|
});
|
|
});
|
|
|
|
describe('plugin-folder shadowing (CA-PLH-015)', () => {
|
|
// Title set by the scanner: `plugin.json "<field>" path shadows the default <dir>/ folder`.
|
|
const SHADOW_RE = /shadows the default/i;
|
|
|
|
it('flags a manifest path that shadows the default commands/ folder', async () => {
|
|
resetCounter();
|
|
const result = await scan(SHADOW);
|
|
const shadows = result.findings.filter(f => f.scanner === 'PLH' && SHADOW_RE.test(f.title || ''));
|
|
assert.equal(shadows.length, 1, `Expected exactly one shadow finding, got ${shadows.length}: ${shadows.map(f => f.title).join(' | ')}`);
|
|
const f = shadows[0];
|
|
assert.equal(f.severity, 'medium', 'Shadowed default folder is medium (dead config)');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('commands'), `Title should name the shadowed field: ${f.title}`);
|
|
assert.ok(/plugin\.json$/.test(f.file || ''), `Finding should point at plugin.json: ${f.file}`);
|
|
assert.ok(f.details && f.details.field === 'commands', 'details.field should be the manifest key');
|
|
assert.ok(f.details.ignoredDir === 'commands', `details.ignoredDir should be the default folder: ${f.details.ignoredDir}`);
|
|
});
|
|
|
|
it('does NOT flag a default folder addressed explicitly in the manifest array', async () => {
|
|
resetCounter();
|
|
// agents: ["./agents/", "./more-agents/"] addresses the default agents/ folder → no warning.
|
|
const result = await scan(SHADOW);
|
|
const agentShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /agents/.test(f.title || '')
|
|
);
|
|
assert.equal(agentShadows.length, 0, 'agents/ is addressed explicitly via ./agents/ → not shadowed');
|
|
});
|
|
|
|
it('does NOT flag skills (adds to default, not replace)', async () => {
|
|
resetCounter();
|
|
// skills: "./custom-skills/" ADDS to the default skills/ scan; both load → no shadow.
|
|
const result = await scan(SHADOW);
|
|
const skillShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /skills/.test(f.title || '')
|
|
);
|
|
assert.equal(skillShadows.length, 0, 'skills adds-to-default; never a shadow');
|
|
});
|
|
|
|
it('does NOT flag when the default folder is absent', async () => {
|
|
resetCounter();
|
|
// outputStyles: "./styles/" is declared, but there is no output-styles/ folder → nothing ignored.
|
|
const result = await scan(SHADOW);
|
|
const osShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /output-styles/.test(f.title || '')
|
|
);
|
|
assert.equal(osShadows.length, 0, 'No default output-styles/ folder exists → no shadow');
|
|
});
|
|
|
|
it('does NOT flag a plugin with no component-path keys', async () => {
|
|
resetCounter();
|
|
// test-plugin has commands/ and agents/ folders but declares no custom paths → nothing shadowed.
|
|
const result = await scan(TEST_PLUGIN);
|
|
const shadows = result.findings.filter(f => f.scanner === 'PLH' && SHADOW_RE.test(f.title || ''));
|
|
assert.equal(shadows.length, 0, 'No manifest path keys → no shadow findings');
|
|
});
|
|
});
|
|
|
|
describe('skills:-array entry validation (CA-PLH-016)', () => {
|
|
// Title set by the scanner: `plugin.json "skills" entry <problem>: <entry>`.
|
|
const SKILLS_RE = /^plugin\.json "skills" entry/;
|
|
|
|
it('flags one finding per bad entry (file, missing, escape, non-string) and none for a valid dir', async () => {
|
|
resetCounter();
|
|
// skills: ["./valid-skill/", "./a-file.md", "./missing-dir/", "../escape", 42]
|
|
const result = await scan(SKILLS_ARR);
|
|
const bad = result.findings.filter(f => f.scanner === 'PLH' && SKILLS_RE.test(f.title || ''));
|
|
assert.equal(bad.length, 4, `Expected 4 bad-entry findings, got ${bad.length}: ${bad.map(f => f.title).join(' | ')}`);
|
|
for (const f of bad) {
|
|
assert.equal(f.severity, 'medium', `skills entry problem is medium: ${f.title}`);
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(/plugin\.json$/.test(f.file || ''), `Finding should point at plugin.json: ${f.file}`);
|
|
assert.ok(f.details && f.details.field === 'skills', 'details.field should be "skills"');
|
|
}
|
|
const problems = bad.map(f => f.details.problem).sort();
|
|
assert.deepEqual(
|
|
problems,
|
|
['escapes-root', 'non-string', 'not-a-directory', 'not-found'],
|
|
`Each problem type should appear once; got ${problems.join(',')}`
|
|
);
|
|
});
|
|
|
|
it('does NOT flag the valid skill directory', async () => {
|
|
resetCounter();
|
|
const result = await scan(SKILLS_ARR);
|
|
const validFlagged = result.findings.some(f =>
|
|
f.scanner === 'PLH' && SKILLS_RE.test(f.title || '') && /valid-skill/.test(f.title || '')
|
|
);
|
|
assert.equal(validFlagged, false, './valid-skill/ is an existing directory → not flagged');
|
|
});
|
|
|
|
it('does NOT flag a plugin with no skills: key', async () => {
|
|
resetCounter();
|
|
// test-plugin declares no skills: field.
|
|
const result = await scan(TEST_PLUGIN);
|
|
const skillsFindings = result.findings.filter(f => f.scanner === 'PLH' && SKILLS_RE.test(f.title || ''));
|
|
assert.equal(skillsFindings.length, 0, 'No skills: key → no skills-entry findings');
|
|
});
|
|
});
|
|
|
|
describe('finding format', () => {
|
|
it('findings have standard fields', async () => {
|
|
resetCounter();
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
assert.ok(result.findings.length > 0);
|
|
const f = result.findings[0];
|
|
assert.ok(f.id.startsWith('CA-PLH-'));
|
|
assert.equal(f.scanner, 'PLH');
|
|
assert.ok(['critical', 'high', 'medium', 'low', 'info'].includes(f.severity));
|
|
assert.ok(f.title);
|
|
assert.ok(f.description);
|
|
});
|
|
});
|
|
|
|
describe('PLH — plugin agent declares fields Claude Code ignores (E)', () => {
|
|
// Hermetic temp fixture: the path-guard blocks committing .claude-plugin/.
|
|
// Plugin subagents silently ignore hooks/mcpServers/permissionMode frontmatter
|
|
// (code.claude.com/docs sub-agents, V15) — dead config; permissionMode = false security.
|
|
let tmpRoot;
|
|
let result;
|
|
|
|
async function writePlugin(root, agentExtraFrontmatter) {
|
|
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
|
await mkdir(join(root, 'agents'), { recursive: true });
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'plugin.json'),
|
|
JSON.stringify({ name: 'demo', description: 'demo plugin for tests', version: '1.0.0' }, null, 2) + '\n',
|
|
'utf8',
|
|
);
|
|
await writeFile(
|
|
join(root, 'agents', 'doer.md'),
|
|
`---\nname: doer\ndescription: does things\n${agentExtraFrontmatter}---\n\n# Doer\n\nBody.\n`,
|
|
'utf8',
|
|
);
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-plh-agentdead-'));
|
|
await writePlugin(tmpRoot, 'permissionMode: plan\nhooks: present\nmcpServers: present\n');
|
|
result = await scan(tmpRoot);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
it('flags permissionMode as medium (false security)', () => {
|
|
const f = result.findings.find(x =>
|
|
x.scanner === 'PLH' && (x.evidence || '').startsWith('permissionMode:'));
|
|
assert.ok(f, `expected a permissionMode finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
|
assert.equal(f.severity, 'medium');
|
|
});
|
|
|
|
it('flags hooks and mcpServers as low (dead config)', () => {
|
|
for (const key of ['hooks', 'mcpServers']) {
|
|
const f = result.findings.find(x =>
|
|
x.scanner === 'PLH' && (x.evidence || '').startsWith(`${key}:`));
|
|
assert.ok(f, `expected a ${key} finding`);
|
|
assert.equal(f.severity, 'low', `${key} should be low`);
|
|
}
|
|
});
|
|
|
|
it('does NOT flag a clean agent (name + description only)', async () => {
|
|
resetCounter();
|
|
const clean = await mkdtemp(join(tmpdir(), 'ca-plh-agentclean-'));
|
|
try {
|
|
await writePlugin(clean, '');
|
|
const r = await scan(clean);
|
|
const dead = r.findings.filter(x =>
|
|
x.scanner === 'PLH' && /which Claude Code ignores/.test(x.title || ''));
|
|
assert.equal(dead.length, 0,
|
|
`clean agent should have no ignored-field findings; got: ${dead.map(x => x.title).join(' | ')}`);
|
|
} finally {
|
|
await rm(clean, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Økt #46 — `plugin-health` dogfood. Four defects, all found by running the
|
|
// command as written and comparing against a fasit registered BEFORE the run.
|
|
//
|
|
// F1 (M-BUG-21, third arm): the arg loop ended in
|
|
// `else if (!args[i].startsWith('-')) targetPath = args[i]` with no
|
|
// unknown-flag branch, so `--output-file /tmp/x.json` was dropped silently and
|
|
// `/tmp/x.json` became the scan target. Worse than in drift-cli: scanning a
|
|
// non-existent path yields "No plugins found" (info) and exit 0 — an
|
|
// apparently GREEN answer, not an error.
|
|
//
|
|
// F3 (stderr-only class): default mode wrote the report to STDERR only, so
|
|
// commands/plugin-health.md ("... 2>/dev/null" + "Read stdout output (JSON)")
|
|
// captured zero bytes. There was no --output-file at all (ux-rules rule 2).
|
|
//
|
|
// F5/F7: per-plugin data (name/commandCount/agentCount) and the grade formula
|
|
// never left scan(); cross-plugin findings were flattened into `findings` with
|
|
// no marker. commands/plugin-health.md mandates a
|
|
// `| Plugin | Grade | Commands | Agents |` table plus a separate Cross-Plugin
|
|
// section — both unbuildable, so the command had to fabricate them.
|
|
//
|
|
// F9: `.claude-plugin/marketplace.json` was reported as "Unknown file". It is
|
|
// the documented location for a marketplace catalog
|
|
// (code.claude.com/docs/en/plugin-marketplaces), and with `"source": "./"` one
|
|
// repo is legitimately both plugin and marketplace.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const PLH_CLI = resolve(__dirname, '../../scanners/plugin-health-scanner.mjs');
|
|
const PLH_RUN = { encoding: 'utf-8', timeout: 20000 };
|
|
|
|
function plhSpawn(args) {
|
|
const res = spawnSync('node', [PLH_CLI, ...args], PLH_RUN);
|
|
return { status: res.status, stdout: String(res.stdout || ''), stderr: String(res.stderr || '') };
|
|
}
|
|
|
|
function plhExpectFailure(args) {
|
|
const { status, stderr } = plhSpawn(args);
|
|
return { status, stderr };
|
|
}
|
|
|
|
describe('plugin-health-scanner argument validation (F1 / M-BUG-21 third arm)', () => {
|
|
it('rejects an unknown flag instead of swallowing its value as the scan target', () => {
|
|
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--bogus', 'some-value', '--json']);
|
|
assert.equal(status, 3, 'unknown flag must fail loudly, not scan "some-value"');
|
|
assert.match(stderr, /unknown option/i);
|
|
assert.match(stderr, /--bogus/);
|
|
});
|
|
|
|
it('rejects --output-file without a value', () => {
|
|
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--output-file']);
|
|
assert.equal(status, 3);
|
|
assert.match(stderr, /--output-file/);
|
|
assert.match(stderr, /requires a value/i);
|
|
});
|
|
|
|
it('still accepts a bare path as the scan target', () => {
|
|
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--json']);
|
|
assert.equal(status, 0);
|
|
assert.equal(JSON.parse(stdout).files_scanned, 1);
|
|
});
|
|
});
|
|
|
|
describe('plugin-health-scanner --output-file (F3 / ux-rules rule 2)', () => {
|
|
let dir;
|
|
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-out-')); });
|
|
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
|
|
|
it('writes the payload to the file and keeps default-mode stdout empty', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
|
assert.equal(status, 0);
|
|
assert.equal(stdout, '', 'default mode must not print to stdout (ux-rules rule 1)');
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.equal(payload.scanner, 'PLH');
|
|
assert.equal(payload.files_scanned, 1);
|
|
});
|
|
|
|
it('carries humanizer fields the command renders (F4)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([BROKEN_PLUGIN, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(payload.findings.length > 0, 'broken-plugin must produce findings');
|
|
for (const f of payload.findings) {
|
|
assert.ok(f.userImpactCategory, `finding ${f.id} missing userImpactCategory`);
|
|
assert.ok(f.userActionLanguage, `finding ${f.id} missing userActionLanguage`);
|
|
}
|
|
});
|
|
|
|
it('exposes per-plugin rows with grade, score and component counts (F5)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(Array.isArray(payload.plugins), 'payload must carry a plugins array');
|
|
assert.equal(payload.plugins.length, 1);
|
|
const p = payload.plugins[0];
|
|
assert.equal(p.name, 'test-plugin');
|
|
assert.ok(typeof p.commandCount === 'number');
|
|
assert.ok(typeof p.agentCount === 'number');
|
|
assert.ok(typeof p.score === 'number');
|
|
assert.match(p.grade, /^[ABCDF]$/);
|
|
});
|
|
|
|
it('separates cross-plugin findings from per-plugin findings (F7)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([DUP_NAME, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(Array.isArray(payload.cross_plugin_findings), 'payload must carry cross_plugin_findings');
|
|
assert.ok(payload.cross_plugin_findings.length > 0, 'duplicate-plugin-name must yield a namespace collision');
|
|
for (const f of payload.cross_plugin_findings) {
|
|
assert.equal(f.crossPlugin, true, 'cross-plugin findings must be marked');
|
|
}
|
|
const perPlugin = payload.findings.filter(f => f.crossPlugin !== true);
|
|
assert.ok(
|
|
perPlugin.length + payload.cross_plugin_findings.length === payload.findings.length,
|
|
'cross_plugin_findings must be a subset of findings, not a parallel universe'
|
|
);
|
|
});
|
|
|
|
it('leaves --raw and --json byte-stable (no new keys on the frozen envelope)', () => {
|
|
const raw = JSON.parse(plhSpawn([TEST_PLUGIN, '--raw']).stdout);
|
|
const json = JSON.parse(plhSpawn([TEST_PLUGIN, '--json']).stdout);
|
|
for (const env of [raw, json]) {
|
|
assert.deepEqual(
|
|
Object.keys(env).sort(),
|
|
['counts', 'duration_ms', 'files_scanned', 'findings', 'scanner', 'status'],
|
|
'frozen v5.0.0 envelope must not gain keys'
|
|
);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('.claude-plugin/marketplace.json is not an unknown file (F9)', () => {
|
|
let dir;
|
|
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-mp-')); });
|
|
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
|
|
|
async function writeMarketplacePlugin(root, extraFiles = {}) {
|
|
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'plugin.json'),
|
|
JSON.stringify({ name: 'mp-plugin', description: 'd', version: '1.0.0' })
|
|
);
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'marketplace.json'),
|
|
JSON.stringify({ name: 'cat', owner: { name: 'x' }, plugins: [] })
|
|
);
|
|
await writeFile(join(root, 'CLAUDE.md'), '# mp-plugin\n');
|
|
for (const [name, body] of Object.entries(extraFiles)) {
|
|
await writeFile(join(root, '.claude-plugin', name), body);
|
|
}
|
|
}
|
|
|
|
it('does not flag marketplace.json (documented catalog location)', async () => {
|
|
resetCounter();
|
|
await writeMarketplacePlugin(dir);
|
|
const result = await scan(dir);
|
|
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
|
assert.equal(unknown.length, 0,
|
|
`marketplace.json is documented; got: ${unknown.map(f => f.file).join(' | ')}`);
|
|
});
|
|
|
|
it('still flags a genuinely unexpected file', async () => {
|
|
resetCounter();
|
|
await writeMarketplacePlugin(dir, { 'notes.txt': 'scratch' });
|
|
const result = await scan(dir);
|
|
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
|
assert.equal(unknown.length, 1);
|
|
assert.match(unknown[0].file, /notes\.txt$/);
|
|
});
|
|
});
|