BREAKING CHANGE: the {NNN} in CA-{SCANNER}-{NNN} identifies the check that
produced the finding. It used to be the finding's position in that scanner's
output for that run, which made it unstable across CONFIGURATIONS, not just
across releases as STATE framed it. Measured on two fixtures: "No custom
subagents" was CA-GAP-007 on minimal-project and CA-GAP-004 on healthy-project.
A user who fixed an unrelated earlier gap silently renumbered every later one,
so a .config-audit-ignore pin retargeted to a neighbouring finding with no
version change at all.
Second measured arm: README already documented the opposite scheme. It and the
scanner headers describe ~20 numbers as check codes (CA-SKL-003 = oversized
body, CA-PLH-015 = folder shadowing, CA-TOK-006 = schema deferral), and the
counter could only produce those in the all-fire case -- source-order positions
are 4, 3 and 8. The documentation described the scheme; the implementation was
what was wrong. Every published number is preserved by construction and pinned
exhaustively in tests/lib/finding-codes.test.mjs.
scanners/lib/finding-codes.mjs is the single authority. Every finding() call
passes a `code`; an undeclared or missing one THROWS. No counter fallback --
that would reproduce D1's findGapId -> 'unknown' silent degradation and let a
half-converted scanner ship IDs that look valid. findingCounter/resetCounter
are deleted outright, not left as no-ops. Retirement is now a mechanism:
RETIRED_CODES tombstones a withdrawn key so its number is never reissued,
seeded with GAP t3_8 -- the D1 removal that opened this chunk.
IDs are consequently NOT unique per finding: one check failing in three files
emits three findings sharing an ID. That inverts which consumer is correct, so
every f.id/findingId site was classified before the change. diff-engine and
most of fix-engine already keyed on scanner+title+file (drift was never lying);
fix-engine's verification did not, and keyed on the ID alone -- fixing one of
two sibling instances marked both fixed, and the untouched one, still present
in the re-scan, was reported as a REGRESSION. Red test first, then keyed on
(findingId, file), which both planFixes and applyFixes already carry.
plugin-health's crossIds Set was measured and is a clean negative: cross
findings are allFindings.slice(crossPluginStart) and codes 18/19 are emitted
only in that tail, so the partition holds by construction.
unknownSuppressions() reports a pin that names no declared check, in the
--output-file payload (ux-rules rule 2 -- a stderr-only warning is invisible to
the commands) and only when one exists, so a clean config is byte-identical.
That is what makes the break safe: a stale pin goes loud instead of dying quiet.
Frozen tests/snapshots/v5.0.0/ untouched on disk. IDs are masked out of that
comparison (mask-finding-ids.mjs) rather than re-derived -- re-deriving
positional IDs would assert the retired scheme against itself, and #58's
isGapEntry off-by-one is the measured example of that misfiring. The dead
re-derivation is removed from strip-retired-gap.mjs. default-output snapshots
re-approved after confirming the diff is IDs and nothing else.
Guards, each seen red against its own defect: a missing code (scanner errors
out mid-sweep), an orphan declaration, a resurrected retired key, and a
documented ID naming no check. The sweep asserts the union across all 16
scanners, never per scanner -- a per-scanner assertion goes green on a partial
conversion.
Fasit written before implementation: docs/mbug28-id-semantics-fasit.local.md,
including one correction made before running (CML has 12 checks over 13 call
sites -- the anchored and calibrated char-budget arms are one check, which a
repeated-title sweep found and my call-site count had missed).
Suite 1535 -> 1573, 0 failing.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MyqCQKK2ornJ1jFWwqx17E
563 lines
26 KiB
JavaScript
563 lines
26 KiB
JavaScript
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdtemp, mkdir, writeFile, rm, readFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { scan, discoverPlugins } from '../../scanners/plugin-health-scanner.mjs';
|
|
import { findingId } from '../../scanners/lib/finding-codes.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const FIXTURES = resolve(__dirname, '../fixtures');
|
|
const TEST_PLUGIN = resolve(FIXTURES, 'test-plugin');
|
|
const BROKEN_PLUGIN = resolve(FIXTURES, 'broken-plugin');
|
|
const DUP_NAME = resolve(FIXTURES, 'duplicate-plugin-name');
|
|
const DUP_CMD = resolve(FIXTURES, 'duplicate-command-name');
|
|
const SHADOW = resolve(FIXTURES, 'plugin-shadow-folder');
|
|
const SKILLS_ARR = resolve(FIXTURES, 'plugin-skills-array');
|
|
const SECTION_COV = resolve(FIXTURES, 'plugin-section-coverage');
|
|
|
|
describe('discoverPlugins', () => {
|
|
it('discovers a single plugin when pointed at plugin dir', async () => {
|
|
const plugins = await discoverPlugins(TEST_PLUGIN);
|
|
assert.equal(plugins.length, 1);
|
|
assert.ok(plugins[0].endsWith('test-plugin'));
|
|
});
|
|
|
|
it('discovers multiple plugins in parent dir', async () => {
|
|
const plugins = await discoverPlugins(FIXTURES);
|
|
// Should find test-plugin and broken-plugin (both have .claude-plugin/plugin.json)
|
|
assert.ok(plugins.length >= 2, `Expected >=2, got ${plugins.length}`);
|
|
});
|
|
|
|
it('returns empty array for dir with no plugins', async () => {
|
|
const plugins = await discoverPlugins(resolve(FIXTURES, 'empty-project'));
|
|
assert.equal(plugins.length, 0);
|
|
});
|
|
});
|
|
|
|
describe('scan on valid test-plugin', () => {
|
|
it('returns ok status', async () => {
|
|
const result = await scan(TEST_PLUGIN);
|
|
assert.equal(result.scanner, 'PLH');
|
|
assert.equal(result.status, 'ok');
|
|
});
|
|
|
|
it('finds commands and agents', async () => {
|
|
const result = await scan(TEST_PLUGIN);
|
|
assert.ok(result.files_scanned >= 1, 'Should scan at least 1 plugin');
|
|
// Valid plugin should have few or no findings
|
|
const criticals = result.findings.filter(f => f.severity === 'critical');
|
|
assert.equal(criticals.length, 0, 'Valid plugin should have no critical findings');
|
|
});
|
|
|
|
it('no findings for missing plugin.json fields', async () => {
|
|
const result = await scan(TEST_PLUGIN);
|
|
// Anchor on PLH + a title-substring stable across humanizer rewrites.
|
|
// Raw: "Missing required field in plugin.json: <field>". Humanized: "A plugin's manifest is missing a required field".
|
|
const missingFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /(missing.{0,40}(field|manifest))|(manifest.{0,40}missing)/i.test(f.title || '')
|
|
);
|
|
assert.equal(missingFields.length, 0, 'All required fields present in test-plugin');
|
|
});
|
|
|
|
it('no findings for missing CLAUDE.md sections', async () => {
|
|
const result = await scan(TEST_PLUGIN);
|
|
// Raw: "CLAUDE.md missing '<name>' section". Humanized: "A plugin's instructions file is missing a recommended section".
|
|
const missingSections = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '')
|
|
);
|
|
assert.equal(missingSections.length, 0, 'All sections present in test-plugin CLAUDE.md');
|
|
});
|
|
});
|
|
|
|
describe('CLAUDE.md section findings track present components', () => {
|
|
it('flags a missing section only for components the plugin actually ships', async () => {
|
|
const result = await scan(SECTION_COV);
|
|
// section-coverage ships commands/ but no agents/ or hooks, and its CLAUDE.md omits the
|
|
// Commands section. Per the component-aware rule: flag the present component's missing
|
|
// section, never require docs for absent components.
|
|
const sectionFindings = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /missing.{0,40}section/i.test(f.title || '')
|
|
);
|
|
assert.ok(sectionFindings.some(f => /command/i.test(f.title)),
|
|
'missing Commands section must be flagged when commands/ exists');
|
|
assert.ok(!sectionFindings.some(f => /agent/i.test(f.title)),
|
|
'agents section must not be flagged (no agents/)');
|
|
assert.ok(!sectionFindings.some(f => /hook/i.test(f.title)),
|
|
'hooks section must not be flagged (no hooks)');
|
|
});
|
|
});
|
|
|
|
describe('scan on broken-plugin', () => {
|
|
it('detects missing plugin.json fields', async () => {
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// One check ("missing required field"), two instances (description, version).
|
|
const missingFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && f.id === findingId('PLH', 'missing-required-field')
|
|
);
|
|
assert.ok(missingFields.length >= 2, 'Should detect missing description and version');
|
|
});
|
|
|
|
it('detects missing CLAUDE.md', async () => {
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
const missingMd = result.findings.filter(f => f.scanner === 'PLH' && f.id === findingId('PLH', 'missing-claude-md'));
|
|
assert.equal(missingMd.length, 1, 'Should detect missing CLAUDE.md');
|
|
});
|
|
|
|
it('detects command without frontmatter', async () => {
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
const noFrontmatter = result.findings.filter(f => f.scanner === 'PLH' && f.id === findingId('PLH', 'command-missing-frontmatter'));
|
|
assert.equal(noFrontmatter.length, 1, 'Should detect command without frontmatter');
|
|
});
|
|
|
|
it('flags missing required agent field (description) but not optional model/tools', async () => {
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
// Per CC sub-agents docs: only `name` and `description` are required; `model` and `tools`
|
|
// are optional (inherit / all-tools by default). bad-agent.md has `name` only.
|
|
const agentFields = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /Agent missing frontmatter field/.test(f.title || '')
|
|
);
|
|
assert.ok(agentFields.some(f => /description/.test(f.title)), 'missing required `description` must be flagged');
|
|
assert.ok(!agentFields.some(f => /\b(model|tools)\b/.test(f.title)),
|
|
`optional model/tools must not be flagged, got: ${agentFields.map(f => f.title).join('; ')}`);
|
|
});
|
|
});
|
|
|
|
describe('scan with no plugins', () => {
|
|
it('returns info finding for empty directory', async () => {
|
|
const result = await scan(resolve(FIXTURES, 'empty-project'));
|
|
assert.equal(result.findings.length, 1);
|
|
assert.equal(result.findings[0].id, findingId('PLH', 'no-plugins-found'));
|
|
assert.equal(result.findings[0].scanner, 'PLH');
|
|
assert.equal(result.findings[0].severity, 'info');
|
|
});
|
|
});
|
|
|
|
describe('cross-plugin command conflict detection', () => {
|
|
it('scans fixtures dir and reports findings for all plugins', async () => {
|
|
const result = await scan(FIXTURES);
|
|
assert.equal(result.scanner, 'PLH');
|
|
assert.ok(result.files_scanned >= 2, 'Should scan multiple plugins');
|
|
});
|
|
});
|
|
|
|
describe('plugin namespace collision detection', () => {
|
|
const COLLISION_RE = /namespace collision/i;
|
|
|
|
it('flags two plugins that declare the same name', async () => {
|
|
const result = await scan(DUP_NAME);
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 1, `Expected exactly one namespace collision, got ${collisions.length}`);
|
|
const f = collisions[0];
|
|
assert.equal(f.severity, 'medium', 'Namespace collision is medium severity');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('dup'), `Title should name the colliding namespace: ${f.title}`);
|
|
assert.ok(f.details && Array.isArray(f.details.namespaces), 'Should carry details.namespaces');
|
|
assert.equal(f.details.namespaces.length, 2, 'Two plugins collide on "dup"');
|
|
for (const ns of f.details.namespaces) {
|
|
assert.equal(ns.name, 'dup');
|
|
assert.ok(ns.source.startsWith('plugin:'), `source should be plugin-scoped: ${ns.source}`);
|
|
assert.ok(ns.path, 'each namespace entry carries a path');
|
|
}
|
|
});
|
|
|
|
it('excludes name-less plugins from the collision map', async () => {
|
|
const result = await scan(DUP_NAME);
|
|
// gamma + delta declare no name; they must NOT form an undefined/empty collision.
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 1, 'Only the real "dup" collision; name-less plugins are ignored');
|
|
assert.ok(
|
|
!collisions.some(f => /undefined|""|''|null/.test(f.title)),
|
|
'No collision finding for an empty/undefined name'
|
|
);
|
|
});
|
|
|
|
it('does not flag a single plugin as a collision', async () => {
|
|
const result = await scan(TEST_PLUGIN);
|
|
const collisions = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && COLLISION_RE.test(f.title || '')
|
|
);
|
|
assert.equal(collisions.length, 0, 'A single plugin must not collide with itself');
|
|
});
|
|
});
|
|
|
|
describe('cross-plugin command name ambiguity (COL-level)', () => {
|
|
const CMD_RE = /used by multiple plugins/i;
|
|
|
|
it('flags a command name shared across different plugin namespaces as low', async () => {
|
|
const result = await scan(DUP_CMD);
|
|
const amb = result.findings.filter(f => f.scanner === 'PLH' && CMD_RE.test(f.title || ''));
|
|
assert.equal(amb.length, 1, `Expected one command-ambiguity finding, got ${amb.length}`);
|
|
const f = amb[0];
|
|
assert.equal(f.severity, 'low', 'Namespaced commands are ambiguity (low), not a hard conflict (high)');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('shared-cmd'), `Title should name the command: ${f.title}`);
|
|
assert.ok(f.details && Array.isArray(f.details.namespaces), 'Should carry details.namespaces');
|
|
assert.equal(f.details.namespaces.length, 2);
|
|
});
|
|
|
|
it('labels plugins by declared name, not folder basename', async () => {
|
|
const result = await scan(DUP_CMD);
|
|
const f = result.findings.find(x => x.scanner === 'PLH' && CMD_RE.test(x.title || ''));
|
|
const sources = f.details.namespaces.map(n => n.source).sort();
|
|
// Folders are one/two; declared names are plugin-one/plugin-two.
|
|
assert.deepEqual(sources, ['plugin:plugin-one', 'plugin:plugin-two']);
|
|
});
|
|
|
|
it('does not emit a high-severity command conflict (legacy behavior removed)', async () => {
|
|
const result = await scan(DUP_CMD);
|
|
const high = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && /command name conflict/i.test(f.title || '') && f.severity === 'high'
|
|
);
|
|
assert.equal(high.length, 0, 'The old high-severity command-conflict finding must be gone');
|
|
});
|
|
|
|
it('does not flag a shared command WITHIN a colliding namespace (namespace-collision covers it)', async () => {
|
|
// alpha + beta both declare name "dup" and both ship a "hello" command.
|
|
const result = await scan(DUP_NAME);
|
|
const amb = result.findings.filter(f => f.scanner === 'PLH' && CMD_RE.test(f.title || ''));
|
|
assert.equal(amb.length, 0, 'Same namespace = one /dup:hello; the namespace collision is the right signal');
|
|
});
|
|
});
|
|
|
|
describe('plugin-folder shadowing (CA-PLH-015)', () => {
|
|
// Title set by the scanner: `plugin.json "<field>" path shadows the default <dir>/ folder`.
|
|
const SHADOW_RE = /shadows the default/i;
|
|
|
|
it('flags a manifest path that shadows the default commands/ folder', async () => {
|
|
const result = await scan(SHADOW);
|
|
const shadows = result.findings.filter(f => f.scanner === 'PLH' && SHADOW_RE.test(f.title || ''));
|
|
assert.equal(shadows.length, 1, `Expected exactly one shadow finding, got ${shadows.length}: ${shadows.map(f => f.title).join(' | ')}`);
|
|
const f = shadows[0];
|
|
assert.equal(f.severity, 'medium', 'Shadowed default folder is medium (dead config)');
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(f.title.includes('commands'), `Title should name the shadowed field: ${f.title}`);
|
|
assert.ok(/plugin\.json$/.test(f.file || ''), `Finding should point at plugin.json: ${f.file}`);
|
|
assert.ok(f.details && f.details.field === 'commands', 'details.field should be the manifest key');
|
|
assert.ok(f.details.ignoredDir === 'commands', `details.ignoredDir should be the default folder: ${f.details.ignoredDir}`);
|
|
});
|
|
|
|
it('does NOT flag a default folder addressed explicitly in the manifest array', async () => {
|
|
// agents: ["./agents/", "./more-agents/"] addresses the default agents/ folder → no warning.
|
|
const result = await scan(SHADOW);
|
|
const agentShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /agents/.test(f.title || '')
|
|
);
|
|
assert.equal(agentShadows.length, 0, 'agents/ is addressed explicitly via ./agents/ → not shadowed');
|
|
});
|
|
|
|
it('does NOT flag skills (adds to default, not replace)', async () => {
|
|
// skills: "./custom-skills/" ADDS to the default skills/ scan; both load → no shadow.
|
|
const result = await scan(SHADOW);
|
|
const skillShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /skills/.test(f.title || '')
|
|
);
|
|
assert.equal(skillShadows.length, 0, 'skills adds-to-default; never a shadow');
|
|
});
|
|
|
|
it('does NOT flag when the default folder is absent', async () => {
|
|
// outputStyles: "./styles/" is declared, but there is no output-styles/ folder → nothing ignored.
|
|
const result = await scan(SHADOW);
|
|
const osShadows = result.findings.filter(f =>
|
|
f.scanner === 'PLH' && SHADOW_RE.test(f.title || '') && /output-styles/.test(f.title || '')
|
|
);
|
|
assert.equal(osShadows.length, 0, 'No default output-styles/ folder exists → no shadow');
|
|
});
|
|
|
|
it('does NOT flag a plugin with no component-path keys', async () => {
|
|
// test-plugin has commands/ and agents/ folders but declares no custom paths → nothing shadowed.
|
|
const result = await scan(TEST_PLUGIN);
|
|
const shadows = result.findings.filter(f => f.scanner === 'PLH' && SHADOW_RE.test(f.title || ''));
|
|
assert.equal(shadows.length, 0, 'No manifest path keys → no shadow findings');
|
|
});
|
|
});
|
|
|
|
describe('skills:-array entry validation (CA-PLH-016)', () => {
|
|
// Title set by the scanner: `plugin.json "skills" entry <problem>: <entry>`.
|
|
const SKILLS_RE = /^plugin\.json "skills" entry/;
|
|
|
|
it('flags one finding per bad entry (file, missing, escape, non-string) and none for a valid dir', async () => {
|
|
// skills: ["./valid-skill/", "./a-file.md", "./missing-dir/", "../escape", 42]
|
|
const result = await scan(SKILLS_ARR);
|
|
const bad = result.findings.filter(f => f.scanner === 'PLH' && SKILLS_RE.test(f.title || ''));
|
|
assert.equal(bad.length, 4, `Expected 4 bad-entry findings, got ${bad.length}: ${bad.map(f => f.title).join(' | ')}`);
|
|
for (const f of bad) {
|
|
assert.equal(f.severity, 'medium', `skills entry problem is medium: ${f.title}`);
|
|
assert.equal(f.category, 'plugin-hygiene');
|
|
assert.ok(/plugin\.json$/.test(f.file || ''), `Finding should point at plugin.json: ${f.file}`);
|
|
assert.ok(f.details && f.details.field === 'skills', 'details.field should be "skills"');
|
|
}
|
|
const problems = bad.map(f => f.details.problem).sort();
|
|
assert.deepEqual(
|
|
problems,
|
|
['escapes-root', 'non-string', 'not-a-directory', 'not-found'],
|
|
`Each problem type should appear once; got ${problems.join(',')}`
|
|
);
|
|
});
|
|
|
|
it('does NOT flag the valid skill directory', async () => {
|
|
const result = await scan(SKILLS_ARR);
|
|
const validFlagged = result.findings.some(f =>
|
|
f.scanner === 'PLH' && SKILLS_RE.test(f.title || '') && /valid-skill/.test(f.title || '')
|
|
);
|
|
assert.equal(validFlagged, false, './valid-skill/ is an existing directory → not flagged');
|
|
});
|
|
|
|
it('does NOT flag a plugin with no skills: key', async () => {
|
|
// test-plugin declares no skills: field.
|
|
const result = await scan(TEST_PLUGIN);
|
|
const skillsFindings = result.findings.filter(f => f.scanner === 'PLH' && SKILLS_RE.test(f.title || ''));
|
|
assert.equal(skillsFindings.length, 0, 'No skills: key → no skills-entry findings');
|
|
});
|
|
});
|
|
|
|
describe('finding format', () => {
|
|
it('findings have standard fields', async () => {
|
|
const result = await scan(BROKEN_PLUGIN);
|
|
assert.ok(result.findings.length > 0);
|
|
const f = result.findings[0];
|
|
assert.ok(f.id.startsWith('CA-PLH-'));
|
|
assert.equal(f.scanner, 'PLH');
|
|
assert.ok(['critical', 'high', 'medium', 'low', 'info'].includes(f.severity));
|
|
assert.ok(f.title);
|
|
assert.ok(f.description);
|
|
});
|
|
});
|
|
|
|
describe('PLH — plugin agent declares fields Claude Code ignores (E)', () => {
|
|
// Hermetic temp fixture: the path-guard blocks committing .claude-plugin/.
|
|
// Plugin subagents silently ignore hooks/mcpServers/permissionMode frontmatter
|
|
// (code.claude.com/docs sub-agents, V15) — dead config; permissionMode = false security.
|
|
let tmpRoot;
|
|
let result;
|
|
|
|
async function writePlugin(root, agentExtraFrontmatter) {
|
|
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
|
await mkdir(join(root, 'agents'), { recursive: true });
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'plugin.json'),
|
|
JSON.stringify({ name: 'demo', description: 'demo plugin for tests', version: '1.0.0' }, null, 2) + '\n',
|
|
'utf8',
|
|
);
|
|
await writeFile(
|
|
join(root, 'agents', 'doer.md'),
|
|
`---\nname: doer\ndescription: does things\n${agentExtraFrontmatter}---\n\n# Doer\n\nBody.\n`,
|
|
'utf8',
|
|
);
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
tmpRoot = await mkdtemp(join(tmpdir(), 'ca-plh-agentdead-'));
|
|
await writePlugin(tmpRoot, 'permissionMode: plan\nhooks: present\nmcpServers: present\n');
|
|
result = await scan(tmpRoot);
|
|
});
|
|
|
|
afterEach(async () => {
|
|
if (tmpRoot) await rm(tmpRoot, { recursive: true, force: true });
|
|
});
|
|
|
|
it('flags permissionMode as medium (false security)', () => {
|
|
const f = result.findings.find(x =>
|
|
x.scanner === 'PLH' && (x.evidence || '').startsWith('permissionMode:'));
|
|
assert.ok(f, `expected a permissionMode finding; got: ${result.findings.map(x => x.title).join(' | ')}`);
|
|
assert.equal(f.severity, 'medium');
|
|
});
|
|
|
|
it('flags hooks and mcpServers as low (dead config)', () => {
|
|
for (const key of ['hooks', 'mcpServers']) {
|
|
const f = result.findings.find(x =>
|
|
x.scanner === 'PLH' && (x.evidence || '').startsWith(`${key}:`));
|
|
assert.ok(f, `expected a ${key} finding`);
|
|
assert.equal(f.severity, 'low', `${key} should be low`);
|
|
}
|
|
});
|
|
|
|
it('does NOT flag a clean agent (name + description only)', async () => {
|
|
const clean = await mkdtemp(join(tmpdir(), 'ca-plh-agentclean-'));
|
|
try {
|
|
await writePlugin(clean, '');
|
|
const r = await scan(clean);
|
|
const dead = r.findings.filter(x =>
|
|
x.scanner === 'PLH' && /which Claude Code ignores/.test(x.title || ''));
|
|
assert.equal(dead.length, 0,
|
|
`clean agent should have no ignored-field findings; got: ${dead.map(x => x.title).join(' | ')}`);
|
|
} finally {
|
|
await rm(clean, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Økt #46 — `plugin-health` dogfood. Four defects, all found by running the
|
|
// command as written and comparing against a fasit registered BEFORE the run.
|
|
//
|
|
// F1 (M-BUG-21, third arm): the arg loop ended in
|
|
// `else if (!args[i].startsWith('-')) targetPath = args[i]` with no
|
|
// unknown-flag branch, so `--output-file /tmp/x.json` was dropped silently and
|
|
// `/tmp/x.json` became the scan target. Worse than in drift-cli: scanning a
|
|
// non-existent path yields "No plugins found" (info) and exit 0 — an
|
|
// apparently GREEN answer, not an error.
|
|
//
|
|
// F3 (stderr-only class): default mode wrote the report to STDERR only, so
|
|
// commands/plugin-health.md ("... 2>/dev/null" + "Read stdout output (JSON)")
|
|
// captured zero bytes. There was no --output-file at all (ux-rules rule 2).
|
|
//
|
|
// F5/F7: per-plugin data (name/commandCount/agentCount) and the grade formula
|
|
// never left scan(); cross-plugin findings were flattened into `findings` with
|
|
// no marker. commands/plugin-health.md mandates a
|
|
// `| Plugin | Grade | Commands | Agents |` table plus a separate Cross-Plugin
|
|
// section — both unbuildable, so the command had to fabricate them.
|
|
//
|
|
// F9: `.claude-plugin/marketplace.json` was reported as "Unknown file". It is
|
|
// the documented location for a marketplace catalog
|
|
// (code.claude.com/docs/en/plugin-marketplaces), and with `"source": "./"` one
|
|
// repo is legitimately both plugin and marketplace.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const PLH_CLI = resolve(__dirname, '../../scanners/plugin-health-scanner.mjs');
|
|
const PLH_RUN = { encoding: 'utf-8', timeout: 20000 };
|
|
|
|
function plhSpawn(args) {
|
|
const res = spawnSync('node', [PLH_CLI, ...args], PLH_RUN);
|
|
return { status: res.status, stdout: String(res.stdout || ''), stderr: String(res.stderr || '') };
|
|
}
|
|
|
|
function plhExpectFailure(args) {
|
|
const { status, stderr } = plhSpawn(args);
|
|
return { status, stderr };
|
|
}
|
|
|
|
describe('plugin-health-scanner argument validation (F1 / M-BUG-21 third arm)', () => {
|
|
it('rejects an unknown flag instead of swallowing its value as the scan target', () => {
|
|
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--bogus', 'some-value', '--json']);
|
|
assert.equal(status, 3, 'unknown flag must fail loudly, not scan "some-value"');
|
|
assert.match(stderr, /unknown option/i);
|
|
assert.match(stderr, /--bogus/);
|
|
});
|
|
|
|
it('rejects --output-file without a value', () => {
|
|
const { status, stderr } = plhExpectFailure([TEST_PLUGIN, '--output-file']);
|
|
assert.equal(status, 3);
|
|
assert.match(stderr, /--output-file/);
|
|
assert.match(stderr, /requires a value/i);
|
|
});
|
|
|
|
it('still accepts a bare path as the scan target', () => {
|
|
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--json']);
|
|
assert.equal(status, 0);
|
|
assert.equal(JSON.parse(stdout).files_scanned, 1);
|
|
});
|
|
});
|
|
|
|
describe('plugin-health-scanner --output-file (F3 / ux-rules rule 2)', () => {
|
|
let dir;
|
|
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-out-')); });
|
|
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
|
|
|
it('writes the payload to the file and keeps default-mode stdout empty', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
const { status, stdout } = plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
|
assert.equal(status, 0);
|
|
assert.equal(stdout, '', 'default mode must not print to stdout (ux-rules rule 1)');
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.equal(payload.scanner, 'PLH');
|
|
assert.equal(payload.files_scanned, 1);
|
|
});
|
|
|
|
it('carries humanizer fields the command renders (F4)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([BROKEN_PLUGIN, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(payload.findings.length > 0, 'broken-plugin must produce findings');
|
|
for (const f of payload.findings) {
|
|
assert.ok(f.userImpactCategory, `finding ${f.id} missing userImpactCategory`);
|
|
assert.ok(f.userActionLanguage, `finding ${f.id} missing userActionLanguage`);
|
|
}
|
|
});
|
|
|
|
it('exposes per-plugin rows with grade, score and component counts (F5)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([TEST_PLUGIN, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(Array.isArray(payload.plugins), 'payload must carry a plugins array');
|
|
assert.equal(payload.plugins.length, 1);
|
|
const p = payload.plugins[0];
|
|
assert.equal(p.name, 'test-plugin');
|
|
assert.ok(typeof p.commandCount === 'number');
|
|
assert.ok(typeof p.agentCount === 'number');
|
|
assert.ok(typeof p.score === 'number');
|
|
assert.match(p.grade, /^[ABCDF]$/);
|
|
});
|
|
|
|
it('separates cross-plugin findings from per-plugin findings (F7)', async () => {
|
|
const out = join(dir, 'plh.json');
|
|
plhSpawn([DUP_NAME, '--output-file', out]);
|
|
const payload = JSON.parse(await readFile(out, 'utf-8'));
|
|
assert.ok(Array.isArray(payload.cross_plugin_findings), 'payload must carry cross_plugin_findings');
|
|
assert.ok(payload.cross_plugin_findings.length > 0, 'duplicate-plugin-name must yield a namespace collision');
|
|
for (const f of payload.cross_plugin_findings) {
|
|
assert.equal(f.crossPlugin, true, 'cross-plugin findings must be marked');
|
|
}
|
|
const perPlugin = payload.findings.filter(f => f.crossPlugin !== true);
|
|
assert.ok(
|
|
perPlugin.length + payload.cross_plugin_findings.length === payload.findings.length,
|
|
'cross_plugin_findings must be a subset of findings, not a parallel universe'
|
|
);
|
|
});
|
|
|
|
it('leaves --raw and --json byte-stable (no new keys on the frozen envelope)', () => {
|
|
const raw = JSON.parse(plhSpawn([TEST_PLUGIN, '--raw']).stdout);
|
|
const json = JSON.parse(plhSpawn([TEST_PLUGIN, '--json']).stdout);
|
|
for (const env of [raw, json]) {
|
|
assert.deepEqual(
|
|
Object.keys(env).sort(),
|
|
['counts', 'duration_ms', 'files_scanned', 'findings', 'scanner', 'status'],
|
|
'frozen v5.0.0 envelope must not gain keys'
|
|
);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('.claude-plugin/marketplace.json is not an unknown file (F9)', () => {
|
|
let dir;
|
|
beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), 'ca-plh-mp-')); });
|
|
afterEach(async () => { await rm(dir, { recursive: true, force: true }); });
|
|
|
|
async function writeMarketplacePlugin(root, extraFiles = {}) {
|
|
await mkdir(join(root, '.claude-plugin'), { recursive: true });
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'plugin.json'),
|
|
JSON.stringify({ name: 'mp-plugin', description: 'd', version: '1.0.0' })
|
|
);
|
|
await writeFile(
|
|
join(root, '.claude-plugin', 'marketplace.json'),
|
|
JSON.stringify({ name: 'cat', owner: { name: 'x' }, plugins: [] })
|
|
);
|
|
await writeFile(join(root, 'CLAUDE.md'), '# mp-plugin\n');
|
|
for (const [name, body] of Object.entries(extraFiles)) {
|
|
await writeFile(join(root, '.claude-plugin', name), body);
|
|
}
|
|
}
|
|
|
|
it('does not flag marketplace.json (documented catalog location)', async () => {
|
|
await writeMarketplacePlugin(dir);
|
|
const result = await scan(dir);
|
|
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
|
assert.equal(unknown.length, 0,
|
|
`marketplace.json is documented; got: ${unknown.map(f => f.file).join(' | ')}`);
|
|
});
|
|
|
|
it('still flags a genuinely unexpected file', async () => {
|
|
await writeMarketplacePlugin(dir, { 'notes.txt': 'scratch' });
|
|
const result = await scan(dir);
|
|
const unknown = result.findings.filter(f => /Unknown file/i.test(f.title || ''));
|
|
assert.equal(unknown.length, 1);
|
|
assert.match(unknown[0].file, /notes\.txt$/);
|
|
});
|
|
});
|