Dogfooding `/config-audit plugin-health` against a fasit registered before the
run: 11 of 12 predictions confirmed, 1 refuted with evidence, 0 deviations.
The command's default path could not produce the report it documents.
M-BUG-21 (third arm): the argument loop ended in
`else if (!args[i].startsWith('-')) targetPath = args[i]` with no unknown-flag
branch, so `--output-file /tmp/x.json` was dropped and its value became the scan
target. Worse than in drift-cli: a non-existent path discovers no plugins, so the
scanner answered "No plugins found" (info) with exit 0 — a reassuring answer, not
an error. Unknown options and a value-less `--output-file` now exit 3.
M-BUG-33: the scanner had no `--output-file` and its default-mode report goes to
stderr, which `commands/plugin-health.md` discards with `2>/dev/null` before
telling the agent to read stdout. Zero bytes captured.
M-BUG-34: per-plugin rows and the grade formula never left `scan()` — the only
grade code, `formatPluginHealthReport`, had no caller — and cross-plugin findings
were flattened behind a `category` they share with per-plugin findings. The
mandated table and Cross-Plugin section were unbuildable, so the command had to
fabricate them. `scanDetailed()` now returns them; `scan()`'s frozen v5.0.0
envelope is unchanged by construction.
M-BUG-35: `.claude-plugin/marketplace.json` was flagged as an unknown file. It is
the documented catalog location, and `"source": "./"` makes the repo root its own
plugin, so one `.claude-plugin/` legitimately holds both.
Also: `commands/posture.md` ran both optional scanners in default mode under
`2>/dev/null` and read stdout — the same class as feature-gap.md:133 in the fix
chunk. A CLI-side flag fix does not close its callers.
Tests 1420 -> 1432, red first. Frozen v5.0.0 snapshots untouched.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XhhZ8zg1amR7YCAPqiZWdt
160 lines
6.3 KiB
JavaScript
160 lines
6.3 KiB
JavaScript
/**
|
||
* Wave 5 Step 14 — Group B command-template shape tests.
|
||
*
|
||
* Verifies that the 6 audit/analysis command templates in Group B have the
|
||
* correct structural shape after the humanizer integration:
|
||
*
|
||
* - All 6 files: contain a Bash invocation block, reference the Read tool,
|
||
* and contain the `--raw` flag (or the literal `"$ARGUMENTS"` string).
|
||
*
|
||
* - Findings-rendering files (drift.md, plugin-health.md, config-audit.md,
|
||
* discover.md, analyze.md): reference at least one of
|
||
* `userImpactCategory|userActionLanguage|relevanceContext`, and do NOT
|
||
* contain hardcoded grade-prose tables of the form `[ABCDF]\s+grade\s+is`.
|
||
*
|
||
* - status.md: phase-label table is present, the machine field name
|
||
* `current_phase` is preserved (machine contract), and at least one
|
||
* humanized phase label appears ("Looking at your config files",
|
||
* "Working out what to recommend", "Putting together your action plan",
|
||
* "Making the changes", "Double-checking everything worked").
|
||
*
|
||
* - Anchor must-contains from plan line 575–579:
|
||
* - config-audit.md: contains userImpactCategory|userActionLanguage
|
||
* - drift.md: contains --raw OR humanized
|
||
*/
|
||
|
||
import { test } from 'node:test';
|
||
import { strict as assert } from 'node:assert';
|
||
import { readFile } from 'node:fs/promises';
|
||
import { resolve, dirname } from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||
const COMMANDS_DIR = resolve(__dirname, '..', '..', 'commands');
|
||
|
||
const GROUP_B_FILES = [
|
||
'drift.md',
|
||
'plugin-health.md',
|
||
'config-audit.md',
|
||
'discover.md',
|
||
'analyze.md',
|
||
'status.md',
|
||
];
|
||
|
||
const FINDINGS_RENDERING_FILES = [
|
||
'drift.md',
|
||
'plugin-health.md',
|
||
'config-audit.md',
|
||
'discover.md',
|
||
'analyze.md',
|
||
];
|
||
|
||
const HUMANIZED_FIELD_REGEX = /userImpactCategory|userActionLanguage|relevanceContext/;
|
||
const RAW_OR_ARGUMENTS_REGEX = /--raw|"\$ARGUMENTS"/;
|
||
const HARDCODED_GRADE_PROSE_REGEX = /[ABCDF]\s+grade\s+is/;
|
||
const BASH_BLOCK_REGEX = /```bash\b/;
|
||
const READ_TOOL_REGEX = /\bRead\s+tool\b|allowed-tools:.*\bRead\b/;
|
||
|
||
const HUMANIZED_PHASE_LABELS = [
|
||
'Looking at your config files',
|
||
'Working out what to recommend',
|
||
'Asking what you',
|
||
'Putting together your action plan',
|
||
'Making the changes',
|
||
'Double-checking everything worked',
|
||
];
|
||
|
||
async function readCommand(name) {
|
||
return await readFile(resolve(COMMANDS_DIR, name), 'utf-8');
|
||
}
|
||
|
||
test('Group B: every file contains a Bash invocation block', async () => {
|
||
for (const name of GROUP_B_FILES) {
|
||
const content = await readCommand(name);
|
||
assert.match(content, BASH_BLOCK_REGEX, `${name} missing bash block`);
|
||
}
|
||
});
|
||
|
||
test('Group B: every file references the Read tool', async () => {
|
||
for (const name of GROUP_B_FILES) {
|
||
const content = await readCommand(name);
|
||
assert.match(content, READ_TOOL_REGEX, `${name} missing Read tool reference`);
|
||
}
|
||
});
|
||
|
||
test('Group B: every file contains --raw or "$ARGUMENTS" (pass-through plumbing)', async () => {
|
||
for (const name of GROUP_B_FILES) {
|
||
const content = await readCommand(name);
|
||
assert.match(content, RAW_OR_ARGUMENTS_REGEX, `${name} missing --raw / $ARGUMENTS plumbing`);
|
||
}
|
||
});
|
||
|
||
test('Group B findings-renderers: reference at least one humanized field', async () => {
|
||
for (const name of FINDINGS_RENDERING_FILES) {
|
||
const content = await readCommand(name);
|
||
assert.match(
|
||
content,
|
||
HUMANIZED_FIELD_REGEX,
|
||
`${name} must reference userImpactCategory, userActionLanguage, or relevanceContext`,
|
||
);
|
||
}
|
||
});
|
||
|
||
test('Group B findings-renderers: no hardcoded grade-prose tables', async () => {
|
||
for (const name of FINDINGS_RENDERING_FILES) {
|
||
const content = await readCommand(name);
|
||
assert.doesNotMatch(
|
||
content,
|
||
HARDCODED_GRADE_PROSE_REGEX,
|
||
`${name} contains a hardcoded "[grade] grade is..." prose table — humanizer owns grade vocabulary now`,
|
||
);
|
||
}
|
||
});
|
||
|
||
test('Group B anchor: config-audit.md references userImpactCategory|userActionLanguage', async () => {
|
||
const content = await readCommand('config-audit.md');
|
||
assert.match(content, /userImpactCategory|userActionLanguage/);
|
||
});
|
||
|
||
test('Group B anchor: drift.md references --raw or humanized', async () => {
|
||
const content = await readCommand('drift.md');
|
||
assert.match(content, /--raw|humanized/);
|
||
});
|
||
|
||
test('status.md: preserves current_phase machine field and adds humanized phase labels', async () => {
|
||
const content = await readCommand('status.md');
|
||
// Machine contract preserved
|
||
assert.match(content, /\bcurrent_phase\b/, 'status.md must keep current_phase as machine field');
|
||
// At least 3 of the 6 humanized phase labels appear
|
||
const present = HUMANIZED_PHASE_LABELS.filter(label => content.includes(label));
|
||
assert.ok(
|
||
present.length >= 3,
|
||
`status.md must include at least 3 humanized phase labels; found ${present.length}: ${present.join(', ')}`,
|
||
);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Økt #46 — ux-rules rule 2 for the plugin-health scanner.
|
||
//
|
||
// plugin-health.md passed the humanized-field assertion above while the data it
|
||
// names was unreachable: the scanner had no --output-file, and its default-mode
|
||
// report went to stderr, which the command discards with `2>/dev/null`. A .md
|
||
// contract test that only greps for prose cannot catch that — these assert the
|
||
// plumbing that makes the prose true.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test('plugin-health.md invokes the scanner with --output-file (ux-rules rule 2)', async () => {
|
||
const content = await readCommand('plugin-health.md');
|
||
const call = content.split('\n').find(l => l.includes('plugin-health-scanner.mjs'));
|
||
assert.ok(call, 'plugin-health.md must invoke plugin-health-scanner.mjs');
|
||
assert.match(call, /--output-file/, 'scanner call must write to a file, not stdout/stderr');
|
||
});
|
||
|
||
test('posture.md invokes the plugin-health and drift scanners with --output-file', async () => {
|
||
const content = await readCommand('posture.md');
|
||
for (const scanner of ['plugin-health-scanner.mjs', 'drift-cli.mjs']) {
|
||
const call = content.split('\n').find(l => l.includes(`scanners/${scanner}`));
|
||
assert.ok(call, `posture.md must invoke ${scanner}`);
|
||
assert.match(call, /--output-file/, `${scanner} call in posture.md discards its output`);
|
||
}
|
||
});
|