import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { readFile, writeFile, unlink, mkdir, access } from 'node:fs/promises'; import { homedir } from 'node:os'; const exec = promisify(execFile); const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO = resolve(__dirname, '../..'); const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium'); const BROKEN_PLUGIN = resolve(REPO, 'tests/fixtures/broken-plugin'); const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines'); const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json'); /** * Run a CLI subprocess and return stdout/stderr regardless of exit code * (some CLIs exit non-zero on findings — we still need their output). */ async function runCli(cliPath, args, env = {}) { try { const { stdout, stderr } = await exec('node', [cliPath, ...args], { timeout: 60000, cwd: REPO, env: { ...process.env, ...env }, maxBuffer: 10 * 1024 * 1024, }); return { stdout: stdout || '', stderr: stderr || '', code: 0 }; } catch (err) { return { stdout: err.stdout || '', stderr: err.stderr || '', code: err.code ?? 1, }; } } /** Strip time-varying duration_ms / Xms occurrences for snapshot comparison. */ function normalizeTokenHotspotsPayload(p) { const out = JSON.parse(JSON.stringify(p)); out.duration_ms = 0; return out; } function normalizeManifestOutput(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; } return out; } function normalizeWhatsActiveOutput(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; if (out.meta.gitRoot) out.meta.gitRoot = ''; if (out.meta.projectKey) out.meta.projectKey = ''; } return out; } function normalizePluginHealthOutput(o) { const out = JSON.parse(JSON.stringify(o)); out.duration_ms = 0; return out; } function normalizeDriftOutput(o) { // Drift result has no time fields; just round-trip through JSON. return JSON.parse(JSON.stringify(o)); } // ============================================================================ // token-hotspots-cli // ============================================================================ describe('token-hotspots-cli humanizer (Step 7)', () => { const CLI = resolve(REPO, 'scanners/token-hotspots-cli.mjs'); const SNAPSHOT = resolve(REPO, 'tests/snapshots/v5.0.0/token-hotspots.json'); it('--json: payload.findings byte-equal v5.0.0 snapshot', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual( normalizeTokenHotspotsPayload(actual), normalizeTokenHotspotsPayload(expected), ); }); it('--raw: payload.findings byte-equal v5.0.0 snapshot', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual( normalizeTokenHotspotsPayload(actual), normalizeTokenHotspotsPayload(expected), ); }); it('default: payload.findings include humanizer fields when findings exist', async () => { const { stdout } = await runCli(CLI, [FIXTURE]); const actual = JSON.parse(stdout); if (actual.findings.length === 0) return; for (const f of actual.findings) { assert.equal(typeof f.userImpactCategory, 'string', `${f.id}: default mode must add userImpactCategory`); assert.equal(typeof f.userActionLanguage, 'string', `${f.id}: default mode must add userActionLanguage`); assert.equal(typeof f.relevanceContext, 'string', `${f.id}: default mode must add relevanceContext`); } }); it('--json: payload.findings do NOT carry humanizer fields', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const actual = JSON.parse(stdout); for (const f of actual.findings) { assert.equal(f.userImpactCategory, undefined, `${f.id}: --json must not add userImpactCategory`); } }); }); // ============================================================================ // plugin-health-scanner // // NOTE: plugin-health scans the plugin root (not the fixture path), so its // findings reflect the current marketplace state — snapshot frozen at Wave 0 // no longer matches as new plugins are added. We verify mode-equivalence // (--json == --raw) instead. // ============================================================================ describe('plugin-health-scanner humanizer (Step 7)', () => { const CLI = resolve(REPO, 'scanners/plugin-health-scanner.mjs'); it('--json and --raw produce byte-identical stdout (both bypass humanizer)', async () => { const { stdout: jsonOut } = await runCli(CLI, [FIXTURE, '--json']); const { stdout: rawOut } = await runCli(CLI, [FIXTURE, '--raw']); assert.deepStrictEqual( normalizePluginHealthOutput(JSON.parse(jsonOut)), normalizePluginHealthOutput(JSON.parse(rawOut)), ); }); it('--json output preserves v5.0.0 finding shape (no humanizer fields)', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const actual = JSON.parse(stdout); for (const f of actual.findings || []) { assert.equal(f.userImpactCategory, undefined, `${f.id}: --json must not add userImpactCategory`); } }); it('default mode renders to stderr (humanized when findings exist)', async () => { const { stderr: defaultStderr } = await runCli(CLI, [BROKEN_PLUGIN]); const { stderr: rawStderr } = await runCli(CLI, [BROKEN_PLUGIN, '--raw']); // --raw suppresses prose stderr (machine mode); default emits humanized prose. // Just verify both run without crash; humanization assertion is best-effort // because broken-plugin may produce no PLH-translated findings. assert.ok(typeof defaultStderr === 'string'); assert.ok(typeof rawStderr === 'string'); }); }); // ============================================================================ // drift-cli // ============================================================================ describe('drift-cli humanizer (Step 7)', () => { const CLI = resolve(REPO, 'scanners/drift-cli.mjs'); const SNAPSHOT = resolve(REPO, 'tests/snapshots/v5.0.0/drift.json'); async function ensureBaseline() { try { await access(DEFAULT_BASELINE); return true; } catch { // Try to save one try { await mkdir(BASELINE_DIR, { recursive: true }); await runCli(CLI, [FIXTURE, '--save']); await access(DEFAULT_BASELINE); return true; } catch { return false; } } } it('--json: diff byte-equal v5.0.0 snapshot', async () => { const ok = await ensureBaseline(); if (!ok) { // SKIP — baseline cannot be created in this environment. return; } const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual( normalizeDriftOutput(actual), normalizeDriftOutput(expected), ); }); it('--raw: diff byte-equal v5.0.0 snapshot', async () => { const ok = await ensureBaseline(); if (!ok) return; const { stdout } = await runCli(CLI, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual( normalizeDriftOutput(actual), normalizeDriftOutput(expected), ); }); it('default: stderr report differs from --raw stderr when findings exist', async () => { const ok = await ensureBaseline(); if (!ok) return; const { stderr: defaultStderr } = await runCli(CLI, [FIXTURE]); const { stderr: rawStderr } = await runCli(CLI, [FIXTURE, '--raw']); // If there are findings whose titles get humanized, default stderr differs from raw. // If no humanizable titles in this fixture, both can match — just verify no crash. assert.ok(typeof defaultStderr === 'string'); assert.ok(typeof rawStderr === 'string'); }); }); // ============================================================================ // manifest // // NOTE: manifest scans the active config cascade (env-dependent), so the // frozen v5.0.0 snapshot drifts as the marketplace changes. We verify // --json == --raw == default (no-op for inventory) instead. // ============================================================================ describe('manifest humanizer (Step 7) — no-op for --raw', () => { const CLI = resolve(REPO, 'scanners/manifest.mjs'); it('--json and --raw produce byte-identical output', async () => { const { stdout: jsonOut } = await runCli(CLI, [FIXTURE, '--json']); const { stdout: rawOut } = await runCli(CLI, [FIXTURE, '--raw']); assert.deepStrictEqual( normalizeManifestOutput(JSON.parse(jsonOut)), normalizeManifestOutput(JSON.parse(rawOut)), ); }); it('default and --raw produce structurally identical output (inventory CLI)', async () => { const { stdout: defaultOut } = await runCli(CLI, [FIXTURE]); const { stdout: rawOut } = await runCli(CLI, [FIXTURE, '--raw']); assert.deepStrictEqual( normalizeManifestOutput(JSON.parse(defaultOut)), normalizeManifestOutput(JSON.parse(rawOut)), ); }); it('preserves v5.0.0 envelope shape', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const out = JSON.parse(stdout); assert.ok(out.meta); assert.ok(Array.isArray(out.sources)); assert.equal(typeof out.total, 'number'); }); }); // ============================================================================ // whats-active // // NOTE: whats-active scans the active config (env-dependent). Frozen snapshot // drifts; we verify mode-equivalence instead. // ============================================================================ describe('whats-active humanizer (Step 7) — no-op for --raw', () => { const CLI = resolve(REPO, 'scanners/whats-active.mjs'); it('--json and --raw produce byte-identical output', async () => { const { stdout: jsonOut } = await runCli(CLI, [FIXTURE, '--json']); const { stdout: rawOut } = await runCli(CLI, [FIXTURE, '--raw']); assert.deepStrictEqual( normalizeWhatsActiveOutput(JSON.parse(jsonOut)), normalizeWhatsActiveOutput(JSON.parse(rawOut)), ); }); it('default and --raw produce structurally identical output (inventory CLI)', async () => { const { stdout: defaultOut } = await runCli(CLI, [FIXTURE]); const { stdout: rawOut } = await runCli(CLI, [FIXTURE, '--raw']); assert.deepStrictEqual( normalizeWhatsActiveOutput(JSON.parse(defaultOut)), normalizeWhatsActiveOutput(JSON.parse(rawOut)), ); }); it('preserves v5.0.0 envelope shape', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const out = JSON.parse(stdout); assert.ok(out.meta); assert.ok(out.claudeMd); assert.ok(Array.isArray(out.plugins)); assert.ok(Array.isArray(out.skills)); }); }); // ============================================================================ // fix-cli // ============================================================================ describe('fix-cli humanizer (Step 7)', () => { const CLI = resolve(REPO, 'scanners/fix-cli.mjs'); const SNAPSHOT = resolve(REPO, 'tests/snapshots/v5.0.0/fix-cli.json'); it('--json: stdout JSON byte-equal v5.0.0 snapshot', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--json']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual(actual, expected); }); it('--raw: stdout JSON byte-equal v5.0.0 snapshot', async () => { const { stdout } = await runCli(CLI, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(SNAPSHOT, 'utf-8')); assert.deepStrictEqual(actual, expected); }); it('default mode stderr differs from --raw stderr when findings have humanizer translations', async () => { const { stderr: defaultStderr } = await runCli(CLI, [FIXTURE]); const { stderr: rawStderr } = await runCli(CLI, [FIXTURE, '--raw']); // 20 manual findings in fixture; many have GAP translations → stderr differs. assert.notEqual(defaultStderr, rawStderr, 'fix-cli default stderr must differ from --raw stderr when humanizer translates titles'); }); });