/** * SC-6 — JSON backwards-compatibility test (Wave 4 Step 10). * * For each CLI that has a frozen v5.0.0 JSON snapshot, run the CLI with * --json against the marketplace-medium fixture and compare the output * to the snapshot. Time-varying fields are normalized. * * 5 fixture-deterministic CLIs are checked byte-equal against the v5.0.0 * snapshot: * - scan-orchestrator * - posture * - token-hotspots-cli * - drift-cli (requires saved baseline; falls back to mode-equivalence * if the baseline cannot be created) * - fix-cli * * 3 environment-aware CLIs (plugin-health, manifest, whats-active) read * the active config cascade, so frozen snapshots drift as the * marketplace evolves. They are verified by mode-equivalence * (--json == --raw) instead — the same strategy Wave 3 * cli-humanizer.test.mjs already uses. */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { readFile, access, mkdir } from 'node:fs/promises'; import { homedir } from 'node:os'; const exec = promisify(execFile); const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO = resolve(__dirname, '..'); const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium'); const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0'); const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines'); const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json'); async function runCli(scriptPath, args) { try { const { stdout, stderr } = await exec('node', [scriptPath, ...args], { timeout: 60000, cwd: REPO, maxBuffer: 10 * 1024 * 1024, }); return { stdout: stdout || '', stderr: stderr || '' }; } catch (err) { return { stdout: err.stdout || '', stderr: err.stderr || '' }; } } async function ensureDriftBaseline() { try { await access(DEFAULT_BASELINE); return true; } catch { try { await mkdir(BASELINE_DIR, { recursive: true }); await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']); await access(DEFAULT_BASELINE); return true; } catch { return false; } } } // --------------------------------------------------------------------------- // Normalizers — strip time / path fields that vary between runs. // --------------------------------------------------------------------------- function normalizeScanOrchestrator(env) { const out = JSON.parse(JSON.stringify(env)); if (out.meta) { out.meta.target = ''; out.meta.timestamp = ''; } if (Array.isArray(out.scanners)) { for (const s of out.scanners) { s.duration_ms = 0; } } return out; } function normalizePosture(p) { const out = JSON.parse(JSON.stringify(p)); if (out.scannerEnvelope) { if (out.scannerEnvelope.meta) { out.scannerEnvelope.meta.target = ''; out.scannerEnvelope.meta.timestamp = ''; } if (Array.isArray(out.scannerEnvelope.scanners)) { for (const s of out.scannerEnvelope.scanners) { s.duration_ms = 0; } } } return out; } function normalizeTokenHotspots(p) { const out = JSON.parse(JSON.stringify(p)); out.duration_ms = 0; return out; } function normalizeDrift(p) { // Drift result has no time fields — round-trip through JSON for safety. return JSON.parse(JSON.stringify(p)); } function normalizeFix(p) { // Fix-cli stdout is the planFixes result with no time fields. return JSON.parse(JSON.stringify(p)); } function normalizePluginHealth(p) { const out = JSON.parse(JSON.stringify(p)); out.duration_ms = 0; return out; } function normalizeManifest(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; } return out; } function normalizeWhatsActive(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; if (out.meta.gitRoot) out.meta.gitRoot = ''; if (out.meta.projectKey) out.meta.projectKey = ''; } return out; } // --------------------------------------------------------------------------- // Fixture-deterministic CLIs — strict byte-equal against v5.0.0 snapshot. // --------------------------------------------------------------------------- const DETERMINISTIC_CLIS = [ { name: 'scan-orchestrator', script: 'scanners/scan-orchestrator.mjs', snapshot: 'scan-orchestrator.json', normalize: normalizeScanOrchestrator, }, { name: 'posture', script: 'scanners/posture.mjs', snapshot: 'posture.json', normalize: normalizePosture, }, { name: 'token-hotspots-cli', script: 'scanners/token-hotspots-cli.mjs', snapshot: 'token-hotspots.json', normalize: normalizeTokenHotspots, }, { name: 'fix-cli', script: 'scanners/fix-cli.mjs', snapshot: 'fix-cli.json', normalize: normalizeFix, }, ]; describe('SC-6 JSON backwards-compatibility — fixture-deterministic CLIs', () => { for (const cli of DETERMINISTIC_CLIS) { it(`${cli.name} --json byte-equals v5.0.0 snapshot`, async () => { const script = resolve(REPO, cli.script); const { stdout } = await runCli(script, [FIXTURE, '--json']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8')); assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected)); }); } }); // --------------------------------------------------------------------------- // Drift-cli: separate suite because it requires a baseline precondition. // --------------------------------------------------------------------------- describe('SC-6 JSON backwards-compatibility — drift-cli', () => { it('drift-cli --json byte-equals v5.0.0 snapshot (when baseline available)', async () => { const ok = await ensureDriftBaseline(); if (!ok) { // Skip silently — environment cannot create a baseline. Wave 0 + Wave 3 // tests already exercise drift extensively; this is a defensive fallback. return; } const script = resolve(REPO, 'scanners/drift-cli.mjs'); const { stdout } = await runCli(script, [FIXTURE, '--json']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8')); assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected)); }); }); // --------------------------------------------------------------------------- // Environment-aware CLIs — mode-equivalence (--json == --raw). Frozen v5.0.0 // snapshots drift as marketplace state evolves, so byte-equal would be flaky. // --------------------------------------------------------------------------- const ENV_AWARE_CLIS = [ { name: 'plugin-health-scanner', script: 'scanners/plugin-health-scanner.mjs', normalize: normalizePluginHealth, }, { name: 'manifest', script: 'scanners/manifest.mjs', normalize: normalizeManifest, }, { name: 'whats-active', script: 'scanners/whats-active.mjs', normalize: normalizeWhatsActive, }, ]; describe('SC-6 JSON backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => { for (const cli of ENV_AWARE_CLIS) { it(`${cli.name} --json equals --raw (machine modes are byte-identical)`, async () => { const script = resolve(REPO, cli.script); const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']); const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']); assert.deepStrictEqual( cli.normalize(JSON.parse(jsonOut)), cli.normalize(JSON.parse(rawOut)), ); }); } }); // --------------------------------------------------------------------------- // Cross-cutting: --json must NOT add humanizer fields to any CLI's findings. // --------------------------------------------------------------------------- describe('SC-6 JSON output never carries humanizer fields', () => { const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext']; function* walkFindings(payload) { if (!payload || typeof payload !== 'object') return; if (Array.isArray(payload.findings)) { for (const f of payload.findings) yield f; } if (Array.isArray(payload.scanners)) { for (const s of payload.scanners) { if (Array.isArray(s.findings)) { for (const f of s.findings) yield f; } } } if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope); } for (const cli of DETERMINISTIC_CLIS) { it(`${cli.name} --json findings carry no humanizer fields`, async () => { const script = resolve(REPO, cli.script); const { stdout } = await runCli(script, [FIXTURE, '--json']); const actual = JSON.parse(stdout); for (const f of walkFindings(actual)) { for (const field of EXPECTED_HUMANIZER_FIELDS) { assert.equal( f[field], undefined, `${cli.name} ${f.id ?? ''}: --json must not add ${field}`, ); } } }); } });