/** * SC-7 — --raw backwards-compatibility test (Wave 4 Step 11). * * Mirror of tests/json-backcompat.test.mjs but exercises the --raw flag, * the explicit "v5.0.0 verbatim" escape hatch documented in Wave 3. * * 4 fixture-deterministic CLIs (scan-orchestrator, posture, * token-hotspots-cli, fix-cli) plus drift-cli are checked byte-equal * against tests/snapshots/v5.0.0/.json (with time fields * normalized). * * 3 environment-aware CLIs (plugin-health, manifest, whats-active) are * checked for mode-equivalence (--raw equals --json), matching the * established Wave 3 strategy. * * Posture additionally asserts its --raw stderr scorecard matches the * verbatim v5.0.0 stderr capture in tests/snapshots/v5.0.0-stderr/ * posture.txt, with (Xms) duration markers normalized to (0ms). */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { readFile, access, mkdir } from 'node:fs/promises'; import { homedir } from 'node:os'; const exec = promisify(execFile); const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO = resolve(__dirname, '..'); const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium'); const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0'); const STDERR_SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0-stderr'); const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines'); const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json'); async function runCli(scriptPath, args) { try { const { stdout, stderr } = await exec('node', [scriptPath, ...args], { timeout: 60000, cwd: REPO, maxBuffer: 10 * 1024 * 1024, }); return { stdout: stdout || '', stderr: stderr || '' }; } catch (err) { return { stdout: err.stdout || '', stderr: err.stderr || '' }; } } async function ensureDriftBaseline() { try { await access(DEFAULT_BASELINE); return true; } catch { try { await mkdir(BASELINE_DIR, { recursive: true }); await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']); await access(DEFAULT_BASELINE); return true; } catch { return false; } } } // --------------------------------------------------------------------------- // Normalizers — same as json-backcompat to keep the contracts aligned. // `claudeMdEstimatedTokens` is stripped because walkClaudeMdCascade walks // upward from the fixture into this plugin's own CLAUDE.md; any docs edit // here ripples into it even when scanner internals are unchanged. // --------------------------------------------------------------------------- function stripAncestorDerived(envOrEnvelope) { if (Array.isArray(envOrEnvelope?.scanners)) { for (const s of envOrEnvelope.scanners) { if (s?.activeConfig && 'claudeMdEstimatedTokens' in s.activeConfig) { s.activeConfig.claudeMdEstimatedTokens = ''; } } } } function normalizeScanOrchestrator(env) { const out = JSON.parse(JSON.stringify(env)); if (out.meta) { out.meta.target = ''; out.meta.timestamp = ''; } if (Array.isArray(out.scanners)) { for (const s of out.scanners) { s.duration_ms = 0; } } stripAncestorDerived(out); return out; } function normalizePosture(p) { const out = JSON.parse(JSON.stringify(p)); if (out.scannerEnvelope) { if (out.scannerEnvelope.meta) { out.scannerEnvelope.meta.target = ''; out.scannerEnvelope.meta.timestamp = ''; } if (Array.isArray(out.scannerEnvelope.scanners)) { for (const s of out.scannerEnvelope.scanners) { s.duration_ms = 0; } } stripAncestorDerived(out.scannerEnvelope); } return out; } function normalizeTokenHotspots(p) { const out = JSON.parse(JSON.stringify(p)); out.duration_ms = 0; return out; } function normalizeDrift(p) { return JSON.parse(JSON.stringify(p)); } function normalizeFix(p) { return JSON.parse(JSON.stringify(p)); } function normalizePluginHealth(p) { const out = JSON.parse(JSON.stringify(p)); out.duration_ms = 0; return out; } function normalizeManifest(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; } return out; } function normalizeWhatsActive(o) { const out = JSON.parse(JSON.stringify(o)); if (out.meta) { out.meta.repoPath = ''; out.meta.generatedAt = ''; out.meta.durationMs = 0; if (out.meta.gitRoot) out.meta.gitRoot = ''; if (out.meta.projectKey) out.meta.projectKey = ''; } return out; } /** Normalize Xms duration markers in stderr prose for verbatim comparison. */ function normalizeStderrDurations(s) { return s.replace(/\(\d+ms\)/g, '(0ms)'); } // --------------------------------------------------------------------------- // Fixture-deterministic CLIs — strict byte-equal --raw vs v5.0.0 snapshot. // --------------------------------------------------------------------------- const DETERMINISTIC_CLIS = [ { name: 'scan-orchestrator', script: 'scanners/scan-orchestrator.mjs', snapshot: 'scan-orchestrator.json', normalize: normalizeScanOrchestrator, }, { name: 'posture', script: 'scanners/posture.mjs', snapshot: 'posture.json', normalize: normalizePosture, }, { name: 'token-hotspots-cli', script: 'scanners/token-hotspots-cli.mjs', snapshot: 'token-hotspots.json', normalize: normalizeTokenHotspots, }, { name: 'fix-cli', script: 'scanners/fix-cli.mjs', snapshot: 'fix-cli.json', normalize: normalizeFix, }, ]; describe('SC-7 --raw backwards-compatibility — fixture-deterministic CLIs', () => { for (const cli of DETERMINISTIC_CLIS) { it(`${cli.name} --raw byte-equals v5.0.0 snapshot`, async () => { const script = resolve(REPO, cli.script); const { stdout } = await runCli(script, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8')); assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected)); }); } }); // --------------------------------------------------------------------------- // Drift-cli with baseline precondition. // --------------------------------------------------------------------------- describe('SC-7 --raw backwards-compatibility — drift-cli', () => { it('drift-cli --raw byte-equals v5.0.0 snapshot (when baseline available)', async () => { const ok = await ensureDriftBaseline(); if (!ok) return; const script = resolve(REPO, 'scanners/drift-cli.mjs'); const { stdout } = await runCli(script, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8')); assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected)); }); }); // --------------------------------------------------------------------------- // Environment-aware CLIs — mode-equivalence. // --------------------------------------------------------------------------- const ENV_AWARE_CLIS = [ { name: 'plugin-health-scanner', script: 'scanners/plugin-health-scanner.mjs', normalize: normalizePluginHealth, }, { name: 'manifest', script: 'scanners/manifest.mjs', normalize: normalizeManifest, }, { name: 'whats-active', script: 'scanners/whats-active.mjs', normalize: normalizeWhatsActive, }, ]; describe('SC-7 --raw backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => { for (const cli of ENV_AWARE_CLIS) { it(`${cli.name} --raw equals --json (machine modes are byte-identical)`, async () => { const script = resolve(REPO, cli.script); const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']); const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']); assert.deepStrictEqual( cli.normalize(JSON.parse(rawOut)), cli.normalize(JSON.parse(jsonOut)), ); }); } }); // --------------------------------------------------------------------------- // Posture stderr scorecard — verbatim v5.0.0 in --raw mode. // --------------------------------------------------------------------------- describe('SC-7 --raw posture stderr scorecard verbatim', () => { it('posture --raw stderr matches tests/snapshots/v5.0.0-stderr/posture.txt (modulo Xms)', async () => { const script = resolve(REPO, 'scanners/posture.mjs'); const { stderr } = await runCli(script, [FIXTURE, '--raw']); const expected = await readFile(resolve(STDERR_SNAPSHOT_DIR, 'posture.txt'), 'utf8'); assert.equal( normalizeStderrDurations(stderr.trim()), normalizeStderrDurations(expected.trim()), 'posture --raw stderr must reproduce the v5.0.0 scorecard verbatim (apart from durations)', ); }); }); // --------------------------------------------------------------------------- // Cross-cutting: --raw must NOT add humanizer fields anywhere. // --------------------------------------------------------------------------- describe('SC-7 --raw output never carries humanizer fields', () => { const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext']; function* walkFindings(payload) { if (!payload || typeof payload !== 'object') return; if (Array.isArray(payload.findings)) { for (const f of payload.findings) yield f; } if (Array.isArray(payload.scanners)) { for (const s of payload.scanners) { if (Array.isArray(s.findings)) { for (const f of s.findings) yield f; } } } if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope); } for (const cli of DETERMINISTIC_CLIS) { it(`${cli.name} --raw findings carry no humanizer fields`, async () => { const script = resolve(REPO, cli.script); const { stdout } = await runCli(script, [FIXTURE, '--raw']); const actual = JSON.parse(stdout); for (const f of walkFindings(actual)) { for (const field of EXPECTED_HUMANIZER_FIELDS) { assert.equal( f[field], undefined, `${cli.name} ${f.id ?? ''}: --raw must not add ${field}`, ); } } }); } });