From 20b867adc1450a33f811a2c660c20059d5bc8ed6 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 1 May 2026 18:20:04 +0200 Subject: [PATCH] test(humanizer): --raw backwards-compatibility test (SC-7) [skip-docs] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 11 of v5.1.0 humanizer Wave 4. Adds tests/raw-backcompat.test.mjs mirroring the SC-6 contract for the --raw flag — the explicit "v5.0.0 verbatim" escape hatch. - 4 fixture-deterministic CLIs (scan-orchestrator, posture, token-hotspots-cli, fix-cli) get strict byte-equal against tests/snapshots/v5.0.0/.json with time fields normalized. - drift-cli is checked under the same contract guarded by ensureDriftBaseline. - 3 environment-aware CLIs (plugin-health, manifest, whats-active) are checked for mode-equivalence (--raw equals --json). - Posture additionally asserts its --raw stderr scorecard reproduces tests/snapshots/v5.0.0-stderr/posture.txt verbatim, modulo (Xms) duration markers normalized to (0ms). - Cross-cutting suite asserts --raw findings carry no humanizer fields on any CLI. Tests: 751 to 764 (+13 SC-7 cases). Full suite passes. Co-Authored-By: Claude Opus 4.7 --- .../tests/raw-backcompat.test.mjs | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 plugins/config-audit/tests/raw-backcompat.test.mjs diff --git a/plugins/config-audit/tests/raw-backcompat.test.mjs b/plugins/config-audit/tests/raw-backcompat.test.mjs new file mode 100644 index 0000000..f066431 --- /dev/null +++ b/plugins/config-audit/tests/raw-backcompat.test.mjs @@ -0,0 +1,298 @@ +/** + * SC-7 — --raw backwards-compatibility test (Wave 4 Step 11). + * + * Mirror of tests/json-backcompat.test.mjs but exercises the --raw flag, + * the explicit "v5.0.0 verbatim" escape hatch documented in Wave 3. + * + * 4 fixture-deterministic CLIs (scan-orchestrator, posture, + * token-hotspots-cli, fix-cli) plus drift-cli are checked byte-equal + * against tests/snapshots/v5.0.0/.json (with time fields + * normalized). + * + * 3 environment-aware CLIs (plugin-health, manifest, whats-active) are + * checked for mode-equivalence (--raw equals --json), matching the + * established Wave 3 strategy. + * + * Posture additionally asserts its --raw stderr scorecard matches the + * verbatim v5.0.0 stderr capture in tests/snapshots/v5.0.0-stderr/ + * posture.txt, with (Xms) duration markers normalized to (0ms). + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { readFile, access, mkdir } from 'node:fs/promises'; +import { homedir } from 'node:os'; + +const exec = promisify(execFile); +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO = resolve(__dirname, '..'); +const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium'); +const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0'); +const STDERR_SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0-stderr'); +const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines'); +const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json'); + +async function runCli(scriptPath, args) { + try { + const { stdout, stderr } = await exec('node', [scriptPath, ...args], { + timeout: 60000, + cwd: REPO, + maxBuffer: 10 * 1024 * 1024, + }); + return { stdout: stdout || '', stderr: stderr || '' }; + } catch (err) { + return { stdout: err.stdout || '', stderr: err.stderr || '' }; + } +} + +async function ensureDriftBaseline() { + try { + await access(DEFAULT_BASELINE); + return true; + } catch { + try { + await mkdir(BASELINE_DIR, { recursive: true }); + await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']); + await access(DEFAULT_BASELINE); + return true; + } catch { + return false; + } + } +} + +// --------------------------------------------------------------------------- +// Normalizers — same as json-backcompat to keep the contracts aligned. +// --------------------------------------------------------------------------- + +function normalizeScanOrchestrator(env) { + const out = JSON.parse(JSON.stringify(env)); + if (out.meta) { + out.meta.target = ''; + out.meta.timestamp = ''; + } + if (Array.isArray(out.scanners)) { + for (const s of out.scanners) { + s.duration_ms = 0; + } + } + return out; +} + +function normalizePosture(p) { + const out = JSON.parse(JSON.stringify(p)); + if (out.scannerEnvelope) { + if (out.scannerEnvelope.meta) { + out.scannerEnvelope.meta.target = ''; + out.scannerEnvelope.meta.timestamp = ''; + } + if (Array.isArray(out.scannerEnvelope.scanners)) { + for (const s of out.scannerEnvelope.scanners) { + s.duration_ms = 0; + } + } + } + return out; +} + +function normalizeTokenHotspots(p) { + const out = JSON.parse(JSON.stringify(p)); + out.duration_ms = 0; + return out; +} + +function normalizeDrift(p) { + return JSON.parse(JSON.stringify(p)); +} + +function normalizeFix(p) { + return JSON.parse(JSON.stringify(p)); +} + +function normalizePluginHealth(p) { + const out = JSON.parse(JSON.stringify(p)); + out.duration_ms = 0; + return out; +} + +function normalizeManifest(o) { + const out = JSON.parse(JSON.stringify(o)); + if (out.meta) { + out.meta.repoPath = ''; + out.meta.generatedAt = ''; + out.meta.durationMs = 0; + } + return out; +} + +function normalizeWhatsActive(o) { + const out = JSON.parse(JSON.stringify(o)); + if (out.meta) { + out.meta.repoPath = ''; + out.meta.generatedAt = ''; + out.meta.durationMs = 0; + if (out.meta.gitRoot) out.meta.gitRoot = ''; + if (out.meta.projectKey) out.meta.projectKey = ''; + } + return out; +} + +/** Normalize Xms duration markers in stderr prose for verbatim comparison. */ +function normalizeStderrDurations(s) { + return s.replace(/\(\d+ms\)/g, '(0ms)'); +} + +// --------------------------------------------------------------------------- +// Fixture-deterministic CLIs — strict byte-equal --raw vs v5.0.0 snapshot. +// --------------------------------------------------------------------------- + +const DETERMINISTIC_CLIS = [ + { + name: 'scan-orchestrator', + script: 'scanners/scan-orchestrator.mjs', + snapshot: 'scan-orchestrator.json', + normalize: normalizeScanOrchestrator, + }, + { + name: 'posture', + script: 'scanners/posture.mjs', + snapshot: 'posture.json', + normalize: normalizePosture, + }, + { + name: 'token-hotspots-cli', + script: 'scanners/token-hotspots-cli.mjs', + snapshot: 'token-hotspots.json', + normalize: normalizeTokenHotspots, + }, + { + name: 'fix-cli', + script: 'scanners/fix-cli.mjs', + snapshot: 'fix-cli.json', + normalize: normalizeFix, + }, +]; + +describe('SC-7 --raw backwards-compatibility — fixture-deterministic CLIs', () => { + for (const cli of DETERMINISTIC_CLIS) { + it(`${cli.name} --raw byte-equals v5.0.0 snapshot`, async () => { + const script = resolve(REPO, cli.script); + const { stdout } = await runCli(script, [FIXTURE, '--raw']); + const actual = JSON.parse(stdout); + const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8')); + assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected)); + }); + } +}); + +// --------------------------------------------------------------------------- +// Drift-cli with baseline precondition. +// --------------------------------------------------------------------------- + +describe('SC-7 --raw backwards-compatibility — drift-cli', () => { + it('drift-cli --raw byte-equals v5.0.0 snapshot (when baseline available)', async () => { + const ok = await ensureDriftBaseline(); + if (!ok) return; + const script = resolve(REPO, 'scanners/drift-cli.mjs'); + const { stdout } = await runCli(script, [FIXTURE, '--raw']); + const actual = JSON.parse(stdout); + const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8')); + assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected)); + }); +}); + +// --------------------------------------------------------------------------- +// Environment-aware CLIs — mode-equivalence. +// --------------------------------------------------------------------------- + +const ENV_AWARE_CLIS = [ + { + name: 'plugin-health-scanner', + script: 'scanners/plugin-health-scanner.mjs', + normalize: normalizePluginHealth, + }, + { + name: 'manifest', + script: 'scanners/manifest.mjs', + normalize: normalizeManifest, + }, + { + name: 'whats-active', + script: 'scanners/whats-active.mjs', + normalize: normalizeWhatsActive, + }, +]; + +describe('SC-7 --raw backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => { + for (const cli of ENV_AWARE_CLIS) { + it(`${cli.name} --raw equals --json (machine modes are byte-identical)`, async () => { + const script = resolve(REPO, cli.script); + const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']); + const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']); + assert.deepStrictEqual( + cli.normalize(JSON.parse(rawOut)), + cli.normalize(JSON.parse(jsonOut)), + ); + }); + } +}); + +// --------------------------------------------------------------------------- +// Posture stderr scorecard — verbatim v5.0.0 in --raw mode. +// --------------------------------------------------------------------------- + +describe('SC-7 --raw posture stderr scorecard verbatim', () => { + it('posture --raw stderr matches tests/snapshots/v5.0.0-stderr/posture.txt (modulo Xms)', async () => { + const script = resolve(REPO, 'scanners/posture.mjs'); + const { stderr } = await runCli(script, [FIXTURE, '--raw']); + const expected = await readFile(resolve(STDERR_SNAPSHOT_DIR, 'posture.txt'), 'utf8'); + assert.equal( + normalizeStderrDurations(stderr.trim()), + normalizeStderrDurations(expected.trim()), + 'posture --raw stderr must reproduce the v5.0.0 scorecard verbatim (apart from durations)', + ); + }); +}); + +// --------------------------------------------------------------------------- +// Cross-cutting: --raw must NOT add humanizer fields anywhere. +// --------------------------------------------------------------------------- + +describe('SC-7 --raw output never carries humanizer fields', () => { + const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext']; + + function* walkFindings(payload) { + if (!payload || typeof payload !== 'object') return; + if (Array.isArray(payload.findings)) { + for (const f of payload.findings) yield f; + } + if (Array.isArray(payload.scanners)) { + for (const s of payload.scanners) { + if (Array.isArray(s.findings)) { + for (const f of s.findings) yield f; + } + } + } + if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope); + } + + for (const cli of DETERMINISTIC_CLIS) { + it(`${cli.name} --raw findings carry no humanizer fields`, async () => { + const script = resolve(REPO, cli.script); + const { stdout } = await runCli(script, [FIXTURE, '--raw']); + const actual = JSON.parse(stdout); + for (const f of walkFindings(actual)) { + for (const field of EXPECTED_HUMANIZER_FIELDS) { + assert.equal( + f[field], + undefined, + `${cli.name} ${f.id ?? ''}: --raw must not add ${field}`, + ); + } + } + }); + } +});