test(humanizer): --raw backwards-compatibility test (SC-7) [skip-docs]
Step 11 of v5.1.0 humanizer Wave 4. Adds tests/raw-backcompat.test.mjs mirroring the SC-6 contract for the --raw flag — the explicit "v5.0.0 verbatim" escape hatch. - 4 fixture-deterministic CLIs (scan-orchestrator, posture, token-hotspots-cli, fix-cli) get strict byte-equal against tests/snapshots/v5.0.0/<cli>.json with time fields normalized. - drift-cli is checked under the same contract guarded by ensureDriftBaseline. - 3 environment-aware CLIs (plugin-health, manifest, whats-active) are checked for mode-equivalence (--raw equals --json). - Posture additionally asserts its --raw stderr scorecard reproduces tests/snapshots/v5.0.0-stderr/posture.txt verbatim, modulo (Xms) duration markers normalized to (0ms). - Cross-cutting suite asserts --raw findings carry no humanizer fields on any CLI. Tests: 751 to 764 (+13 SC-7 cases). Full suite passes. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen
parent
12af13a703
commit
20b867adc1
1 changed files with 298 additions and 0 deletions
298
plugins/config-audit/tests/raw-backcompat.test.mjs
Normal file
298
plugins/config-audit/tests/raw-backcompat.test.mjs
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
/**
|
||||
* SC-7 — --raw backwards-compatibility test (Wave 4 Step 11).
|
||||
*
|
||||
* Mirror of tests/json-backcompat.test.mjs but exercises the --raw flag,
|
||||
* the explicit "v5.0.0 verbatim" escape hatch documented in Wave 3.
|
||||
*
|
||||
* 4 fixture-deterministic CLIs (scan-orchestrator, posture,
|
||||
* token-hotspots-cli, fix-cli) plus drift-cli are checked byte-equal
|
||||
* against tests/snapshots/v5.0.0/<cli>.json (with time fields
|
||||
* normalized).
|
||||
*
|
||||
* 3 environment-aware CLIs (plugin-health, manifest, whats-active) are
|
||||
* checked for mode-equivalence (--raw equals --json), matching the
|
||||
* established Wave 3 strategy.
|
||||
*
|
||||
* Posture additionally asserts its --raw stderr scorecard matches the
|
||||
* verbatim v5.0.0 stderr capture in tests/snapshots/v5.0.0-stderr/
|
||||
* posture.txt, with (Xms) duration markers normalized to (0ms).
|
||||
*/
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { readFile, access, mkdir } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO = resolve(__dirname, '..');
|
||||
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
|
||||
const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0');
|
||||
const STDERR_SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0-stderr');
|
||||
const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines');
|
||||
const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json');
|
||||
|
||||
async function runCli(scriptPath, args) {
|
||||
try {
|
||||
const { stdout, stderr } = await exec('node', [scriptPath, ...args], {
|
||||
timeout: 60000,
|
||||
cwd: REPO,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
return { stdout: stdout || '', stderr: stderr || '' };
|
||||
} catch (err) {
|
||||
return { stdout: err.stdout || '', stderr: err.stderr || '' };
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureDriftBaseline() {
|
||||
try {
|
||||
await access(DEFAULT_BASELINE);
|
||||
return true;
|
||||
} catch {
|
||||
try {
|
||||
await mkdir(BASELINE_DIR, { recursive: true });
|
||||
await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']);
|
||||
await access(DEFAULT_BASELINE);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Normalizers — same as json-backcompat to keep the contracts aligned.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function normalizeScanOrchestrator(env) {
|
||||
const out = JSON.parse(JSON.stringify(env));
|
||||
if (out.meta) {
|
||||
out.meta.target = '<TARGET>';
|
||||
out.meta.timestamp = '<TIMESTAMP>';
|
||||
}
|
||||
if (Array.isArray(out.scanners)) {
|
||||
for (const s of out.scanners) {
|
||||
s.duration_ms = 0;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizePosture(p) {
|
||||
const out = JSON.parse(JSON.stringify(p));
|
||||
if (out.scannerEnvelope) {
|
||||
if (out.scannerEnvelope.meta) {
|
||||
out.scannerEnvelope.meta.target = '<TARGET>';
|
||||
out.scannerEnvelope.meta.timestamp = '<TIMESTAMP>';
|
||||
}
|
||||
if (Array.isArray(out.scannerEnvelope.scanners)) {
|
||||
for (const s of out.scannerEnvelope.scanners) {
|
||||
s.duration_ms = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeTokenHotspots(p) {
|
||||
const out = JSON.parse(JSON.stringify(p));
|
||||
out.duration_ms = 0;
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeDrift(p) {
|
||||
return JSON.parse(JSON.stringify(p));
|
||||
}
|
||||
|
||||
function normalizeFix(p) {
|
||||
return JSON.parse(JSON.stringify(p));
|
||||
}
|
||||
|
||||
function normalizePluginHealth(p) {
|
||||
const out = JSON.parse(JSON.stringify(p));
|
||||
out.duration_ms = 0;
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeManifest(o) {
|
||||
const out = JSON.parse(JSON.stringify(o));
|
||||
if (out.meta) {
|
||||
out.meta.repoPath = '<TARGET>';
|
||||
out.meta.generatedAt = '<TIMESTAMP>';
|
||||
out.meta.durationMs = 0;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function normalizeWhatsActive(o) {
|
||||
const out = JSON.parse(JSON.stringify(o));
|
||||
if (out.meta) {
|
||||
out.meta.repoPath = '<TARGET>';
|
||||
out.meta.generatedAt = '<TIMESTAMP>';
|
||||
out.meta.durationMs = 0;
|
||||
if (out.meta.gitRoot) out.meta.gitRoot = '<GITROOT>';
|
||||
if (out.meta.projectKey) out.meta.projectKey = '<PROJECTKEY>';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Normalize Xms duration markers in stderr prose for verbatim comparison. */
|
||||
function normalizeStderrDurations(s) {
|
||||
return s.replace(/\(\d+ms\)/g, '(0ms)');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture-deterministic CLIs — strict byte-equal --raw vs v5.0.0 snapshot.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DETERMINISTIC_CLIS = [
|
||||
{
|
||||
name: 'scan-orchestrator',
|
||||
script: 'scanners/scan-orchestrator.mjs',
|
||||
snapshot: 'scan-orchestrator.json',
|
||||
normalize: normalizeScanOrchestrator,
|
||||
},
|
||||
{
|
||||
name: 'posture',
|
||||
script: 'scanners/posture.mjs',
|
||||
snapshot: 'posture.json',
|
||||
normalize: normalizePosture,
|
||||
},
|
||||
{
|
||||
name: 'token-hotspots-cli',
|
||||
script: 'scanners/token-hotspots-cli.mjs',
|
||||
snapshot: 'token-hotspots.json',
|
||||
normalize: normalizeTokenHotspots,
|
||||
},
|
||||
{
|
||||
name: 'fix-cli',
|
||||
script: 'scanners/fix-cli.mjs',
|
||||
snapshot: 'fix-cli.json',
|
||||
normalize: normalizeFix,
|
||||
},
|
||||
];
|
||||
|
||||
describe('SC-7 --raw backwards-compatibility — fixture-deterministic CLIs', () => {
|
||||
for (const cli of DETERMINISTIC_CLIS) {
|
||||
it(`${cli.name} --raw byte-equals v5.0.0 snapshot`, async () => {
|
||||
const script = resolve(REPO, cli.script);
|
||||
const { stdout } = await runCli(script, [FIXTURE, '--raw']);
|
||||
const actual = JSON.parse(stdout);
|
||||
const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8'));
|
||||
assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Drift-cli with baseline precondition.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('SC-7 --raw backwards-compatibility — drift-cli', () => {
|
||||
it('drift-cli --raw byte-equals v5.0.0 snapshot (when baseline available)', async () => {
|
||||
const ok = await ensureDriftBaseline();
|
||||
if (!ok) return;
|
||||
const script = resolve(REPO, 'scanners/drift-cli.mjs');
|
||||
const { stdout } = await runCli(script, [FIXTURE, '--raw']);
|
||||
const actual = JSON.parse(stdout);
|
||||
const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8'));
|
||||
assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected));
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Environment-aware CLIs — mode-equivalence.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ENV_AWARE_CLIS = [
|
||||
{
|
||||
name: 'plugin-health-scanner',
|
||||
script: 'scanners/plugin-health-scanner.mjs',
|
||||
normalize: normalizePluginHealth,
|
||||
},
|
||||
{
|
||||
name: 'manifest',
|
||||
script: 'scanners/manifest.mjs',
|
||||
normalize: normalizeManifest,
|
||||
},
|
||||
{
|
||||
name: 'whats-active',
|
||||
script: 'scanners/whats-active.mjs',
|
||||
normalize: normalizeWhatsActive,
|
||||
},
|
||||
];
|
||||
|
||||
describe('SC-7 --raw backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => {
|
||||
for (const cli of ENV_AWARE_CLIS) {
|
||||
it(`${cli.name} --raw equals --json (machine modes are byte-identical)`, async () => {
|
||||
const script = resolve(REPO, cli.script);
|
||||
const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']);
|
||||
const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']);
|
||||
assert.deepStrictEqual(
|
||||
cli.normalize(JSON.parse(rawOut)),
|
||||
cli.normalize(JSON.parse(jsonOut)),
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Posture stderr scorecard — verbatim v5.0.0 in --raw mode.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('SC-7 --raw posture stderr scorecard verbatim', () => {
|
||||
it('posture --raw stderr matches tests/snapshots/v5.0.0-stderr/posture.txt (modulo Xms)', async () => {
|
||||
const script = resolve(REPO, 'scanners/posture.mjs');
|
||||
const { stderr } = await runCli(script, [FIXTURE, '--raw']);
|
||||
const expected = await readFile(resolve(STDERR_SNAPSHOT_DIR, 'posture.txt'), 'utf8');
|
||||
assert.equal(
|
||||
normalizeStderrDurations(stderr.trim()),
|
||||
normalizeStderrDurations(expected.trim()),
|
||||
'posture --raw stderr must reproduce the v5.0.0 scorecard verbatim (apart from durations)',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cross-cutting: --raw must NOT add humanizer fields anywhere.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('SC-7 --raw output never carries humanizer fields', () => {
|
||||
const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext'];
|
||||
|
||||
function* walkFindings(payload) {
|
||||
if (!payload || typeof payload !== 'object') return;
|
||||
if (Array.isArray(payload.findings)) {
|
||||
for (const f of payload.findings) yield f;
|
||||
}
|
||||
if (Array.isArray(payload.scanners)) {
|
||||
for (const s of payload.scanners) {
|
||||
if (Array.isArray(s.findings)) {
|
||||
for (const f of s.findings) yield f;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope);
|
||||
}
|
||||
|
||||
for (const cli of DETERMINISTIC_CLIS) {
|
||||
it(`${cli.name} --raw findings carry no humanizer fields`, async () => {
|
||||
const script = resolve(REPO, cli.script);
|
||||
const { stdout } = await runCli(script, [FIXTURE, '--raw']);
|
||||
const actual = JSON.parse(stdout);
|
||||
for (const f of walkFindings(actual)) {
|
||||
for (const field of EXPECTED_HUMANIZER_FIELDS) {
|
||||
assert.equal(
|
||||
f[field],
|
||||
undefined,
|
||||
`${cli.name} ${f.id ?? '<no-id>'}: --raw must not add ${field}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue