config-audit/tests/json-backcompat.test.mjs

/**
 * SC-6 — JSON backwards-compatibility test (Wave 4 Step 10).
 *
 * For each CLI that has a frozen v5.0.0 JSON snapshot, run the CLI with
 * --json against the marketplace-medium fixture and compare the output
 * to the snapshot. Time-varying fields are normalized.
 *
 * 5 fixture-deterministic CLIs are checked byte-equal against the v5.0.0
 * snapshot:
 *   - scan-orchestrator
 *   - posture
 *   - token-hotspots-cli
 *   - drift-cli (requires saved baseline; falls back to mode-equivalence
 *                if the baseline cannot be created)
 *   - fix-cli
 *
 * 3 environment-aware CLIs (plugin-health, manifest, whats-active) read
 * the active config cascade, so frozen snapshots drift as the
 * marketplace evolves. They are verified by mode-equivalence
 * (--json == --raw) instead — the same strategy Wave 3
 * cli-humanizer.test.mjs already uses.
 */
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { readFile, writeFile, access, mkdir } from 'node:fs/promises';
import { homedir } from 'node:os';
import { hermeticEnv, HERMETIC_HOME } from './helpers/hermetic-home.mjs';
import { stripHotspotLoadPattern } from './helpers/strip-hotspot-load-pattern.mjs';
import { stripAddedScanners, stripAddedScannerStderr } from './helpers/strip-added-scanner.mjs';
import { stripRetiredGap } from './helpers/strip-retired-gap.mjs';
import { maskFindingIds } from './helpers/mask-finding-ids.mjs';

const exec = promisify(execFile);
const __dirname = dirname(fileURLToPath(import.meta.url));
const REPO = resolve(__dirname, '..');
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0');
// Baseline lives under the hermetic HOME so drift output is isolated from
// the developer's ~/.config-audit state, matching the scan env below.
const BASELINE_DIR = resolve(HERMETIC_HOME, '.config-audit/baselines');
const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json');
const SEED = process.env.SEED_SNAPSHOT === '1';

async function runCli(scriptPath, args) {
  try {
    const { stdout, stderr } = await exec('node', [scriptPath, ...args], {
      timeout: 60000,
      cwd: REPO,
      maxBuffer: 10 * 1024 * 1024,
      env: hermeticEnv(),
    });
    return { stdout: stdout || '', stderr: stderr || '' };
  } catch (err) {
    return { stdout: err.stdout || '', stderr: err.stderr || '' };
  }
}

async function ensureDriftBaseline() {
  try {
    await access(DEFAULT_BASELINE);
    return true;
  } catch {
    try {
      await mkdir(BASELINE_DIR, { recursive: true });
      await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']);
      await access(DEFAULT_BASELINE);
      return true;
    } catch {
      return false;
    }
  }
}

// ---------------------------------------------------------------------------
// Normalizers — strip time / path / ancestor-derived fields that vary
// independently of scanner internals. `claudeMdEstimatedTokens` is computed
// by walking the FS cascade upward from the fixture; any edit to this
// plugin's own CLAUDE.md ripples into it, even though scanner behavior is
// unchanged. The byte-stability contract covers scanner output shape, not
// the size of ancestor input docs.
// ---------------------------------------------------------------------------

function stripAncestorDerived(envOrEnvelope) {
  if (Array.isArray(envOrEnvelope?.scanners)) {
    for (const s of envOrEnvelope.scanners) {
      if (s?.activeConfig && 'claudeMdEstimatedTokens' in s.activeConfig) {
        s.activeConfig.claudeMdEstimatedTokens = '<ANCESTOR_DERIVED>';
      }
    }
  }
}

function normalizeScanOrchestrator(env) {
  const out = JSON.parse(JSON.stringify(env));
  if (out.meta) {
    out.meta.target = '<TARGET>';
    out.meta.timestamp = '<TIMESTAMP>';
  }
  if (Array.isArray(out.scanners)) {
    for (const s of out.scanners) {
      s.duration_ms = 0;
    }
  }
  stripAncestorDerived(out);
  return maskFindingIds(stripRetiredGap(stripAddedScanners(stripHotspotLoadPattern(out))));
}

function normalizePosture(p) {
  const out = JSON.parse(JSON.stringify(p));
  if (out.scannerEnvelope) {
    if (out.scannerEnvelope.meta) {
      out.scannerEnvelope.meta.target = '<TARGET>';
      out.scannerEnvelope.meta.timestamp = '<TIMESTAMP>';
    }
    if (Array.isArray(out.scannerEnvelope.scanners)) {
      for (const s of out.scannerEnvelope.scanners) {
        s.duration_ms = 0;
      }
    }
    stripAncestorDerived(out.scannerEnvelope);
  }
  return maskFindingIds(stripRetiredGap(stripAddedScanners(stripHotspotLoadPattern(out))));
}

function normalizeTokenHotspots(p) {
  const out = JSON.parse(JSON.stringify(p));
  out.duration_ms = 0;
  return maskFindingIds(stripHotspotLoadPattern(out));
}

function normalizeDrift(p) {
  // Drift result has no time fields — round-trip through JSON for safety.
  return maskFindingIds(stripRetiredGap(JSON.parse(JSON.stringify(p))));
}

function normalizeFix(p) {
  // Fix-cli stdout is the planFixes result with no time fields.
  return maskFindingIds(stripRetiredGap(JSON.parse(JSON.stringify(p))));
}

function normalizePluginHealth(p) {
  const out = JSON.parse(JSON.stringify(p));
  out.duration_ms = 0;
  return maskFindingIds(out);
}

function normalizeManifest(o) {
  const out = JSON.parse(JSON.stringify(o));
  if (out.meta) {
    out.meta.repoPath = '<TARGET>';
    out.meta.generatedAt = '<TIMESTAMP>';
    out.meta.durationMs = 0;
  }
  return out;
}

function normalizeWhatsActive(o) {
  const out = JSON.parse(JSON.stringify(o));
  if (out.meta) {
    out.meta.repoPath = '<TARGET>';
    out.meta.generatedAt = '<TIMESTAMP>';
    out.meta.durationMs = 0;
    if (out.meta.gitRoot) out.meta.gitRoot = '<GITROOT>';
    if (out.meta.projectKey) out.meta.projectKey = '<PROJECTKEY>';
  }
  return out;
}

// ---------------------------------------------------------------------------
// Fixture-deterministic CLIs — strict byte-equal against v5.0.0 snapshot.
// ---------------------------------------------------------------------------

const DETERMINISTIC_CLIS = [
  {
    name: 'scan-orchestrator',
    script: 'scanners/scan-orchestrator.mjs',
    snapshot: 'scan-orchestrator.json',
    normalize: normalizeScanOrchestrator,
  },
  {
    name: 'posture',
    script: 'scanners/posture.mjs',
    snapshot: 'posture.json',
    normalize: normalizePosture,
  },
  {
    name: 'token-hotspots-cli',
    script: 'scanners/token-hotspots-cli.mjs',
    snapshot: 'token-hotspots.json',
    normalize: normalizeTokenHotspots,
  },
  {
    name: 'fix-cli',
    script: 'scanners/fix-cli.mjs',
    snapshot: 'fix-cli.json',
    normalize: normalizeFix,
  },
];

describe('SC-6 JSON backwards-compatibility — fixture-deterministic CLIs', () => {
  for (const cli of DETERMINISTIC_CLIS) {
    it(`${cli.name} --json byte-equals v5.0.0 snapshot`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout } = await runCli(script, [FIXTURE, '--json']);
      const actual = JSON.parse(stdout);
      const snapshotPath = resolve(SNAPSHOT_DIR, cli.snapshot);
      if (SEED) {
        await writeFile(snapshotPath, JSON.stringify(actual, null, 2) + '\n', 'utf8');
        return;
      }
      const expected = JSON.parse(await readFile(snapshotPath, 'utf8'));
      assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected));
    });
  }
});

// ---------------------------------------------------------------------------
// Drift-cli: separate suite because it requires a baseline precondition.
// ---------------------------------------------------------------------------

describe('SC-6 JSON backwards-compatibility — drift-cli', () => {
  it('drift-cli --json byte-equals v5.0.0 snapshot (when baseline available)', async () => {
    const ok = await ensureDriftBaseline();
    if (!ok) {
      // Skip silently — environment cannot create a baseline. Wave 0 + Wave 3
      // tests already exercise drift extensively; this is a defensive fallback.
      return;
    }
    const script = resolve(REPO, 'scanners/drift-cli.mjs');
    const { stdout } = await runCli(script, [FIXTURE, '--json']);
    const actual = JSON.parse(stdout);
    const snapshotPath = resolve(SNAPSHOT_DIR, 'drift.json');
    if (SEED) {
      await writeFile(snapshotPath, JSON.stringify(actual, null, 2) + '\n', 'utf8');
      return;
    }
    const expected = JSON.parse(await readFile(snapshotPath, 'utf8'));
    assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected));
  });
});

// ---------------------------------------------------------------------------
// Environment-aware CLIs — mode-equivalence (--json == --raw). Frozen v5.0.0
// snapshots drift as marketplace state evolves, so byte-equal would be flaky.
// ---------------------------------------------------------------------------

const ENV_AWARE_CLIS = [
  {
    name: 'plugin-health-scanner',
    script: 'scanners/plugin-health-scanner.mjs',
    normalize: normalizePluginHealth,
  },
  {
    name: 'manifest',
    script: 'scanners/manifest.mjs',
    normalize: normalizeManifest,
  },
  {
    name: 'whats-active',
    script: 'scanners/whats-active.mjs',
    normalize: normalizeWhatsActive,
  },
];

describe('SC-6 JSON backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => {
  for (const cli of ENV_AWARE_CLIS) {
    it(`${cli.name} --json equals --raw (machine modes are byte-identical)`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']);
      const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']);
      assert.deepStrictEqual(
        cli.normalize(JSON.parse(jsonOut)),
        cli.normalize(JSON.parse(rawOut)),
      );
    });
  }
});

// ---------------------------------------------------------------------------
// Cross-cutting: --json must NOT add humanizer fields to any CLI's findings.
// ---------------------------------------------------------------------------

describe('SC-6 JSON output never carries humanizer fields', () => {
  const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext'];

  function* walkFindings(payload) {
    if (!payload || typeof payload !== 'object') return;
    if (Array.isArray(payload.findings)) {
      for (const f of payload.findings) yield f;
    }
    if (Array.isArray(payload.scanners)) {
      for (const s of payload.scanners) {
        if (Array.isArray(s.findings)) {
          for (const f of s.findings) yield f;
        }
      }
    }
    if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope);
  }

  for (const cli of DETERMINISTIC_CLIS) {
    it(`${cli.name} --json findings carry no humanizer fields`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout } = await runCli(script, [FIXTURE, '--json']);
      const actual = JSON.parse(stdout);
      for (const f of walkFindings(actual)) {
        for (const field of EXPECTED_HUMANIZER_FIELDS) {
          assert.equal(
            f[field],
            undefined,
            `${cli.name} ${f.id ?? '<no-id>'}: --json must not add ${field}`,
          );
        }
      }
    });
  }
});