config-audit/tests/raw-backcompat.test.mjs

/**
 * SC-7 — --raw backwards-compatibility test (Wave 4 Step 11).
 *
 * Mirror of tests/json-backcompat.test.mjs but exercises the --raw flag,
 * the explicit "v5.0.0 verbatim" escape hatch documented in Wave 3.
 *
 * 4 fixture-deterministic CLIs (scan-orchestrator, posture,
 * token-hotspots-cli, fix-cli) plus drift-cli are checked byte-equal
 * against tests/snapshots/v5.0.0/<cli>.json (with time fields
 * normalized).
 *
 * 3 environment-aware CLIs (plugin-health, manifest, whats-active) are
 * checked for mode-equivalence (--raw equals --json), matching the
 * established Wave 3 strategy.
 *
 * Posture additionally asserts its --raw stderr scorecard matches the
 * verbatim v5.0.0 stderr capture in tests/snapshots/v5.0.0-stderr/
 * posture.txt, with (Xms) duration markers normalized to (0ms).
 */
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { readFile, writeFile, access, mkdir } from 'node:fs/promises';
import { homedir } from 'node:os';
import { hermeticEnv, HERMETIC_HOME } from './helpers/hermetic-home.mjs';
import { stripHotspotLoadPattern } from './helpers/strip-hotspot-load-pattern.mjs';
import { stripAddedScanners, stripAddedScannerStderr } from './helpers/strip-added-scanner.mjs';
import { stripRetiredGap, maskGapTallyStderr } from './helpers/strip-retired-gap.mjs';
import { maskFindingIds } from './helpers/mask-finding-ids.mjs';

const exec = promisify(execFile);
const __dirname = dirname(fileURLToPath(import.meta.url));
const REPO = resolve(__dirname, '..');
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0');
const STDERR_SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0-stderr');
// Baseline under the hermetic HOME so drift is isolated from ~/.config-audit.
const BASELINE_DIR = resolve(HERMETIC_HOME, '.config-audit/baselines');
const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json');
const SEED = process.env.SEED_SNAPSHOT === '1';

async function runCli(scriptPath, args) {
  try {
    const { stdout, stderr } = await exec('node', [scriptPath, ...args], {
      timeout: 60000,
      cwd: REPO,
      maxBuffer: 10 * 1024 * 1024,
      env: hermeticEnv(),
    });
    return { stdout: stdout || '', stderr: stderr || '' };
  } catch (err) {
    return { stdout: err.stdout || '', stderr: err.stderr || '' };
  }
}

async function ensureDriftBaseline() {
  try {
    await access(DEFAULT_BASELINE);
    return true;
  } catch {
    try {
      await mkdir(BASELINE_DIR, { recursive: true });
      await runCli(resolve(REPO, 'scanners/drift-cli.mjs'), [FIXTURE, '--save']);
      await access(DEFAULT_BASELINE);
      return true;
    } catch {
      return false;
    }
  }
}

// ---------------------------------------------------------------------------
// Normalizers — same as json-backcompat to keep the contracts aligned.
// `claudeMdEstimatedTokens` is stripped because walkClaudeMdCascade walks
// upward from the fixture into this plugin's own CLAUDE.md; any docs edit
// here ripples into it even when scanner internals are unchanged.
// ---------------------------------------------------------------------------

function stripAncestorDerived(envOrEnvelope) {
  if (Array.isArray(envOrEnvelope?.scanners)) {
    for (const s of envOrEnvelope.scanners) {
      if (s?.activeConfig && 'claudeMdEstimatedTokens' in s.activeConfig) {
        s.activeConfig.claudeMdEstimatedTokens = '<ANCESTOR_DERIVED>';
      }
    }
  }
}

function normalizeScanOrchestrator(env) {
  const out = JSON.parse(JSON.stringify(env));
  if (out.meta) {
    out.meta.target = '<TARGET>';
    out.meta.timestamp = '<TIMESTAMP>';
  }
  if (Array.isArray(out.scanners)) {
    for (const s of out.scanners) {
      s.duration_ms = 0;
    }
  }
  stripAncestorDerived(out);
  return maskFindingIds(stripRetiredGap(stripAddedScanners(stripHotspotLoadPattern(out))));
}

function normalizePosture(p) {
  const out = JSON.parse(JSON.stringify(p));
  if (out.scannerEnvelope) {
    if (out.scannerEnvelope.meta) {
      out.scannerEnvelope.meta.target = '<TARGET>';
      out.scannerEnvelope.meta.timestamp = '<TIMESTAMP>';
    }
    if (Array.isArray(out.scannerEnvelope.scanners)) {
      for (const s of out.scannerEnvelope.scanners) {
        s.duration_ms = 0;
      }
    }
    stripAncestorDerived(out.scannerEnvelope);
  }
  return maskFindingIds(stripRetiredGap(stripAddedScanners(stripHotspotLoadPattern(out))));
}

function normalizeTokenHotspots(p) {
  const out = JSON.parse(JSON.stringify(p));
  out.duration_ms = 0;
  return maskFindingIds(stripHotspotLoadPattern(out));
}

function normalizeDrift(p) {
  return maskFindingIds(stripRetiredGap(JSON.parse(JSON.stringify(p))));
}

function normalizeFix(p) {
  return maskFindingIds(stripRetiredGap(JSON.parse(JSON.stringify(p))));
}

function normalizePluginHealth(p) {
  const out = JSON.parse(JSON.stringify(p));
  out.duration_ms = 0;
  return maskFindingIds(out);
}

function normalizeManifest(o) {
  const out = JSON.parse(JSON.stringify(o));
  if (out.meta) {
    out.meta.repoPath = '<TARGET>';
    out.meta.generatedAt = '<TIMESTAMP>';
    out.meta.durationMs = 0;
  }
  return out;
}

function normalizeWhatsActive(o) {
  const out = JSON.parse(JSON.stringify(o));
  if (out.meta) {
    out.meta.repoPath = '<TARGET>';
    out.meta.generatedAt = '<TIMESTAMP>';
    out.meta.durationMs = 0;
    if (out.meta.gitRoot) out.meta.gitRoot = '<GITROOT>';
    if (out.meta.projectKey) out.meta.projectKey = '<PROJECTKEY>';
  }
  return out;
}

/** Normalize Xms duration markers in stderr prose for verbatim comparison. */
function normalizeStderrDurations(s) {
  return s.replace(/\(\d+ms\)/g, '(0ms)');
}

// ---------------------------------------------------------------------------
// Fixture-deterministic CLIs — strict byte-equal --raw vs v5.0.0 snapshot.
// ---------------------------------------------------------------------------

const DETERMINISTIC_CLIS = [
  {
    name: 'scan-orchestrator',
    script: 'scanners/scan-orchestrator.mjs',
    snapshot: 'scan-orchestrator.json',
    normalize: normalizeScanOrchestrator,
  },
  {
    name: 'posture',
    script: 'scanners/posture.mjs',
    snapshot: 'posture.json',
    normalize: normalizePosture,
  },
  {
    name: 'token-hotspots-cli',
    script: 'scanners/token-hotspots-cli.mjs',
    snapshot: 'token-hotspots.json',
    normalize: normalizeTokenHotspots,
  },
  {
    name: 'fix-cli',
    script: 'scanners/fix-cli.mjs',
    snapshot: 'fix-cli.json',
    normalize: normalizeFix,
  },
];

describe('SC-7 --raw backwards-compatibility — fixture-deterministic CLIs', () => {
  for (const cli of DETERMINISTIC_CLIS) {
    it(`${cli.name} --raw byte-equals v5.0.0 snapshot`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout } = await runCli(script, [FIXTURE, '--raw']);
      const actual = JSON.parse(stdout);
      const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8'));
      assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected));
    });
  }
});

// ---------------------------------------------------------------------------
// Drift-cli with baseline precondition.
// ---------------------------------------------------------------------------

describe('SC-7 --raw backwards-compatibility — drift-cli', () => {
  it('drift-cli --raw byte-equals v5.0.0 snapshot (when baseline available)', async () => {
    const ok = await ensureDriftBaseline();
    if (!ok) return;
    const script = resolve(REPO, 'scanners/drift-cli.mjs');
    const { stdout } = await runCli(script, [FIXTURE, '--raw']);
    const actual = JSON.parse(stdout);
    const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8'));
    assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected));
  });
});

// ---------------------------------------------------------------------------
// Environment-aware CLIs — mode-equivalence.
// ---------------------------------------------------------------------------

const ENV_AWARE_CLIS = [
  {
    name: 'plugin-health-scanner',
    script: 'scanners/plugin-health-scanner.mjs',
    normalize: normalizePluginHealth,
  },
  {
    name: 'manifest',
    script: 'scanners/manifest.mjs',
    normalize: normalizeManifest,
  },
  {
    name: 'whats-active',
    script: 'scanners/whats-active.mjs',
    normalize: normalizeWhatsActive,
  },
];

describe('SC-7 --raw backwards-compatibility — environment-aware CLIs (mode-equivalence)', () => {
  for (const cli of ENV_AWARE_CLIS) {
    it(`${cli.name} --raw equals --json (machine modes are byte-identical)`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout: rawOut } = await runCli(script, [FIXTURE, '--raw']);
      const { stdout: jsonOut } = await runCli(script, [FIXTURE, '--json']);
      assert.deepStrictEqual(
        cli.normalize(JSON.parse(rawOut)),
        cli.normalize(JSON.parse(jsonOut)),
      );
    });
  }
});

// ---------------------------------------------------------------------------
// Posture stderr scorecard — verbatim v5.0.0 in --raw mode.
// ---------------------------------------------------------------------------

describe('SC-7 --raw posture stderr scorecard verbatim', () => {
  it('posture --raw stderr matches tests/snapshots/v5.0.0-stderr/posture.txt (modulo Xms)', async () => {
    const script = resolve(REPO, 'scanners/posture.mjs');
    const { stderr } = await runCli(script, [FIXTURE, '--raw']);
    const stderrSnapshotPath = resolve(STDERR_SNAPSHOT_DIR, 'posture.txt');
    if (SEED) {
      await writeFile(stderrSnapshotPath, stderr, 'utf8');
      return;
    }
    const expected = await readFile(stderrSnapshotPath, 'utf8');
    assert.equal(
      normalizeStderrDurations(maskGapTallyStderr(stripAddedScannerStderr(stderr.trim()))),
      normalizeStderrDurations(maskGapTallyStderr(stripAddedScannerStderr(expected.trim()))),
      'posture --raw stderr must reproduce the v5.0.0 scorecard verbatim (apart from durations)',
    );
  });
});

// ---------------------------------------------------------------------------
// Cross-cutting: --raw must NOT add humanizer fields anywhere.
// ---------------------------------------------------------------------------

describe('SC-7 --raw output never carries humanizer fields', () => {
  const EXPECTED_HUMANIZER_FIELDS = ['userImpactCategory', 'userActionLanguage', 'relevanceContext'];

  function* walkFindings(payload) {
    if (!payload || typeof payload !== 'object') return;
    if (Array.isArray(payload.findings)) {
      for (const f of payload.findings) yield f;
    }
    if (Array.isArray(payload.scanners)) {
      for (const s of payload.scanners) {
        if (Array.isArray(s.findings)) {
          for (const f of s.findings) yield f;
        }
      }
    }
    if (payload.scannerEnvelope) yield* walkFindings(payload.scannerEnvelope);
  }

  for (const cli of DETERMINISTIC_CLIS) {
    it(`${cli.name} --raw findings carry no humanizer fields`, async () => {
      const script = resolve(REPO, cli.script);
      const { stdout } = await runCli(script, [FIXTURE, '--raw']);
      const actual = JSON.parse(stdout);
      for (const f of walkFindings(actual)) {
        for (const field of EXPECTED_HUMANIZER_FIELDS) {
          assert.equal(
            f[field],
            undefined,
            `${cli.name} ${f.id ?? '<no-id>'}: --raw must not add ${field}`,
          );
        }
      }
    });
  }
});