open/config-audit
1
0
Fork 0
Code Issues Releases 1 Activity

test(snapshots): make byte/snapshot tests hermetic + re-seed baseline

Browse source 
The COL collision-scanner and the CLAUDE.md cascade resolve ~/.claude from
process.env.HOME (active-config-reader). Snapshot/byte CLIs were spawned with
the developer's real HOME, so they picked up installed plugins/skills and the
user CLAUDE.md — making the v5.0.0 + default-output snapshots machine- and
time-dependent. They were seeded 2026-05-01 with COL=1 (a real ~/.claude skill
collision) and drifted to COL=0 after the polyrepo split: 26 pre-existing
failures unrelated to Batch 1.

Fix (test-only, no production change):
- tests/helpers/hermetic-home.mjs — empty temp HOME, mirroring the pattern
  collision.test.mjs already uses for the COL unit test.
- 7 harnesses spawn CLIs (or call lint()) under the hermetic HOME, so output
  depends only on committed fixtures. Determinism verified across runs.
- Re-seeded all snapshots under hermetic HOME via SEED_SNAPSHOT/UPDATE_SNAPSHOT
  (added a SEED guard to the frozen v5.0.0 byte tests). Snapshots now reflect
  the fixture alone (COL=0, fixture-only activeConfig counts).
- Also re-seeded the unused env-aware snapshots (manifest/whats-active/
  plugin-health), which had baked dozens of real ~/.claude skill/plugin names
  into the committed repo — privacy cleanup.

Full suite: 812/812 green, stable across 3 runs.
This commit is contained in:
Kjell Tore Guttormsen 2026-06-18 12:26:00 +02:00
commit 8216fb4175
20 changed files with 326 additions and 3386 deletions
Download patch file Download diff file Expand all files Collapse all files

23
tests/json-backcompat.test.mjs
View file

@ -26,16 +26,20 @@ import { resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { readFile, access, mkdir } from 'node:fs/promises';
import { readFile, writeFile, access, mkdir } from 'node:fs/promises';
import { homedir } from 'node:os';
import { hermeticEnv, HERMETIC_HOME } from './helpers/hermetic-home.mjs';
const exec = promisify(execFile);
const __dirname = dirname(fileURLToPath(import.meta.url));
const REPO = resolve(__dirname, '..');
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
const SNAPSHOT_DIR = resolve(REPO, 'tests/snapshots/v5.0.0');
const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines');
// Baseline lives under the hermetic HOME so drift output is isolated from
// the developer's ~/.config-audit state, matching the scan env below.
const BASELINE_DIR = resolve(HERMETIC_HOME, '.config-audit/baselines');
const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json');
const SEED = process.env.SEED_SNAPSHOT === '1';
async function runCli(scriptPath, args) {
try {
@ -43,6 +47,7 @@ async function runCli(scriptPath, args) {
timeout: 60000,
cwd: REPO,
maxBuffer: 10 * 1024 * 1024,
env: hermeticEnv(),
});
return { stdout: stdout || '', stderr: stderr || '' };
} catch (err) {
@ -198,7 +203,12 @@ describe('SC-6 JSON backwards-compatibility — fixture-deterministic CLIs', ()
const script = resolve(REPO, cli.script);
const { stdout } = await runCli(script, [FIXTURE, '--json']);
const actual = JSON.parse(stdout);
const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, cli.snapshot), 'utf8'));
const snapshotPath = resolve(SNAPSHOT_DIR, cli.snapshot);
if (SEED) {
await writeFile(snapshotPath, JSON.stringify(actual, null, 2) + '\n', 'utf8');
return;
}
const expected = JSON.parse(await readFile(snapshotPath, 'utf8'));
assert.deepStrictEqual(cli.normalize(actual), cli.normalize(expected));
});
}
@ -219,7 +229,12 @@ describe('SC-6 JSON backwards-compatibility — drift-cli', () => {
const script = resolve(REPO, 'scanners/drift-cli.mjs');
const { stdout } = await runCli(script, [FIXTURE, '--json']);
const actual = JSON.parse(stdout);
const expected = JSON.parse(await readFile(resolve(SNAPSHOT_DIR, 'drift.json'), 'utf8'));
const snapshotPath = resolve(SNAPSHOT_DIR, 'drift.json');
if (SEED) {
await writeFile(snapshotPath, JSON.stringify(actual, null, 2) + '\n', 'utf8');
return;
}
const expected = JSON.parse(await readFile(snapshotPath, 'utf8'));
assert.deepStrictEqual(normalizeDrift(actual), normalizeDrift(expected));
});
});