test(snapshots): make byte/snapshot tests hermetic + re-seed baseline
The COL collision-scanner and the CLAUDE.md cascade resolve ~/.claude from process.env.HOME (active-config-reader). Snapshot/byte CLIs were spawned with the developer's real HOME, so they picked up installed plugins/skills and the user CLAUDE.md — making the v5.0.0 + default-output snapshots machine- and time-dependent. They were seeded 2026-05-01 with COL=1 (a real ~/.claude skill collision) and drifted to COL=0 after the polyrepo split: 26 pre-existing failures unrelated to Batch 1. Fix (test-only, no production change): - tests/helpers/hermetic-home.mjs — empty temp HOME, mirroring the pattern collision.test.mjs already uses for the COL unit test. - 7 harnesses spawn CLIs (or call lint()) under the hermetic HOME, so output depends only on committed fixtures. Determinism verified across runs. - Re-seeded all snapshots under hermetic HOME via SEED_SNAPSHOT/UPDATE_SNAPSHOT (added a SEED guard to the frozen v5.0.0 byte tests). Snapshots now reflect the fixture alone (COL=0, fixture-only activeConfig counts). - Also re-seeded the unused env-aware snapshots (manifest/whats-active/ plugin-health), which had baked dozens of real ~/.claude skill/plugin names into the committed repo — privacy cleanup. Full suite: 812/812 green, stable across 3 runs.
This commit is contained in:
parent
4b94da0f11
commit
8216fb4175
20 changed files with 326 additions and 3386 deletions
|
|
@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { readFile, writeFile, unlink, mkdir, access } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { hermeticEnv, HERMETIC_HOME } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -13,7 +13,7 @@ const REPO = resolve(__dirname, '../..');
|
|||
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
|
||||
const BROKEN_PLUGIN = resolve(REPO, 'tests/fixtures/broken-plugin');
|
||||
|
||||
const BASELINE_DIR = resolve(homedir(), '.config-audit/baselines');
|
||||
const BASELINE_DIR = resolve(HERMETIC_HOME, '.config-audit/baselines');
|
||||
const DEFAULT_BASELINE = resolve(BASELINE_DIR, 'default.json');
|
||||
|
||||
/**
|
||||
|
|
@ -25,7 +25,7 @@ async function runCli(cliPath, args, env = {}) {
|
|||
const { stdout, stderr } = await exec('node', [cliPath, ...args], {
|
||||
timeout: 60000,
|
||||
cwd: REPO,
|
||||
env: { ...process.env, ...env },
|
||||
env: hermeticEnv(env),
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
return { stdout: stdout || '', stderr: stderr || '', code: 0 };
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import assert from 'node:assert/strict';
|
|||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { lint } from '../lint-default-output.mjs';
|
||||
import { withHermeticHome } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const REPO = resolve(__dirname, '../..');
|
||||
|
|
@ -10,7 +11,9 @@ const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-medium');
|
|||
|
||||
describe('SC-3 forbidden-words lint (default-output)', () => {
|
||||
it('produces no tier1 or tier3 violations across the 6 prose CLIs', async () => {
|
||||
const { failures, warnings } = await lint(FIXTURE);
|
||||
// lint() spawns the prose CLIs inheriting process.env; HOME isolation keeps
|
||||
// the COL collision scanner off the developer's real ~/.claude.
|
||||
const { failures, warnings } = await withHermeticHome(() => lint(FIXTURE));
|
||||
const failureSummary = failures
|
||||
.map((f) => `[${f.cli}] tier${f.tier} "${f.word}" × ${f.count}`)
|
||||
.join('\n ');
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { readFile, unlink } from 'node:fs/promises';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -49,6 +50,7 @@ async function runPosture(flags) {
|
|||
const proc = await exec('node', [CLI, FIXTURE, ...flags], {
|
||||
timeout: 60000,
|
||||
cwd: REPO,
|
||||
env: hermeticEnv(),
|
||||
}).catch(err => err); // posture exits non-zero on findings — capture either way
|
||||
return {
|
||||
stdout: proc.stdout || '',
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { readFile, unlink } from 'node:fs/promises';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
|
@ -44,6 +45,7 @@ async function runOrchestrator(flags) {
|
|||
await exec('node', [CLI, FIXTURE, '--output-file', out, ...flags], {
|
||||
timeout: 60000,
|
||||
cwd: REPO,
|
||||
env: hermeticEnv(),
|
||||
});
|
||||
const written = await readFile(out, 'utf-8');
|
||||
return JSON.parse(written);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue