test: isolate HOME in all CLI-spawning tests (close leak class)
Follow-up to the posture-grade-stability fix in 66433fe. Audited every
test that spawns a CLI and found more of the same class: tests running
HOME-scoped scanners (SKL/COL) or the CLAUDE.md cascade against the
developer's real ~/.claude instead of an isolated HOME.
Fixed (env: hermeticEnv()):
- posture.test.mjs — runs full posture (SKL/COL/cascade); twin of
the posture-grade-stability leak, masked only
because its asserts are structural/relative
- drift-cli.test.mjs — ACTIVE bug: the CLI wrote baselines into the
real ~/.claude during the run (pollution); now
isolated, and afterEach cleanup wrapped in
withHermeticHome so it looks in the same HOME
- token-hotspots-cli.test.mjs — scan-orchestrator run executes SKL/COL on
real HOME; TOK reads the HOME cascade
- accurate-tokens.test.mjs — TOK reads the HOME cascade (kept the
ANTHROPIC_API_KEY deletion)
Proven safe, left as-is (no HOME-scoped scan affecting assertions, no
HOME writes): post-edit-verify.test.mjs (fast-path early-returns only),
fix-cli.test.mjs (output byte-identical real vs empty HOME — fixable
findings are project-local HKV/RUL/SET, never SKL/COL),
lint-default-output (caller already uses withHermeticHome).
Suite 875/875, no snapshot drift. No test regressed under isolation,
confirming none had a hidden real-HOME dependency.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ter3E2JSi1Khgmuf2kady8
This commit is contained in:
parent
66433fee48
commit
325182ddc9
4 changed files with 34 additions and 34 deletions
|
|
@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
|
|
@ -14,7 +15,8 @@ const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-large');
|
|||
|
||||
describe('--accurate-tokens (no API key)', () => {
|
||||
it('skips API calibration and reports calibration.skipped === "no-api-key"', async () => {
|
||||
const env = { ...process.env };
|
||||
// Isolate HOME (TOK reads the CLAUDE.md cascade) while dropping the API key.
|
||||
const env = hermeticEnv();
|
||||
delete env.ANTHROPIC_API_KEY;
|
||||
const { stdout, stderr } = await exec(
|
||||
'node',
|
||||
|
|
@ -30,6 +32,7 @@ describe('--accurate-tokens (no API key)', () => {
|
|||
const { stdout } = await exec('node', [CLI, FIXTURE, '--json'], {
|
||||
timeout: 30000,
|
||||
cwd: REPO,
|
||||
env: hermeticEnv(),
|
||||
});
|
||||
const json = JSON.parse(stdout);
|
||||
assert.equal(json.calibration, undefined);
|
||||
|
|
|
|||
|
|
@ -4,24 +4,28 @@ import { resolve, join } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { deleteBaseline } from '../../scanners/lib/baseline.mjs';
|
||||
import { hermeticEnv, withHermeticHome } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const FIXTURES = resolve(__dirname, '../fixtures');
|
||||
const HEALTHY = resolve(FIXTURES, 'healthy-project');
|
||||
const DRIFT_CLI = resolve(__dirname, '../../scanners/drift-cli.mjs');
|
||||
|
||||
// Isolate HOME: drift runs a full scan (HOME-scoped SKL/COL) AND writes its
|
||||
// baselines under ~/.claude. Without this, the CLI would pollute the real
|
||||
// ~/.claude during the run. See tests/helpers/hermetic-home.mjs.
|
||||
const RUN = { encoding: 'utf-8', timeout: 30000, env: hermeticEnv() };
|
||||
|
||||
const TEST_BASELINE = `_drift_test_${Date.now()}`;
|
||||
|
||||
afterEach(async () => {
|
||||
await deleteBaseline(TEST_BASELINE);
|
||||
// Cleanup must look in the SAME hermetic HOME the CLI wrote to.
|
||||
await withHermeticHome(() => deleteBaseline(TEST_BASELINE));
|
||||
});
|
||||
|
||||
describe('drift-cli --save', () => {
|
||||
it('saves a baseline and confirms', () => {
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE, '--json'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE, '--json'], RUN);
|
||||
const output = JSON.parse(result);
|
||||
assert.equal(output.saved, true);
|
||||
assert.equal(output.name, TEST_BASELINE);
|
||||
|
|
@ -32,15 +36,9 @@ describe('drift-cli --save', () => {
|
|||
describe('drift-cli --list', () => {
|
||||
it('lists baselines including saved one', async () => {
|
||||
// Save first
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], RUN);
|
||||
|
||||
const result = execFileSync('node', [DRIFT_CLI, '--list', '--json'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
const result = execFileSync('node', [DRIFT_CLI, '--list', '--json'], RUN);
|
||||
const output = JSON.parse(result);
|
||||
assert.ok(Array.isArray(output.baselines));
|
||||
const found = output.baselines.find(b => b.name === TEST_BASELINE);
|
||||
|
|
@ -51,16 +49,10 @@ describe('drift-cli --list', () => {
|
|||
describe('drift-cli compare', () => {
|
||||
it('outputs valid JSON with --json flag', () => {
|
||||
// Save baseline first
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], RUN);
|
||||
|
||||
// Compare same fixture against itself
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', TEST_BASELINE, '--json'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', TEST_BASELINE, '--json'], RUN);
|
||||
const diff = JSON.parse(result);
|
||||
assert.ok('newFindings' in diff);
|
||||
assert.ok('resolvedFindings' in diff);
|
||||
|
|
@ -71,15 +63,9 @@ describe('drift-cli compare', () => {
|
|||
});
|
||||
|
||||
it('shows stable trend when comparing same fixture', () => {
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--save', '--name', TEST_BASELINE], RUN);
|
||||
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', TEST_BASELINE, '--json'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
const result = execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', TEST_BASELINE, '--json'], RUN);
|
||||
const diff = JSON.parse(result);
|
||||
assert.equal(diff.summary.trend, 'stable');
|
||||
assert.equal(diff.summary.newCount, 0);
|
||||
|
|
@ -88,10 +74,7 @@ describe('drift-cli compare', () => {
|
|||
|
||||
it('exits with code 1 when baseline not found', () => {
|
||||
assert.throws(() => {
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', `nonexistent_${Date.now()}`, '--json'], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 30000,
|
||||
});
|
||||
execFileSync('node', [DRIFT_CLI, HEALTHY, '--baseline', `nonexistent_${Date.now()}`, '--json'], RUN);
|
||||
}, (err) => {
|
||||
assert.equal(err.status, 1);
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
|
|
@ -11,9 +12,13 @@ const FIXTURES = resolve(__dirname, '../fixtures');
|
|||
const POSTURE_BIN = resolve(__dirname, '../../scanners/posture.mjs');
|
||||
|
||||
async function runPosture(args) {
|
||||
// Isolate HOME: posture runs the HOME-scoped SKL/COL scanners + CLAUDE.md
|
||||
// cascade, so a real ~/.claude would leak the developer's plugins/skills
|
||||
// into grades and counts. See tests/helpers/hermetic-home.mjs.
|
||||
const { stdout, stderr } = await exec('node', [POSTURE_BIN, ...args], {
|
||||
timeout: 30000,
|
||||
cwd: resolve(__dirname, '../..'),
|
||||
env: hermeticEnv(),
|
||||
});
|
||||
return { stdout, stderr };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { readFile, unlink } from 'node:fs/promises';
|
||||
import { hermeticEnv } from '../helpers/hermetic-home.mjs';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
|
|
@ -12,12 +13,16 @@ const REPO = resolve(__dirname, '../..');
|
|||
const CLI = resolve(REPO, 'scanners/token-hotspots-cli.mjs');
|
||||
const ORCH = resolve(REPO, 'scanners/scan-orchestrator.mjs');
|
||||
const FIXTURE = resolve(REPO, 'tests/fixtures/marketplace-large');
|
||||
// Isolate HOME: the scan-orchestrator run executes HOME-scoped SKL/COL, and
|
||||
// TOK reads the CLAUDE.md cascade (which includes ~/.claude). See hermetic-home.mjs.
|
||||
const ENV = hermeticEnv();
|
||||
|
||||
describe('token-hotspots-cli', () => {
|
||||
it('returns valid JSON with hotspots.length >= 3', async () => {
|
||||
const { stdout } = await exec('node', [CLI, FIXTURE, '--json'], {
|
||||
timeout: 30000,
|
||||
cwd: REPO,
|
||||
env: ENV,
|
||||
});
|
||||
const json = JSON.parse(stdout);
|
||||
assert.equal(json.scanner, 'TOK');
|
||||
|
|
@ -33,6 +38,7 @@ describe('token-hotspots-cli', () => {
|
|||
await exec('node', [CLI, FIXTURE, '--output-file', out], {
|
||||
timeout: 30000,
|
||||
cwd: REPO,
|
||||
env: ENV,
|
||||
});
|
||||
const written = await readFile(out, 'utf-8');
|
||||
const json = JSON.parse(written);
|
||||
|
|
@ -47,6 +53,7 @@ describe('token-hotspots-cli', () => {
|
|||
const { stdout } = await exec('node', [CLI, FIXTURE, '--json'], {
|
||||
timeout: 30000,
|
||||
cwd: REPO,
|
||||
env: ENV,
|
||||
});
|
||||
const json = JSON.parse(stdout);
|
||||
assert.equal(json.telemetry_recipe_path, undefined,
|
||||
|
|
@ -57,6 +64,7 @@ describe('token-hotspots-cli', () => {
|
|||
const { stdout } = await exec('node', [CLI, FIXTURE, '--json', '--with-telemetry-recipe'], {
|
||||
timeout: 30000,
|
||||
cwd: REPO,
|
||||
env: ENV,
|
||||
});
|
||||
const json = JSON.parse(stdout);
|
||||
assert.equal(typeof json.telemetry_recipe_path, 'string');
|
||||
|
|
@ -75,6 +83,7 @@ describe('scan-orchestrator integration — TOK hotspots survive envelope', () =
|
|||
await exec('node', [ORCH, FIXTURE, '--output-file', out], {
|
||||
timeout: 60000,
|
||||
cwd: REPO,
|
||||
env: ENV,
|
||||
});
|
||||
const written = await readFile(out, 'utf-8');
|
||||
const envelope = JSON.parse(written);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue