test(ci): give policy.json ci.failOn / ci.compact a real test
The "--fail-on via policy.json" block wrote a policy.json into a tmp root it never scanned and passed --fail-on on the CLI, so ci.failOn / ci.compact had no test at all. The rewrite scans a copy of grade-a-project as the process's own working tree (the only place policy.json is honored since v8.1.0): WARNING with 0 critical exits 1 by default, and exits 0 only if `ci.failOn: 'critical'` is read from the policy. Measured: each branch of main() mutated away turns exactly its own test red (fail 1), real code 4/4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
7c5514e5dd
commit
c4471617a4
1 changed files with 44 additions and 27 deletions
|
|
@ -1,12 +1,13 @@
|
||||||
// ci-integration.test.mjs — Tests for --fail-on and --compact CI flags
|
// ci-integration.test.mjs — Tests for --fail-on and --compact CI flags
|
||||||
import { describe, it, afterEach, after } from 'node:test';
|
import { describe, it, beforeEach, afterEach, after } from 'node:test';
|
||||||
import { spawn } from 'node:child_process';
|
import { spawn } from 'node:child_process';
|
||||||
import { strict as assert } from 'node:assert';
|
import { strict as assert } from 'node:assert';
|
||||||
import { resolve, dirname } from 'node:path';
|
import { resolve, dirname, join } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { mkdirSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
|
import { mkdirSync, writeFileSync, rmSync, readFileSync, existsSync, cpSync } from 'node:fs';
|
||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { materializeTree } from '../helpers/payload-trees.mjs';
|
import { materializeTree } from '../helpers/payload-trees.mjs';
|
||||||
|
import { mkOwnTreeDir } from '../helpers/own-tree.mjs';
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
const ORCHESTRATOR = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
|
const ORCHESTRATOR = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
|
||||||
|
|
@ -16,11 +17,12 @@ after(poisonedTree.cleanup);
|
||||||
const POISONED = poisonedTree.dir;
|
const POISONED = poisonedTree.dir;
|
||||||
const CLEAN = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');
|
const CLEAN = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');
|
||||||
|
|
||||||
function run(args, timeout = 120000) {
|
function run(args, timeout = 120000, cwd = undefined) {
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
const chunks = [];
|
const chunks = [];
|
||||||
const errChunks = [];
|
const errChunks = [];
|
||||||
const child = spawn('node', [ORCHESTRATOR, ...args], {
|
const child = spawn('node', [ORCHESTRATOR, ...args], {
|
||||||
|
cwd,
|
||||||
timeout,
|
timeout,
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
});
|
});
|
||||||
|
|
@ -117,36 +119,51 @@ describe('--fail-on + --compact combined', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('--fail-on via policy.json', () => {
|
// v8.1.1: the policy file is honored only for the caller's own working tree
|
||||||
const policyRoot = resolve(tmpdir(), `llm-security-policy-ci-${Date.now()}`);
|
// (scanners/lib/own-working-tree.mjs), so these run the orchestrator with the
|
||||||
const policyDir = resolve(policyRoot, '.llm-security');
|
// fixture as its cwd and '.' as its target. grade-a-project is WARNING with 0
|
||||||
|
// critical (default exit 1), so `ci.failOn: 'critical'` read from the policy
|
||||||
|
// is the only way to get exit 0 — the test cannot pass without reading it.
|
||||||
|
describe('--fail-on / --compact via policy.json (own working tree)', () => {
|
||||||
|
let ownDir;
|
||||||
|
|
||||||
|
function writePolicy(ci) {
|
||||||
|
mkdirSync(join(ownDir, '.llm-security'), { recursive: true });
|
||||||
|
writeFileSync(join(ownDir, '.llm-security', 'policy.json'), JSON.stringify({ ci }));
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
ownDir = mkOwnTreeDir('ci-policy-own-');
|
||||||
|
cpSync(CLEAN, ownDir, { recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
try { rmSync(policyRoot, { recursive: true }); } catch {}
|
rmSync(ownDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('known-positive: without policy the fixture exits 1 (WARNING, no critical)', async () => {
|
||||||
|
const { code, stdout } = await run(['.'], 120000, ownDir);
|
||||||
|
const env = JSON.parse(stdout);
|
||||||
|
assert.equal(env.aggregate.counts.critical, 0);
|
||||||
|
assert.equal(code, 1, 'WARNING verdict without --fail-on should exit 1');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('reads failOn from policy.json ci section', async () => {
|
it('reads failOn from policy.json ci section', async () => {
|
||||||
// Create a dir with policy + a file that triggers findings
|
writePolicy({ failOn: 'critical' });
|
||||||
mkdirSync(policyDir, { recursive: true });
|
const { code } = await run(['.'], 120000, ownDir);
|
||||||
writeFileSync(resolve(policyDir, 'policy.json'), JSON.stringify({
|
assert.equal(code, 0, 'policy ci.failOn: critical with no critical findings should exit 0');
|
||||||
ci: { failOn: 'low' },
|
|
||||||
}));
|
|
||||||
// Scan grade-a fixture but pass policyRoot — policy is loaded from target
|
|
||||||
// Actually: policy is loaded from args.target, so we scan the policyRoot itself
|
|
||||||
// It will find few/no findings but the policy failOn is set
|
|
||||||
const { code } = await run([CLEAN, '--fail-on', 'low']);
|
|
||||||
// grade-a has LOW findings → exit 1
|
|
||||||
assert.equal(code, 1, 'should exit 1 — low findings with --fail-on low');
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('CLI --fail-on overrides policy.json', async () => {
|
it('CLI --fail-on overrides policy.json', async () => {
|
||||||
mkdirSync(policyDir, { recursive: true });
|
writePolicy({ failOn: 'critical' });
|
||||||
writeFileSync(resolve(policyDir, 'policy.json'), JSON.stringify({
|
const { code } = await run(['.', '--fail-on', 'medium'], 120000, ownDir);
|
||||||
ci: { failOn: 'critical' },
|
assert.equal(code, 1, 'CLI --fail-on medium should override policy failOn: critical');
|
||||||
}));
|
});
|
||||||
// Policy says critical-only, but CLI says low — CLI wins
|
|
||||||
// We test by scanning the clean fixture with CLI --fail-on low
|
it('reads compact from policy.json ci section', async () => {
|
||||||
const { code } = await run([CLEAN, '--fail-on', 'low']);
|
writePolicy({ compact: true });
|
||||||
assert.equal(code, 1, 'CLI --fail-on low should override policy failOn: critical');
|
const { stdout } = await run(['.'], 120000, ownDir);
|
||||||
|
assert.ok(stdout.includes('Verdict:'), 'policy ci.compact: true should print the compact summary');
|
||||||
|
assert.throws(() => JSON.parse(stdout), 'compact output should not be the JSON envelope');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue