llm-security/tests/scanners/ci-integration.test.mjs

// ci-integration.test.mjs — Tests for --fail-on and --compact CI flags
import { describe, it, beforeEach, afterEach, after } from 'node:test';
import { spawn } from 'node:child_process';
import { strict as assert } from 'node:assert';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdirSync, writeFileSync, rmSync, readFileSync, existsSync, cpSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { materializeTree } from '../helpers/payload-trees.mjs';
import { mkOwnTreeDir } from '../helpers/own-tree.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ORCHESTRATOR = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
const CLI = resolve(__dirname, '../../bin/llm-security.mjs');
const poisonedTree = materializeTree('memory-scan/poisoned-project');
after(poisonedTree.cleanup);
const POISONED = poisonedTree.dir;
const CLEAN = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');

function run(args, timeout = 120000, cwd = undefined) {
  return new Promise((resolve) => {
    const chunks = [];
    const errChunks = [];
    const child = spawn('node', [ORCHESTRATOR, ...args], {
      cwd,
      timeout,
      stdio: ['ignore', 'pipe', 'pipe'],
    });
    child.stdout.on('data', (c) => chunks.push(c));
    child.stderr.on('data', (c) => errChunks.push(c));
    child.on('close', (code) => {
      resolve({
        code: code ?? 1,
        stdout: Buffer.concat(chunks).toString(),
        stderr: Buffer.concat(errChunks).toString(),
      });
    });
  });
}

describe('--fail-on flag', () => {
  it('exit 0 when --fail-on critical and no critical findings', async () => {
    const { code } = await run([CLEAN, '--fail-on', 'critical']);
    assert.equal(code, 0, 'should exit 0 — no critical findings in clean fixture');
  });

  it('exit 1 when --fail-on critical and critical findings exist', async () => {
    const { code } = await run([POISONED, '--fail-on', 'critical']);
    assert.equal(code, 1, 'should exit 1 — poisoned fixture has critical findings');
  });

  it('exit 1 when --fail-on high and high findings exist', async () => {
    const { code } = await run([POISONED, '--fail-on', 'high']);
    assert.equal(code, 1, 'should exit 1 — poisoned fixture has high findings');
  });

  it('exit 1 when --fail-on medium and medium findings exist', async () => {
    const { code } = await run([CLEAN, '--fail-on', 'medium']);
    // grade-a-project produces medium from taint/toxic-flow
    const { code: code2 } = await run([POISONED, '--fail-on', 'medium']);
    assert.equal(code2, 1, 'should exit 1 — poisoned fixture has medium+ findings');
  });

  it('preserves existing exit codes without --fail-on', async () => {
    const { code } = await run([POISONED]);
    // Poisoned project produces BLOCK verdict → exit 2
    assert.equal(code, 2, 'should exit 2 (BLOCK verdict) without --fail-on');
  });

  it('rejects invalid --fail-on value', async () => {
    const { code, stderr } = await run(['.', '--fail-on', 'invalid']);
    assert.equal(code, 1, 'should exit 1 for invalid severity');
    assert.ok(stderr.includes('--fail-on must be one of'), 'should print validation error');
  });
});

describe('--compact flag', () => {
  it('outputs one-liner format to stdout (not JSON)', async () => {
    const { stdout } = await run([POISONED, '--compact']);
    assert.ok(!stdout.includes('"scanners"'), 'should not contain JSON envelope key');
    assert.ok(stdout.includes('[CRITICAL]') || stdout.includes('[HIGH]'), 'should contain severity prefix');
    assert.ok(stdout.includes('---'), 'should contain summary separator');
    assert.ok(stdout.includes('Verdict:'), 'should contain verdict summary line');
  });

  it('writes full JSON to --output-file, compact aggregate to stdout', async () => {
    const tmpFile = resolve(tmpdir(), `llm-security-ci-test-${Date.now()}.json`);
    try {
      const { stdout } = await run([POISONED, '--compact', '--output-file', tmpFile]);
      assert.ok(existsSync(tmpFile), 'output file should exist');
      const content = JSON.parse(readFileSync(tmpFile, 'utf8'));
      assert.ok(content.scanners, 'file should contain full JSON with scanners key');
      const stdoutParsed = JSON.parse(stdout);
      assert.ok(stdoutParsed.aggregate, 'stdout should contain compact aggregate JSON');
    } finally {
      if (existsSync(tmpFile)) rmSync(tmpFile);
    }
  });

  it('with --output-file writes one-liner findings to stderr', async () => {
    const tmpFile = resolve(tmpdir(), `llm-security-ci-test-${Date.now()}.json`);
    try {
      const { stderr } = await run([POISONED, '--compact', '--output-file', tmpFile]);
      assert.ok(
        stderr.includes('[CRITICAL]') || stderr.includes('[HIGH]'),
        'stderr should contain one-liner findings in compact+output-file mode'
      );
    } finally {
      if (existsSync(tmpFile)) rmSync(tmpFile);
    }
  });
});

describe('--fail-on + --compact combined', () => {
  it('exit 0 with compact output when below threshold', async () => {
    const { code, stdout } = await run([CLEAN, '--fail-on', 'critical', '--compact']);
    assert.equal(code, 0, 'should exit 0 — no critical findings');
    assert.ok(stdout.includes('Verdict:'), 'should still show compact summary');
  });
});

// v8.1.1: the policy file is honored only for the caller's own working tree
// (scanners/lib/own-working-tree.mjs), so these run the orchestrator with the
// fixture as its cwd and '.' as its target. grade-a-project is WARNING with 0
// critical (default exit 1), so `ci.failOn: 'critical'` read from the policy
// is the only way to get exit 0 — the test cannot pass without reading it.
describe('--fail-on / --compact via policy.json (own working tree)', () => {
  let ownDir;

  function writePolicy(ci) {
    mkdirSync(join(ownDir, '.llm-security'), { recursive: true });
    writeFileSync(join(ownDir, '.llm-security', 'policy.json'), JSON.stringify({ ci }));
  }

  beforeEach(() => {
    ownDir = mkOwnTreeDir('ci-policy-own-');
    cpSync(CLEAN, ownDir, { recursive: true });
  });

  afterEach(() => {
    rmSync(ownDir, { recursive: true, force: true });
  });

  it('known-positive: without policy the fixture exits 1 (WARNING, no critical)', async () => {
    const { code, stdout } = await run(['.'], 120000, ownDir);
    const env = JSON.parse(stdout);
    assert.equal(env.aggregate.counts.critical, 0);
    assert.equal(code, 1, 'WARNING verdict without --fail-on should exit 1');
  });

  it('reads failOn from policy.json ci section', async () => {
    writePolicy({ failOn: 'critical' });
    const { code } = await run(['.'], 120000, ownDir);
    assert.equal(code, 0, 'policy ci.failOn: critical with no critical findings should exit 0');
  });

  it('CLI --fail-on overrides policy.json', async () => {
    writePolicy({ failOn: 'critical' });
    const { code } = await run(['.', '--fail-on', 'medium'], 120000, ownDir);
    assert.equal(code, 1, 'CLI --fail-on medium should override policy failOn: critical');
  });

  it('reads compact from policy.json ci section', async () => {
    writePolicy({ compact: true });
    const { stdout } = await run(['.'], 120000, ownDir);
    assert.ok(stdout.includes('Verdict:'), 'policy ci.compact: true should print the compact summary');
    assert.throws(() => JSON.parse(stdout), 'compact output should not be the JSON envelope');
  });
});