llm-security/tests/scanners/signature-scanner-custom-rules.test.mjs

// signature-scanner-custom-rules.test.mjs — Regression for #36 (LOW, v7.8.3).
//
// The scanner hardcoded knowledge/signatures.json and never read the documented
// `sig.custom_rules_path` policy option (while it DID read the sibling
// `enabled_families`), so operators could not supply custom signatures despite
// the policy-loader default advertising the key. Custom rules supplied via
// policy.json must be loaded and merged; a missing/invalid file must fail
// gracefully (built-in ruleset still applies, status stays ok).

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { join } from 'node:path';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/signature-scanner.mjs';

/** Write a policy.json under dir/.llm-security. */
function writePolicy(dir, policy) {
  mkdirSync(join(dir, '.llm-security'), { recursive: true });
  writeFileSync(join(dir, '.llm-security', 'policy.json'), JSON.stringify(policy));
}

describe('signature-scanner: custom_rules_path (#36)', () => {
  it('loads and applies custom rules supplied via policy', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'custom-sigs.json' } });
      writeFileSync(join(dir, 'custom-sigs.json'), JSON.stringify({
        rules: [{
          id: 'CUSTOM-WS-001',
          family: 'webshell',
          severity: 'high',
          pattern: 'EVILCUSTOMMARKER_[0-9]+',
          description: 'Operator-supplied custom webshell marker',
        }],
      }));
      writeFileSync(join(dir, 'payload.txt'), 'prefix EVILCUSTOMMARKER_42 suffix\n');

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const custom = result.findings.find(f => f.evidence && f.evidence.includes('CUSTOM-WS-001'));
      assert.ok(
        custom,
        `expected the custom rule CUSTOM-WS-001 to fire, got: ${result.findings.map(f => f.evidence).join('; ') || '(none)'}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('custom rules merge with (not replace) the built-in ruleset', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'custom-sigs.json' } });
      writeFileSync(join(dir, 'custom-sigs.json'), JSON.stringify({
        rules: [{
          id: 'CUSTOM-WS-002',
          family: 'webshell',
          severity: 'high',
          pattern: 'EVILCUSTOMMARKER_[0-9]+',
          description: 'Operator-supplied custom webshell marker',
        }],
      }));
      // A built-in webshell signature target
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(
        builtin,
        `built-in webshell signature should still fire alongside custom rules, got: ${result.findings.map(f => f.file).join('; ') || '(none)'}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fails gracefully when custom_rules_path points at a missing file', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'does-not-exist.json' } });
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok', 'missing custom ruleset must not error the scan');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(builtin, 'built-in ruleset should still apply when custom file is missing');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fails gracefully when the custom ruleset is invalid JSON', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'broken.json' } });
      writeFileSync(join(dir, 'broken.json'), '{ not json');
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok', 'invalid custom ruleset must not error the scan');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(builtin, 'built-in ruleset should still apply when custom file is invalid');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});