llm-security/tests/scanners/signature-scanner-custom-rules.test.mjs

// signature-scanner-custom-rules.test.mjs — Regression for #36 (LOW, v7.8.3).
//
// The scanner hardcoded knowledge/signatures.json and never read the documented
// `sig.custom_rules_path` policy option (while it DID read the sibling
// `enabled_families`), so operators could not supply custom signatures despite
// the policy-loader default advertising the key. Custom rules supplied via
// policy.json must be loaded and merged; a missing/invalid file must fail
// gracefully (built-in ruleset still applies, status stays ok).

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { join } from 'node:path';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/signature-scanner.mjs';

/** Write a policy.json under dir/.llm-security. */
function writePolicy(dir, policy) {
  mkdirSync(join(dir, '.llm-security'), { recursive: true });
  writeFileSync(join(dir, '.llm-security', 'policy.json'), JSON.stringify(policy));
}

describe('signature-scanner: custom_rules_path (#36)', () => {
  it('loads and applies custom rules supplied via policy', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'custom-sigs.json' } });
      writeFileSync(join(dir, 'custom-sigs.json'), JSON.stringify({
        rules: [{
          id: 'CUSTOM-WS-001',
          family: 'webshell',
          severity: 'high',
          pattern: 'EVILCUSTOMMARKER_[0-9]+',
          description: 'Operator-supplied custom webshell marker',
        }],
      }));
      writeFileSync(join(dir, 'payload.txt'), 'prefix EVILCUSTOMMARKER_42 suffix\n');

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const custom = result.findings.find(f => f.evidence && f.evidence.includes('CUSTOM-WS-001'));
      assert.ok(
        custom,
        `expected the custom rule CUSTOM-WS-001 to fire, got: ${result.findings.map(f => f.evidence).join('; ') || '(none)'}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('custom rules merge with (not replace) the built-in ruleset', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'custom-sigs.json' } });
      writeFileSync(join(dir, 'custom-sigs.json'), JSON.stringify({
        rules: [{
          id: 'CUSTOM-WS-002',
          family: 'webshell',
          severity: 'high',
          pattern: 'EVILCUSTOMMARKER_[0-9]+',
          description: 'Operator-supplied custom webshell marker',
        }],
      }));
      // A built-in webshell signature target
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(
        builtin,
        `built-in webshell signature should still fire alongside custom rules, got: ${result.findings.map(f => f.file).join('; ') || '(none)'}`,
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fails gracefully when custom_rules_path points at a missing file', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'does-not-exist.json' } });
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok', 'missing custom ruleset must not error the scan');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(builtin, 'built-in ruleset should still apply when custom file is missing');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('drops a custom rule whose pattern is not a string, instead of coercing it', async () => {
    // Same coercion bug as tests/lib/malware-signatures.test.mjs, exercised
    // through the operator-facing path: JSON.parse of a JSON object pattern
    // yields a JS object, and `new RegExp(pattern, 'i')` does not throw on it —
    // it stringifies via ToString to the literal "[object Object]" and
    // compiles THAT as a regex. The leading/trailing brackets make it a
    // character class, not a literal match: `[object Object]` matches any
    // single occurrence of o/b/j/e/c/t/space, which is nearly every file. So
    // the rule must be dropped by compileRules's type check, not left for the
    // compile try/catch, which never sees an error here.
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'custom-sigs.json' } });
      writeFileSync(join(dir, 'custom-sigs.json'), JSON.stringify({
        rules: [{
          id: 'CUSTOM-OBJ-001',
          family: 'webshell',
          severity: 'high',
          pattern: { source: 'EVILCUSTOMMARKER_[0-9]+' },
          description: 'Malformed: pattern is an object, not a string',
        }],
      }));

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok');
      const bad = result.findings.find(f => f.evidence && f.evidence.includes('CUSTOM-OBJ-001'));
      assert.equal(
        bad, undefined,
        'a rule with a non-string pattern must be dropped, not coerced into a matching regex',
      );
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  it('fails gracefully when the custom ruleset is invalid JSON', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
    try {
      writePolicy(dir, { sig: { custom_rules_path: 'broken.json' } });
      writeFileSync(join(dir, 'broken.json'), '{ not json');
      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");

      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      assert.equal(result.status, 'ok', 'invalid custom ruleset must not error the scan');
      const builtin = result.findings.find(f => f.file === 'shell.php');
      assert.ok(builtin, 'built-in ruleset should still apply when custom file is invalid');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});