open/llm-security
1
0
Fork 0
Code Issues Releases 6 Activity

test(llm-security): store the malicious-skill demo encoded, materialize at run time

Browse source 
v8.1.0 S2. examples/malicious-skill-demo/evil-project-health/ (7 files,
30 Unicode Tag chars, a base64 exfil blob) is now one archive,
evil-project-health.archive.json: rot13 text, every codepoint above U+007E
stored as a number, sha256 of each retired file recorded. materialize.mjs
writes it to a temp dir (CLI prints the path); run-demo.sh materializes
and deletes it itself; the six scanner tests that scanned the tree use it.
payload-trees.test.mjs asserts byte identity (mutation-checked).

av-surface: b 8->6, c 1->0, d 2->1. Demo 13/13 before and after. All
scanners report identical findings except git-forensics: it used to scan
this repository's own history (21 findings, none about the demo) and now
reports skipped in a temp dir, which git.test.mjs already accepts.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-22 13:21:22 +02:00
commit b3c47330e1
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
18 changed files with 639 additions and 409 deletions
Download patch file Download diff file Expand all files Collapse all files

10
tests/scanners/entropy.test.mjs
View file

@ -3,16 +3,16 @@
// - ENCODED_CONFIG: base64 blob in SKILL.fixture.md
// - auth_credential: high-entropy hardcoded credential in telemetry.mjs
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/entropy-scanner.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
describe('entropy-scanner integration', () => {
let discovery;

7
tests/scanners/git.test.mjs
View file

@ -10,15 +10,18 @@
// - Correct structure of the scanner result envelope
// - All findings (if any) have the DS-GIT- prefix
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/git-forensics.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
// The plugin root — may or may not be a standalone git repo
const PLUGIN_ROOT = resolve(__dirname, '../..');

10
tests/scanners/network.test.mjs
View file

@ -12,16 +12,16 @@
// We do NOT assert on DNS resolution — it is network-dependent.
// Only URL pattern detection (Phase 1–2 of the scanner) is tested.
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/network-mapper.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
describe('network-mapper integration', () => {
let discovery;

9
tests/scanners/permission.test.mjs
View file

@ -8,15 +8,16 @@
//
// So the fixture IS detected as a plugin and the scanner should return status 'ok'.
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/permission-mapper.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
describe('permission-mapper integration', () => {
beforeEach(() => {

10
tests/scanners/taint.test.mjs
View file

@ -8,16 +8,16 @@
//
// The taint-tracer uses heuristic analysis (~70% recall), so we require >= 3 detections.
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/taint-tracer.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
describe('taint-tracer integration', () => {
let discovery;

10
tests/scanners/unicode.test.mjs
View file

@ -4,16 +4,16 @@
// - Unicode Tag block codepoints (steganographic hidden message) in SKILL.fixture.md
// - BIDI override characters in SKILL.fixture.md
import { describe, it, beforeEach } from 'node:test';
import { describe, it, beforeEach, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/unicode-scanner.mjs';
import { materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
const __dirname = fileURLToPath(new URL('.', import.meta.url));
const FIXTURE = resolve(__dirname, '../../examples/malicious-skill-demo/evil-project-health');
// v8.1.0 (S2): the demo tree is stored encoded and materialized at test time.
const { dir: FIXTURE, cleanup } = materializeArchive();
after(cleanup);
describe('unicode-scanner integration', () => {
let discovery;