open/llm-security
1
0
Fork 0
Code Issues Releases 6 Activity

test(llm-security): store the malicious-skill demo encoded, materialize at run time

Browse source 
v8.1.0 S2. examples/malicious-skill-demo/evil-project-health/ (7 files,
30 Unicode Tag chars, a base64 exfil blob) is now one archive,
evil-project-health.archive.json: rot13 text, every codepoint above U+007E
stored as a number, sha256 of each retired file recorded. materialize.mjs
writes it to a temp dir (CLI prints the path); run-demo.sh materializes
and deletes it itself; the six scanner tests that scanned the tree use it.
payload-trees.test.mjs asserts byte identity (mutation-checked).

av-surface: b 8->6, c 1->0, d 2->1. Demo 13/13 before and after. All
scanners report identical findings except git-forensics: it used to scan
this repository's own history (21 findings, none about the demo) and now
reports skipped in a temp dir, which git.test.mjs already accepts.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-22 13:21:22 +02:00
commit b3c47330e1
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
18 changed files with 639 additions and 409 deletions
Download patch file Download diff file Expand all files Collapse all files

33
tests/helpers/payload-trees.test.mjs
View file

@ -1,12 +1,14 @@
// payload-trees.test.mjs — the materialized trees are byte-identical to the
// fixtures that used to sit on disk (sha256 measured before deletion, S1).
// fixtures that used to sit on disk (sha256 measured before deletion, S1),
// and so are the encoded example archives (S2).
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, readdirSync } from 'node:fs';
import { join, relative } from 'node:path';
import { join, relative, basename } from 'node:path';
import { createHash } from 'node:crypto';
import { PAYLOAD_TREES, materializeTree } from './payload-trees.mjs';
import { DEMO_ARCHIVE, readArchive, materializeArchive } from '../../examples/malicious-skill-demo/materialize.mjs';
function walk(dir) {
return readdirSync(dir, { withFileTypes: true, recursive: true })
@ -31,3 +33,30 @@ describe('payload-trees: materialized bytes match the retired on-disk fixtures',
});
}
});
// S2 (2026-09-22): the example trees are encoded archives next to their
// examples; same check, sha256 measured from the retired on-disk files.
const EXAMPLE_ARCHIVES = {
'malicious-skill-demo/evil-project-health': DEMO_ARCHIVE,
};
describe('example archives: materialized bytes match the retired on-disk trees', () => {
for (const [name, archivePath] of Object.entries(EXAMPLE_ARCHIVES)) {
it(name, () => {
const { files } = readArchive(archivePath);
const { dir, cleanup } = materializeArchive(archivePath);
try {
assert.equal(basename(dir), basename(name), 'leaf directory name changed');
const written = walk(dir).map(f => relative(dir, f)).sort();
assert.deepEqual(written, Object.keys(files).sort(), 'file set changed');
for (const [rel, { sha256 }] of Object.entries(files)) {
assert.match(sha256, /^[0-9a-f]{64}$/, `${name}/${rel} has no recorded sha256`);
const got = createHash('sha256').update(readFileSync(join(dir, rel))).digest('hex');
assert.equal(got, sha256, `${name}/${rel} differs from the retired fixture`);
}
} finally {
cleanup();
}
});
}
});