Kjell Tore Guttormsen
b3c47330e1
v8.1.0 S2. examples/malicious-skill-demo/evil-project-health/ (7 files, 30 Unicode Tag chars, a base64 exfil blob) is now one archive, evil-project-health.archive.json: rot13 text, every codepoint above U+007E stored as a number, sha256 of each retired file recorded. materialize.mjs writes it to a temp dir (CLI prints the path); run-demo.sh materializes and deletes it itself; the six scanner tests that scanned the tree use it. payload-trees.test.mjs asserts byte identity (mutation-checked). av-surface: b 8->6, c 1->0, d 2->1. Demo 13/13 before and after. All scanners report identical findings except git-forensics: it used to scan this repository's own history (21 findings, none about the demo) and now reports skipped in a temp dir, which git.test.mjs already accepts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
128 lines
3.7 KiB
Bash
Executable file
128 lines
3.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# Malicious Skill Demo - Regression test for the deep-scan pipeline
|
|
# NOTE: Unix/macOS only. Requires bash. Not available on Windows without WSL.
|
|
#
|
|
# Usage:
|
|
# cd plugins/llm-security
|
|
# ./examples/malicious-skill-demo/run-demo.sh
|
|
#
|
|
# Expected: BLOCK verdict, 40+ findings across 7 scanners, exit code 2
|
|
#
|
|
# The demo plugin is not stored on disk as files (v8.1.0: antivirus products
|
|
# quarantine it). It lives encoded in evil-project-health.archive.json and is
|
|
# materialized into a temp dir for the run, then deleted.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PLUGIN_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
OUTPUT_FILE="$(mktemp)"
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
TARGET=""
|
|
cleanup() {
|
|
rm -f "$OUTPUT_FILE"
|
|
if [ -n "$TARGET" ]; then rm -rf "$(dirname "$TARGET")"; fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
TARGET="$(node "$SCRIPT_DIR/materialize.mjs")"
|
|
|
|
assert() {
|
|
local desc="$1" result="$2"
|
|
if [ "$result" -eq 0 ]; then
|
|
echo " PASS: $desc"
|
|
PASS=$((PASS + 1))
|
|
else
|
|
echo " FAIL: $desc"
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
echo "=== LLM Security Deep-Scan Regression Test ==="
|
|
echo ""
|
|
echo "Target: $TARGET"
|
|
echo "Scanners: $PLUGIN_DIR/scanners/scan-orchestrator.mjs"
|
|
echo ""
|
|
|
|
if [ ! -d "$TARGET" ]; then
|
|
echo "ERROR: Target directory not found: $TARGET"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "$PLUGIN_DIR/scanners/scan-orchestrator.mjs" ]; then
|
|
echo "ERROR: Scan orchestrator not found. Run from the llm-security plugin root."
|
|
exit 1
|
|
fi
|
|
|
|
echo "Running 7 deterministic scanners..."
|
|
echo ""
|
|
|
|
# Run scan, capture exit code
|
|
node "$PLUGIN_DIR/scanners/scan-orchestrator.mjs" "$TARGET" --output-file "$OUTPUT_FILE" 2>/dev/null
|
|
SCAN_EXIT=$?
|
|
|
|
echo ""
|
|
echo "--- Assertions ---"
|
|
|
|
# 1. Exit code should be 2 (BLOCK)
|
|
[ "$SCAN_EXIT" -eq 2 ]
|
|
assert "Exit code is 2 (BLOCK)" $?
|
|
|
|
# 2. Output file exists and has content
|
|
[ -s "$OUTPUT_FILE" ]
|
|
assert "Output file has content" $?
|
|
|
|
# 3. Parse JSON and check verdict
|
|
VERDICT=$(node -e "const d=JSON.parse(require('fs').readFileSync('$OUTPUT_FILE','utf-8')); console.log(d.aggregate.verdict)" 2>/dev/null)
|
|
[ "$VERDICT" = "BLOCK" ]
|
|
assert "Verdict is BLOCK" $?
|
|
|
|
# 4. Total findings >= 40
|
|
TOTAL=$(node -e "const d=JSON.parse(require('fs').readFileSync('$OUTPUT_FILE','utf-8')); console.log(d.aggregate.total_findings)" 2>/dev/null)
|
|
[ "$TOTAL" -ge 40 ]
|
|
assert "Total findings >= 40 (got: $TOTAL)" $?
|
|
|
|
# 5. Risk score >= 90
|
|
SCORE=$(node -e "const d=JSON.parse(require('fs').readFileSync('$OUTPUT_FILE','utf-8')); console.log(d.aggregate.risk_score)" 2>/dev/null)
|
|
[ "$SCORE" -ge 90 ]
|
|
assert "Risk score >= 90 (got: $SCORE)" $?
|
|
|
|
# 6. All 7 scanner prefixes present
|
|
for PREFIX in UNI ENT PRM DEP TNT GIT NET; do
|
|
HAS=$(node -e "
|
|
const d=JSON.parse(require('fs').readFileSync('$OUTPUT_FILE','utf-8'));
|
|
const has = Object.values(d.scanners).some(s => s.scanner && s.scanner.toLowerCase().includes('${PREFIX}'.toLowerCase()));
|
|
if (!has) {
|
|
// Check findings for the prefix
|
|
const inFindings = Object.values(d.scanners).some(s => s.findings && s.findings.some(f => f.id && f.id.startsWith('DS-${PREFIX}-')));
|
|
console.log(inFindings);
|
|
} else {
|
|
console.log(has);
|
|
}
|
|
" 2>/dev/null)
|
|
[ "$HAS" = "true" ]
|
|
assert "Scanner $PREFIX present in output" $?
|
|
done
|
|
|
|
# 7. At least one CRITICAL finding
|
|
CRITS=$(node -e "const d=JSON.parse(require('fs').readFileSync('$OUTPUT_FILE','utf-8')); console.log(d.aggregate.counts.critical)" 2>/dev/null)
|
|
[ "$CRITS" -ge 1 ]
|
|
assert "At least 1 CRITICAL finding (got: $CRITS)" $?
|
|
|
|
echo ""
|
|
echo "--- Results ---"
|
|
echo " Passed: $PASS"
|
|
echo " Failed: $FAIL"
|
|
echo " Total: $((PASS + FAIL))"
|
|
echo ""
|
|
|
|
if [ "$FAIL" -eq 0 ]; then
|
|
echo "=== ALL ASSERTIONS PASSED ==="
|
|
exit 0
|
|
else
|
|
echo "=== $FAIL ASSERTION(S) FAILED ==="
|
|
exit 1
|
|
fi
|