// trigger-scanner.test.mjs — Tests for the TRG trigger/activation-abuse scanner. // Fixtures in tests/fixtures/trigger-scan/: // - clean/ : a scoped, specifically-described skill (0 findings expected) // - poisoned/ : a built-in-shadowing command (read), a broad+baiting skill (run), // and an obfuscated-baiting agent (zero-width space inside "anything") import { describe, it, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; import { resolve, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { resetCounter } from '../../scanners/lib/output.mjs'; import { discoverFiles } from '../../scanners/lib/file-discovery.mjs'; import { scan } from '../../scanners/trigger-scanner.mjs'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/clean'); const POISONED_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/poisoned'); // --------------------------------------------------------------------------- // Clean — scoped skill, no trigger abuse // --------------------------------------------------------------------------- describe('trigger-scanner: clean', () => { let discovery; beforeEach(async () => { resetCounter(); discovery = await discoverFiles(CLEAN_FIXTURE); }); it('returns status ok', async () => { const result = await scan(CLEAN_FIXTURE, discovery); assert.equal(result.status, 'ok'); }); it('scans at least one trigger file', async () => { const result = await scan(CLEAN_FIXTURE, discovery); assert.ok(result.files_scanned >= 1, `Expected >= 1 file scanned, got ${result.files_scanned}`); }); it('produces 0 findings for a scoped skill', async () => { const result = await scan(CLEAN_FIXTURE, discovery); assert.equal( result.findings.length, 0, `Expected 0 findings, got ${result.findings.length}: ${result.findings.map(f => f.title).join('; ')}`, ); }); }); // --------------------------------------------------------------------------- // Poisoned — shadow + baiting + broad + obfuscated baiting // --------------------------------------------------------------------------- describe('trigger-scanner: poisoned', () => { let discovery; beforeEach(async () => { resetCounter(); discovery = await discoverFiles(POISONED_FIXTURE); }); it('returns status ok', async () => { const result = await scan(POISONED_FIXTURE, discovery); assert.equal(result.status, 'ok'); }); it('detects multiple trigger-abuse findings', async () => { const result = await scan(POISONED_FIXTURE, discovery); assert.ok( result.findings.length >= 3, `Expected >= 3 findings, got ${result.findings.length}: ${result.findings.map(f => `${f.title} [${f.severity}]`).join('; ')}`, ); }); it('all findings carry DS-TRG- ids and scanner TRG', async () => { const result = await scan(POISONED_FIXTURE, discovery); const wrong = result.findings.filter(f => !f.id.startsWith('DS-TRG-') || f.scanner !== 'TRG'); assert.equal(wrong.length, 0, `Wrong id/scanner: ${wrong.map(f => `${f.id}/${f.scanner}`).join(', ')}`); }); it('first finding id is DS-TRG-001', async () => { const result = await scan(POISONED_FIXTURE, discovery); assert.equal(result.findings[0].id, 'DS-TRG-001'); }); it('fires TRG-shadow on a built-in name collision (read)', async () => { const result = await scan(POISONED_FIXTURE, discovery); const shadow = result.findings.filter(f => /shadow/i.test(f.title)); assert.ok(shadow.length >= 1, `Expected a shadow finding, got: ${result.findings.map(f => f.title).join('; ')}`); assert.ok(shadow.some(f => f.file.includes('read.md')), 'shadow finding should reference read.md'); }); it('fires TRG-baiting on maximally-activating phrases', async () => { const result = await scan(POISONED_FIXTURE, discovery); const baiting = result.findings.filter(f => /baiting/i.test(f.title)); assert.ok(baiting.length >= 1, `Expected a baiting finding, got: ${result.findings.map(f => f.title).join('; ')}`); }); it('flags obfuscated baiting (zero-width space inside the phrase)', async () => { const result = await scan(POISONED_FIXTURE, discovery); const obf = result.findings.find(f => /baiting/i.test(f.title) && f.file.includes('obfuscated-bait.md')); assert.ok(obf, `Expected an obfuscated baiting finding on obfuscated-bait.md, got: ${result.findings.map(f => `${f.title}@${f.file}`).join('; ')}`); }); it('escalates broad-name + universal-claim to HIGH', async () => { const result = await scan(POISONED_FIXTURE, discovery); const broad = result.findings.filter(f => /broad/i.test(f.title)); assert.ok(broad.length >= 1, `Expected a broad-trigger finding, got: ${result.findings.map(f => f.title).join('; ')}`); assert.ok(broad.every(f => f.severity === 'high'), 'broad-trigger findings should be HIGH severity'); }); it('all findings have required fields', async () => { const result = await scan(POISONED_FIXTURE, discovery); for (const f of result.findings) { assert.ok(f.id, 'missing id'); assert.equal(f.scanner, 'TRG', `${f.id} scanner should be TRG`); assert.ok(f.severity, `${f.id} missing severity`); assert.ok(f.title, `${f.id} missing title`); assert.ok(f.description, `${f.id} missing description`); assert.ok(f.file, `${f.id} missing file`); assert.ok(f.owasp, `${f.id} missing owasp`); assert.ok(f.recommendation, `${f.id} missing recommendation`); } }); it('severity counts sum to total findings', async () => { const result = await scan(POISONED_FIXTURE, discovery); const { counts } = result; const total = counts.critical + counts.high + counts.medium + counts.low + counts.info; assert.equal(total, result.findings.length); }); }); // --------------------------------------------------------------------------- // Wiring — OWASP map + orchestrator registration (Step 2) // --------------------------------------------------------------------------- describe('trigger-scanner: OWASP map registration', () => { it('TRG is present in all four OWASP maps with the right codes', async () => { const { OWASP_MAP, OWASP_AGENTIC_MAP, OWASP_SKILLS_MAP, OWASP_MCP_MAP } = await import('../../scanners/lib/severity.mjs'); assert.deepEqual(OWASP_MAP.TRG, ['LLM06']); assert.deepEqual(OWASP_AGENTIC_MAP.TRG, []); assert.deepEqual(OWASP_SKILLS_MAP.TRG, ['AST04']); assert.deepEqual(OWASP_MCP_MAP.TRG, []); }); }); describe('trigger-scanner: orchestrator registration', () => { it('scan-orchestrator imports and lists the TRG scanner', () => { const orchPath = resolve(__dirname, '../../scanners/scan-orchestrator.mjs'); const text = readFileSync(orchPath, 'utf8'); assert.match(text, /import\s+\{\s*scan as trgScan\s*\}\s+from\s+['"]\.\/trigger-scanner\.mjs['"]/); assert.match(text, /\{\s*name:\s*'trg',\s*fn:\s*trgScan\s*\}/); }); }); // --------------------------------------------------------------------------- // Policy — overriding baiting_phrases changes detection // --------------------------------------------------------------------------- describe('trigger-scanner: policy override', () => { it('baiting_phrases from .llm-security/policy.json replaces the defaults', async () => { const dir = mkdtempSync(join(tmpdir(), 'trg-policy-')); try { mkdirSync(join(dir, '.llm-security'), { recursive: true }); writeFileSync( join(dir, '.llm-security', 'policy.json'), JSON.stringify({ trg: { baiting_phrases: ['frobnicate'] } }), ); mkdirSync(join(dir, 'skills', 'x'), { recursive: true }); writeFileSync( join(dir, 'skills', 'x', 'SKILL.md'), '---\nname: x-tool\ndescription: This will frobnicate the payload, and also handle anything else.\n---\n\n# x-tool\n', ); resetCounter(); const discovery = await discoverFiles(dir); const result = await scan(dir, discovery); const baiting = result.findings.filter(f => /baiting/i.test(f.title)); assert.ok(baiting.length >= 1, 'the policy-provided phrase should trigger baiting'); assert.ok(baiting.some(f => /frobnicate/i.test(f.evidence)), 'should match the overridden phrase'); // The default phrase list (which includes "anything") was replaced, so it must not match. assert.ok(!baiting.some(f => /anything/i.test(f.evidence)), 'default phrases must be replaced, not merged'); } finally { rmSync(dir, { recursive: true, force: true }); } }); });