llm-security/tests/scanners/trigger-scanner.test.mjs

// trigger-scanner.test.mjs — Tests for the TRG trigger/activation-abuse scanner.
// Fixtures in tests/fixtures/trigger-scan/:
//   - clean/    : a scoped, specifically-described skill (0 findings expected)
//   - poisoned/ : a built-in-shadowing command (read), a broad+baiting skill (run),
//                 and an obfuscated-baiting agent (zero-width space inside "anything")

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/trigger-scanner.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/clean');
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/poisoned');

// ---------------------------------------------------------------------------
// Clean — scoped skill, no trigger abuse
// ---------------------------------------------------------------------------

describe('trigger-scanner: clean', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(CLEAN_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('scans at least one trigger file', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.ok(result.files_scanned >= 1, `Expected >= 1 file scanned, got ${result.files_scanned}`);
  });

  it('produces 0 findings for a scoped skill', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(
      result.findings.length, 0,
      `Expected 0 findings, got ${result.findings.length}: ${result.findings.map(f => f.title).join('; ')}`,
    );
  });
});

// ---------------------------------------------------------------------------
// Poisoned — shadow + baiting + broad + obfuscated baiting
// ---------------------------------------------------------------------------

describe('trigger-scanner: poisoned', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(POISONED_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('detects multiple trigger-abuse findings', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.ok(
      result.findings.length >= 3,
      `Expected >= 3 findings, got ${result.findings.length}: ${result.findings.map(f => `${f.title} [${f.severity}]`).join('; ')}`,
    );
  });

  it('all findings carry DS-TRG- ids and scanner TRG', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const wrong = result.findings.filter(f => !f.id.startsWith('DS-TRG-') || f.scanner !== 'TRG');
    assert.equal(wrong.length, 0, `Wrong id/scanner: ${wrong.map(f => `${f.id}/${f.scanner}`).join(', ')}`);
  });

  it('first finding id is DS-TRG-001', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.findings[0].id, 'DS-TRG-001');
  });

  it('fires TRG-shadow on a built-in name collision (read)', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const shadow = result.findings.filter(f => /shadow/i.test(f.title));
    assert.ok(shadow.length >= 1, `Expected a shadow finding, got: ${result.findings.map(f => f.title).join('; ')}`);
    assert.ok(shadow.some(f => f.file.includes('read.md')), 'shadow finding should reference read.md');
  });

  it('fires TRG-baiting on maximally-activating phrases', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const baiting = result.findings.filter(f => /baiting/i.test(f.title));
    assert.ok(baiting.length >= 1, `Expected a baiting finding, got: ${result.findings.map(f => f.title).join('; ')}`);
  });

  it('flags obfuscated baiting (zero-width space inside the phrase)', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const obf = result.findings.find(f => /baiting/i.test(f.title) && f.file.includes('obfuscated-bait.md'));
    assert.ok(obf, `Expected an obfuscated baiting finding on obfuscated-bait.md, got: ${result.findings.map(f => `${f.title}@${f.file}`).join('; ')}`);
  });

  it('escalates broad-name + universal-claim to HIGH', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const broad = result.findings.filter(f => /broad/i.test(f.title));
    assert.ok(broad.length >= 1, `Expected a broad-trigger finding, got: ${result.findings.map(f => f.title).join('; ')}`);
    assert.ok(broad.every(f => f.severity === 'high'), 'broad-trigger findings should be HIGH severity');
  });

  it('all findings have required fields', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    for (const f of result.findings) {
      assert.ok(f.id, 'missing id');
      assert.equal(f.scanner, 'TRG', `${f.id} scanner should be TRG`);
      assert.ok(f.severity, `${f.id} missing severity`);
      assert.ok(f.title, `${f.id} missing title`);
      assert.ok(f.description, `${f.id} missing description`);
      assert.ok(f.file, `${f.id} missing file`);
      assert.ok(f.owasp, `${f.id} missing owasp`);
      assert.ok(f.recommendation, `${f.id} missing recommendation`);
    }
  });

  it('severity counts sum to total findings', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const { counts } = result;
    const total = counts.critical + counts.high + counts.medium + counts.low + counts.info;
    assert.equal(total, result.findings.length);
  });
});