llm-security/tests/scanners/trigger-scanner.test.mjs

// trigger-scanner.test.mjs — Tests for the TRG trigger/activation-abuse scanner.
// Fixtures in tests/fixtures/trigger-scan/:
//   - clean/    : a scoped, specifically-described skill (0 findings expected)
//   - poisoned/ : a built-in-shadowing command (read), a broad+baiting skill (run),
//                 and an obfuscated-baiting agent (zero-width space inside "anything")

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { discoverFiles } from '../../scanners/lib/file-discovery.mjs';
import { scan } from '../../scanners/trigger-scanner.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const CLEAN_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/clean');
const POISONED_FIXTURE = resolve(__dirname, '../fixtures/trigger-scan/poisoned');

// ---------------------------------------------------------------------------
// Clean — scoped skill, no trigger abuse
// ---------------------------------------------------------------------------

describe('trigger-scanner: clean', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(CLEAN_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('scans at least one trigger file', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.ok(result.files_scanned >= 1, `Expected >= 1 file scanned, got ${result.files_scanned}`);
  });

  it('produces 0 findings for a scoped skill', async () => {
    const result = await scan(CLEAN_FIXTURE, discovery);
    assert.equal(
      result.findings.length, 0,
      `Expected 0 findings, got ${result.findings.length}: ${result.findings.map(f => f.title).join('; ')}`,
    );
  });
});

// ---------------------------------------------------------------------------
// Poisoned — shadow + baiting + broad + obfuscated baiting
// ---------------------------------------------------------------------------

describe('trigger-scanner: poisoned', () => {
  let discovery;

  beforeEach(async () => {
    resetCounter();
    discovery = await discoverFiles(POISONED_FIXTURE);
  });

  it('returns status ok', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.status, 'ok');
  });

  it('detects multiple trigger-abuse findings', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.ok(
      result.findings.length >= 3,
      `Expected >= 3 findings, got ${result.findings.length}: ${result.findings.map(f => `${f.title} [${f.severity}]`).join('; ')}`,
    );
  });

  it('all findings carry DS-TRG- ids and scanner TRG', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const wrong = result.findings.filter(f => !f.id.startsWith('DS-TRG-') || f.scanner !== 'TRG');
    assert.equal(wrong.length, 0, `Wrong id/scanner: ${wrong.map(f => `${f.id}/${f.scanner}`).join(', ')}`);
  });

  it('first finding id is DS-TRG-001', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    assert.equal(result.findings[0].id, 'DS-TRG-001');
  });

  it('fires TRG-shadow on a built-in name collision (read)', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const shadow = result.findings.filter(f => /shadow/i.test(f.title));
    assert.ok(shadow.length >= 1, `Expected a shadow finding, got: ${result.findings.map(f => f.title).join('; ')}`);
    assert.ok(shadow.some(f => f.file.includes('read.md')), 'shadow finding should reference read.md');
  });

  it('fires TRG-baiting on maximally-activating phrases', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const baiting = result.findings.filter(f => /baiting/i.test(f.title));
    assert.ok(baiting.length >= 1, `Expected a baiting finding, got: ${result.findings.map(f => f.title).join('; ')}`);
  });

  it('flags obfuscated baiting (zero-width space inside the phrase)', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const obf = result.findings.find(f => /baiting/i.test(f.title) && f.file.includes('obfuscated-bait.md'));
    assert.ok(obf, `Expected an obfuscated baiting finding on obfuscated-bait.md, got: ${result.findings.map(f => `${f.title}@${f.file}`).join('; ')}`);
  });

  it('escalates broad-name + universal-claim to HIGH', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const broad = result.findings.filter(f => /broad/i.test(f.title));
    assert.ok(broad.length >= 1, `Expected a broad-trigger finding, got: ${result.findings.map(f => f.title).join('; ')}`);
    assert.ok(broad.every(f => f.severity === 'high'), 'broad-trigger findings should be HIGH severity');
  });

  it('all findings have required fields', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    for (const f of result.findings) {
      assert.ok(f.id, 'missing id');
      assert.equal(f.scanner, 'TRG', `${f.id} scanner should be TRG`);
      assert.ok(f.severity, `${f.id} missing severity`);
      assert.ok(f.title, `${f.id} missing title`);
      assert.ok(f.description, `${f.id} missing description`);
      assert.ok(f.file, `${f.id} missing file`);
      assert.ok(f.owasp, `${f.id} missing owasp`);
      assert.ok(f.recommendation, `${f.id} missing recommendation`);
    }
  });

  it('severity counts sum to total findings', async () => {
    const result = await scan(POISONED_FIXTURE, discovery);
    const { counts } = result;
    const total = counts.critical + counts.high + counts.medium + counts.low + counts.info;
    assert.equal(total, result.findings.length);
  });
});

// ---------------------------------------------------------------------------
// Wiring — OWASP map + orchestrator registration (Step 2)
// ---------------------------------------------------------------------------

describe('trigger-scanner: OWASP map registration', () => {
  it('TRG is present in all four OWASP maps with the right codes', async () => {
    const { OWASP_MAP, OWASP_AGENTIC_MAP, OWASP_SKILLS_MAP, OWASP_MCP_MAP } =
      await import('../../scanners/lib/severity.mjs');
    assert.deepEqual(OWASP_MAP.TRG, ['LLM06']);
    assert.deepEqual(OWASP_AGENTIC_MAP.TRG, []);
    assert.deepEqual(OWASP_SKILLS_MAP.TRG, ['AST04']);
    assert.deepEqual(OWASP_MCP_MAP.TRG, []);
  });
});

describe('trigger-scanner: orchestrator registration', () => {
  it('scan-orchestrator imports and lists the TRG scanner', () => {
    const orchPath = resolve(__dirname, '../../scanners/scan-orchestrator.mjs');
    const text = readFileSync(orchPath, 'utf8');
    assert.match(text, /import\s+\{\s*scan as trgScan\s*\}\s+from\s+['"]\.\/trigger-scanner\.mjs['"]/);
    assert.match(text, /\{\s*name:\s*'trg',\s*fn:\s*trgScan\s*\}/);
  });
});

// ---------------------------------------------------------------------------
// Policy — overriding baiting_phrases changes detection
// ---------------------------------------------------------------------------

describe('trigger-scanner: policy override', () => {
  it('baiting_phrases from .llm-security/policy.json replaces the defaults', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'trg-policy-'));
    try {
      mkdirSync(join(dir, '.llm-security'), { recursive: true });
      writeFileSync(
        join(dir, '.llm-security', 'policy.json'),
        JSON.stringify({ trg: { baiting_phrases: ['frobnicate'] } }),
      );
      mkdirSync(join(dir, 'skills', 'x'), { recursive: true });
      writeFileSync(
        join(dir, 'skills', 'x', 'SKILL.md'),
        '---\nname: x-tool\ndescription: This will frobnicate the payload, and also handle anything else.\n---\n\n# x-tool\n',
      );
      resetCounter();
      const discovery = await discoverFiles(dir);
      const result = await scan(dir, discovery);
      const baiting = result.findings.filter(f => /baiting/i.test(f.title));
      assert.ok(baiting.length >= 1, 'the policy-provided phrase should trigger baiting');
      assert.ok(baiting.some(f => /frobnicate/i.test(f.evidence)), 'should match the overridden phrase');
      // The default phrase list (which includes "anything") was replaced, so it must not match.
      assert.ok(!baiting.some(f => /anything/i.test(f.evidence)), 'default phrases must be replaced, not merged');
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });
});