ktg-plugin-marketplace/plugins/llm-security/tests/scanners/posture.test.mjs

// posture.test.mjs — Tests for the deterministic posture scanner
// Tests against fixtures in tests/fixtures/posture-scan/ with:
//   - grade-a-project: full security config (hooks, settings, CLAUDE.md) → Grade A
//   - grade-f-project: dangerous flags, poisoned memory, no hooks → Grade F

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { resetCounter } from '../../scanners/lib/output.mjs';
import { scan } from '../../scanners/posture-scanner.mjs';

const __dirname = fileURLToPath(new URL('.', import.meta.url));
const GRADE_A_FIXTURE = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');
const GRADE_F_FIXTURE = resolve(__dirname, '../fixtures/posture-scan/grade-f-project');

// ---------------------------------------------------------------------------
// Grade A project — well-configured, should get Grade A
// ---------------------------------------------------------------------------

describe('posture-scanner: grade-a-project', () => {
  let result;

  beforeEach(async () => {
    resetCounter();
    result = await scan(GRADE_A_FIXTURE);
  });

  it('returns status ok', () => {
    assert.equal(result.status, 'ok');
  });

  it('has scanner name', () => {
    assert.equal(result.scanner, 'posture-scanner');
  });

  it('has version', () => {
    assert.ok(result.version, 'Expected version string');
  });

  it('produces Grade A', () => {
    assert.equal(result.scoring.grade, 'A');
  });

  it('has 16 categories assessed', () => {
    assert.equal(result.categories.length, 16);
  });

  it('has low risk score', () => {
    assert.ok(result.risk.score <= 25, `Expected risk score <= 25, got ${result.risk.score}`);
  });

  it('verdict is ALLOW or WARNING', () => {
    assert.ok(
      result.risk.verdict === 'ALLOW' || result.risk.verdict === 'WARNING',
      `Expected ALLOW or WARNING, got ${result.risk.verdict}`,
    );
  });

  it('deny-first configuration is PASS', () => {
    const cat = result.categories.find(c => c.id === 1);
    assert.equal(cat.status, 'PASS');
  });

  it('secrets protection is PASS', () => {
    const cat = result.categories.find(c => c.id === 2);
    assert.equal(cat.status, 'PASS');
  });

  it('path guarding is PASS', () => {
    const cat = result.categories.find(c => c.id === 3);
    assert.equal(cat.status, 'PASS');
  });

  it('MCP trust is N/A (no MCP servers)', () => {
    const cat = result.categories.find(c => c.id === 4);
    assert.equal(cat.status, 'N_A');
  });

  it('destructive blocking is PASS', () => {
    const cat = result.categories.find(c => c.id === 5);
    assert.equal(cat.status, 'PASS');
  });

  it('sandbox configuration is PASS', () => {
    const cat = result.categories.find(c => c.id === 6);
    assert.equal(cat.status, 'PASS');
  });

  it('cognitive state security is PASS', () => {
    const cat = result.categories.find(c => c.id === 10);
    assert.equal(cat.status, 'PASS');
  });

  it('zero critical findings', () => {
    assert.equal(result.counts.critical, 0);
  });

  it('pass rate >= 0.89', () => {
    assert.ok(result.scoring.pass_rate >= 0.89, `Expected pass_rate >= 0.89, got ${result.scoring.pass_rate}`);
  });

  it('duration_ms is a non-negative number', () => {
    assert.ok(typeof result.duration_ms === 'number');
    assert.ok(result.duration_ms >= 0);
  });

  it('timestamp is ISO format', () => {
    assert.ok(result.timestamp.match(/^\d{4}-\d{2}-\d{2}T/), 'Expected ISO timestamp');
  });

  it('each category has required fields', () => {
    for (const cat of result.categories) {
      assert.ok(cat.id, 'Category missing id');
      assert.ok(cat.name, 'Category missing name');
      assert.ok(cat.owasp, 'Category missing owasp');
      assert.ok(cat.status, 'Category missing status');
      assert.ok(typeof cat.findings_count === 'number', 'Category missing findings_count');
      assert.ok(Array.isArray(cat.evidence), 'Category missing evidence array');
    }
  });

  // v5.0 new categories
  it('prompt injection hardening is PASS', () => {
    const cat = result.categories.find(c => c.id === 11);
    assert.equal(cat.status, 'PASS');
  });

  it('Rule of Two is PASS', () => {
    const cat = result.categories.find(c => c.id === 12);
    assert.equal(cat.status, 'PASS');
  });

  it('long-horizon monitoring is PASS', () => {
    const cat = result.categories.find(c => c.id === 13);
    assert.equal(cat.status, 'PASS');
  });

  it('new categories have correct names', () => {
    const cat11 = result.categories.find(c => c.id === 11);
    const cat12 = result.categories.find(c => c.id === 12);
    const cat13 = result.categories.find(c => c.id === 13);
    assert.equal(cat11.name, 'Prompt Injection Hardening');
    assert.equal(cat12.name, 'Rule of Two');
    assert.equal(cat13.name, 'Long-Horizon Monitoring');
  });

  it('new categories have OWASP mappings', () => {
    const cat11 = result.categories.find(c => c.id === 11);
    const cat12 = result.categories.find(c => c.id === 12);
    const cat13 = result.categories.find(c => c.id === 13);
    assert.ok(cat11.owasp.includes('LLM01'), 'Cat 11 should map to LLM01');
    assert.ok(cat12.owasp.includes('ASI02'), 'Cat 12 should map to ASI02');
    assert.ok(cat13.owasp.includes('ASI06'), 'Cat 13 should map to ASI06');
  });

  // v6.0 compliance categories
  it('EU AI Act Compliance category exists', () => {
    const cat = result.categories.find(c => c.id === 14);
    assert.ok(cat, 'Category 14 should exist');
    assert.equal(cat.name, 'EU AI Act Compliance');
  });

  it('NIST AI RMF Alignment category exists', () => {
    const cat = result.categories.find(c => c.id === 15);
    assert.ok(cat, 'Category 15 should exist');
    assert.equal(cat.name, 'NIST AI RMF Alignment');
  });

  it('ISO 42001 Readiness category exists', () => {
    const cat = result.categories.find(c => c.id === 16);
    assert.ok(cat, 'Category 16 should exist');
    assert.equal(cat.name, 'ISO 42001 Readiness');
  });

  it('compliance categories are PARTIAL for grade-a (has hooks+config but no reports/tests)', () => {
    for (const id of [14, 15, 16]) {
      const cat = result.categories.find(c => c.id === id);
      assert.ok(
        cat.status === 'PASS' || cat.status === 'PARTIAL',
        `Category ${id} (${cat.name}) should be PASS or PARTIAL, got ${cat.status}`,
      );
    }
  });

  it('compliance categories have Governance OWASP mapping', () => {
    for (const id of [14, 15, 16]) {
      const cat = result.categories.find(c => c.id === id);
      assert.ok(cat.owasp, `Category ${id} should have owasp mapping`);
    }
  });
});

// ---------------------------------------------------------------------------
// Grade F project — dangerous config, poisoned memory → Grade F
// ---------------------------------------------------------------------------

describe('posture-scanner: grade-f-project', () => {
  let result;

  beforeEach(async () => {
    resetCounter();
    result = await scan(GRADE_F_FIXTURE);
  });

  it('returns status ok', () => {
    assert.equal(result.status, 'ok');
  });

  it('produces Grade F', () => {
    assert.equal(result.scoring.grade, 'F');
  });

  it('has high risk score (>= 40)', () => {
    assert.ok(result.risk.score >= 40, `Expected risk >= 40, got ${result.risk.score}`);
  });

  it('verdict is BLOCK or WARNING', () => {
    assert.ok(
      result.risk.verdict === 'BLOCK' || result.risk.verdict === 'WARNING',
      `Expected BLOCK or WARNING, got ${result.risk.verdict}`,
    );
  });

  it('deny-first configuration is FAIL', () => {
    const cat = result.categories.find(c => c.id === 1);
    assert.equal(cat.status, 'FAIL');
  });

  it('secrets protection is FAIL', () => {
    const cat = result.categories.find(c => c.id === 2);
    assert.equal(cat.status, 'FAIL');
  });

  it('destructive blocking is FAIL', () => {
    const cat = result.categories.find(c => c.id === 5);
    assert.equal(cat.status, 'FAIL');
  });

  it('sandbox configuration is FAIL', () => {
    const cat = result.categories.find(c => c.id === 6);
    assert.equal(cat.status, 'FAIL');
  });

  it('cognitive state security is FAIL', () => {
    const cat = result.categories.find(c => c.id === 10);
    assert.equal(cat.status, 'FAIL');
  });

  it('has multiple FAIL categories', () => {
    const fails = result.categories.filter(c => c.status === 'FAIL');
    assert.ok(fails.length >= 7, `Expected >= 7 FAIL categories, got ${fails.length}`);
  });

  it('has critical findings', () => {
    assert.ok(result.counts.critical >= 1, `Expected >= 1 critical findings, got ${result.counts.critical}`);
  });

  it('has high findings', () => {
    assert.ok(result.counts.high >= 1, `Expected >= 1 high findings, got ${result.counts.high}`);
  });

  it('findings have PST scanner prefix', () => {
    const wrongPrefix = result.findings.filter(f => !f.id.startsWith('DS-PST-'));
    assert.equal(
      wrongPrefix.length, 0,
      `All findings should have DS-PST- prefix. Wrong: ${wrongPrefix.map(f => f.id).join(', ')}`,
    );
  });

  it('findings have required fields', () => {
    for (const f of result.findings) {
      assert.ok(f.id, `Finding missing id`);
      assert.ok(f.scanner === 'PST', `Finding ${f.id} scanner should be PST, got ${f.scanner}`);
      assert.ok(f.severity, `Finding ${f.id} missing severity`);
      assert.ok(f.title, `Finding ${f.id} missing title`);
      assert.ok(f.owasp, `Finding ${f.id} missing owasp`);
    }
  });

  it('finding IDs are sequential starting from DS-PST-001', () => {
    if (result.findings.length === 0) return;
    assert.equal(result.findings[0].id, 'DS-PST-001');
  });

  it('severity counts sum to total findings', () => {
    const { counts } = result;
    const total = counts.critical + counts.high + counts.medium + counts.low + counts.info;
    assert.equal(total, result.findings.length, 'Severity counts should sum to total findings');
  });

  it('pass_rate < 0.33', () => {
    assert.ok(result.scoring.pass_rate < 0.33, `Expected pass_rate < 0.33, got ${result.scoring.pass_rate}`);
  });

  // v5.0 new categories
  it('prompt injection hardening is FAIL', () => {
    const cat = result.categories.find(c => c.id === 11);
    assert.equal(cat.status, 'FAIL');
  });

  it('Rule of Two is FAIL', () => {
    const cat = result.categories.find(c => c.id === 12);
    assert.equal(cat.status, 'FAIL');
  });

  it('long-horizon monitoring is FAIL', () => {
    const cat = result.categories.find(c => c.id === 13);
    assert.equal(cat.status, 'FAIL');
  });

  // v6.0 compliance categories — grade-f has no security config → FAIL
  it('EU AI Act Compliance is FAIL', () => {
    const cat = result.categories.find(c => c.id === 14);
    assert.ok(cat, 'Category 14 should exist');
    assert.equal(cat.status, 'FAIL');
  });

  it('NIST AI RMF Alignment is FAIL', () => {
    const cat = result.categories.find(c => c.id === 15);
    assert.ok(cat, 'Category 15 should exist');
    assert.equal(cat.status, 'FAIL');
  });

  it('ISO 42001 Readiness is FAIL', () => {
    const cat = result.categories.find(c => c.id === 16);
    assert.ok(cat, 'Category 16 should exist');
    assert.equal(cat.status, 'FAIL');
  });
});

// ---------------------------------------------------------------------------
// Scanner interface compliance
// ---------------------------------------------------------------------------

describe('posture-scanner: interface', () => {
  it('scan() accepts a path string and returns a result', async () => {
    resetCounter();
    const result = await scan(GRADE_A_FIXTURE);
    assert.ok(result);
    assert.equal(typeof result, 'object');
  });

  it('result has scoring block', async () => {
    resetCounter();
    const result = await scan(GRADE_A_FIXTURE);
    assert.ok(result.scoring);
    assert.ok(typeof result.scoring.pass === 'number');
    assert.ok(typeof result.scoring.partial === 'number');
    assert.ok(typeof result.scoring.fail === 'number');
    assert.ok(typeof result.scoring.na === 'number');
    assert.ok(typeof result.scoring.applicable === 'number');
    assert.ok(typeof result.scoring.score === 'number');
    assert.ok(typeof result.scoring.pass_rate === 'number');
    assert.ok(typeof result.scoring.grade === 'string');
  });

  it('result has risk block', async () => {
    resetCounter();
    const result = await scan(GRADE_A_FIXTURE);
    assert.ok(result.risk);
    assert.ok(typeof result.risk.score === 'number');
    assert.ok(typeof result.risk.band === 'string');
    assert.ok(typeof result.risk.verdict === 'string');
  });

  it('result has counts block', async () => {
    resetCounter();
    const result = await scan(GRADE_A_FIXTURE);
    assert.ok(result.counts);
    assert.ok(typeof result.counts.critical === 'number');
    assert.ok(typeof result.counts.high === 'number');
    assert.ok(typeof result.counts.medium === 'number');
    assert.ok(typeof result.counts.low === 'number');
    assert.ok(typeof result.counts.info === 'number');
  });

  it('completes in under 2 seconds', async () => {
    resetCounter();
    const start = Date.now();
    await scan(GRADE_A_FIXTURE);
    const elapsed = Date.now() - start;
    assert.ok(elapsed < 2000, `Expected < 2000ms, took ${elapsed}ms`);
  });
});