330 lines
11 KiB
JavaScript
330 lines
11 KiB
JavaScript
// posture.test.mjs — Tests for the deterministic posture scanner
|
|
// Tests against fixtures in tests/fixtures/posture-scan/ with:
|
|
// - grade-a-project: full security config (hooks, settings, CLAUDE.md) → Grade A
|
|
// - grade-f-project: dangerous flags, poisoned memory, no hooks → Grade F
|
|
|
|
import { describe, it, beforeEach } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { resetCounter } from '../../scanners/lib/output.mjs';
|
|
import { scan } from '../../scanners/posture-scanner.mjs';
|
|
|
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
const GRADE_A_FIXTURE = resolve(__dirname, '../fixtures/posture-scan/grade-a-project');
|
|
const GRADE_F_FIXTURE = resolve(__dirname, '../fixtures/posture-scan/grade-f-project');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Grade A project — well-configured, should get Grade A
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('posture-scanner: grade-a-project', () => {
|
|
let result;
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
result = await scan(GRADE_A_FIXTURE);
|
|
});
|
|
|
|
it('returns status ok', () => {
|
|
assert.equal(result.status, 'ok');
|
|
});
|
|
|
|
it('has scanner name', () => {
|
|
assert.equal(result.scanner, 'posture-scanner');
|
|
});
|
|
|
|
it('has version', () => {
|
|
assert.ok(result.version, 'Expected version string');
|
|
});
|
|
|
|
it('produces Grade A', () => {
|
|
assert.equal(result.scoring.grade, 'A');
|
|
});
|
|
|
|
it('has 13 categories assessed', () => {
|
|
assert.equal(result.categories.length, 13);
|
|
});
|
|
|
|
it('has low risk score', () => {
|
|
assert.ok(result.risk.score <= 25, `Expected risk score <= 25, got ${result.risk.score}`);
|
|
});
|
|
|
|
it('verdict is ALLOW or WARNING', () => {
|
|
assert.ok(
|
|
result.risk.verdict === 'ALLOW' || result.risk.verdict === 'WARNING',
|
|
`Expected ALLOW or WARNING, got ${result.risk.verdict}`,
|
|
);
|
|
});
|
|
|
|
it('deny-first configuration is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 1);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('secrets protection is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 2);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('path guarding is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 3);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('MCP trust is N/A (no MCP servers)', () => {
|
|
const cat = result.categories.find(c => c.id === 4);
|
|
assert.equal(cat.status, 'N_A');
|
|
});
|
|
|
|
it('destructive blocking is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 5);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('sandbox configuration is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 6);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('cognitive state security is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 10);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('zero critical findings', () => {
|
|
assert.equal(result.counts.critical, 0);
|
|
});
|
|
|
|
it('pass rate >= 0.89', () => {
|
|
assert.ok(result.scoring.pass_rate >= 0.89, `Expected pass_rate >= 0.89, got ${result.scoring.pass_rate}`);
|
|
});
|
|
|
|
it('duration_ms is a non-negative number', () => {
|
|
assert.ok(typeof result.duration_ms === 'number');
|
|
assert.ok(result.duration_ms >= 0);
|
|
});
|
|
|
|
it('timestamp is ISO format', () => {
|
|
assert.ok(result.timestamp.match(/^\d{4}-\d{2}-\d{2}T/), 'Expected ISO timestamp');
|
|
});
|
|
|
|
it('each category has required fields', () => {
|
|
for (const cat of result.categories) {
|
|
assert.ok(cat.id, 'Category missing id');
|
|
assert.ok(cat.name, 'Category missing name');
|
|
assert.ok(cat.owasp, 'Category missing owasp');
|
|
assert.ok(cat.status, 'Category missing status');
|
|
assert.ok(typeof cat.findings_count === 'number', 'Category missing findings_count');
|
|
assert.ok(Array.isArray(cat.evidence), 'Category missing evidence array');
|
|
}
|
|
});
|
|
|
|
// v5.0 new categories
|
|
it('prompt injection hardening is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 11);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('Rule of Two is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 12);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('long-horizon monitoring is PASS', () => {
|
|
const cat = result.categories.find(c => c.id === 13);
|
|
assert.equal(cat.status, 'PASS');
|
|
});
|
|
|
|
it('new categories have correct names', () => {
|
|
const cat11 = result.categories.find(c => c.id === 11);
|
|
const cat12 = result.categories.find(c => c.id === 12);
|
|
const cat13 = result.categories.find(c => c.id === 13);
|
|
assert.equal(cat11.name, 'Prompt Injection Hardening');
|
|
assert.equal(cat12.name, 'Rule of Two');
|
|
assert.equal(cat13.name, 'Long-Horizon Monitoring');
|
|
});
|
|
|
|
it('new categories have OWASP mappings', () => {
|
|
const cat11 = result.categories.find(c => c.id === 11);
|
|
const cat12 = result.categories.find(c => c.id === 12);
|
|
const cat13 = result.categories.find(c => c.id === 13);
|
|
assert.ok(cat11.owasp.includes('LLM01'), 'Cat 11 should map to LLM01');
|
|
assert.ok(cat12.owasp.includes('ASI02'), 'Cat 12 should map to ASI02');
|
|
assert.ok(cat13.owasp.includes('ASI06'), 'Cat 13 should map to ASI06');
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Grade F project — dangerous config, poisoned memory → Grade F
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('posture-scanner: grade-f-project', () => {
|
|
let result;
|
|
|
|
beforeEach(async () => {
|
|
resetCounter();
|
|
result = await scan(GRADE_F_FIXTURE);
|
|
});
|
|
|
|
it('returns status ok', () => {
|
|
assert.equal(result.status, 'ok');
|
|
});
|
|
|
|
it('produces Grade F', () => {
|
|
assert.equal(result.scoring.grade, 'F');
|
|
});
|
|
|
|
it('has high risk score (>= 40)', () => {
|
|
assert.ok(result.risk.score >= 40, `Expected risk >= 40, got ${result.risk.score}`);
|
|
});
|
|
|
|
it('verdict is BLOCK or WARNING', () => {
|
|
assert.ok(
|
|
result.risk.verdict === 'BLOCK' || result.risk.verdict === 'WARNING',
|
|
`Expected BLOCK or WARNING, got ${result.risk.verdict}`,
|
|
);
|
|
});
|
|
|
|
it('deny-first configuration is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 1);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('secrets protection is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 2);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('destructive blocking is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 5);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('sandbox configuration is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 6);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('cognitive state security is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 10);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('has multiple FAIL categories', () => {
|
|
const fails = result.categories.filter(c => c.status === 'FAIL');
|
|
assert.ok(fails.length >= 7, `Expected >= 7 FAIL categories, got ${fails.length}`);
|
|
});
|
|
|
|
it('has critical findings', () => {
|
|
assert.ok(result.counts.critical >= 1, `Expected >= 1 critical findings, got ${result.counts.critical}`);
|
|
});
|
|
|
|
it('has high findings', () => {
|
|
assert.ok(result.counts.high >= 1, `Expected >= 1 high findings, got ${result.counts.high}`);
|
|
});
|
|
|
|
it('findings have PST scanner prefix', () => {
|
|
const wrongPrefix = result.findings.filter(f => !f.id.startsWith('DS-PST-'));
|
|
assert.equal(
|
|
wrongPrefix.length, 0,
|
|
`All findings should have DS-PST- prefix. Wrong: ${wrongPrefix.map(f => f.id).join(', ')}`,
|
|
);
|
|
});
|
|
|
|
it('findings have required fields', () => {
|
|
for (const f of result.findings) {
|
|
assert.ok(f.id, `Finding missing id`);
|
|
assert.ok(f.scanner === 'PST', `Finding ${f.id} scanner should be PST, got ${f.scanner}`);
|
|
assert.ok(f.severity, `Finding ${f.id} missing severity`);
|
|
assert.ok(f.title, `Finding ${f.id} missing title`);
|
|
assert.ok(f.owasp, `Finding ${f.id} missing owasp`);
|
|
}
|
|
});
|
|
|
|
it('finding IDs are sequential starting from DS-PST-001', () => {
|
|
if (result.findings.length === 0) return;
|
|
assert.equal(result.findings[0].id, 'DS-PST-001');
|
|
});
|
|
|
|
it('severity counts sum to total findings', () => {
|
|
const { counts } = result;
|
|
const total = counts.critical + counts.high + counts.medium + counts.low + counts.info;
|
|
assert.equal(total, result.findings.length, 'Severity counts should sum to total findings');
|
|
});
|
|
|
|
it('pass_rate < 0.33', () => {
|
|
assert.ok(result.scoring.pass_rate < 0.33, `Expected pass_rate < 0.33, got ${result.scoring.pass_rate}`);
|
|
});
|
|
|
|
// v5.0 new categories
|
|
it('prompt injection hardening is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 11);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('Rule of Two is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 12);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
|
|
it('long-horizon monitoring is FAIL', () => {
|
|
const cat = result.categories.find(c => c.id === 13);
|
|
assert.equal(cat.status, 'FAIL');
|
|
});
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scanner interface compliance
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('posture-scanner: interface', () => {
|
|
it('scan() accepts a path string and returns a result', async () => {
|
|
resetCounter();
|
|
const result = await scan(GRADE_A_FIXTURE);
|
|
assert.ok(result);
|
|
assert.equal(typeof result, 'object');
|
|
});
|
|
|
|
it('result has scoring block', async () => {
|
|
resetCounter();
|
|
const result = await scan(GRADE_A_FIXTURE);
|
|
assert.ok(result.scoring);
|
|
assert.ok(typeof result.scoring.pass === 'number');
|
|
assert.ok(typeof result.scoring.partial === 'number');
|
|
assert.ok(typeof result.scoring.fail === 'number');
|
|
assert.ok(typeof result.scoring.na === 'number');
|
|
assert.ok(typeof result.scoring.applicable === 'number');
|
|
assert.ok(typeof result.scoring.score === 'number');
|
|
assert.ok(typeof result.scoring.pass_rate === 'number');
|
|
assert.ok(typeof result.scoring.grade === 'string');
|
|
});
|
|
|
|
it('result has risk block', async () => {
|
|
resetCounter();
|
|
const result = await scan(GRADE_A_FIXTURE);
|
|
assert.ok(result.risk);
|
|
assert.ok(typeof result.risk.score === 'number');
|
|
assert.ok(typeof result.risk.band === 'string');
|
|
assert.ok(typeof result.risk.verdict === 'string');
|
|
});
|
|
|
|
it('result has counts block', async () => {
|
|
resetCounter();
|
|
const result = await scan(GRADE_A_FIXTURE);
|
|
assert.ok(result.counts);
|
|
assert.ok(typeof result.counts.critical === 'number');
|
|
assert.ok(typeof result.counts.high === 'number');
|
|
assert.ok(typeof result.counts.medium === 'number');
|
|
assert.ok(typeof result.counts.low === 'number');
|
|
assert.ok(typeof result.counts.info === 'number');
|
|
});
|
|
|
|
it('completes in under 2 seconds', async () => {
|
|
resetCounter();
|
|
const start = Date.now();
|
|
await scan(GRADE_A_FIXTURE);
|
|
const elapsed = Date.now() - start;
|
|
assert.ok(elapsed < 2000, `Expected < 2000ms, took ${elapsed}ms`);
|
|
});
|
|
});
|