feat(governance): add structured JSONL audit trail with SIEM-ready schema
New audit-trail.mjs writes structured events to LLM_SECURITY_AUDIT_LOG path. Integrated into post-session-guard at 6 warning emission points: trifecta, escalation-after-input, data flow, volume threshold, slow-burn, behavioral drift. No-op when env var not set — zero overhead for existing users. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
2116e702df
commit
269c14445c
3 changed files with 241 additions and 0 deletions
|
|
@ -42,6 +42,7 @@ import { tmpdir } from 'node:os';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import { extractMcpServer } from '../../scanners/lib/mcp-description-cache.mjs';
|
import { extractMcpServer } from '../../scanners/lib/mcp-description-cache.mjs';
|
||||||
import { jensenShannonDivergence, buildDistribution } from '../../scanners/lib/distribution-stats.mjs';
|
import { jensenShannonDivergence, buildDistribution } from '../../scanners/lib/distribution-stats.mjs';
|
||||||
|
import { writeAuditEvent } from '../../scanners/lib/audit-trail.mjs';
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Constants
|
// Constants
|
||||||
|
|
@ -799,6 +800,14 @@ if (!(classes.length === 1 && (classes[0] === 'neutral' || classes[0] === 'deleg
|
||||||
const sensitiveExfil = checkSensitiveExfil(window);
|
const sensitiveExfil = checkSensitiveExfil(window);
|
||||||
messages.push(formatWarning(evidence, mcpInfo, sensitiveExfil));
|
messages.push(formatWarning(evidence, mcpInfo, sensitiveExfil));
|
||||||
appendEntry(stateFile, { type: 'warning', ts: Date.now() });
|
appendEntry(stateFile, { type: 'warning', ts: Date.now() });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'trifecta_warning',
|
||||||
|
severity: mcpInfo.concentrated || sensitiveExfil ? 'critical' : 'high',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { evidence, mcp_concentrated: mcpInfo.concentrated, sensitive_exfil: sensitiveExfil },
|
||||||
|
owasp: ['ASI01', 'ASI02', 'LLM01'],
|
||||||
|
action_taken: TRIFECTA_MODE === 'block' && (mcpInfo.concentrated || sensitiveExfil) ? 'blocked' : 'warned',
|
||||||
|
});
|
||||||
|
|
||||||
// --- Rule of Two: Block mode ---
|
// --- Rule of Two: Block mode ---
|
||||||
// Block for high-confidence trifecta: MCP-concentrated OR sensitive path + exfil
|
// Block for high-confidence trifecta: MCP-concentrated OR sensitive path + exfil
|
||||||
|
|
@ -823,6 +832,14 @@ if (classes.includes('delegation')) {
|
||||||
if (escalation.detected && !hasEscalationWarning(window)) {
|
if (escalation.detected && !hasEscalationWarning(window)) {
|
||||||
messages.push(formatEscalationWarning(detail, escalation.inputDetail));
|
messages.push(formatEscalationWarning(detail, escalation.inputDetail));
|
||||||
appendEntry(stateFile, { type: 'escalation_warning', ts: Date.now() });
|
appendEntry(stateFile, { type: 'escalation_warning', ts: Date.now() });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'escalation_after_input',
|
||||||
|
severity: 'medium',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { tool: detail, input_source: escalation.inputDetail },
|
||||||
|
owasp: ['ASI01'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -840,6 +857,14 @@ if (!(classes.length === 1 && classes[0] === 'neutral')) {
|
||||||
if (detected) {
|
if (detected) {
|
||||||
messages.push(formatDataFlowWarning(evidence, flowLink.sourceEntries));
|
messages.push(formatDataFlowWarning(evidence, flowLink.sourceEntries));
|
||||||
appendEntry(stateFile, { type: 'data_flow_warning', ts: Date.now() });
|
appendEntry(stateFile, { type: 'data_flow_warning', ts: Date.now() });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'data_flow_trifecta',
|
||||||
|
severity: 'high',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { evidence, flow_sources: flowLink.sourceEntries.length },
|
||||||
|
owasp: ['ASI01', 'ASI02'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -855,6 +880,14 @@ if (outputSize > 0) {
|
||||||
if (totalVolume >= bytes && !hasVolumeWarning(allEntries, bytes)) {
|
if (totalVolume >= bytes && !hasVolumeWarning(allEntries, bytes)) {
|
||||||
messages.push(formatVolumeWarning(totalVolume, label, severity));
|
messages.push(formatVolumeWarning(totalVolume, label, severity));
|
||||||
appendEntry(stateFile, { type: 'volume_warning', ts: Date.now(), threshold: bytes });
|
appendEntry(stateFile, { type: 'volume_warning', ts: Date.now(), threshold: bytes });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'volume_threshold',
|
||||||
|
severity: severity.toLowerCase(),
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { total_bytes: totalVolume, threshold: label },
|
||||||
|
owasp: ['ASI02'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
break; // only emit highest unwarned threshold
|
break; // only emit highest unwarned threshold
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -869,6 +902,14 @@ if (outputSize > 0) {
|
||||||
if (slowBurn.detected && !hasSlowBurnWarning(longWindow)) {
|
if (slowBurn.detected && !hasSlowBurnWarning(longWindow)) {
|
||||||
messages.push(formatSlowBurnWarning(slowBurn.spread));
|
messages.push(formatSlowBurnWarning(slowBurn.spread));
|
||||||
appendEntry(stateFile, { type: 'slow_burn_warning', ts: Date.now() });
|
appendEntry(stateFile, { type: 'slow_burn_warning', ts: Date.now() });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'slow_burn_trifecta',
|
||||||
|
severity: 'medium',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { spread: slowBurn.spread },
|
||||||
|
owasp: ['ASI06', 'ASI08'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Behavioral drift: JSD on tool distribution (first vs last DRIFT_SAMPLE_SIZE)
|
// Behavioral drift: JSD on tool distribution (first vs last DRIFT_SAMPLE_SIZE)
|
||||||
|
|
@ -876,6 +917,14 @@ if (outputSize > 0) {
|
||||||
if (drift.drifted && !hasDriftWarning(longWindow)) {
|
if (drift.drifted && !hasDriftWarning(longWindow)) {
|
||||||
messages.push(formatDriftWarning(drift.jsd, drift.firstTools, drift.lastTools));
|
messages.push(formatDriftWarning(drift.jsd, drift.firstTools, drift.lastTools));
|
||||||
appendEntry(stateFile, { type: 'drift_warning', ts: Date.now() });
|
appendEntry(stateFile, { type: 'drift_warning', ts: Date.now() });
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'behavioral_drift',
|
||||||
|
severity: 'medium',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { jsd: drift.jsd, first_tools: drift.firstTools, last_tools: drift.lastTools },
|
||||||
|
owasp: ['ASI06', 'ASI08'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
83
plugins/llm-security/scanners/lib/audit-trail.mjs
Normal file
83
plugins/llm-security/scanners/lib/audit-trail.mjs
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
// audit-trail.mjs — Structured JSONL audit trail writer
|
||||||
|
// Writes SIEM-ready events to the path specified by LLM_SECURITY_AUDIT_LOG.
|
||||||
|
// No-op when env var is not set. Zero external dependencies.
|
||||||
|
|
||||||
|
import { appendFileSync, writeFileSync, accessSync, constants } from 'node:fs';
|
||||||
|
import { dirname } from 'node:path';
|
||||||
|
|
||||||
|
let auditPath = null;
|
||||||
|
let initialized = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize audit trail. Validates the path is writable on first call.
|
||||||
|
* @returns {boolean} true if audit trail is enabled and writable
|
||||||
|
*/
|
||||||
|
function initAuditTrail() {
|
||||||
|
if (initialized) return auditPath !== null;
|
||||||
|
initialized = true;
|
||||||
|
|
||||||
|
const envPath = process.env.LLM_SECURITY_AUDIT_LOG;
|
||||||
|
if (!envPath) return false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Ensure parent directory exists and is writable
|
||||||
|
const dir = dirname(envPath);
|
||||||
|
accessSync(dir, constants.W_OK);
|
||||||
|
// Touch file if it doesn't exist
|
||||||
|
try { accessSync(envPath); } catch { writeFileSync(envPath, ''); }
|
||||||
|
auditPath = envPath;
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
process.stderr.write(`[llm-security] Audit trail path not writable: ${envPath} (${err.message})\n`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write a structured audit event as one JSON line.
|
||||||
|
* No-op when LLM_SECURITY_AUDIT_LOG is not set.
|
||||||
|
*
|
||||||
|
* @param {object} event
|
||||||
|
* @param {string} event.event_type - e.g. trifecta_warning, injection_detected
|
||||||
|
* @param {string} event.severity - critical|high|medium|low|info
|
||||||
|
* @param {string} event.source - hook or scanner name
|
||||||
|
* @param {object} [event.details] - event-specific payload
|
||||||
|
* @param {string[]} [event.owasp] - OWASP categories
|
||||||
|
* @param {string} [event.action_taken] - blocked|warned|allowed
|
||||||
|
*/
|
||||||
|
export function writeAuditEvent(event) {
|
||||||
|
if (!initAuditTrail()) return;
|
||||||
|
|
||||||
|
const entry = {
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
session_id: String(process.ppid || process.pid),
|
||||||
|
event_type: event.event_type || 'unknown',
|
||||||
|
severity: event.severity || 'info',
|
||||||
|
source: event.source || 'unknown',
|
||||||
|
details: event.details || {},
|
||||||
|
owasp: event.owasp || [],
|
||||||
|
action_taken: event.action_taken || 'warned',
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
appendFileSync(auditPath, JSON.stringify(entry) + '\n');
|
||||||
|
} catch (err) {
|
||||||
|
process.stderr.write(`[llm-security] Audit trail write failed: ${err.message}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether audit trail is enabled (for guard clauses in hooks).
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isAuditEnabled() {
|
||||||
|
return initAuditTrail();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset internal state (for testing only).
|
||||||
|
*/
|
||||||
|
export function _resetForTest() {
|
||||||
|
auditPath = null;
|
||||||
|
initialized = false;
|
||||||
|
}
|
||||||
109
plugins/llm-security/tests/lib/audit-trail.test.mjs
Normal file
109
plugins/llm-security/tests/lib/audit-trail.test.mjs
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
// audit-trail.test.mjs — Tests for structured JSONL audit trail
|
||||||
|
|
||||||
|
import { describe, it, beforeEach, afterEach } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { writeFileSync, readFileSync, unlinkSync, existsSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { writeAuditEvent, isAuditEnabled, _resetForTest } from '../../scanners/lib/audit-trail.mjs';
|
||||||
|
|
||||||
|
const TEST_LOG = join(tmpdir(), `llm-security-audit-test-${Date.now()}.jsonl`);
|
||||||
|
|
||||||
|
describe('audit-trail', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
_resetForTest();
|
||||||
|
// Clean up test file
|
||||||
|
try { unlinkSync(TEST_LOG); } catch {}
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
_resetForTest();
|
||||||
|
delete process.env.LLM_SECURITY_AUDIT_LOG;
|
||||||
|
try { unlinkSync(TEST_LOG); } catch {}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is disabled when env var not set', () => {
|
||||||
|
delete process.env.LLM_SECURITY_AUDIT_LOG;
|
||||||
|
assert.equal(isAuditEnabled(), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is enabled when env var is set to writable path', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = TEST_LOG;
|
||||||
|
assert.equal(isAuditEnabled(), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no-op when env var not set', () => {
|
||||||
|
delete process.env.LLM_SECURITY_AUDIT_LOG;
|
||||||
|
writeAuditEvent({ event_type: 'test', severity: 'info', source: 'test' });
|
||||||
|
assert.equal(existsSync(TEST_LOG), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('writes valid JSONL when enabled', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = TEST_LOG;
|
||||||
|
writeAuditEvent({
|
||||||
|
event_type: 'trifecta_warning',
|
||||||
|
severity: 'high',
|
||||||
|
source: 'post-session-guard',
|
||||||
|
details: { window_size: 20 },
|
||||||
|
owasp: ['ASI01', 'ASI02'],
|
||||||
|
action_taken: 'warned',
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = readFileSync(TEST_LOG, 'utf8').trim();
|
||||||
|
const entry = JSON.parse(content);
|
||||||
|
|
||||||
|
assert.equal(entry.event_type, 'trifecta_warning');
|
||||||
|
assert.equal(entry.severity, 'high');
|
||||||
|
assert.equal(entry.source, 'post-session-guard');
|
||||||
|
assert.deepEqual(entry.owasp, ['ASI01', 'ASI02']);
|
||||||
|
assert.equal(entry.action_taken, 'warned');
|
||||||
|
assert.ok(entry.timestamp.match(/^\d{4}-\d{2}-\d{2}T/), 'Expected ISO timestamp');
|
||||||
|
assert.ok(entry.session_id, 'Expected session_id');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('appends multiple events as separate lines', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = TEST_LOG;
|
||||||
|
writeAuditEvent({ event_type: 'event1', severity: 'info', source: 'test' });
|
||||||
|
writeAuditEvent({ event_type: 'event2', severity: 'medium', source: 'test' });
|
||||||
|
writeAuditEvent({ event_type: 'event3', severity: 'high', source: 'test' });
|
||||||
|
|
||||||
|
const lines = readFileSync(TEST_LOG, 'utf8').trim().split('\n');
|
||||||
|
assert.equal(lines.length, 3);
|
||||||
|
|
||||||
|
const e1 = JSON.parse(lines[0]);
|
||||||
|
const e3 = JSON.parse(lines[2]);
|
||||||
|
assert.equal(e1.event_type, 'event1');
|
||||||
|
assert.equal(e3.event_type, 'event3');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('events contain all required fields', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = TEST_LOG;
|
||||||
|
writeAuditEvent({ event_type: 'test', severity: 'info', source: 'test-hook' });
|
||||||
|
|
||||||
|
const entry = JSON.parse(readFileSync(TEST_LOG, 'utf8').trim());
|
||||||
|
const required = ['timestamp', 'session_id', 'event_type', 'severity', 'source', 'details', 'owasp', 'action_taken'];
|
||||||
|
for (const field of required) {
|
||||||
|
assert.ok(field in entry, `Missing required field: ${field}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('provides defaults for optional fields', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = TEST_LOG;
|
||||||
|
writeAuditEvent({ event_type: 'minimal' });
|
||||||
|
|
||||||
|
const entry = JSON.parse(readFileSync(TEST_LOG, 'utf8').trim());
|
||||||
|
assert.equal(entry.severity, 'info');
|
||||||
|
assert.equal(entry.source, 'unknown');
|
||||||
|
assert.deepEqual(entry.details, {});
|
||||||
|
assert.deepEqual(entry.owasp, []);
|
||||||
|
assert.equal(entry.action_taken, 'warned');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not crash on invalid path', () => {
|
||||||
|
process.env.LLM_SECURITY_AUDIT_LOG = '/nonexistent/dir/audit.jsonl';
|
||||||
|
// Should not throw — gracefully logs to stderr
|
||||||
|
assert.doesNotThrow(() => {
|
||||||
|
writeAuditEvent({ event_type: 'test', severity: 'info', source: 'test' });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Add table
Add a link
Reference in a new issue