feat: initial open marketplace with llm-security, config-audit, ultraplan-local
This commit is contained in:
Kjell Tore Guttormsen
commit
f93d6abdae
380 changed files with 65935 additions and 0 deletions
288
plugins/config-audit/tests/lib/diff-engine.test.mjs
Normal file
288
plugins/config-audit/tests/lib/diff-engine.test.mjs
Normal file
|
|
@ -0,0 +1,288 @@
|
|||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { diffEnvelopes, formatDiffReport } from '../../scanners/lib/diff-engine.mjs';
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function makeFinding(scanner, title, severity = 'medium', file = null) {
|
||||
return {
|
||||
id: `CA-${scanner}-001`,
|
||||
scanner,
|
||||
severity,
|
||||
title,
|
||||
description: `Description for ${title}`,
|
||||
file,
|
||||
line: null,
|
||||
evidence: null,
|
||||
category: null,
|
||||
recommendation: null,
|
||||
autoFixable: false,
|
||||
};
|
||||
}
|
||||
|
||||
function makeScannerResult(scanner, findings) {
|
||||
const counts = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
|
||||
for (const f of findings) {
|
||||
if (counts[f.severity] !== undefined) counts[f.severity]++;
|
||||
}
|
||||
return {
|
||||
scanner,
|
||||
status: 'ok',
|
||||
files_scanned: 3,
|
||||
duration_ms: 10,
|
||||
findings,
|
||||
counts,
|
||||
};
|
||||
}
|
||||
|
||||
function makeEnvelope(scannerResults) {
|
||||
const aggregate = { critical: 0, high: 0, medium: 0, low: 0, info: 0 };
|
||||
let total = 0;
|
||||
for (const r of scannerResults) {
|
||||
for (const sev of Object.keys(aggregate)) {
|
||||
aggregate[sev] += (r.counts[sev] || 0);
|
||||
}
|
||||
total += r.findings.length;
|
||||
}
|
||||
return {
|
||||
meta: { target: '/test', timestamp: new Date().toISOString(), version: '2.0.0', tool: 'config-audit' },
|
||||
scanners: scannerResults,
|
||||
aggregate: { total_findings: total, counts: aggregate, risk_score: 0, risk_band: 'Low', verdict: 'PASS', scanners_ok: scannerResults.length, scanners_error: 0, scanners_skipped: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
// ========================================
|
||||
// diffEnvelopes
|
||||
// ========================================
|
||||
describe('diffEnvelopes', () => {
|
||||
it('identifies new findings', () => {
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'New issue', 'high', 'CLAUDE.md')]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.newFindings.length, 1);
|
||||
assert.equal(diff.newFindings[0].title, 'New issue');
|
||||
assert.equal(diff.resolvedFindings.length, 0);
|
||||
});
|
||||
|
||||
it('identifies resolved findings', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'Old issue', 'high', 'CLAUDE.md')]),
|
||||
]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.resolvedFindings.length, 1);
|
||||
assert.equal(diff.resolvedFindings[0].title, 'Old issue');
|
||||
assert.equal(diff.newFindings.length, 0);
|
||||
});
|
||||
|
||||
it('identifies unchanged findings', () => {
|
||||
const f = makeFinding('CML', 'Persistent issue', 'medium', 'CLAUDE.md');
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [f])]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [{ ...f }])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.unchangedFindings.length, 1);
|
||||
assert.equal(diff.newFindings.length, 0);
|
||||
assert.equal(diff.resolvedFindings.length, 0);
|
||||
});
|
||||
|
||||
it('detects moved findings (same title, different file)', () => {
|
||||
const baseFinding = makeFinding('CML', 'Moved issue', 'high', 'old-file.md');
|
||||
const currFinding = makeFinding('CML', 'Moved issue', 'high', 'new-file.md');
|
||||
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [baseFinding])]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [currFinding])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.movedFindings.length, 1);
|
||||
assert.equal(diff.movedFindings[0].from.file, 'old-file.md');
|
||||
assert.equal(diff.movedFindings[0].to.file, 'new-file.md');
|
||||
assert.equal(diff.newFindings.length, 0);
|
||||
assert.equal(diff.resolvedFindings.length, 0);
|
||||
});
|
||||
|
||||
it('calculates score delta', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [
|
||||
makeFinding('CML', 'A', 'high', 'a.md'),
|
||||
makeFinding('CML', 'B', 'high', 'a.md'),
|
||||
makeFinding('CML', 'C', 'high', 'a.md'),
|
||||
]),
|
||||
]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.ok(diff.scoreChange.delta > 0, 'Score should improve when findings are resolved');
|
||||
assert.equal(typeof diff.scoreChange.before.grade, 'string');
|
||||
assert.equal(typeof diff.scoreChange.after.grade, 'string');
|
||||
});
|
||||
|
||||
it('calculates area changes', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'X', 'low', 'a.md')]),
|
||||
makeScannerResult('SET', []),
|
||||
]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', []),
|
||||
makeScannerResult('SET', [makeFinding('SET', 'Y', 'low', 'b.json')]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.ok(diff.areaChanges.length >= 2);
|
||||
const cmlChange = diff.areaChanges.find(a => a.name === 'CLAUDE.md');
|
||||
assert.ok(cmlChange, 'Should have CLAUDE.md area change');
|
||||
assert.ok(cmlChange.delta > 0, 'CLAUDE.md should improve');
|
||||
});
|
||||
|
||||
it('detects improving trend', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [
|
||||
makeFinding('CML', 'A', 'high', 'a.md'),
|
||||
makeFinding('CML', 'B', 'high', 'a.md'),
|
||||
]),
|
||||
]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.summary.trend, 'improving');
|
||||
});
|
||||
|
||||
it('detects degrading trend', () => {
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [
|
||||
makeFinding('CML', 'A', 'high', 'a.md'),
|
||||
makeFinding('CML', 'B', 'high', 'a.md'),
|
||||
]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.summary.trend, 'degrading');
|
||||
});
|
||||
|
||||
it('detects stable trend (same findings)', () => {
|
||||
const f = makeFinding('CML', 'Same', 'medium', 'a.md');
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [f])]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [{ ...f }])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.summary.trend, 'stable');
|
||||
});
|
||||
|
||||
it('handles empty baseline', () => {
|
||||
const baseline = makeEnvelope([]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'New', 'low', 'a.md')]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.newFindings.length, 1);
|
||||
assert.equal(diff.resolvedFindings.length, 0);
|
||||
assert.equal(diff.summary.totalBefore, 0);
|
||||
});
|
||||
|
||||
it('handles identical envelopes (all unchanged)', () => {
|
||||
const f1 = makeFinding('CML', 'Issue A', 'medium', 'a.md');
|
||||
const f2 = makeFinding('SET', 'Issue B', 'low', 'b.json');
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [f1]),
|
||||
makeScannerResult('SET', [f2]),
|
||||
]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [{ ...f1 }]),
|
||||
makeScannerResult('SET', [{ ...f2 }]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.unchangedFindings.length, 2);
|
||||
assert.equal(diff.newFindings.length, 0);
|
||||
assert.equal(diff.resolvedFindings.length, 0);
|
||||
assert.equal(diff.movedFindings.length, 0);
|
||||
assert.equal(diff.summary.trend, 'stable');
|
||||
});
|
||||
|
||||
it('summary has correct counts', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [
|
||||
makeFinding('CML', 'Keep', 'low', 'a.md'),
|
||||
makeFinding('CML', 'Resolve', 'high', 'b.md'),
|
||||
]),
|
||||
]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [
|
||||
makeFinding('CML', 'Keep', 'low', 'a.md'),
|
||||
makeFinding('CML', 'Brand new', 'medium', 'c.md'),
|
||||
]),
|
||||
]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.summary.totalBefore, 2);
|
||||
assert.equal(diff.summary.totalAfter, 2);
|
||||
assert.equal(diff.summary.newCount, 1);
|
||||
assert.equal(diff.summary.resolvedCount, 1);
|
||||
});
|
||||
|
||||
it('handles findings with null file gracefully', () => {
|
||||
const f = makeFinding('CML', 'No file', 'info', null);
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [f])]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [{ ...f }])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.unchangedFindings.length, 1);
|
||||
});
|
||||
|
||||
it('multiple findings with same key are matched correctly', () => {
|
||||
const f1 = makeFinding('CML', 'Duplicate', 'low', 'a.md');
|
||||
const f2 = makeFinding('CML', 'Duplicate', 'low', 'a.md');
|
||||
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [f1, f2])]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [{ ...f1 }])]);
|
||||
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
assert.equal(diff.unchangedFindings.length, 1);
|
||||
assert.equal(diff.resolvedFindings.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
// ========================================
|
||||
// formatDiffReport
|
||||
// ========================================
|
||||
describe('formatDiffReport', () => {
|
||||
it('returns a non-empty string', () => {
|
||||
const diff = diffEnvelopes(makeEnvelope([]), makeEnvelope([]));
|
||||
const report = formatDiffReport(diff);
|
||||
assert.ok(report.length > 0);
|
||||
assert.equal(typeof report, 'string');
|
||||
});
|
||||
|
||||
it('contains header', () => {
|
||||
const diff = diffEnvelopes(makeEnvelope([]), makeEnvelope([]));
|
||||
const report = formatDiffReport(diff);
|
||||
assert.ok(report.includes('Config-Audit Drift Report'));
|
||||
});
|
||||
|
||||
it('shows trend', () => {
|
||||
const baseline = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'X', 'high', 'a.md')]),
|
||||
]);
|
||||
const current = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
const report = formatDiffReport(diff);
|
||||
assert.ok(report.includes('Improving'));
|
||||
});
|
||||
|
||||
it('lists new findings', () => {
|
||||
const baseline = makeEnvelope([makeScannerResult('CML', [])]);
|
||||
const current = makeEnvelope([
|
||||
makeScannerResult('CML', [makeFinding('CML', 'Fresh issue', 'high', 'x.md')]),
|
||||
]);
|
||||
const diff = diffEnvelopes(baseline, current);
|
||||
const report = formatDiffReport(diff);
|
||||
assert.ok(report.includes('Fresh issue'));
|
||||
assert.ok(report.includes('New findings'));
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue