feat(llm-security)!: v7.0.0 commit 6 — tests, docs, version bump
Final commit in the trustworthy-scoring series. Bundles verdict cutoff
alignment, the last suite of tests, and all documentation touch-points
that quote version numbers or describe v7.0.0 behaviour.
Verdict/band co-monotonicity
- `scanners/lib/severity.mjs` — verdict cutoffs moved from 61/21 to 65/15
so `BLOCK >= 65`, `WARNING >= 15` locks onto the v2 riskBand() boundaries.
Prevents "BLOCK / Medium band" contradictions under the v2 formula.
Scanner hardening (bug fixes from v7.0.0 testing)
- `scanners/entropy-scanner.mjs` — `policy_source` now uses
`existsSync('.llm-security/policy.json')` instead of value-based check.
Old heuristic always reported 'policy.json' because DEFAULT_POLICY now
carries an `entropy.thresholds` section.
- `scanners/lib/file-discovery.mjs` — `.sass` and GPU shader extensions
(`.glsl, .frag, .vert, .shader, .wgsl`) added to TEXT_EXTENSIONS. Without
this, shader files were invisible to file-discovery, so they were never
counted as skipped by the entropy-scanner extension filter.
Tests
- `tests/scanners/entropy-context.test.mjs` (new, 24 tests) — A. File-ext
skip (4), B. Line-level rules 11-17 (8), C. Policy overrides (3).
Fixtures generate 80-char base64 payloads at runtime via
`crypto.randomBytes` to dodge the plugin's own pre-edit credential hook
on the test source.
- `tests/lib/severity.test.mjs` — rewritten with v2 scoring table (70
tests total, was 52).
- `tests/lib/output.test.mjs:243` — "1 critical = score 80" under v2
(was 25 under v1).
- Full suite: 1485/1485 green (was 1461).
Docs
- `CHANGELOG.md` — v7.0.0 entry with BREAKING CHANGES section.
- `README.md` (plugin + marketplace root) — version badge, history table,
plugin-card version string, test count.
- `CLAUDE.md` — header version, "v7.0.0 — Trustworthy scoring" summary
paragraph at the top.
- `docs/security-hardening-guide.md` — new section 6 "Calibration & false
positives" documenting v2 formula, context-aware entropy scanner,
typosquat allowlist, and §6.4 tuning workflow. Existing "Recommended
baseline" section renumbered to §7.
Version bump
- `6.6.0 -> 7.0.0` across package.json, .claude-plugin/plugin.json,
scanners/ide-extension-scanner.mjs VERSION const, README badge,
CLAUDE.md header, marketplace root README card.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen
parent
915aca69e4
commit
6f86de937a
14 changed files with 515 additions and 85 deletions
|
|
@ -234,13 +234,13 @@ describe('envelope', () => {
|
|||
});
|
||||
|
||||
it('computes correct risk_score from aggregated counts', () => {
|
||||
// 1 critical = score 25
|
||||
// v2 formula (v7.0.0+): 1 critical = score 80 (70 + log2(2)*10 = 80)
|
||||
const f = finding({ scanner: 'ENT', severity: 'critical', title: 'C', description: 'x' });
|
||||
const scanners = {
|
||||
entropy: scannerResult('entropy-scanner', 'ok', [f], 5, 30),
|
||||
};
|
||||
const result = envelope('/project', scanners, 30);
|
||||
assert.equal(result.aggregate.risk_score, 25);
|
||||
assert.equal(result.aggregate.risk_score, 80);
|
||||
});
|
||||
|
||||
it('returns BLOCK verdict when critical finding present', () => {
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ describe('SEVERITY', () => {
|
|||
// riskScore
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('riskScore', () => {
|
||||
describe('riskScore (v2 — severity-dominated log-scaled, v7.0.0+)', () => {
|
||||
it('returns 0 when all counts are zero', () => {
|
||||
assert.equal(riskScore({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }), 0);
|
||||
});
|
||||
|
|
@ -55,37 +55,75 @@ describe('riskScore', () => {
|
|||
assert.equal(riskScore({}), 0);
|
||||
});
|
||||
|
||||
it('returns 25 for one critical finding (weight=25)', () => {
|
||||
assert.equal(riskScore({ critical: 1 }), 25);
|
||||
});
|
||||
|
||||
it('returns 100 (capped) for four critical findings (4*25=100)', () => {
|
||||
assert.equal(riskScore({ critical: 4 }), 100);
|
||||
});
|
||||
|
||||
it('caps at 100 even if raw score would exceed it', () => {
|
||||
assert.equal(riskScore({ critical: 10, high: 10 }), 100);
|
||||
});
|
||||
|
||||
it('returns 10 for one high finding (weight=10)', () => {
|
||||
assert.equal(riskScore({ high: 1 }), 10);
|
||||
});
|
||||
|
||||
it('returns 4 for one medium finding (weight=4)', () => {
|
||||
assert.equal(riskScore({ medium: 1 }), 4);
|
||||
});
|
||||
|
||||
it('returns 1 for one low finding (weight=1)', () => {
|
||||
assert.equal(riskScore({ low: 1 }), 1);
|
||||
});
|
||||
|
||||
it('returns 0 for info-only findings (weight=0)', () => {
|
||||
it('returns 0 for info-only findings (info tier is non-scoring)', () => {
|
||||
assert.equal(riskScore({ info: 100 }), 0);
|
||||
});
|
||||
|
||||
it('returns correct sum for mixed counts', () => {
|
||||
// 1*25 + 2*10 + 3*4 + 4*1 + 5*0 = 25+20+12+4+0 = 61
|
||||
assert.equal(riskScore({ critical: 1, high: 2, medium: 3, low: 4, info: 5 }), 61);
|
||||
// --- Low tier: 1 + log2(n+1)*3, capped at 11 ---
|
||||
it('returns 4 for one low finding', () => {
|
||||
assert.equal(riskScore({ low: 1 }), 4);
|
||||
});
|
||||
|
||||
it('returns 11 for twenty low findings (tier-capped)', () => {
|
||||
assert.equal(riskScore({ low: 20 }), 11);
|
||||
});
|
||||
|
||||
// --- Medium tier: 15 + log2(n+1)*5, capped at 35 ---
|
||||
it('returns 20 for one medium finding (tier base + log scale)', () => {
|
||||
assert.equal(riskScore({ medium: 1 }), 20);
|
||||
});
|
||||
|
||||
it('returns 28 for five medium findings', () => {
|
||||
assert.equal(riskScore({ medium: 5 }), 28);
|
||||
});
|
||||
|
||||
it('returns 29 for six medium findings (still inside Medium band)', () => {
|
||||
assert.equal(riskScore({ medium: 6 }), 29);
|
||||
});
|
||||
|
||||
// --- High tier: 40 + log2(n+1)*8, capped at 65 ---
|
||||
it('returns 48 for one high finding', () => {
|
||||
assert.equal(riskScore({ high: 1 }), 48);
|
||||
});
|
||||
|
||||
it('returns 64 for seven high findings (just below Critical band)', () => {
|
||||
assert.equal(riskScore({ high: 7 }), 64);
|
||||
});
|
||||
|
||||
it('returns 65 when high tier saturates — many high + many medium', () => {
|
||||
// 17 high + 136 medium (hyperframes-like) → high-tier dominates, cap 65
|
||||
assert.equal(riskScore({ high: 17, medium: 136 }), 65);
|
||||
});
|
||||
|
||||
// --- Critical tier: 70 + log2(n+1)*10, capped at 95 ---
|
||||
it('returns 80 for one critical finding', () => {
|
||||
assert.equal(riskScore({ critical: 1 }), 80);
|
||||
});
|
||||
|
||||
it('returns 86 for two critical findings (enters Extreme band)', () => {
|
||||
assert.equal(riskScore({ critical: 2 }), 86);
|
||||
});
|
||||
|
||||
it('returns 93 for four critical findings', () => {
|
||||
assert.equal(riskScore({ critical: 4 }), 93);
|
||||
});
|
||||
|
||||
it('returns 95 for ten critical findings (tier-capped)', () => {
|
||||
assert.equal(riskScore({ critical: 10 }), 95);
|
||||
});
|
||||
|
||||
it('does not exceed 100 even with huge critical counts', () => {
|
||||
assert.ok(riskScore({ critical: 1000 }) <= 100);
|
||||
});
|
||||
|
||||
it('critical dominates high — mixed critical+high scored at critical tier', () => {
|
||||
// {critical:1, high:2} → critical tier: 70 + log2(2)*10 = 80
|
||||
assert.equal(riskScore({ critical: 1, high: 2, medium: 3, low: 4, info: 5 }), 80);
|
||||
});
|
||||
|
||||
it('high dominates medium — {high:1, medium:100} scored at high tier', () => {
|
||||
// 40 + log2(2)*8 = 48
|
||||
assert.equal(riskScore({ high: 1, medium: 100 }), 48);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -93,7 +131,7 @@ describe('riskScore', () => {
|
|||
// verdict
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('verdict', () => {
|
||||
describe('verdict (v7.0.0 — co-monotonic with riskBand)', () => {
|
||||
it('returns ALLOW for zero findings', () => {
|
||||
assert.equal(verdict({ critical: 0, high: 0, medium: 0, low: 0, info: 0 }), 'ALLOW');
|
||||
});
|
||||
|
|
@ -102,37 +140,36 @@ describe('verdict', () => {
|
|||
assert.equal(verdict({}), 'ALLOW');
|
||||
});
|
||||
|
||||
it('returns BLOCK when critical >= 1', () => {
|
||||
it('returns BLOCK when critical >= 1 (score=80)', () => {
|
||||
assert.equal(verdict({ critical: 1 }), 'BLOCK');
|
||||
});
|
||||
|
||||
it('returns BLOCK when score >= 61 (even with no critical)', () => {
|
||||
// Need score >= 61 without critical: 7 high = 70 >= 61
|
||||
assert.equal(verdict({ high: 7 }), 'BLOCK');
|
||||
it('returns BLOCK when score >= 65 without critical (17 high + 136 medium = 65)', () => {
|
||||
assert.equal(verdict({ high: 17, medium: 136 }), 'BLOCK');
|
||||
});
|
||||
|
||||
it('returns BLOCK for score exactly 61', () => {
|
||||
// 1 critical + 2 high + 3 medium + 4 low = 25+20+12+4 = 61
|
||||
assert.equal(verdict({ critical: 1, high: 2, medium: 3, low: 4 }), 'BLOCK');
|
||||
it('returns WARNING for 7 high findings (score=64, Critical band boundary not crossed)', () => {
|
||||
assert.equal(verdict({ high: 7 }), 'WARNING');
|
||||
});
|
||||
|
||||
it('returns WARNING when high >= 1 (and no critical)', () => {
|
||||
assert.equal(verdict({ high: 1 }), 'WARNING');
|
||||
});
|
||||
|
||||
it('returns WARNING when score >= 21 (even with no high or critical)', () => {
|
||||
// 6 medium = 24 >= 21; no critical or high
|
||||
it('returns WARNING for 1 medium (score=20, inside Medium band)', () => {
|
||||
assert.equal(verdict({ medium: 1 }), 'WARNING');
|
||||
});
|
||||
|
||||
it('returns WARNING for 6 medium (score=29)', () => {
|
||||
assert.equal(verdict({ medium: 6 }), 'WARNING');
|
||||
});
|
||||
|
||||
it('returns WARNING for score exactly 21 (no high or critical)', () => {
|
||||
// Smallest score >= 21 from low only would need 21 low, but medium is easier:
|
||||
// 5 medium + 1 low = 20+1 = 21
|
||||
assert.equal(verdict({ medium: 5, low: 1 }), 'WARNING');
|
||||
it('returns ALLOW for 20 low findings (score=11, firmly Low band)', () => {
|
||||
assert.equal(verdict({ low: 20 }), 'ALLOW');
|
||||
});
|
||||
|
||||
it('returns ALLOW for score of 20 (low only, no high/critical)', () => {
|
||||
assert.equal(verdict({ low: 20 }), 'ALLOW');
|
||||
it('returns ALLOW for 1 low finding (score=4)', () => {
|
||||
assert.equal(verdict({ low: 1 }), 'ALLOW');
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -140,56 +177,56 @@ describe('verdict', () => {
|
|||
// riskBand
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('riskBand', () => {
|
||||
describe('riskBand (v7.0.0 cutoffs: 14/39/64/84)', () => {
|
||||
it('returns Low for score 0', () => {
|
||||
assert.equal(riskBand(0), 'Low');
|
||||
});
|
||||
|
||||
it('returns Low for score 20 (boundary)', () => {
|
||||
assert.equal(riskBand(20), 'Low');
|
||||
it('returns Low for score 14 (upper boundary)', () => {
|
||||
assert.equal(riskBand(14), 'Low');
|
||||
});
|
||||
|
||||
it('returns Medium for score 21', () => {
|
||||
assert.equal(riskBand(21), 'Medium');
|
||||
it('returns Medium for score 15 (Medium tier start)', () => {
|
||||
assert.equal(riskBand(15), 'Medium');
|
||||
});
|
||||
|
||||
it('returns Medium for score 25', () => {
|
||||
assert.equal(riskBand(25), 'Medium');
|
||||
it('returns Medium for score 20 (one medium finding)', () => {
|
||||
assert.equal(riskBand(20), 'Medium');
|
||||
});
|
||||
|
||||
it('returns Medium for score 40 (boundary)', () => {
|
||||
assert.equal(riskBand(40), 'Medium');
|
||||
it('returns Medium for score 39 (upper boundary)', () => {
|
||||
assert.equal(riskBand(39), 'Medium');
|
||||
});
|
||||
|
||||
it('returns High for score 41', () => {
|
||||
assert.equal(riskBand(41), 'High');
|
||||
it('returns High for score 40 (High tier start — one high finding is 48)', () => {
|
||||
assert.equal(riskBand(40), 'High');
|
||||
});
|
||||
|
||||
it('returns High for score 50', () => {
|
||||
assert.equal(riskBand(50), 'High');
|
||||
it('returns High for score 48 (one high finding)', () => {
|
||||
assert.equal(riskBand(48), 'High');
|
||||
});
|
||||
|
||||
it('returns High for score 60 (boundary)', () => {
|
||||
assert.equal(riskBand(60), 'High');
|
||||
it('returns High for score 64 (seven high findings, upper boundary)', () => {
|
||||
assert.equal(riskBand(64), 'High');
|
||||
});
|
||||
|
||||
it('returns Critical for score 61', () => {
|
||||
assert.equal(riskBand(61), 'Critical');
|
||||
it('returns Critical for score 65 (many high without critical)', () => {
|
||||
assert.equal(riskBand(65), 'Critical');
|
||||
});
|
||||
|
||||
it('returns Critical for score 75', () => {
|
||||
assert.equal(riskBand(75), 'Critical');
|
||||
});
|
||||
|
||||
it('returns Critical for score 80 (boundary)', () => {
|
||||
it('returns Critical for score 80 (one critical finding)', () => {
|
||||
assert.equal(riskBand(80), 'Critical');
|
||||
});
|
||||
|
||||
it('returns Extreme for score 81', () => {
|
||||
assert.equal(riskBand(81), 'Extreme');
|
||||
it('returns Critical for score 84 (upper boundary)', () => {
|
||||
assert.equal(riskBand(84), 'Critical');
|
||||
});
|
||||
|
||||
it('returns Extreme for score 95', () => {
|
||||
it('returns Extreme for score 85 (two critical findings reach here)', () => {
|
||||
assert.equal(riskBand(85), 'Extreme');
|
||||
});
|
||||
|
||||
it('returns Extreme for score 95 (ten critical findings, tier-capped)', () => {
|
||||
assert.equal(riskBand(95), 'Extreme');
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue