feat(disposition): separate the assessment axis from the action
`decide` returned a `Disposition` — WARN / QUARANTINE_REVIEW / FAIL_SECURE —
which names an ACTION. But BRIEF design principle 4 says the library reports
and the pipeline decides, and disposition.py admitted the gap in its own
docstring: "It imposes no blocking of its own." So we returned an action we
cannot enforce, having discarded the judgement that produced it. A consumer
wanting different behaviour had to reinterpret the action itself — which is
why a consumer ends up pinning our GRADING: the action was all they got.
`Risk` (NONE/LOW/ELEVATED/SEVERE) now carries that judgement, and
`Policy.action_map` lets a caller map it to their own action. Both overlays
move the assessment rather than the action, so a custom map cannot silently
drop the compound escalation or the quarantine floor. `guard`'s fail-closed
path pins both axes and deliberately bypasses the map: downgrading SEVERE
means "I accept this class of finding", never "I accept a crashed scanner".
`DispositionResult.assessment` is required with no default. `Risk.NONE` is the
natural-looking default and the wrong one — a site that forgot the field would
report clean, and the axis would fail open.
MEASURED ADDITIVE, not assumed:
- 703 -> 715 tests, no existing test changed
- coverage matrix 128/128 recall, 6/6 documented gaps still hold
- the PRESET_USER_UPLOAD grading table locked in 0.3.1 re-measured row by
row: ordinary link/image/autolink/refdef -> warn on BOTH doors, unchanged
Both locked consumer promises in docs/PLAN-v1.md were checked against that
measurement and neither fires: the grading is untouched (linkedin-studio), and
the relative-target asymmetry is untouched (llm-ingestion-okf).
Scope held to disposition, per PLAN-v1.md:380. Version stays 0.4.0; the 0.5.0
bump lands in its release commit with all five version surfaces at once —
that is the fix for the defect where the v0.4.0 tag carried a 0.3.4 README.
Records limitation 32: `Severity` still carries disposition intent on the
DETECTION side, which this change does not address and cannot without moving
the grading.
This commit is contained in:
parent
3c56d50e05
commit
de097110d2
7 changed files with 329 additions and 38 deletions
|
|
@ -5,10 +5,12 @@ import pytest
|
|||
|
||||
from llm_ingestion_guard.report import Finding, Report, Severity, Source
|
||||
from llm_ingestion_guard.disposition import (
|
||||
DEFAULT_ACTION_MAP,
|
||||
Disposition,
|
||||
DispositionResult,
|
||||
Policy,
|
||||
Provenance,
|
||||
Risk,
|
||||
Trust,
|
||||
decide,
|
||||
guard,
|
||||
|
|
@ -258,3 +260,99 @@ def test_floor_and_escalation_compose_to_fail_secure():
|
|||
_finding(severity=Severity.MEDIUM, label="entropy:base64", detector="entropy"),
|
||||
)
|
||||
assert decide(report, PRESET_USER_UPLOAD).disposition is Disposition.FAIL_SECURE
|
||||
|
||||
|
||||
# --- 0.5.0 axis separation: assessment (how dangerous) vs disposition (what to
|
||||
# --- do). PLAN-v1.md:294/:380 — change DISPOSITION, never the grading.
|
||||
#
|
||||
# `Disposition` is three *actions*, but BRIEF principle 4 says the library
|
||||
# reports and the *pipeline* decides. `decide` therefore returned an action it
|
||||
# cannot enforce, and a consumer wanting a different action had to re-derive it
|
||||
# from that action — the assessment which produced it was already gone. That is
|
||||
# why a consumer ends up pinning the *grading*: the action is all they get. The
|
||||
# assessment axis hands them the input instead of the verdict.
|
||||
|
||||
|
||||
def test_result_carries_an_assessment_distinct_from_the_disposition():
|
||||
# The payoff: a clean report and a LOW-severity report are BOTH `WARN`
|
||||
# today, and indistinguishable without re-reading the report. The assessment
|
||||
# axis separates them while leaving the action identical.
|
||||
clean = decide(_report(), UNTRUSTED)
|
||||
low = decide(_report(_finding(severity=Severity.LOW, label="active:markdown-link")),
|
||||
UNTRUSTED)
|
||||
|
||||
assert clean.disposition is Disposition.WARN
|
||||
assert low.disposition is Disposition.WARN # action: identical
|
||||
assert clean.assessment is Risk.NONE
|
||||
assert low.assessment is Risk.LOW # assessment: distinct
|
||||
|
||||
|
||||
def test_default_action_map_reproduces_todays_outcomes():
|
||||
# The default mapping is a no-op by construction: this is the contract that
|
||||
# keeps `PRESET_USER_UPLOAD` grading untouched (locked promise 1).
|
||||
assert DEFAULT_ACTION_MAP == {
|
||||
Risk.NONE: Disposition.WARN,
|
||||
Risk.LOW: Disposition.WARN,
|
||||
Risk.ELEVATED: Disposition.QUARANTINE_REVIEW,
|
||||
Risk.SEVERE: Disposition.FAIL_SECURE,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("severity,policy,provenance,expected_risk", [
|
||||
(Severity.CRITICAL, TRUSTED, Provenance.PROSE, Risk.SEVERE),
|
||||
(Severity.HIGH, TRUSTED, Provenance.PROSE, Risk.LOW),
|
||||
(Severity.HIGH, TRUSTED, Provenance.CODE_FENCE, Risk.SEVERE),
|
||||
(Severity.HIGH, UNTRUSTED, Provenance.PROSE, Risk.SEVERE),
|
||||
(Severity.MEDIUM, TRUSTED, Provenance.PROSE, Risk.LOW),
|
||||
(Severity.MEDIUM, UNTRUSTED, Provenance.PROSE, Risk.ELEVATED),
|
||||
(Severity.LOW, UNTRUSTED, Provenance.PROSE, Risk.LOW),
|
||||
])
|
||||
def test_assessment_tracks_danger_given_trust(severity, policy, provenance, expected_risk):
|
||||
# The assessment is trust-aware, exactly as BRIEF §4.7 describes: the *same*
|
||||
# hit is a different assessment in prose vs a code fence — not merely a
|
||||
# different action on one shared assessment.
|
||||
result = decide(_report(_finding(severity=severity)), policy, provenance=provenance)
|
||||
assert result.assessment is expected_risk
|
||||
assert result.disposition is DEFAULT_ACTION_MAP[expected_risk]
|
||||
|
||||
|
||||
def test_custom_action_map_changes_the_action_not_the_assessment():
|
||||
# THE point of the separation. A consumer that wants to hold for review
|
||||
# rather than block says so in the policy, and the assessment it was derived
|
||||
# from is unchanged — so they never have to pin our grading to get their
|
||||
# behaviour.
|
||||
report = _report(_finding(severity=Severity.HIGH))
|
||||
strict = Policy(trust=Trust.UNTRUSTED)
|
||||
lenient = Policy(trust=Trust.UNTRUSTED, action_map={
|
||||
**DEFAULT_ACTION_MAP,
|
||||
Risk.SEVERE: Disposition.QUARANTINE_REVIEW,
|
||||
})
|
||||
|
||||
assert decide(report, strict).disposition is Disposition.FAIL_SECURE
|
||||
assert decide(report, lenient).disposition is Disposition.QUARANTINE_REVIEW
|
||||
assert decide(report, strict).assessment is Risk.SEVERE
|
||||
assert decide(report, lenient).assessment is Risk.SEVERE
|
||||
|
||||
|
||||
def test_overlays_escalate_the_assessment_not_only_the_action():
|
||||
# Compound escalation and the quarantine floor are assessment-level moves;
|
||||
# if they only moved the action, a custom action_map would silently drop
|
||||
# them. Two MEDIUM findings untrusted: ELEVATED escalated to SEVERE.
|
||||
report = _report(
|
||||
_finding(severity=Severity.MEDIUM, label="lexicon:config"),
|
||||
_finding(severity=Severity.MEDIUM, label="entropy:base64", detector="entropy"),
|
||||
)
|
||||
result = decide(report, PRESET_USER_UPLOAD)
|
||||
assert result.assessment is Risk.SEVERE
|
||||
assert result.disposition is Disposition.FAIL_SECURE
|
||||
|
||||
|
||||
def test_guard_fails_closed_with_a_severe_assessment():
|
||||
# The fail-closed path must not leave the assessment unset, or a consumer
|
||||
# mapping on the assessment alone would read a scanner crash as clean.
|
||||
def boom() -> Report:
|
||||
raise RuntimeError("detector exploded")
|
||||
|
||||
result = guard(boom, PRESET_USER_UPLOAD)
|
||||
assert result.disposition is Disposition.FAIL_SECURE
|
||||
assert result.assessment is Risk.SEVERE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue