Kjell Tore Guttormsen
6e9b8168e3
v0.3.0 made the untrusted upload path unusable: measured on both doors, an ordinary remote image fail_secure'd and an ordinary link/autolink/refdef quarantined, so only documents without external references persisted. Two independent defects compounded; neither fix works alone: 1. `markdown-image: HIGH` fired on any external image. The exfil primitive is a URL that moves bytes outward, not an image. `is_ordinary_url` now grades on shape - http(s)/protocol-relative, no query, no userinfo, no percent-escape, no opaque host label or path segment -> LOW; anything data-carrying keeps the carrier's severity. raw-html and data: URIs stay HIGH unconditionally. Opacity reuses entropy's primitives; floors calibrated against real doc URLs (worst legit token H=4.08, exfil segments 4.36-4.54) and frozen in calibration. 2. The quarantine_default floor fired on ANY finding, a premise that broke when every ordinary link became a finding. It now fires at MEDIUM+ - a no-op for every detector that shipped before 0.3.0 (no LOW/INFO exists), which is what makes this a patch rather than a minor. The corpus blind spot that let this pass 522 green tests is closed: the FP corpus carries realistic markdown and is asserted on the OUTPUT gate under PRESET_USER_UPLOAD, with a counter-corpus of exfil-shaped URLs that must still block. Beaconing and short opaque segments are conceded in LIMITATIONS and asserted by the coverage matrix rather than papered over. No new public API; no new preset (0.4.0 work); allow_reserved default unchanged.
122 lines
4.8 KiB
Python
122 lines
4.8 KiB
Python
"""test_calibration — freeze the shared calibration surface (Session D).
|
|
|
|
Session D consolidated every tunable threshold into
|
|
``llm_ingestion_guard.calibration`` so the Node port can mirror *exactly* the
|
|
same numbers. These tests are the frozen contract in two halves:
|
|
|
|
1. the raw values themselves (the tuple the port shares), and
|
|
2. the proof that each detector actually *sources* its threshold from here —
|
|
so the freeze is a live single-source-of-truth, not a dead copy that can
|
|
silently drift from the value the code uses.
|
|
|
|
Changing a calibration number is a deliberate recalibration: it must break a
|
|
test here first.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from llm_ingestion_guard import calibration as cal
|
|
from llm_ingestion_guard.report import Severity
|
|
|
|
|
|
# --- frozen raw values ------------------------------------------------------
|
|
|
|
def test_entropy_thresholds_frozen():
|
|
assert (cal.ENTROPY_CRITICAL_H, cal.ENTROPY_CRITICAL_LEN) == (5.4, 128)
|
|
assert (cal.ENTROPY_HIGH_H, cal.ENTROPY_HIGH_LEN) == (5.1, 64)
|
|
assert (cal.ENTROPY_MEDIUM_H, cal.ENTROPY_MEDIUM_LEN) == (4.7, 40)
|
|
|
|
|
|
def test_entropy_shape_floors_frozen():
|
|
assert cal.ENTROPY_BASE64_FLOOR_LEN == 100
|
|
assert cal.ENTROPY_HEX_FLOOR_LEN == 64
|
|
|
|
|
|
def test_lexicon_selfsafety_frozen():
|
|
assert cal.MAX_SCAN_CHARS == 1_000_000
|
|
assert cal.ROT13_MIN_LEN == 40
|
|
|
|
|
|
def test_cognitive_load_lengths_frozen():
|
|
assert cal.COGNITIVE_LOAD_MIN_LEN == 2500
|
|
assert cal.COGNITIVE_LOAD_TAIL_START == 2000
|
|
|
|
|
|
def test_disposition_rank_frozen():
|
|
assert cal.DISPOSITION_RANK == {
|
|
"warn": 0,
|
|
"quarantine_review": 1,
|
|
"fail_secure": 2,
|
|
}
|
|
|
|
|
|
def test_active_content_severity_frozen():
|
|
assert cal.ACTIVE_CONTENT_SEVERITY == {
|
|
"markdown-image": Severity.HIGH,
|
|
"markdown-link": Severity.MEDIUM,
|
|
"reference-link": Severity.MEDIUM,
|
|
"autolink": Severity.MEDIUM,
|
|
"raw-html": Severity.HIGH,
|
|
"data-uri": Severity.HIGH,
|
|
}
|
|
|
|
|
|
def test_url_shape_thresholds_frozen():
|
|
# 0.3.1: severity grades on URL shape. These floors sit above every
|
|
# legitimate documentation URL token measured on 2026-07-25 (worst: H=4.08)
|
|
# and below the base64/hex payload segments an exfil path uses (4.36-4.54).
|
|
assert cal.ACTIVE_CONTENT_ORDINARY_SEVERITY is Severity.LOW
|
|
assert (cal.URL_OPAQUE_ENTROPY_H, cal.URL_OPAQUE_MIN_LEN) == (4.4, 24)
|
|
assert cal.URL_OPAQUE_HEX_MIN_LEN == 32
|
|
|
|
|
|
def test_no_detector_emitted_low_before_the_url_shape_change():
|
|
"""The floor change (any finding -> MEDIUM+) is only honest as a *patch* if
|
|
nothing that shipped before it emitted LOW — otherwise it would silently
|
|
loosen an existing consumer's gate. The lexicon is the only table-driven
|
|
severity source; assert it still holds no LOW/INFO pattern."""
|
|
from llm_ingestion_guard.lexicon import load_lexicon
|
|
assert not [p for p in load_lexicon()
|
|
if p.severity in (Severity.LOW, Severity.INFO)]
|
|
|
|
|
|
# --- binding: each detector reads its threshold from calibration ------------
|
|
# The freeze is meaningful only if the modules actually READ these values. An
|
|
# import alias binds the SAME object, so identity (`is`) proves the single
|
|
# source of truth rather than a coincidental equal copy.
|
|
|
|
def test_entropy_module_sources_from_calibration():
|
|
from llm_ingestion_guard import entropy
|
|
assert entropy._CRITICAL_H is cal.ENTROPY_CRITICAL_H
|
|
assert entropy._CRITICAL_LEN is cal.ENTROPY_CRITICAL_LEN
|
|
assert entropy._HIGH_H is cal.ENTROPY_HIGH_H
|
|
assert entropy._HIGH_LEN is cal.ENTROPY_HIGH_LEN
|
|
assert entropy._MEDIUM_H is cal.ENTROPY_MEDIUM_H
|
|
assert entropy._MEDIUM_LEN is cal.ENTROPY_MEDIUM_LEN
|
|
assert entropy._BASE64_FLOOR_LEN is cal.ENTROPY_BASE64_FLOOR_LEN
|
|
assert entropy._HEX_FLOOR_LEN is cal.ENTROPY_HEX_FLOOR_LEN
|
|
|
|
|
|
def test_lexicon_module_sources_from_calibration():
|
|
from llm_ingestion_guard import lexicon
|
|
assert lexicon.MAX_SCAN_CHARS is cal.MAX_SCAN_CHARS
|
|
assert lexicon._ROT13_MIN_LEN is cal.ROT13_MIN_LEN
|
|
|
|
|
|
def test_disposition_module_sources_from_calibration():
|
|
from llm_ingestion_guard import disposition
|
|
from llm_ingestion_guard.disposition import Disposition
|
|
# Enum-keyed rank reconstructed from calibration's value-keyed source.
|
|
assert disposition._DISPOSITION_RANK == {
|
|
Disposition.WARN: 0,
|
|
Disposition.QUARANTINE_REVIEW: 1,
|
|
Disposition.FAIL_SECURE: 2,
|
|
}
|
|
|
|
|
|
def test_active_content_module_sources_from_calibration():
|
|
from llm_ingestion_guard import active_content
|
|
assert active_content._SEVERITY is cal.ACTIVE_CONTENT_SEVERITY
|
|
assert active_content._ORDINARY_SEVERITY is cal.ACTIVE_CONTENT_ORDINARY_SEVERITY
|
|
assert active_content._OPAQUE_H is cal.URL_OPAQUE_ENTROPY_H
|
|
assert active_content._OPAQUE_MIN_LEN is cal.URL_OPAQUE_MIN_LEN
|
|
assert active_content._OPAQUE_HEX_LEN is cal.URL_OPAQUE_HEX_MIN_LEN
|