Kjell Tore Guttormsen
cff043787d
The output gate claimed LLM10 self-safety on the grounds that its patterns have no nested quantifiers. True, and irrelevant: nesting is not what makes these blow up. A run in front of a REQUIRED literal, reachable from a short anchor, is enough -- crafted input repeats the anchor and never supplies the literal, so every start position rescans the tail. Quadratic, not exponential, and the max_scan_chars cap does not help: it bounds the input, and quadratic work on a bounded input is still hours. Measured, not argued. `<a:` x 100_000 took 23.4s in AUTOLINK_RE alone; the composed gate on that payload took 458.7s, extrapolating to ~5.7 hours at the 1_000_000-char input the gate itself accepts. Size-matched ordinary prose runs 0.31s, so the separation is 18x-660x -- unlike the blob in the neighbouring test, which is the *faster* side of prose and never exercised backtracking. Two fixes, chosen per pattern rather than uniformly: - active_content + lexicon JSON (15 runs): exclude the character that opens the pattern's own anchor (`[` for markdown, `<` for tags), so a run cannot reach past the next start position and the per-start costs telescope. Verified to cost no recall: long URLs, long alt text, and `<` inside a quoted attribute all still match. Bounding instead would have been linear too but wrong here -- the content is attacker-controlled, so padding past a bound would be a one-line bypass of the EchoLeak class this table exists to catch. - connstr egress (4 runs): bound the password at MAX_CONNSTR_VALUE. The exclusion fix is unavailable -- the anchor character is `/` and passwords containing `/` are the common case (measured: they match today). The residual miss is a credential over 256 chars; a token that long is still caught by egress:jwt-token. hybrid-xss:script-tag had neither option: its run is the script BODY, which may legitimately contain `<`. It now matches the opening tag and drops the `</script>` requirement. That also closes a fail-open -- `<script>alert(1)` unclosed was silently missed -- at the cost of flagging prose that merely mentions `<script>`, now documented. Found by the composed-gate test staying red after every individual scanner was already linear: the lexicon's six html-obfuscation patterns were the remaining 813x. A per-scanner test alone would have shipped that. 662 passed (was 642), and faster than before the fix.
138 lines
5.4 KiB
Python
138 lines
5.4 KiB
Python
"""test_calibration — freeze the shared calibration surface (Session D).
|
|
|
|
Session D consolidated every tunable threshold into
|
|
``llm_ingestion_guard.calibration`` so the Node port can mirror *exactly* the
|
|
same numbers. These tests are the frozen contract in two halves:
|
|
|
|
1. the raw values themselves (the tuple the port shares), and
|
|
2. the proof that each detector actually *sources* its threshold from here —
|
|
so the freeze is a live single-source-of-truth, not a dead copy that can
|
|
silently drift from the value the code uses.
|
|
|
|
Changing a calibration number is a deliberate recalibration: it must break a
|
|
test here first.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from llm_ingestion_guard import calibration as cal
|
|
from llm_ingestion_guard.report import Severity
|
|
|
|
|
|
# --- frozen raw values ------------------------------------------------------
|
|
|
|
def test_entropy_thresholds_frozen():
|
|
assert (cal.ENTROPY_CRITICAL_H, cal.ENTROPY_CRITICAL_LEN) == (5.4, 128)
|
|
assert (cal.ENTROPY_HIGH_H, cal.ENTROPY_HIGH_LEN) == (5.1, 64)
|
|
assert (cal.ENTROPY_MEDIUM_H, cal.ENTROPY_MEDIUM_LEN) == (4.7, 40)
|
|
|
|
|
|
def test_entropy_shape_floors_frozen():
|
|
assert cal.ENTROPY_BASE64_FLOOR_LEN == 100
|
|
assert cal.ENTROPY_HEX_FLOOR_LEN == 64
|
|
|
|
|
|
def test_lexicon_selfsafety_frozen():
|
|
assert cal.MAX_SCAN_CHARS == 1_000_000
|
|
assert cal.ROT13_MIN_LEN == 40
|
|
|
|
|
|
def test_output_selfsafety_frozen():
|
|
assert cal.MAX_CONNSTR_VALUE == 256
|
|
|
|
|
|
def test_cognitive_load_lengths_frozen():
|
|
assert cal.COGNITIVE_LOAD_MIN_LEN == 2500
|
|
assert cal.COGNITIVE_LOAD_TAIL_START == 2000
|
|
|
|
|
|
def test_disposition_rank_frozen():
|
|
assert cal.DISPOSITION_RANK == {
|
|
"warn": 0,
|
|
"quarantine_review": 1,
|
|
"fail_secure": 2,
|
|
}
|
|
|
|
|
|
def test_active_content_severity_frozen():
|
|
assert cal.ACTIVE_CONTENT_SEVERITY == {
|
|
"markdown-image": Severity.HIGH,
|
|
"markdown-link": Severity.MEDIUM,
|
|
"reference-link": Severity.MEDIUM,
|
|
"autolink": Severity.MEDIUM,
|
|
"raw-html": Severity.HIGH,
|
|
"data-uri": Severity.HIGH,
|
|
}
|
|
|
|
|
|
def test_url_shape_thresholds_frozen():
|
|
# 0.3.1: severity grades on URL shape. These floors sit above every
|
|
# legitimate documentation URL token measured on 2026-07-25 (worst: H=4.08)
|
|
# and below the base64/hex payload segments an exfil path uses (4.36-4.54).
|
|
assert cal.ACTIVE_CONTENT_ORDINARY_SEVERITY is Severity.LOW
|
|
assert (cal.URL_OPAQUE_ENTROPY_H, cal.URL_OPAQUE_MIN_LEN) == (4.4, 24)
|
|
assert cal.URL_OPAQUE_HEX_MIN_LEN == 32
|
|
|
|
|
|
def test_no_detector_emitted_low_before_the_url_shape_change():
|
|
"""The floor change (any finding -> MEDIUM+) is only honest as a *patch* if
|
|
nothing that shipped before it emitted LOW — otherwise it would silently
|
|
loosen an existing consumer's gate. The lexicon is the only table-driven
|
|
severity source; assert it still holds no LOW/INFO pattern."""
|
|
from llm_ingestion_guard.lexicon import load_lexicon
|
|
assert not [p for p in load_lexicon()
|
|
if p.severity in (Severity.LOW, Severity.INFO)]
|
|
|
|
|
|
# --- binding: each detector reads its threshold from calibration ------------
|
|
# The freeze is meaningful only if the modules actually READ these values. An
|
|
# import alias binds the SAME object, so identity (`is`) proves the single
|
|
# source of truth rather than a coincidental equal copy.
|
|
|
|
def test_entropy_module_sources_from_calibration():
|
|
from llm_ingestion_guard import entropy
|
|
assert entropy._CRITICAL_H is cal.ENTROPY_CRITICAL_H
|
|
assert entropy._CRITICAL_LEN is cal.ENTROPY_CRITICAL_LEN
|
|
assert entropy._HIGH_H is cal.ENTROPY_HIGH_H
|
|
assert entropy._HIGH_LEN is cal.ENTROPY_HIGH_LEN
|
|
assert entropy._MEDIUM_H is cal.ENTROPY_MEDIUM_H
|
|
assert entropy._MEDIUM_LEN is cal.ENTROPY_MEDIUM_LEN
|
|
assert entropy._BASE64_FLOOR_LEN is cal.ENTROPY_BASE64_FLOOR_LEN
|
|
assert entropy._HEX_FLOOR_LEN is cal.ENTROPY_HEX_FLOOR_LEN
|
|
|
|
|
|
def test_lexicon_module_sources_from_calibration():
|
|
from llm_ingestion_guard import lexicon
|
|
assert lexicon.MAX_SCAN_CHARS is cal.MAX_SCAN_CHARS
|
|
assert lexicon._ROT13_MIN_LEN is cal.ROT13_MIN_LEN
|
|
|
|
|
|
def test_output_module_sources_from_calibration():
|
|
# The bound is baked into the compiled patterns, so `is` on a module
|
|
# attribute cannot prove sourcing here -- assert the compiled regex carries
|
|
# the calibrated number instead.
|
|
from llm_ingestion_guard import output
|
|
assert output.MAX_CONNSTR_VALUE is cal.MAX_CONNSTR_VALUE
|
|
connstr = [p for p in output._SECRET_PATTERNS if p.id.endswith("-connstr")]
|
|
assert len(connstr) == 4
|
|
for pattern in connstr:
|
|
assert f"{{1,{cal.MAX_CONNSTR_VALUE}}}" in pattern.regex.pattern
|
|
|
|
|
|
def test_disposition_module_sources_from_calibration():
|
|
from llm_ingestion_guard import disposition
|
|
from llm_ingestion_guard.disposition import Disposition
|
|
# Enum-keyed rank reconstructed from calibration's value-keyed source.
|
|
assert disposition._DISPOSITION_RANK == {
|
|
Disposition.WARN: 0,
|
|
Disposition.QUARANTINE_REVIEW: 1,
|
|
Disposition.FAIL_SECURE: 2,
|
|
}
|
|
|
|
|
|
def test_active_content_module_sources_from_calibration():
|
|
from llm_ingestion_guard import active_content
|
|
assert active_content._SEVERITY is cal.ACTIVE_CONTENT_SEVERITY
|
|
assert active_content._ORDINARY_SEVERITY is cal.ACTIVE_CONTENT_ORDINARY_SEVERITY
|
|
assert active_content._OPAQUE_H is cal.URL_OPAQUE_ENTROPY_H
|
|
assert active_content._OPAQUE_MIN_LEN is cal.URL_OPAQUE_MIN_LEN
|
|
assert active_content._OPAQUE_HEX_LEN is cal.URL_OPAQUE_HEX_MIN_LEN
|