llm-ingestion-pipeline-security/src/llm_ingestion_guard/calibration.py

"""calibration — the one place every tunable threshold lives.

Every detector in this package is calibrated by a handful of numbers: entropy
floors, an input-size cap, minimum lengths, and two small severity/rank tables.
Scattered across four modules, those numbers are impossible to audit and — the
concrete driver here — impossible for a port to mirror *exactly*. A parallel
Node/TypeScript implementation of this gate must classify byte-for-byte the same
way, which means it must share the same constants. This module is that shared
contract: one documented surface the port copies verbatim.

**This module holds values, never logic.** It depends only on
:mod:`~llm_ingestion_guard.report` (for the :class:`Severity` enum used by the
active-content table) and is imported by ``entropy``, ``lexicon``,
``disposition`` and ``active_content`` — a leaf in the dependency graph, so no
import cycle is possible.

Changing any number here is a deliberate recalibration, not a refactor:
``tests/test_calibration.py`` freezes these values and asserts each detector
actually sources its threshold from here.
"""
from __future__ import annotations

from .report import Severity

# --- entropy: length-calibrated Shannon-entropy tiers -----------------------
# Bits-per-char floor paired with a minimum length, because the achievable
# entropy maximum is length-dependent (a short base64 string cannot reach the
# entropy of a long one). Empirically calibrated against the seed scanner:
# plaintext prose H ~3.5-4.2; base64 len64 H ~5.2; base64 len128 H ~5.6.
ENTROPY_CRITICAL_H, ENTROPY_CRITICAL_LEN = 5.4, 128
ENTROPY_HIGH_H, ENTROPY_HIGH_LEN = 5.1, 64
ENTROPY_MEDIUM_H, ENTROPY_MEDIUM_LEN = 4.7, 40

# Shape floors: structured encodings suspicious by size even when their entropy
# sits below the classification floor. The hex floor is the *only* path that
# classifies a hex blob (a 16-symbol alphabet caps entropy at log2(16)=4.0,
# below the 4.7 MEDIUM floor).
ENTROPY_BASE64_FLOOR_LEN = 100
ENTROPY_HEX_FLOOR_LEN = 64

# --- lexicon: self-safety + variant thresholds ------------------------------
# Input-size cap (OWASP LLM06:2026): large enough for a real ingested document;
# beyond it the scanner reads the prefix and flags, so every sub-scanner sees a
# bounded input. Note what this cap does NOT buy: bounded input is only bounded
# runtime if the patterns are linear in it. A quadratic pattern turns this cap
# into hours of work, which is what crafted input against the output path was
# measured to do before the ReDoS fix (see active_content's pattern-table note).
MAX_SCAN_CHARS = 1_000_000

# --- transform surfaces: input cap ------------------------------------------
# The same size, and deliberately NOT the same constant, because the two caps
# buy different things and may need to move apart. MAX_SCAN_CHARS truncates: a
# scanner returns findings, so reading the prefix costs *detection* on the tail
# and nothing else. `sanitize` / `fence` / `neutralize` return content, so the
# equivalent move would hand back either a shortened document (silent data loss)
# or an untransformed tail (a bypass an attacker positions the payload into).
# They reject instead — see contract.OversizeInputError. Held at 1M so a
# document accepted by the input path is one the scanners can also read whole.
MAX_INPUT_CHARS = 1_000_000

# --- output: secret-egress self-safety --------------------------------------
# Longest password a connection-string pattern will match. A bound is required
# (not merely nice) because the run sits in front of a mandatory `@`: unbounded,
# crafted input repeating `redis://:` and never supplying the `@` makes every
# start position rescan the tail — quadratic. Excluding the anchor character the
# way the active-content table does is not available here, since that character
# is `/` and passwords containing `/` are the common case.
# 256 is generous for a password and cheap to scan; the residual miss is a
# credential longer than this, which for the realistic case (a token used as a
# DB password) is still caught by the jwt-token / high-specificity patterns.
MAX_CONNSTR_VALUE = 256

# --- output: hidden markup (A04, A14) ---------------------------------------
# A font below this many CSS pixels is reported as `html-obfuscation:tiny-font`
# (strictly below; pt, em, rem and % are converted at 16px per em). Text under
# 4px cannot be read on any screen, and the published attacks used 1px and
# 0.1px. Measured 2026-10-31: no inline font-size below 8px in any of the 3 505
# benign documents or the 78 OKF concepts, so the threshold does not move the
# false-positive populations.
HIDDEN_FONT_MAX_PX = 4.0

# Largest per-channel difference (0-255) at which a text colour counts as the
# background declared in the same style attribute: `#fefefe` on `#ffffff` is
# camouflage, `#eeeeee` on `#ffffff` (17) is light-grey design.
HIDDEN_COLOUR_TOLERANCE = 8

# Longest style attribute value that is parsed. The bound keeps the attribute
# regex linear (an unclosed quote would otherwise make every `style=` rescan the
# tail); a longer attribute is skipped, which is the residual.
MAX_STYLE_ATTR_CHARS = 2048

# Minimum length before the rot13 variant is scanned — shorter strings hit
# rot13-look-alike false positives.
ROT13_MIN_LEN = 40

# Cognitive-load trap: a CRITICAL pattern found *only* past the first
# COGNITIVE_LOAD_TAIL_START chars of text at least COGNITIVE_LOAD_MIN_LEN long is
# a human-in-the-loop trap (an override buried at the tail of verbose output).
COGNITIVE_LOAD_MIN_LEN = 2500
COGNITIVE_LOAD_TAIL_START = 2000

# --- disposition: gate-decision ordering ------------------------------------
# Rank of each disposition, keyed by its enum *value* string (kept primitive so
# this leaf module needs no import from ``disposition``, which would cycle).
# ``disposition`` rebuilds the enum-keyed map from this. Higher = more severe.
DISPOSITION_RANK = {
    "warn": 0,
    "quarantine_review": 1,
    "fail_secure": 2,
}

# --- assessment: the risk axis (0.5.0 axis separation) ----------------------
# Rank of each risk level, same value-keyed convention as DISPOSITION_RANK.
# ``Risk`` answers *how dangerous is this artifact given its source context*;
# ``Disposition`` answers *what should the pipeline do*. They were one enum
# through 0.4.0, which meant a consumer wanting a different action had to
# re-derive it from the action itself — the assessment was already discarded.
RISK_RANK = {
    "none": 0,
    "low": 1,
    "elevated": 2,
    "severe": 3,
}

# The default risk -> disposition mapping, keyed by both enum *value* strings.
# This map is what keeps the separation additive: it reproduces every
# disposition 0.4.0 rendered, so a caller that ignores the new axis sees no
# change at all. A ``Policy`` may override it; ``None`` means "use this".
#
# NONE and LOW both map to ``warn`` deliberately — that collapse is precisely
# the information 0.4.0 could not express, since a clean document and one
# carrying only low-severity findings were the same single value.
DEFAULT_ACTION_MAP = {
    "none": "warn",
    "low": "warn",
    "elevated": "quarantine_review",
    "severe": "fail_secure",
}

# --- active_content: per-construct severities -------------------------------
# Zero-click auto-fetch / auto-execute constructs are HIGH; click-required ones
# are MEDIUM. Mirrors ``neutralize``'s defang classes. These are the severities
# of a construct whose URL can *carry data outward* — see the shape analysis
# below for the ordinary case.
ACTIVE_CONTENT_SEVERITY = {
    "markdown-image": Severity.HIGH,
    "markdown-link": Severity.MEDIUM,
    "reference-link": Severity.MEDIUM,
    "autolink": Severity.MEDIUM,
    "raw-html": Severity.HIGH,
    # The raw-HTML carrier split (0.7.0). Until 0.6.1 every active tag was HIGH,
    # so the same URL was MEDIUM as `[t](url)` and HIGH as `<a href="url">` — an
    # asymmetry that came from syntax rather than affordance. Following an anchor
    # needs a human, so it grades with the other click-required carriers.
    "raw-html-link": Severity.MEDIUM,
    "data-uri": Severity.HIGH,
}

# --- active_content: URL shape analysis (0.3.1 recalibration) ---------------
# The exfiltration primitive is not "an image" — it is a URL that moves bytes to
# a host the attacker controls. Grading on construct type made
# ``![diagram](https://example.com/arch.png)`` HIGH, which fail-secured ordinary
# documents on the upload preset (measured, v0.3.0). A URL that only *names* a
# remote document is graded ORDINARY instead.
ACTIVE_CONTENT_ORDINARY_SEVERITY = Severity.LOW

# A URL token (host label or path segment) is *opaque* — carried data rather
# than a name — at these floors. Measured 2026-07-25 against real documentation
# URLs (Microsoft Learn, Wikipedia, GitHub raw, regjeringen.no): the worst
# legitimate token scored H=4.08 at length 44, while base64/hex payload segments
# scored 4.36-4.54; random base62 averages 4.23 at length 24. The floor sits
# above every measured legitimate token with margin, because a false positive
# here is what 0.3.1 exists to fix.
URL_OPAQUE_ENTROPY_H, URL_OPAQUE_MIN_LEN = 4.4, 24

# --- active_content: sequence-carried exfiltration (A05, 1.7.0) -------------
# A run of external images can carry a value in its ORDER while every URL in it
# is ordinary (CamoLeak: one image per character from a pre-built URL alphabet).
# `active:markdown-image-sequence` reports a run of at least
# IMAGE_SEQUENCE_MIN_RUN images on one URL template whose varying tokens are
# short symbols, IMAGE_SEQUENCE_SYMBOL_MAX characters or fewer — one byte in hex.
# A longer varying token reads as a name (a gallery), which is the residual.
#
# Measured 2026-10-31 before building: 0 firings at every run length from 3 to 8
# on six benign populations (the three false-positive corpora, 3 505 documents
# and none with an external markdown image; 411 npm READMEs and 166 PyPI
# descriptions, 312 of them with external images; the pinned OKF corpus, 78). The corpora
# therefore do not bind the minimum. 4 is the shortest value worth stealing in
# one run (a four-digit code); the published case is six.
IMAGE_SEQUENCE_MIN_RUN = 4
IMAGE_SEQUENCE_SYMBOL_MAX = 2
# MEDIUM, so the run goes to review rather than to a block: each image on its own
# carries nothing, the reading of the order is an inference from shape, and the
# shape has benign twins (a table of country flags) documented in LIMITATIONS.
ACTIVE_CONTENT_SEQUENCE_SEVERITY = Severity.MEDIUM

# Hex floor for a URL token. Deliberately lower than ENTROPY_HEX_FLOOR_LEN (64):
# in prose a 32-char hex run is usually a checksum, but as a whole path segment
# or host label it is an opaque id — the md5/uuid length an exfil path uses.
URL_OPAQUE_HEX_MIN_LEN = 32