(?:\s+\"[^\"]*\")?)\s*\)" ) MD_LINK_RE = re.compile( r"(?<!!)\[(?P<text>[^\]\[]*)\]$\s*(?P<url>[^)\s\[]+)(?P<title>(?:\s+\"[^\"]*\")?)\s*$" ) # Reference-style link definition: `[label]: destination`. Only fires when the # destination is absolute (has a scheme or is protocol-relative) — a footnote # `[1]: some plain text` is not a link target and is left alone. MD_REFDEF_RE = re.compile( r"(?m)^(?P<pre>[ ]{0,3}\[[^\]\[]+\]:\s*)(?P<url>[A-Za-z][\w+.\-]*:\S+|//\S+)" ) # Angle-bracket autolink: `<scheme:...>`. A URL inside `<...>` cannot contain a # raw `<`, so excluding it costs no recall (verified) and bounds the run. AUTOLINK_RE = re.compile(r"<(?P<url>[A-Za-z][A-Za-z0-9+.\-]*:[^>\s<]+)>") # Standalone `data:` URI in prose (not preceded by a letter/digit -> "metadata:" # is not a match), consuming to the next whitespace / quote / bracket. DATA_URI_RE = re.compile(r"(?<![A-Za-z0-9])data:[^\s'\"<>)]+", re.IGNORECASE) # Raw HTML tag. Attribute values may hold `>` inside quotes, so quoted runs are # consumed atomically. A tag is *active* if it is an inherently-executing element, # carries an event handler, or carries a URL-bearing attribute. The unquoted-char # branch excludes `<` per the ReDoS note above — a raw `<` cannot appear in an # unquoted attribute region anyway, and a `<` inside a QUOTED value is still # consumed by the quoted branches, so this costs no recall (verified). HTML_TAG_RE = re.compile(r"<(?P<slash>/?)(?P<name>[A-Za-z][A-Za-z0-9:-]*)(?P<attrs>(?:[^>\"'<]|\"[^\"]*\"|'[^']*')*)>") _EVENT_ATTR_RE = re.compile(r"\bon[a-z]+\s*=", re.IGNORECASE) _URL_ATTR_RE = re.compile( r"\b(?:src|href|xlink:href|srcset|data|poster|formaction|action|background|cite|codebase|longdesc)\s*=", re.IGNORECASE, ) _ACTIVE_TAGS = frozenset({ "script", "iframe", "object", "embed", "svg", "math", "link", "meta", "base", "form", "img", "input", "button", "video", "audio", "source", "track", "a", "area", "frame", "frameset", "applet", "style", }) # The SCANNER's name set. `<base>`'s only affordance is its `href`, which the # URL-attribute branch still catches; `<base />` without one is inert. The mutator # keeps the full set — see the module docstring. _SCANNER_ACTIVE_TAGS = _ACTIVE_TAGS - {"base"} # Tags whose entire active affordance IS the URL they name. Carrying no URL # attribute at all, they name no target, so no renderer can fetch or follow them # — `<base />`'s argument (0.6.0) applied to the rest of the name branch. The # shapes this frees, observed inside vendor-harvest's fail_secure documents: # `</a>`, `<Frame>`/`</Frame>`, `<video />`, and `<img alt=...>` with no `src` — # end tags and MDX wrapper components dominate. # Everything else in the name set does something a URL cannot describe — # `<script>` executes its body, `<style>` restyles, `<form>` submits — and stays # active with no attributes at all. _URL_AFFORDANCE_TAGS = frozenset({ "a", "area", "img", "video", "audio", "source", "track", "frame", "frameset", }) # Click-required carriers: following one needs a human, exactly like a markdown # inline link. Everything else the renderer fetches or executes unattended. _LINK_TAGS = frozenset({"a", "area"}) # `_URL_ATTR_RE` above is a presence test and deliberately captures no value. # Reading the value needs the same literal alternation with the value attached, so # no new run shape enters the table: every run here sits in front of a required # literal that the alternation has already anchored. (Self-safety, OWASP LLM10 — # `tests/test_output.py::_REDOS_PAYLOADS` carries the measured row.) _URL_ATTR_VALUE_RE = re.compile( r"\b(?:src|href|xlink:href|srcset|data|poster|formaction|action|background|cite|codebase|longdesc)" r"\s*=\s*(?P<v>\"[^\"]*\"|'[^']*'|[^\s>]+)", re.IGNORECASE, ) # `srcset` holds a comma-separated candidate list, so an attribute value is not # always one URL. Splitting means a relative first candidate cannot mask an # external one behind it. _URL_CANDIDATE_SPLIT_RE = re.compile(r"[,\s]+") def _url_attr_is_external(attrs: str) -> bool: """True if a URL-bearing attribute names an attacker-reachable target. Fail-secure: an attribute ``_URL_ATTR_RE`` saw but whose value cannot be read here counts as external, so a gap between the two patterns over-blocks rather than under-blocks. """ seen = False for m in _URL_ATTR_VALUE_RE.finditer(attrs): seen = True value = m.group("v") if value[:1] in "\"'": value = value[1:-1] if any(_has_external_target(c) for c in _URL_CANDIDATE_SPLIT_RE.split(value.strip()) if c): return True return not seen def active_tag_class(name: str, attrs: str) -> str | None: """The active-content class a raw tag belongs to, or ``None`` if it is inert. ``"raw-html-link"`` is the click-required carrier class; ``"raw-html"`` is everything the renderer acts on unattended. The event-handler test runs FIRST, before the name test, so an ``<a onclick=...>`` is graded as the execute-class carrier it is rather than downgraded with the anchors. """ lowered = name.lower() if _EVENT_ATTR_RE.search(attrs): return "raw-html" # Presence, not a readable value: a URL attribute whose value this module # cannot resolve must keep the tag active, mirroring `_url_attr_is_external`'s # fail-secure gap. The corpora carry 0 of these today — empirical, not # structural, so the predicate must not depend on that holding. has_url_attr = bool(_URL_ATTR_RE.search(attrs)) if lowered in _SCANNER_ACTIVE_TAGS: if lowered in _URL_AFFORDANCE_TAGS and not has_url_attr: return None return "raw-html-link" if lowered in _LINK_TAGS else "raw-html" # A name outside the active set is active only through its URL attribute, and # stays on the HIGH side: its rendering is unknown and `href` is not the only # URL attribute it may carry. Measured cost of that conservatism: one # document per wiki corpus. if has_url_attr and _url_attr_is_external(attrs): return "raw-html" return None def is_active_tag(name: str, attrs: str) -> bool: """True if a tag is active for the SCANNER, in either carrier class. Kept as a separate symbol because ``docs/rawhtml-census.py`` patches it to measure a candidate predicate, and consumers import it by name. """ return active_tag_class(name, attrs) is not None def is_defangable_tag(name: str, attrs: str) -> bool: """True if the MUTATOR should defang a tag — deliberately broader than :func:`is_active_tag`: any URL attribute, and the full name set. Over-defanging costs nothing here (``neutralize`` is opt-in and blocks no disposition), while under-defanging would hand a human a live construct. """ return bool( name.lower() in _ACTIVE_TAGS or _EVENT_ATTR_RE.search(attrs) or _URL_ATTR_RE.search(attrs) ) # Absolute (`scheme:`) or protocol-relative (`//`) URL — an attacker-reachable # target. Relative paths resolve against the rendering host and carry no # exfiltration affordance, so the scanner leaves them alone. _EXTERNAL_URL_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9+.\-]*:|//)") def _has_external_target(url: str) -> bool: return bool(_EXTERNAL_URL_RE.match(url)) def _always(url: str) -> bool: # REFDEF is absolute-only by regex; AUTOLINK carries a scheme by # construction; a `data:` URI is its own scheme. return True # --- URL shape: can this URL carry data outward? ----------------------------- # Only http(s) and protocol-relative URLs have an "ordinary" form. Every other # scheme (javascript:, data:, file:, ftp:, ...) is active or fetches out-of-band # on its own terms and never grades down. _ORDINARY_SCHEME_RE = re.compile(r"^(?:https?://|//)", re.IGNORECASE) # Host labels and path segments: the separators that delimit a *name*. A token # that survives this split and still looks like a blob is carried data. _URL_TOKEN_RE = re.compile(r"[/._\-~+,;:=&$!*'()]+") def _is_opaque(token: str) -> bool: """True if a URL token looks like carried data rather than a name. Three reused ``entropy`` signals, cheapest first: base64 that decodes to printable text (the encoding an exfil path actually uses), a hex id at the URL-token floor, and — as a backstop for random-looking tokens that are neither — length-paired Shannon entropy. """ if try_decode_base64(token) is not None: return True if len(token) >= _OPAQUE_HEX_LEN and is_hex_blob(token): return True return len(token) >= _OPAQUE_MIN_LEN and shannon_entropy(token) >= _OPAQUE_H def is_ordinary_url(url: str) -> bool: """True if ``url`` merely *names* a remote document, carrying nothing outward. Ordinary means all of: an http(s) or protocol-relative scheme, no query, no userinfo, no percent-escapes, and no opaque host label or path segment. The fragment is deliberately excluded from the test: it is never sent to the server, so it cannot carry data to the host that a renderer auto-fetches — ``…/overview#prerequisites`` is the single most common shape in real documentation. Percent-escapes count as carrying, which grades a legitimate ``%20`` in a path as data-carrying; that false positive is accepted and documented (``docs/LIMITATIONS.md``) because obfuscated encoding is a core exfil primitive and the ambiguous case belongs on the review side. """ if not _ORDINARY_SCHEME_RE.match(url): return False try: parts = urlsplit(url) except ValueError: # malformed authority (bad IPv6, bad port) -> never ordinary return False if parts.query or parts.username or parts.password: return False # `netloc`, not `hostname`: the latter lowercases, which would destroy the # mixed case a base64 payload smuggled into a subdomain depends on. Userinfo # is already rejected above, so what is left is host[:port]. named = parts.netloc + parts.path if "%" in named: return False return not any(_is_opaque(token) for token in _URL_TOKEN_RE.split(named) if token) # Per-construct severities (_SEVERITY, imported above) live in `calibration` — # zero-click auto-fetch/execute -> HIGH, click-required -> MEDIUM — the Node port # shares them. def scan_active_content( text: str, source: Source = Source.OUTPUT, max_scan_chars: int = MAX_SCAN_CHARS, ) -> Report: """Report active-content constructs with an external target in ``text``. Report-only (design principles 3 & 4): the input is never mutated and no disposition is rendered here. Labels are ``active:<class>``; severities mirror ``neutralize``'s (image / raw-html / data-uri HIGH, links MEDIUM). Self-safety (OWASP LLM10): the scanned length is capped once, and an ``active:oversize-input`` finding announces that the tail went unread. It truncates rather than raising the way the transform surfaces do — what a detector shortens is its own coverage, not the caller's content. Reached through :func:`~llm_ingestion_guard.output.scan_output` the text is already under that surface's cap, so the flag is raised once, there. """ report = Report() if len(text) > max_scan_chars: report.add(Finding( label="active:oversize-input", severity=Severity.MEDIUM, source=source, detector="active_content", count=len(text), owasp="LLM10", evidence=f"input {len(text)} chars exceeds cap {max_scan_chars}; scanned prefix only", )) text = text[:max_scan_chars] def _flag(cls: str, hits: list[tuple[str, bool]]) -> None: """Report one finding for ``cls``, graded by its *worst* member. A class collapses to a single finding, so an exfil-shaped URL hiding behind an ordinary one must set both the severity and the evidence — otherwise the report would show an innocent URL next to a HIGH verdict. """ carrying = [evidence for evidence, ordinary in hits if not ordinary] report.add(Finding( label=f"active:{cls}", severity=_SEVERITY[cls] if carrying else _ORDINARY_SEVERITY, source=source, detector="active_content", count=len(hits), evidence=redact(carrying[0] if carrying else hits[0][0]), owasp="LLM05", )) masked = text def _scan(pattern: re.Pattern[str], url_group, keep) -> list[tuple[str, bool]]: """Collect ``(defanged url, is_ordinary)`` for kept matches; mask every match with spaces (same length, so line structure and later offsets survive).""" nonlocal masked hits: list[tuple[str, bool]] = [] def _sub(m: re.Match[str]) -> str: url = m.group(url_group) if keep(url): hits.append((defang_url(url), is_ordinary_url(url))) return " " * len(m.group(0)) masked = pattern.sub(_sub, masked) return hits # Pass order mirrors neutralize: images first (consumes the leading `!`), # then links, refdefs, autolinks, raw HTML, and standalone data: URIs. imgs = _scan(MD_IMAGE_RE, "url", _has_external_target) if imgs: _flag("markdown-image", imgs) links = _scan(MD_LINK_RE, "url", _has_external_target) if links: _flag("markdown-link", links) refs = _scan(MD_REFDEF_RE, "url", _always) if refs: _flag("reference-link", refs) autos = _scan(AUTOLINK_RE, "url", _always) if autos: _flag("autolink", autos) # Raw HTML is active whatever its URL looks like (an event handler needs no # URL at all), so every tag is flagged as carrying — no ordinary form. The # two carrier classes are collected separately: a document holding both a # `<script>` and an `<a href>` must not lose the anchor behind the script, # nor grade the script down to the anchor's severity. html: dict[str, list[tuple[str, bool]]] = {"raw-html": [], "raw-html-link": []} def _tag(m: re.Match[str]) -> str: cls = active_tag_class(m.group("name"), m.group("attrs") or "") if cls is None: return m.group(0) html[cls].append( (URL_IN_TEXT_RE.sub(lambda u: defang_url(u.group(0)), m.group(0)), False)) return " " * len(m.group(0)) masked = HTML_TAG_RE.sub(_tag, masked) for cls in ("raw-html", "raw-html-link"): if html[cls]: _flag(cls, html[cls]) # A `data:` URI carries its own payload; `is_ordinary_url` rejects the scheme # outright, so this stays HIGH through the same path as the rest. datas = _scan(DATA_URI_RE, 0, _always) if datas:

"""active_content — report-only detection of active content (the EchoLeak class). Query-time guardrails guard the answer; this guards the *persisted artifact*. Active-content constructs in persisted text become an exfiltration channel the moment a renderer touches them: a markdown image URL is auto-fetched zero-click (the EchoLeak class, CVE-2025-32711), a link invites the click, raw active HTML executes. ``lexicon`` and ``entropy`` cannot see these carriers — they are neither injection strings nor high-entropy blobs — so this detector is the gate's coverage for OWASP LLM05 (Improper Output Handling). This module is the canonical home of the active-content pattern table. Two consumers share it: * :func:`scan_active_content` (here) — **report-only**: findings feed ``scan_output`` and thence disposition; the text is never touched. * :func:`~llm_ingestion_guard.neutralize.neutralize` — the separate, opt-in **mutator** that defangs the same constructs for human audit. One deliberate asymmetry between the two: the scanner flags a construct only when its URL is absolute or protocol-relative. A relative in-document link has no attacker-reachable endpoint, and flagging it would silently over-block legitimate wiki/OKF content (design principle 5) — cross-linking is those formats' core mechanism. ``neutralize`` keeps its broader defang-anything behavior: it is opt-in, and bracketed dots in a relative path are auditable, not blocking. The two predicates are therefore separate symbols — :func:`is_active_tag` for the scanner, :func:`is_defangable_tag` for the mutator. They were one symbol until 0.6.0, imported by name across modules, so narrowing the scanner would have moved the mutator silently. **The asymmetry covers raw HTML too** (0.6.0). It previously applied only to the markdown paths: a tag was active if it carried a URL attribute *at all*, so an MDX ```` — a doc-relative route on a name outside the active set — carried HIGH. It now requires an external target, the rule the markdown paths have applied since 0.3.1. ```` left the active *name* set in the same change: its whole affordance is its ``href``, which the URL-attribute branch still catches, while the attribute-less ```` of Azure APIM policy XML has no affordance in any renderer. Measured together rather than one at a time — the classes co-occur — the pair frees 25 of 133 non-WARN documents on the reference corpus and 2 each on the two wiki corpora, at unchanged recall. Method and numbers: ``docs/rawhtml-census.py``; residuals: ``docs/LIMITATIONS.md``. **Raw HTML grades on carrier too, and a tag that names no target is inert** (0.7.0). Two changes that had to ship together, because they co-occur: * the *carrier split* — ````/```` are click-required, so they report as ``active:raw-html-link`` at MEDIUM, the grade the markdown inline link has carried since 0.3.1. Until 0.6.1 the same URL was LOW as ``[t](url)`` and HIGH as ````: an asymmetry produced by syntax, not by affordance. * the *no-URL narrowing* — a tag whose entire affordance IS the URL it names (``_URL_AFFORDANCE_TAGS``), carrying no URL attribute at all, has no affordance in any renderer. This is ````'s argument from 0.6.0 applied to the rest of the name branch, and it frees ````, ````, ``