1
0
Fork 0

docs(limitations): bound two URL-shape false positives with field measurements

Two consumers measured the 0.3.1 URL-shape rule against real corpora on the
same day. Record what they found, and pin the dispositions both of them
inferred wrongly.

Percent-escape FP: zero occurrences across both corpora (0/347 external URLs
in a 527-document vendor-docs corpus, 0/81 in a capture store). This bounds
the shape rather than closing it -- a consumer that slugs filenames from
titles produces %20 systematically, and that corpus is still unmeasured.

Query FP: the over-block that actually occurs. The two corpora hit disjoint
benign populations -- 16/16 publisher-authored utm_* tracking versus 35/35
content identity (?v=, ?channel_id=) where the parameter IS the resource.
An allowlist keyed on tracking-parameter names resolves the first entirely
and the second not at all, so no parameter-level remedy covers both. That
is input to the 0.4.0 axis-separation scope, not a fix here.

Both consumers reported dispositions they had inferred rather than run, and
both were wrong: a query-carrying link is MEDIUM, so it is held or warned,
never hard-failed. tests/test_wiring.py now pins that, plus the active-tag
gate -- counting raw HTML tags overcounts what the gate flags, since
formatting markup is inert and only name/on*=/URL-attr tags are active.

593 passed.
This commit is contained in:
Kjell Tore Guttormsen 2026-07-25 20:56:14 +02:00
commit 956c835d38
2 changed files with 83 additions and 1 deletions

View file

@ -12,6 +12,8 @@ also pin the ``__all__`` export surface a consumer depends on.
"""
from __future__ import annotations
import pytest
import llm_ingestion_guard as g
from llm_ingestion_guard import (
PreparedInput,
@ -125,3 +127,63 @@ def test_screen_output_fails_closed_when_scanner_raises(monkeypatch):
monkeypatch.setattr(g, "scan_output", boom)
result = screen_output("anything", PRESET_TRUSTED_SOURCE)
assert result.disposition is Disposition.FAIL_SECURE
# --- field-measured false positives (consumer corpora, 2026-07-25) ----------
# Two consumers measured the 0.3.1 URL-shape rule against real corpora on the
# same day. Both reported dispositions they had inferred rather than run, and
# both inferences were wrong — so the shapes are pinned here and the numbers
# they bound live in `docs/LIMITATIONS.md`. These characterize *current*
# behaviour: they pass on arrival, and exist so a later change cannot make that
# doc silently untrue.
_FIELD_QUERY_URLS = [
# claude-code-llm-wiki: 16/16 query-carrying external URLs in a 527-document
# vendor-docs corpus were publisher-authored campaign tracking.
("vendor-tracking", "https://claude.com/pricing?utm_source=docs&utm_medium=referral"),
# linkedin-studio: 35/35 in an 81-URL capture store were content identity —
# the parameter *is* the resource, so stripping it does not dereference.
("content-identity-video", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"),
("content-identity-feed", "https://www.youtube.com/feeds/videos.xml?channel_id=UC7cs8q"),
("pagination", "https://www.stortinget.no/no/Saker-og-publikasjoner/?all=true"),
]
@pytest.mark.parametrize("cid,url", _FIELD_QUERY_URLS, ids=[c[0] for c in _FIELD_QUERY_URLS])
def test_benign_query_link_is_held_not_blocked(cid, url):
# A query-carrying *link* is MEDIUM, and MEDIUM never hard-fails: the upload
# preset holds it for a human, the trusted preset lets it pass with a warning.
# Pinned because a consumer read this class as fail-secure and concluded its
# whole corpus was hard-blocked.
upload = screen_output(f"See [pricing]({url}).", PRESET_USER_UPLOAD)
assert upload.disposition is Disposition.QUARANTINE_REVIEW, f"{cid}: {upload.reasons}"
trusted = screen_output(f"See [pricing]({url}).", PRESET_TRUSTED_SOURCE)
assert trusted.disposition is Disposition.WARN, f"{cid}: {trusted.reasons}"
_INERT_VENDOR_DOC_HTML = [
"Line one<br>and more.", "This is <b>bold</b>.", "A <sup>1</sup> footnote.",
"Press <kbd>Cmd</kbd>.", "<details><summary>Expand</summary>body</details>",
"<table><tr><td>cell</td></tr></table>", "<div class=\"note\">note</div>",
"<span style=\"color:red\">red</span>", "<hr>",
]
@pytest.mark.parametrize("text", _INERT_VENDOR_DOC_HTML)
def test_inert_vendor_doc_html_is_not_active(text):
# Counting *raw* HTML tags overcounts what this gate flags: only tags that are
# active by name, by an on*= handler, or by a URL attribute are findings.
# Formatting markup — the bulk of raw HTML in vendor documentation — is inert.
assert screen_output(text, PRESET_USER_UPLOAD).disposition is Disposition.WARN
@pytest.mark.parametrize("text", [
'<a href="https://x.example/p">here</a>', '<img src="https://x.example/a.png">',
'<div onclick="x()">clickme</div>',
])
def test_active_raw_html_still_fails_secure_on_upload(text):
# The other half of the same correction: `a` and `img` are active by name, so
# hand-written links and images in raw HTML *are* caught. The overcount is in
# the formatting tags above, not in a weakened rule.
assert screen_output(text, PRESET_USER_UPLOAD).disposition is Disposition.FAIL_SECURE