"""raw-HTML census — which branch of `active_tag_class` fires, and what a change costs.
`docs/fp-sweep.py` answers *how often* the upload door costs a human. This answers
*why*, for the one detector that drives most of it, and *what a proposed narrowing
would actually buy* — end to end, as a change in `screen_output` disposition, not as
a count of regex hits.
WHY THIS EXISTS. `docs/LIMITATIONS.md` once attributed `active:raw-html` in 52 of
vendor-harvest's 98 non-WARN documents to the relative-URL-attribute over-reach.
Re-measured, that over-reach frees **one** document there. The claim was reasoning,
not measurement, and it stood in a published file for three releases. This script is
the measurement, so the next person changing that detector argues with numbers.
TWO METHOD TRAPS IT EXISTS TO AVOID:
- **Over-reach classes co-occur.** In one corpus, narrowing the URL-attribute branch
alone frees 3 documents and taking `` off the name branch alone frees 13 —
but both together free 25. A document blocked by two classes is freed by neither
alone. Measure candidates you intend to ship *together*, never one at a time.
- **Two of the three published populations are LIVING corpora**, re-harvested by
their owning repo. A before/after split across two sessions mixes the narrowing's
effect with corpus drift. Every candidate here runs against the same corpus state
in one process, and `base` is re-measured rather than quoted from the doc.
The candidates are applied by replacing `active_content.active_tag_class`
in-process, which mirrors a real edit to the *scanner*. Since 0.6.0 that is the
whole story: `neutralize` calls its own `is_defangable_tag`, so patching this
symbol cannot move the mutator. Before 0.6.0 the two shared one symbol and this
caveat read the other way. See the raw-HTML bullets in `docs/LIMITATIONS.md`.
The patch point was `is_active_tag` through 0.6.1, when a candidate could only
answer yes/no. 0.7.0 grades raw HTML on carrier as well, so a candidate returns a
CLASS and `is_active_tag` became a thin wrapper. A boolean patch point would have
left every regrade candidate equal to PRODUCTION — silently, and in the direction
that reads as "no change helps".
The `PRODUCTION` row is the only one that is not a hypothetical: it leaves the
shipped predicate in place. A shipped narrowing must equal its candidate row, and
saying so in the output is what keeps the doc's numbers checkable after the fact.
USAGE — corpus roots are arguments, never hardcoded; the corpora live in private
consumer repos and their paths must not reach a public mirror:
python docs/rawhtml-census.py LABEL=/path/to/corpus [LABEL=/path ...]
[--ext=.md,.txt] [--include=/subtree/]
"""
from __future__ import annotations
import sys
from collections import Counter
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
from llm_ingestion_guard import ( # noqa: E402
PRESET_TRUSTED_SOURCE, PRESET_USER_UPLOAD, Disposition, guard, scan_output,
)
from llm_ingestion_guard import active_content as ac # noqa: E402
BENIGN = Disposition.WARN
BLOCKED = Disposition.FAIL_SECURE
# Both trust tiers, because a change can move them in OPPOSITE directions: the
# carrier split loosens the upload door (HIGH -> MEDIUM is fail_secure ->
# quarantine_review) while tightening the trusted one (one finding becomes two,
# and >=2 findings at MEDIUM+ trip the compound overlay). A single-preset census
# would have reported only the half that flattered the change.
PRESETS = {"upload": PRESET_USER_UPLOAD, "trusted": PRESET_TRUSTED_SOURCE}
# Disposition severity order, for "did this document get strictly worse?".
_TIER = {Disposition.WARN: 0, Disposition.QUARANTINE_REVIEW: 1, Disposition.FAIL_SECURE: 2}
# The row a release is judged against: what consumers are running today.
_SHIPPED_BEFORE_ROW = "A + base-url (0.6.0)"
def has_external_url_attr(attrs: str) -> bool:
"""True if any URL-bearing attribute points at an attacker-reachable target.
Delegates to the shipped reader instead of re-deriving it. This function used
to parse attributes itself, and the copy drifted: it read attribute names with
its own pattern (so `data-src="//evil"` was invisible to it while
`_URL_ATTR_RE` matched it) and treated a value as one URL (so an external
candidate later in a multi-candidate `srcset` was missed). Both shapes made
the `A` rows under-count against the PRODUCTION row printed beside them —
exactly the drift the PRODUCTION row exists to expose. Pinned by
`tests/test_docs_measurement_scripts.py`.
"""
return ac._url_attr_is_external(attrs)
def _variant(*, drop: frozenset[str] = frozenset(), external_only: bool = False,
carrier_split: bool = False, no_url: bool = False):
"""Build an `active_tag_class` replacement.
Candidates return the tag's CLASS (``"raw-html"`` / ``"raw-html-link"``) or
``None``, because since 0.7.0 the raw-HTML pass grades on carrier as well as
activity, and a boolean could not express a regrade. ``drop`` removes names
from the active set, ``external_only`` gates the URL branch, ``carrier_split``
moves the click-required carriers to the link class, and ``no_url`` makes a
URL-affordance tag carrying no URL attribute inert.
"""
keep = frozenset(ac._ACTIVE_TAGS - drop)
def active_tag_class(name: str, attrs: str):
lowered = name.lower()
if ac._EVENT_ATTR_RE.search(attrs):
return "raw-html"
has_url_attr = bool(ac._URL_ATTR_RE.search(attrs))
if lowered in keep:
if no_url and lowered in ac._URL_AFFORDANCE_TAGS and not has_url_attr:
return None
if carrier_split and lowered in ac._LINK_TAGS:
return "raw-html-link"
return "raw-html"
if not has_url_attr:
return None
if external_only and not has_external_url_attr(attrs):
return None
return "raw-html"
return active_tag_class
_INERT = None
CANDIDATES = [
("pre-0.6.0 (no narrowing)", _variant()),
# The URL-attribute branch requires an EXTERNAL target — the rule the markdown
# paths already apply. A relative `href` reaches no attacker-controlled host.
("A: url-attr external-only", _variant(external_only=True)),
# `` off the NAME set. HTML's `` has its whole affordance in `href`,
# which the URL-attribute branch still catches; APIM policy XML's `` is
# attribute-less and has no affordance in any renderer.
("base-url: needs a URL", _variant(drop=frozenset({"base"}))),
("A + base-url (0.6.0)",
_variant(drop=frozenset({"base"}), external_only=True)),
# 0.7.0's pair. C1 REGRADES (a click-required carrier is MEDIUM, not HIGH);
# D NARROWS (a tag whose whole affordance is a URL it does not carry is
# inert). They are listed alone as well as together because they co-occur
# hard: D strips a document's `` and ``, and what is left is the
# `` C1 grades down, so each alone leaves the document blocked by
# the other's residue. Reading either single row as "this change is cheap" is
# the trap this script exists to prevent.
("C1: carrier split (alone)",
_variant(drop=frozenset({"base"}), external_only=True, carrier_split=True)),
("D: no-URL narrowing (alone)",
_variant(drop=frozenset({"base"}), external_only=True, no_url=True)),
("C1 + D (0.7.0)",
_variant(drop=frozenset({"base"}), external_only=True,
carrier_split=True, no_url=True)),
# Not a hypothetical: the shipped predicate, unpatched. `C1 + D` is what 0.7.0
# ships, so these two rows must agree — a mismatch means the code and this
# script have drifted apart and every number below is suspect.
("PRODUCTION (as shipped)", None),
# The CEILING: no narrowing can free more than switching the detector off.
("NONE (ceiling)", lambda name, attrs: _INERT),
]
def documents(root: Path, exts: tuple[str, ...], include: str) -> list[Path]:
files = []
for p in sorted(root.rglob("*")):
if not p.is_file() or p.suffix not in exts:
continue
rel = p.relative_to(root)
if any(part.startswith(".") for part in rel.parts):
continue
if include and include not in f"/{rel}":
continue
files.append(p)
return files
def masked_text(text: str) -> str:
"""Reproduce `scan_active_content`'s masking up to the raw-HTML pass.
Order matters: an image is not also a link, and a construct already consumed
by an earlier pass must not be re-counted as a tag.
"""
masked = text[: ac.MAX_SCAN_CHARS]
for pattern in (ac.MD_IMAGE_RE, ac.MD_LINK_RE, ac.MD_REFDEF_RE, ac.AUTOLINK_RE):
masked = pattern.sub(lambda m: " " * len(m.group(0)), masked)
return masked
def branch_census(texts: list[str]) -> tuple[Counter, Counter, Counter]:
"""Count active-tag occurrences by the branch that fires first."""
names: Counter[str] = Counter()
relative: Counter[str] = Counter()
external: Counter[str] = Counter()
for text in texts:
for m in ac.HTML_TAG_RE.finditer(masked_text(text)):
name, attrs = m.group("name"), m.group("attrs") or ""
if name.lower() in ac._ACTIVE_TAGS:
names[name.lower()] += 1
elif ac._EVENT_ATTR_RE.search(attrs):
names["(on*= handler)"] += 1
elif ac._URL_ATTR_RE.search(attrs):
(external if has_external_url_attr(attrs) else relative)[name] += 1
return names, relative, external
def main() -> None:
exts: tuple[str, ...] = (".md", ".txt")
include = ""
specs = []
for arg in sys.argv[1:]:
if arg.startswith("--ext="):
exts = tuple(e if e.startswith(".") else f".{e}"
for e in arg.split("=", 1)[1].split(","))
elif arg.startswith("--include="):
include = arg.split("=", 1)[1]
else:
specs.append(arg)
if not specs:
print(__doc__)
raise SystemExit(2)
original = ac.active_tag_class
try:
for spec in specs:
if "=" not in spec:
raise SystemExit(f"expected LABEL=PATH, got {spec!r}")
label, _, path = spec.partition("=")
root = Path(path).expanduser()
if not root.is_dir():
raise SystemExit(f"{label}: {root} is not a directory")
texts = [t for t in (p.read_text(encoding="utf-8", errors="replace")
for p in documents(root, exts, include)) if t.strip()]
n = len(texts)
print(f"\n## {label} — {n} documents", flush=True)
base_nonwarn = base_block = None
shipped_before: dict[str, list] = {}
for name, fn in CANDIDATES:
ac.active_tag_class = original if fn is None else fn
# `screen_output(t, policy)` IS `guard(lambda: scan_output(t),
# policy)`. Decomposed by exactly one step here so the scan — the
# expensive part, and identical across trust tiers — runs once per
# document instead of once per tier. Disposition still goes through
# the shipped `guard`, so the fail-closed wrapper is not skipped.
per_preset = {tier: [] for tier in PRESETS}
for t in texts:
try:
report = scan_output(t)
except Exception: # noqa: BLE001 — hand it back to `guard`
report = None
for tier, policy in PRESETS.items():
scan_fn = (lambda: scan_output(t)) if report is None \
else (lambda report=report: report)
per_preset[tier].append(guard(scan_fn, policy).disposition)
dispositions = per_preset["upload"]
non_warn = sum(1 for d in dispositions if d is not BENIGN)
# BOTH metrics, because they answer different questions and a
# REGRADE is invisible to the first one. A narrowing removes the
# finding, so a document can reach WARN; a carrier split only
# lowers the severity, so the document stays non-WARN and merely
# stops being hard-failed. Reporting only `non_warn` would have
# printed "frees 0" for every carrier candidate and read as
# "the split buys nothing" when it converts a hard block into a
# human review — the difference a consumer actually feels.
blocked = sum(1 for d in dispositions if d is BLOCKED)
# Tightening is measured against the row consumers are RUNNING,
# not against the pre-0.6.0 baseline the `frees` column subtracts
# from. "Did this release make anything worse for someone on the
# current version" is a different question from "how much of the
# original over-reach is left", and only the first one belongs in
# a release note.
if name == _SHIPPED_BEFORE_ROW:
shipped_before = per_preset
if base_nonwarn is None:
base_nonwarn, base_block = non_warn, blocked
print(f" {name:>28}: non-WARN {non_warn:4d} ({non_warn / n:5.1%})"
f" fail_secure {blocked:4d}", flush=True)
continue
# A candidate is not free just because it frees documents. Splitting
# one finding into two puts TWO findings at MEDIUM+ in a document
# that had one, which trips the compound overlay — so a change sold
# as a loosening can TIGHTEN a document one tier, and on the trusted
# preset (where nothing was hard-failed to begin with) that is the
# only direction it can move. Reporting `unblocks` without `tightens`
# is a one-sided number.
row = (f" {name:>28}: non-WARN {non_warn:4d} ({non_warn / n:5.1%})"
f" fail_secure {blocked:4d}"
f" frees {base_nonwarn - non_warn:3d} / unblocks "
f"{base_block - blocked:3d}")
if shipped_before:
tightened = {
tier: sum(1 for before, after
in zip(shipped_before[tier], per_preset[tier])
if _TIER[after] > _TIER[before])
for tier in PRESETS
}
row += (f" TIGHTENS vs 0.6.0: upload {tightened['upload']:3d}"
f" trusted {tightened['trusted']:3d}")
print(row, flush=True)
ac.active_tag_class = original
names, relative, external = branch_census(texts)
print(f" name branch : {dict(names.most_common(8))}")
print(f" url-attr relative: {dict(relative.most_common(8))} <- A frees these")
print(f" url-attr external: {dict(external.most_common(8))} <- A keeps these")
finally:
ac.active_tag_class = original
print("\n---")
print("Candidates are measured TOGETHER as well as alone: over-reach classes\n"
"co-occur, and a document blocked by two of them is freed by neither\n"
"alone. Populations are never summed — each has its own denominator.")
if __name__ == "__main__":
main()