llm-ingestion-pipeline-security/docs/rawhtml-census.py

"""raw-HTML census — which branch of `active_tag_class` fires, and what a change costs.

`docs/fp-sweep.py` answers *how often* the upload door costs a human. This answers
*why*, for the one detector that drives most of it, and *what a proposed narrowing
would actually buy* — end to end, as a change in `screen_output` disposition, not as
a count of regex hits.

WHY THIS EXISTS. `docs/LIMITATIONS.md` once attributed `active:raw-html` in 52 of
vendor-harvest's 98 non-WARN documents to the relative-URL-attribute over-reach.
Re-measured, that over-reach frees **one** document there. The claim was reasoning,
not measurement, and it stood in a published file for three releases. This script is
the measurement, so the next person changing that detector argues with numbers.

TWO METHOD TRAPS IT EXISTS TO AVOID:

- **Over-reach classes co-occur.** In one corpus, narrowing the URL-attribute branch
  alone frees 3 documents and taking `<base>` off the name branch alone frees 13 —
  but both together free 25. A document blocked by two classes is freed by neither
  alone. Measure candidates you intend to ship *together*, never one at a time.
- **Two of the three published populations are LIVING corpora**, re-harvested by
  their owning repo. A before/after split across two sessions mixes the narrowing's
  effect with corpus drift. Every candidate here runs against the same corpus state
  in one process, and `base` is re-measured rather than quoted from the doc.

The candidates are applied by replacing `active_content.active_tag_class`
in-process, which mirrors a real edit to the *scanner*. Since 0.6.0 that is the
whole story: `neutralize` calls its own `is_defangable_tag`, so patching this
symbol cannot move the mutator. Before 0.6.0 the two shared one symbol and this
caveat read the other way. See the raw-HTML bullets in `docs/LIMITATIONS.md`.

The patch point was `is_active_tag` through 0.6.1, when a candidate could only
answer yes/no. 0.7.0 grades raw HTML on carrier as well, so a candidate returns a
CLASS and `is_active_tag` became a thin wrapper. A boolean patch point would have
left every regrade candidate equal to PRODUCTION — silently, and in the direction
that reads as "no change helps".

The `PRODUCTION` row is the only one that is not a hypothetical: it leaves the
shipped predicate in place. A shipped narrowing must equal its candidate row, and
saying so in the output is what keeps the doc's numbers checkable after the fact.

USAGE — corpus roots are arguments, never hardcoded; the corpora live in private
consumer repos and their paths must not reach a public mirror:

    python docs/rawhtml-census.py LABEL=/path/to/corpus [LABEL=/path ...]
                                  [--ext=.md,.txt] [--include=/subtree/]
"""
from __future__ import annotations

import sys
from collections import Counter
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))

from llm_ingestion_guard import (  # noqa: E402
    PRESET_TRUSTED_SOURCE, PRESET_USER_UPLOAD, Disposition, guard, scan_output,
)
from llm_ingestion_guard import active_content as ac  # noqa: E402

BENIGN = Disposition.WARN
BLOCKED = Disposition.FAIL_SECURE
# Both trust tiers, because a change can move them in OPPOSITE directions: the
# carrier split loosens the upload door (HIGH -> MEDIUM is fail_secure ->
# quarantine_review) while tightening the trusted one (one finding becomes two,
# and >=2 findings at MEDIUM+ trip the compound overlay). A single-preset census
# would have reported only the half that flattered the change.
PRESETS = {"upload": PRESET_USER_UPLOAD, "trusted": PRESET_TRUSTED_SOURCE}
# Disposition severity order, for "did this document get strictly worse?".
_TIER = {Disposition.WARN: 0, Disposition.QUARANTINE_REVIEW: 1, Disposition.FAIL_SECURE: 2}
# The row a release is judged against: what consumers are running today.
_SHIPPED_BEFORE_ROW = "A + base-url (0.6.0)"

def has_external_url_attr(attrs: str) -> bool:
    """True if any URL-bearing attribute points at an attacker-reachable target.

    Delegates to the shipped reader instead of re-deriving it. This function used
    to parse attributes itself, and the copy drifted: it read attribute names with
    its own pattern (so `data-src="//evil"` was invisible to it while
    `_URL_ATTR_RE` matched it) and treated a value as one URL (so an external
    candidate later in a multi-candidate `srcset` was missed). Both shapes made
    the `A` rows under-count against the PRODUCTION row printed beside them —
    exactly the drift the PRODUCTION row exists to expose. Pinned by
    `tests/test_docs_measurement_scripts.py`.
    """
    return ac._url_attr_is_external(attrs)


def _variant(*, drop: frozenset[str] = frozenset(), external_only: bool = False,
             carrier_split: bool = False, no_url: bool = False):
    """Build an `active_tag_class` replacement.

    Candidates return the tag's CLASS (``"raw-html"`` / ``"raw-html-link"``) or
    ``None``, because since 0.7.0 the raw-HTML pass grades on carrier as well as
    activity, and a boolean could not express a regrade. ``drop`` removes names
    from the active set, ``external_only`` gates the URL branch, ``carrier_split``
    moves the click-required carriers to the link class, and ``no_url`` makes a
    URL-affordance tag carrying no URL attribute inert.
    """
    keep = frozenset(ac._ACTIVE_TAGS - drop)

    def active_tag_class(name: str, attrs: str):
        lowered = name.lower()
        if ac._EVENT_ATTR_RE.search(attrs):
            return "raw-html"
        has_url_attr = bool(ac._URL_ATTR_RE.search(attrs))
        if lowered in keep:
            if no_url and lowered in ac._URL_AFFORDANCE_TAGS and not has_url_attr:
                return None
            if carrier_split and lowered in ac._LINK_TAGS:
                return "raw-html-link"
            return "raw-html"
        if not has_url_attr:
            return None
        if external_only and not has_external_url_attr(attrs):
            return None
        return "raw-html"

    return active_tag_class


_INERT = None

CANDIDATES = [
    ("pre-0.6.0 (no narrowing)", _variant()),
    # The URL-attribute branch requires an EXTERNAL target — the rule the markdown
    # paths already apply. A relative `href` reaches no attacker-controlled host.
    ("A: url-attr external-only", _variant(external_only=True)),
    # `<base>` off the NAME set. HTML's `<base>` has its whole affordance in `href`,
    # which the URL-attribute branch still catches; APIM policy XML's `<base />` is
    # attribute-less and has no affordance in any renderer.
    ("base-url: <base> needs a URL", _variant(drop=frozenset({"base"}))),
    ("A + base-url (0.6.0)",
     _variant(drop=frozenset({"base"}), external_only=True)),
    # 0.7.0's pair. C1 REGRADES (a click-required carrier is MEDIUM, not HIGH);
    # D NARROWS (a tag whose whole affordance is a URL it does not carry is
    # inert). They are listed alone as well as together because they co-occur
    # hard: D strips a document's `</a>` and `<Frame>`, and what is left is the
    # `<a href=...>` C1 grades down, so each alone leaves the document blocked by
    # the other's residue. Reading either single row as "this change is cheap" is
    # the trap this script exists to prevent.
    ("C1: carrier split (alone)",
     _variant(drop=frozenset({"base"}), external_only=True, carrier_split=True)),
    ("D: no-URL narrowing (alone)",
     _variant(drop=frozenset({"base"}), external_only=True, no_url=True)),
    ("C1 + D (0.7.0)",
     _variant(drop=frozenset({"base"}), external_only=True,
              carrier_split=True, no_url=True)),
    # Not a hypothetical: the shipped predicate, unpatched. `C1 + D` is what 0.7.0
    # ships, so these two rows must agree — a mismatch means the code and this
    # script have drifted apart and every number below is suspect.
    ("PRODUCTION (as shipped)", None),
    # The CEILING: no narrowing can free more than switching the detector off.
    ("NONE (ceiling)", lambda name, attrs: _INERT),
]


def documents(root: Path, exts: tuple[str, ...], include: str) -> list[Path]:
    files = []
    for p in sorted(root.rglob("*")):
        if not p.is_file() or p.suffix not in exts:
            continue
        rel = p.relative_to(root)
        if any(part.startswith(".") for part in rel.parts):
            continue
        if include and include not in f"/{rel}":
            continue
        files.append(p)
    return files


def masked_text(text: str) -> str:
    """Reproduce `scan_active_content`'s masking up to the raw-HTML pass.

    Order matters: an image is not also a link, and a construct already consumed
    by an earlier pass must not be re-counted as a tag.
    """
    masked = text[: ac.MAX_SCAN_CHARS]
    for pattern in (ac.MD_IMAGE_RE, ac.MD_LINK_RE, ac.MD_REFDEF_RE, ac.AUTOLINK_RE):
        masked = pattern.sub(lambda m: " " * len(m.group(0)), masked)
    return masked


def branch_census(texts: list[str]) -> tuple[Counter, Counter, Counter]:
    """Count active-tag occurrences by the branch that fires first."""
    names: Counter[str] = Counter()
    relative: Counter[str] = Counter()
    external: Counter[str] = Counter()
    for text in texts:
        for m in ac.HTML_TAG_RE.finditer(masked_text(text)):
            name, attrs = m.group("name"), m.group("attrs") or ""
            if name.lower() in ac._ACTIVE_TAGS:
                names[name.lower()] += 1
            elif ac._EVENT_ATTR_RE.search(attrs):
                names["(on*= handler)"] += 1
            elif ac._URL_ATTR_RE.search(attrs):
                (external if has_external_url_attr(attrs) else relative)[name] += 1
    return names, relative, external


def main() -> None:
    exts: tuple[str, ...] = (".md", ".txt")
    include = ""
    specs = []
    for arg in sys.argv[1:]:
        if arg.startswith("--ext="):
            exts = tuple(e if e.startswith(".") else f".{e}"
                         for e in arg.split("=", 1)[1].split(","))
        elif arg.startswith("--include="):
            include = arg.split("=", 1)[1]
        else:
            specs.append(arg)

    if not specs:
        print(__doc__)
        raise SystemExit(2)

    original = ac.active_tag_class
    try:
        for spec in specs:
            if "=" not in spec:
                raise SystemExit(f"expected LABEL=PATH, got {spec!r}")
            label, _, path = spec.partition("=")
            root = Path(path).expanduser()
            if not root.is_dir():
                raise SystemExit(f"{label}: {root} is not a directory")

            texts = [t for t in (p.read_text(encoding="utf-8", errors="replace")
                                 for p in documents(root, exts, include)) if t.strip()]
            n = len(texts)
            print(f"\n## {label} — {n} documents", flush=True)

            base_nonwarn = base_block = None
            shipped_before: dict[str, list] = {}
            for name, fn in CANDIDATES:
                ac.active_tag_class = original if fn is None else fn
                # `screen_output(t, policy)` IS `guard(lambda: scan_output(t),
                # policy)`. Decomposed by exactly one step here so the scan — the
                # expensive part, and identical across trust tiers — runs once per
                # document instead of once per tier. Disposition still goes through
                # the shipped `guard`, so the fail-closed wrapper is not skipped.
                per_preset = {tier: [] for tier in PRESETS}
                for t in texts:
                    try:
                        report = scan_output(t)
                    except Exception:      # noqa: BLE001 — hand it back to `guard`
                        report = None
                    for tier, policy in PRESETS.items():
                        scan_fn = (lambda: scan_output(t)) if report is None \
                            else (lambda report=report: report)
                        per_preset[tier].append(guard(scan_fn, policy).disposition)
                dispositions = per_preset["upload"]
                non_warn = sum(1 for d in dispositions if d is not BENIGN)
                # BOTH metrics, because they answer different questions and a
                # REGRADE is invisible to the first one. A narrowing removes the
                # finding, so a document can reach WARN; a carrier split only
                # lowers the severity, so the document stays non-WARN and merely
                # stops being hard-failed. Reporting only `non_warn` would have
                # printed "frees 0" for every carrier candidate and read as
                # "the split buys nothing" when it converts a hard block into a
                # human review — the difference a consumer actually feels.
                blocked = sum(1 for d in dispositions if d is BLOCKED)
                # Tightening is measured against the row consumers are RUNNING,
                # not against the pre-0.6.0 baseline the `frees` column subtracts
                # from. "Did this release make anything worse for someone on the
                # current version" is a different question from "how much of the
                # original over-reach is left", and only the first one belongs in
                # a release note.
                if name == _SHIPPED_BEFORE_ROW:
                    shipped_before = per_preset
                if base_nonwarn is None:
                    base_nonwarn, base_block = non_warn, blocked
                    print(f"   {name:>28}: non-WARN {non_warn:4d} ({non_warn / n:5.1%})"
                          f"   fail_secure {blocked:4d}", flush=True)
                    continue
                # A candidate is not free just because it frees documents. Splitting
                # one finding into two puts TWO findings at MEDIUM+ in a document
                # that had one, which trips the compound overlay — so a change sold
                # as a loosening can TIGHTEN a document one tier, and on the trusted
                # preset (where nothing was hard-failed to begin with) that is the
                # only direction it can move. Reporting `unblocks` without `tightens`
                # is a one-sided number.
                row = (f"   {name:>28}: non-WARN {non_warn:4d} ({non_warn / n:5.1%})"
                       f"   fail_secure {blocked:4d}"
                       f"   frees {base_nonwarn - non_warn:3d} / unblocks "
                       f"{base_block - blocked:3d}")
                if shipped_before:
                    tightened = {
                        tier: sum(1 for before, after
                                  in zip(shipped_before[tier], per_preset[tier])
                                  if _TIER[after] > _TIER[before])
                        for tier in PRESETS
                    }
                    row += (f"   TIGHTENS vs 0.6.0: upload {tightened['upload']:3d}"
                            f" trusted {tightened['trusted']:3d}")
                print(row, flush=True)
            ac.active_tag_class = original

            names, relative, external = branch_census(texts)
            print(f"   name branch      : {dict(names.most_common(8))}")
            print(f"   url-attr relative: {dict(relative.most_common(8))}  <- A frees these")
            print(f"   url-attr external: {dict(external.most_common(8))}  <- A keeps these")
    finally:
        ac.active_tag_class = original

    print("\n---")
    print("Candidates are measured TOGETHER as well as alone: over-reach classes\n"
          "co-occur, and a document blocked by two of them is freed by neither\n"
          "alone. Populations are never summed — each has its own denominator.")


if __name__ == "__main__":
    main()