fix(sanitize,okf,active_content): three quadratic patterns, two on the input path
The generalised sweep found what 0.3.2's hand-written rows missed. All three are
the documented class -- a run in front of a required literal that never arrives,
so every start position rescans the tail -- and all three are worse than the
0.3.3 findings, because `sanitize`, `neutralize`, `scan_active_content` and the
okf link graph apply NO input cap. `scan_lexicon`/`scan_output` are the only
entry points that do, so there is no ceiling to extrapolate to.
sanitize._HTML_COMMENT_RE `<!--`*100_000 20.1s, exponent 1.96-2.14
active_content.URL_IN_TEXT_RE `<a `+`A`*100_000 12.99s / 14.9s, exponent ~2.0
okf._MD_LINK_RE `[`*100_000 7.1s, exponent 1.99-2.05
Each fix is the one the pattern's own shape allows, not a copied choice:
- The comment stripper drops the regex for `str.find`. Excluding `<` would lose
every comment containing markup; bounding the run would be a carrier bypass
of the exact construct the stripper exists to remove.
- `URL_IN_TEXT_RE` bounds its scheme run to an RFC 3986 scheme (`{0,63}`).
Bounding is safe *here* only because it is a defanger inside a tag already
flagged `active:raw-html`. A lookbehind was measured too and rejected: it
drops `-http://evil.com`, a one-character evasion. Bounded: 0.185s at 1M.
- `_MD_LINK_RE` excludes `[`, matching `active_content.MD_LINK_RE` exactly,
including the nested-label trade already documented there.
`sanitize` claimed "no catastrophic backtracking" in a comment; that claim was
wrong in the same way `output`'s was before 0.3.2, and is corrected in place.
676 tests (+10), coverage 128/128 + 6/6 gaps, sweep clean across 150 patterns.
The okf destination run gets no row: `[^)\s]+` cannot fail, so a row for it
could never go red.
This commit is contained in:
parent
abbfe5f0fd
commit
73fa1b99ae
9 changed files with 223 additions and 7 deletions
|
|
@ -18,6 +18,8 @@ from __future__ import annotations
|
|||
|
||||
import pytest
|
||||
|
||||
import time
|
||||
|
||||
from llm_ingestion_guard import (
|
||||
scan_active_content,
|
||||
scan_output,
|
||||
|
|
@ -294,3 +296,38 @@ def test_raw_html_counts_end_tags():
|
|||
if f.label == "active:raw-html"]
|
||||
assert len(pair) == 1, "a start/end pair must not split into two findings"
|
||||
assert pair[0].count == 2, f"end tag not counted: {pair[0].count}"
|
||||
|
||||
|
||||
# --- self-safety (OWASP LLM10): the long-attribute arm -----------------------
|
||||
# The `_REDOS_PAYLOADS` rows in test_output.py attack tags that never CLOSE, so
|
||||
# `HTML_TAG_RE` fails and the tag body is never handed on. This arm is the
|
||||
# opposite: the tag closes, and its body is long. `_tag` then runs
|
||||
# `URL_IN_TEXT_RE` over it, whose scheme run sits in front of a required `://`
|
||||
# that never arrives — 12.99s at 100_000 chars through this scanner, exponent
|
||||
# 1.87-2.06 over four doublings, with no input cap on this entry point at all.
|
||||
# Missed by the 0.3.2 sweep because a repeating-unit payload cannot express
|
||||
# "one tag, long body"; found by docs/redos-sweep.py generalised past lexicon.
|
||||
_ATTR_REDOS_N = 100_000
|
||||
|
||||
|
||||
def test_crafted_long_attribute_tag_stays_bounded():
|
||||
payload = "<a " + "A" * _ATTR_REDOS_N + ">"
|
||||
start = time.monotonic()
|
||||
scan_active_content(payload)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
||||
|
||||
def test_url_defanging_survives_the_redos_fix():
|
||||
# Recall parity for the evidence defanger, including the two forms a
|
||||
# lookbehind-based fix would have dropped (`-` / `.` immediately before the
|
||||
# scheme), which is why the scheme run is bounded instead.
|
||||
for raw, expected in (
|
||||
("<a href=http://evil.com>", "hxxp"),
|
||||
("<a href=-http://evil.com>", "hxxp"),
|
||||
("<a href=.http://x.com>", "hxxp"),
|
||||
('<a href="https://a.b/c">', "hxxps"),
|
||||
):
|
||||
report = scan_active_content(raw)
|
||||
evidence = " ".join(f.evidence or "" for f in report.findings)
|
||||
assert expected in evidence, raw
|
||||
assert "http://" not in evidence and "https://" not in evidence, raw
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ empty report; only active-content constructs are ever rewritten. Mutation lives
|
|||
here, kept separate from the report-only output gate (design principles 3 & 4).
|
||||
The transform is pure ``text -> (defanged_text, report)`` — no I/O, no globals.
|
||||
"""
|
||||
import time
|
||||
|
||||
from llm_ingestion_guard.neutralize import neutralize
|
||||
from llm_ingestion_guard.report import Severity, Source
|
||||
|
||||
|
|
@ -139,3 +141,23 @@ def test_prose_with_lone_brackets_and_angles_is_identical():
|
|||
result = neutralize(text)
|
||||
assert result.text == text
|
||||
assert result.report.found is False
|
||||
|
||||
|
||||
# --- self-safety (OWASP LLM10): the long-attribute arm -----------------------
|
||||
# Second call site of the same defect pinned in test_active_content.py: the
|
||||
# defanger runs `URL_IN_TEXT_RE` over each active tag's body. 14.9s at 100_000
|
||||
# chars, exponent 1.91-2.22. `neutralize` applies no input cap either.
|
||||
_ATTR_REDOS_N = 100_000
|
||||
|
||||
|
||||
def test_crafted_long_attribute_tag_stays_bounded():
|
||||
payload = "<a " + "A" * _ATTR_REDOS_N + ">"
|
||||
start = time.monotonic()
|
||||
neutralize(payload)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
||||
|
||||
def test_url_defanging_inside_a_tag_survives_the_redos_fix():
|
||||
result = neutralize("<a href=-http://evil.com>x</a>")
|
||||
assert "hxxp" in result.text
|
||||
assert "http://evil.com" not in result.text
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ OKF spec facts used here (verified against okf/SPEC.md, 2026-07-06):
|
|||
"""
|
||||
import pytest
|
||||
|
||||
import time
|
||||
|
||||
from llm_ingestion_guard.okf import (
|
||||
parse_frontmatter,
|
||||
scan_concept,
|
||||
|
|
@ -640,3 +642,34 @@ def test_t2_constrains_import_not_emission(fm):
|
|||
doc = f"---\nid: x\n{fm}---\n\nbody\n"
|
||||
assert import_bundle({"concepts/x.md": doc}).disposition is Disposition.FAIL_SECURE
|
||||
assert screen_output(doc, PRESET_USER_UPLOAD).disposition is Disposition.WARN
|
||||
|
||||
|
||||
# --- self-safety (OWASP LLM10): ReDoS in the link-graph extractor ------------
|
||||
# `[^\]]*` is a run in front of a REQUIRED `]`: a bundle body that repeats `[`
|
||||
# and never closes it makes every start position rescan the tail. Measured 7.1s
|
||||
# at 100_000 chars, exponent 1.99-2.05 over four doublings, and the link graph
|
||||
# runs over attacker-supplied bundle bodies with no input cap. Found by
|
||||
# docs/redos-sweep.py once it was generalised past the lexicon table; the same
|
||||
# defect in `active_content`'s markdown table was already fixed there the same
|
||||
# way, by excluding the character that opens the pattern's own anchor.
|
||||
_LINK_REDOS_N = 100_000
|
||||
|
||||
|
||||
def test_crafted_link_payload_stays_bounded():
|
||||
start = time.monotonic()
|
||||
extract_link_targets("[" * _LINK_REDOS_N)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
||||
|
||||
# The destination run behind the label gets no row: `[^)\s]+` needs only one
|
||||
# character, so it cannot fail, and a run that cannot fail cannot pay the
|
||||
# per-start rescan. A row for it could never go red — decoration, not a pin.
|
||||
|
||||
|
||||
def test_link_extraction_survives_the_redos_fix():
|
||||
# Recall parity: ordinary links, a label holding brackets it does not close,
|
||||
# and the nested-bracket form the exclusion deliberately gives up on -- the
|
||||
# same trade `active_content.MD_LINK_RE` already makes.
|
||||
assert extract_link_targets("see [x](./a.md) and [y](/b.md)") == ["./a.md", "/b.md"]
|
||||
assert extract_link_targets("[a b](./c.md)") == ["./c.md"]
|
||||
assert extract_link_targets("text [](./t.md)") == ["./i.png"]
|
||||
|
|
|
|||
|
|
@ -436,3 +436,15 @@ def test_gate_is_bounded_on_the_payload_the_first_sweep_missed():
|
|||
start = time.monotonic()
|
||||
scan_output(payload)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
||||
|
||||
def test_gate_is_bounded_on_the_long_attribute_arm():
|
||||
# The composed-gate row for the defect pinned in test_active_content.py and
|
||||
# test_neutralize.py. `scan_output` runs `scan_active_content`, so the gate a
|
||||
# caller actually invokes inherits it. Not expressible as a repeating unit —
|
||||
# the tag has to CLOSE for the body to be handed on — which is exactly why
|
||||
# the unit-table above never covered it.
|
||||
payload = "<a " + "A" * _REDOS_N + ">"
|
||||
start = time.monotonic()
|
||||
scan_output(payload)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ Core invariants (BRIEF §9): clean input returns byte-identical with an all-zero
|
|||
report; the sanitizer only ever *removes* — its output is always a subsequence
|
||||
of the input.
|
||||
"""
|
||||
import time
|
||||
|
||||
from llm_ingestion_guard.sanitize import sanitize
|
||||
from llm_ingestion_guard.report import Severity, Source
|
||||
|
||||
|
|
@ -75,3 +77,42 @@ def test_data_uri_does_not_match_inside_a_word():
|
|||
def test_output_source_is_respected():
|
||||
result = sanitize("xy", source=Source.OUTPUT)
|
||||
assert all(f.source is Source.OUTPUT for f in result.report.findings)
|
||||
|
||||
|
||||
# --- self-safety (OWASP LLM10): ReDoS on the comment stripper ----------------
|
||||
# `sanitize` is step 1 of `prepare_input` -- the first thing every ingested
|
||||
# document hits -- and unlike `scan_lexicon`/`scan_output` it applies NO input
|
||||
# cap, so a quadratic run here has no ceiling at all. `<!--.*?-->` is a lazy run
|
||||
# in front of a REQUIRED literal: crafted input that repeats the opener and never
|
||||
# supplies `-->` makes every start position rescan the tail. Measured 20.1s at
|
||||
# 100_000 chars, exponent 1.96-2.14 over four doublings. Found by
|
||||
# docs/redos-sweep.py once it was generalised past the lexicon table.
|
||||
_REDOS_N = 100_000
|
||||
|
||||
|
||||
def test_crafted_comment_payload_stays_bounded():
|
||||
payload = ("<!--" * (_REDOS_N // 4 + 1))[:_REDOS_N]
|
||||
start = time.monotonic()
|
||||
sanitize(payload)
|
||||
assert time.monotonic() - start < 2.0
|
||||
|
||||
|
||||
def test_legitimate_comment_heavy_document_is_far_under_the_bound():
|
||||
# The bound above only has signal if ordinary comment-dense content is
|
||||
# nowhere near it: this is the same size, 100% closed comments.
|
||||
unit = "<!-- a note -->"
|
||||
payload = (unit * (_REDOS_N // len(unit) + 1))[:_REDOS_N]
|
||||
start = time.monotonic()
|
||||
sanitize(payload)
|
||||
assert time.monotonic() - start < 0.5
|
||||
|
||||
|
||||
def test_comment_stripping_survives_the_redos_fix():
|
||||
# Recall parity for every comment shape the lazy regex used to handle:
|
||||
# nested markup, newlines (the pattern was DOTALL), and an unterminated
|
||||
# comment, which must be left alone rather than swallowed to end-of-input.
|
||||
assert sanitize("a <!-- <b>x</b> --> z").text == "a z"
|
||||
assert sanitize("a <!-- one\ntwo --> z").text == "a z"
|
||||
assert sanitize("a <!-- x --> b <!-- y --> c").text == "a b c"
|
||||
assert sanitize("a <!-- never closed").text == "a <!-- never closed"
|
||||
assert sanitize("a --> b").text == "a --> b"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue