docs(url-shape): make the rule reconstructable, and record what three corpora measured

Three consumers reconstructed is_ordinary_url from prose we sent in coordination
messages and each produced a different wrong number on a real corpus: one omitted
the base64 20-char floor and fired on path words like /blog/; one omitted the
opaque-token condition entirely and undercounted; one computed Shannon entropy
over whole filenames instead of tokens and concluded the 4.4 floor over-blocks
ordinary documents. Same cause each time -- our prose described the rules without
their tokenizer.

docs/URL-SHAPE.md states the algorithm in order, spells out the separator class
and all three length floors, and lists the three reconstruction errors as worked
counter-examples. Its example table is parsed and asserted against the real
predicate by tests/test_url_shape_doc.py, so the reference cannot drift from the
code -- all 18 rows verified load-bearing.

LIMITATIONS.md brought current with the field measurements:

- Percent-escape is no longer zero. Two English corpora measured 0; a 389-file
  Norwegian/Microsoft corpus found 10, all Norwegian (%C3%B8, %C3%A5 are just
  o-slash and a-ring). It is a non-ASCII-language tax, and both zero-measuring
  corpora being English was a sampling bias invisible from inside.
- The query over-block now has THREE disjoint benign populations: utm_* tracking,
  content identity (?v=, ?channel_id=), and Microsoft Learn's ?view= version
  selector. No parameter-level remedy covers any two, which moves this from a
  conclusion to a settled constraint on 0.4.0.
- Legitimate CDN asset ids trip the hex branch permanently; the branch is otherwise
  precise (no other FP in 2401 distinct URLs) and stays.
- Raw HTML with a relative URL attribute is HIGH though it reaches no external
  host, and end tags are counted.
- OKF frontmatter: a one-key block-sequence item is silently misparsed to a string
  where two keys hard-reject, so a pointer can ride past the resource allowlist.
  Consequence: a conformant OKF v0.2 concept cannot traverse door C at all, since
  both backward-breaking migration targets are nested. Fail-secure, but a
  compatibility wall that needs a deliberate parse-safety decision.
- A persist gate cannot cover execution risk, and that boundary is unowned.

New behaviour claims are pinned by tests so a closed concession fails and forces
this doc to be updated. 593 -> 631 passed.

tests/test_active_content.py

@@ -259,3 +259,38 @@ def test_default_source_is_output_and_override_respected():
                for f in scan_active_content(_ECHOLEAK).findings)
     assert all(f.source is Source.INPUT
                for f in scan_active_content(_ECHOLEAK, source=Source.INPUT).findings)
+
+
+# --- raw-HTML over-blocks measured on a vendor-docs corpus (2026-07-26) -------
+# Documented in docs/LIMITATIONS.md. Pinned so the concessions stay honest: a
+# closed over-block should fail here and force the doc to be updated.
+
+@pytest.mark.parametrize("cid,text", [
+    # Fires on the URL-attr branch although the target is a relative doc route,
+    # which cannot reach an attacker-controlled host. `Card` is not in the active
+    # name set — the href alone carries it.
+    ("relative-href-on-inactive-name",
+     '<Card title="Quickstart" icon="play" href="/en/agent-sdk/quickstart">'),
+    # Fires on the *name* branch: names are lower-cased and `frame` is in the
+    # active set (legacy HTML framesets), while `Frame` is a common MDX component.
+    ("mdx-component-named-like-a-tag", "<Frame>"),
+])
+def test_raw_html_overblocks_are_still_high(cid, text):
+    finding = [f for f in scan_active_content(text).findings
+               if f.label == "active:raw-html"]
+    assert len(finding) == 1, f"{cid}: raw-html not reported"
+    assert finding[0].severity is Severity.HIGH, f"{cid}: {finding[0].severity}"
+
+
+def test_raw_html_counts_end_tags():
+    # `</a>` is active by name on its own, so a corpus census counting only opening
+    # tags understates this detector's `count`. The class still collapses to ONE
+    # finding — the count is what moves.
+    solo = [f for f in scan_active_content("</a>").findings
+            if f.label == "active:raw-html"]
+    assert len(solo) == 1 and solo[0].count == 1
+
+    pair = [f for f in scan_active_content('<a href="https://x.example/p">t</a>').findings
+            if f.label == "active:raw-html"]
+    assert len(pair) == 1, "a start/end pair must not split into two findings"
+    assert pair[0].count == 2, f"end tag not counted: {pair[0].count}"

tests/test_okf.py

@@ -516,3 +516,69 @@ def test_okf_adapter_is_exposed_from_package():
 
     assert "okf" in guard.__all__
     assert guard.okf.import_bundle is import_bundle
+
+
+# --- v0.2 frontmatter reach: what the restricted grammar admits (2026-07-26) ---
+# Measured for a consumer planning an additive OKF v0.2 profile. Documented in
+# docs/LIMITATIONS.md; pinned here so the compatibility wall cannot move silently.
+
+_V02_REJECTED = [
+    ("generated (nested)", "generated:\n  at: 2026-07-26T10:00:00Z\n"),
+    ("executor (nested)", "executor:\n  resource: skills/run-on-bq.md\n"),
+    ("attester (nested)", "attester:\n  resource: attesters/sql_equality.py\n"),
+    ("sources (block list of mappings)",
+     "sources:\n  - uri: https://e.com/a\n    kind: doc\n"),
+    ("flow sequence", "tags: [a, b, c]\n"),
+    ("flow mapping", "executor: {resource: skills/run.md}\n"),
+]
+
+
+@pytest.mark.parametrize("cid,fm", _V02_REJECTED, ids=[c[0] for c in _V02_REJECTED])
+def test_v02_nested_and_flow_frontmatter_hard_rejects(cid, fm):
+    # Both of v0.2's backward-breaking migration targets (`generated.at`, `sources`)
+    # are on this list, so a conformant v0.2 concept cannot pass the gate at all.
+    with pytest.raises(OKFFrontmatterError):
+        parse_frontmatter(f"---\nid: x\n{fm}---\n\nbody\n")
+
+
+_V02_ADMITTED = [
+    ("runtime", "runtime: bigquery\n"),
+    ("computation path", "computation: computations/gm.sql\n"),
+    ("status/stale_after", "status: active\nstale_after: 2026-12-01\n"),
+    ("verified bool", "verified: true\n"),
+    ("block sequence of scalars", "tags:\n  - alpha\n  - beta\n"),
+]
+
+
+@pytest.mark.parametrize("cid,fm", _V02_ADMITTED, ids=[c[0] for c in _V02_ADMITTED])
+def test_v02_flat_frontmatter_still_parses(cid, fm):
+    assert parse_frontmatter(f"---\nid: x\n{fm}---\n\nbody\n")[0]["id"] == "x"
+
+
+def test_one_key_block_sequence_item_is_misparsed_as_a_string():
+    # The documented defect: two keys per item hard-reject (loud, safe), but ONE key
+    # parses "successfully" into the wrong type. A consumer reading
+    # frontmatter["sources"][0].get("uri") gets a string, not a mapping.
+    fm, _ = parse_frontmatter(
+        "---\nid: x\nsources:\n  - uri: https://e.com/a\n---\n\nbody\n"
+    )
+    assert fm["sources"] == ["uri: https://e.com/a"], "shape changed — update LIMITATIONS.md"
+    assert not isinstance(fm["sources"][0], dict)
+
+
+def test_relative_resource_pointer_fails_the_allowlist():
+    # A top-level `resource` naming executable code is caught by the https allowlist.
+    for pointer in ("attesters/sql_equality.py", "skills/run-on-bq.md"):
+        with pytest.raises(OKFResourceError):
+            validate_resource_url(pointer)
+
+
+def test_pointer_in_one_key_sequence_reaches_the_consumer_tree():
+    # The security-relevant consequence of the misparse above: the pointer never
+    # touches the top-level `resource` key, so the https allowlist never inspects it
+    # and door C admits the concept. Not conformant OKF — a well-formed bundle will
+    # not produce this shape — but mode-b writes the merged concept verbatim.
+    doc = ("---\nid: x\ntype: Attested Computation\n"
+           "attester:\n  - resource: attesters/sql_equality.py\n---\n\nbody\n")
+    result = import_bundle({"computations/x.md": doc})
+    assert result.disposition is Disposition.WARN, "hole closed — update LIMITATIONS.md"

tests/test_url_shape_doc.py

@@ -0,0 +1,66 @@
+"""The URL-shape doc is executable: every worked example is asserted against code.
+
+`docs/URL-SHAPE.md` exists because three consumers reconstructed `is_ordinary_url`
+from prose and each got a different wrong answer. A reference that can drift from
+the implementation would reproduce exactly that failure, so the worked-examples
+table is parsed out of the document and run through the real predicate here.
+
+A row that disagrees with the code fails this test — whichever of the two is wrong.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+from llm_ingestion_guard.active_content import is_ordinary_url
+from llm_ingestion_guard.entropy import is_base64_like, is_hex_blob
+from llm_ingestion_guard import calibration as cal
+
+_DOC = Path(__file__).resolve().parent.parent / "docs" / "URL-SHAPE.md"
+
+# `| `<url>` | <verdict> | <why> |` — the verdict column is the assertion.
+_ROW_RE = re.compile(r"^\|\s*`([^`]+)`\s*\|\s*(ordinary|carrying)\s*\|", re.MULTILINE)
+
+
+def _rows() -> list[tuple[str, bool]]:
+    text = _DOC.read_text(encoding="utf-8")
+    return [(url, verdict == "ordinary") for url, verdict in _ROW_RE.findall(text)]
+
+
+def test_doc_exists_and_table_was_actually_parsed():
+    # Without this floor a renamed heading or reformatted table would empty the
+    # parametrize list and turn the whole file into a silent pass.
+    rows = _rows()
+    assert len(rows) >= 15, f"only parsed {len(rows)} worked examples from {_DOC.name}"
+    assert any(ordinary for _, ordinary in rows), "no ordinary examples parsed"
+    assert any(not ordinary for _, ordinary in rows), "no carrying examples parsed"
+
+
+@pytest.mark.parametrize("url,expected_ordinary", _rows(), ids=[u for u, _ in _rows()])
+def test_worked_example_matches_implementation(url, expected_ordinary):
+    assert is_ordinary_url(url) is expected_ordinary
+
+
+def test_documented_floors_match_calibration():
+    # The floors table in the doc states three numbers. They are the reason every
+    # reconstruction that omitted them over-fired, so they are pinned to source.
+    text = _DOC.read_text(encoding="utf-8")
+    assert "≥ 20 chars" in text and "≥ 32 chars" in text and "≥ 24 chars" in text
+    assert f"≥ **{cal.URL_OPAQUE_ENTROPY_H}**" in text
+    assert cal.URL_OPAQUE_MIN_LEN == 24 and cal.URL_OPAQUE_HEX_MIN_LEN == 32
+    # The base64 floor lives in `entropy`, not `calibration` — assert behaviourally.
+    assert is_base64_like("A" * 20) and not is_base64_like("A" * 19)
+    assert is_hex_blob("a" * 32) and not is_hex_blob("a" * 31)
+
+
+def test_documented_separator_class_matches_the_tokenizer():
+    # The doc spells out the separator characters because omitting tokenization was
+    # the error that produced the largest wrong number. Keep the two in step.
+    from llm_ingestion_guard.active_content import _URL_TOKEN_RE
+
+    text = _DOC.read_text(encoding="utf-8")
+    assert _URL_TOKEN_RE.pattern in text, "tokenizer regex not quoted verbatim in the doc"
+    for sep in "/._-~+,;:=&$!*'()":
+        assert _URL_TOKEN_RE.split(f"a{sep}b") == ["a", "b"], sep