test(extract): the soft-hyphen door in front of the persist gate (red)

Operator decision 2026-09-18, the alternative chosen verbatim: okf gets
ONE documented normalisation door before the guard, removing U+00AD and
BOOKING the count in the content accounting. The guard is not touched.

The defect it answers, PM's measurement: R761 Prosesskoden:2025 is
refused whole by guard 1.4.0 as `output:zero-width-present` -- an
any-tier carrier, `fail_secure` at every trust level -- over 71 U+00AD
and 0 real zero-width characters. The 71 are Norwegian hyphenation
points inside words. Reproduced here in the small on this tree before
any code moved: three soft hyphens in one markdown file give `0 of 1
extracted document(s) persisted; rejection codes: fail_secure 1` and
exit 1.

Five tests, each red on its claim:

- the source passes the DEFAULT gate, the accounting carries
  `normalised_soft_hyphen: 5` at both levels, and `log.md` says it;
- text preservation as an EXACT invariant and never a share: the
  extracted text of the hyphenated source EQUALS the extracted text of
  the same source written without them;
- the known-negative, U+200B, still `fail_secure` and still in the
  text -- removing a real carrier would be the guard's decision and
  would take a screen away from every consumer;
- a PDF's `source_pages` table is CHARACTER offsets rebuilt from the
  pages while the text comes back through the door, so both must be
  measured over the same rendering or every locator drifts;
- R761's own 71, on the delivery the decision was taken for.

U+00A0 NBSP is untouched: `_ZERO_WIDTH_CPS` is exactly {200B, 200C,
200D, FEFF, 00AD} on guard 1.4.0, read off the installed source, so
the guard has no rule about NBSP and nothing here needs one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_soft_hyphen_door.py

@@ -0,0 +1,141 @@
+"""The soft-hyphen normalisation door, in front of the persist gate.
+
+Operator decision 2026-09-18. R761 Prosesskoden:2025 is refused WHOLE by
+`llm-ingestion-guard` 1.4.0 -- `output:zero-width-present`, HIGH, an any-tier
+carrier and therefore `fail_secure` at every trust level -- because the
+publisher's source carries 71 U+00AD SOFT HYPHEN and 0 real zero-width
+characters (U+200B, U+200C, U+200D, U+FEFF, U+2060 all 0, measured by PM on
+both deliveries). Those 71 are Norwegian hyphenation points inside words:
+`ar[SHY]beider`, `bitu[SHY]men`, `asfalt[SHY]betong`. The verdict is formally
+right and materially a false positive.
+
+Of the three ways out, the operator chose this one: okf removes U+00AD BEFORE
+the guard sees the text and BOOKS the number in the content accounting. The
+guard is not touched, and a real zero-width character is refused exactly as
+before -- which is the known-negative every test here carries.
+
+The three alternatives and why they are not here: weakening the guard is the
+security repo's call and weakens every consumer's screen; delivering a cleaned
+corpus fixes one document and no other; and doing nothing leaves a 701-page
+process code unreadable for the whole chain.
+
+U+00A0 NBSP is NOT touched (6 633 of them in R761). The guard has no rule
+about it -- `_ZERO_WIDTH_CPS` is exactly {200B, 200C, 200D, FEFF, 00AD} on
+1.4.0, measured -- so nothing here has to.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+from llm_ingestion_okf import cli, extract
+
+SHY = "­"
+ZERO_WIDTH = "​"
+
+
+def _build(inbox: Path, out: Path, accounting: Path) -> tuple[int, str]:
+    err = io.StringIO()
+    with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
+        code = cli.main(
+            [
+                "build",
+                str(inbox),
+                "--bundle",
+                str(out),
+                "--bundle-id",
+                "shy",
+                "--okf-version",
+                "0.2",
+                "--accounting",
+                str(accounting),
+            ]
+        )
+    return code, err.getvalue()
+
+
+_SOURCE = (
+    "# Pro{s}sess 84\n\nAr{s}beider med bitu{s}men og asfalt{s}betong.\n\n"
+    "# Krav\n\nTilsvar{s}ende krav gjelder.\n"
+)
+
+
+def test_a_source_with_soft_hyphens_passes_the_default_gate(tmp_path: Path) -> None:
+    """Reproduced on this tree 2026-09-19 before the door existed: three soft
+    hyphens in one markdown file gave `0 of 1 extracted document(s) persisted;
+    rejection codes: fail_secure 1` and exit 1."""
+    inbox = tmp_path / "inbox"
+    inbox.mkdir()
+    (inbox / "notat.md").write_text(_SOURCE.format(s=SHY), encoding="utf-8")
+    accounting = tmp_path / "a.json"
+    code, err = _build(inbox, tmp_path / "b", accounting)
+    assert code == 0, err
+    data = json.loads(accounting.read_text(encoding="utf-8"))
+    assert data["normalised_soft_hyphen"] == 5
+    assert data["documents"][0]["normalised_soft_hyphen"] == 5
+    log = (tmp_path / "b" / "log.md").read_text(encoding="utf-8")
+    assert "5 soft hyphen(s) (U+00AD) removed" in log
+
+
+def test_the_text_is_byte_identical_apart_from_the_removed_characters(tmp_path: Path) -> None:
+    """The door removes the N characters and nothing else. Stated as an EXACT
+    invariant rather than a share: the extracted text with the soft hyphens put
+    back is the extracted text of the source that never had them."""
+    with_shy = extract.extract_document("notat.md", _SOURCE.format(s=SHY).encode("utf-8"))
+    without = extract.extract_document("notat.md", _SOURCE.format(s="").encode("utf-8"))
+    assert with_shy.text == without.text
+    assert with_shy.soft_hyphens == 5
+    assert without.soft_hyphens == 0
+    assert SHY not in with_shy.text
+
+
+def test_a_real_zero_width_character_is_still_refused(tmp_path: Path) -> None:
+    """The known-negative, and the whole reason the door is one character wide.
+    U+200B is a carrier with no typographic job in Norwegian; U+00AD is a
+    hyphenation point. Removing the first would be the guard's decision and
+    would take a screen away from every consumer."""
+    inbox = tmp_path / "inbox"
+    inbox.mkdir()
+    (inbox / "notat.md").write_text(_SOURCE.format(s=ZERO_WIDTH), encoding="utf-8")
+    code, err = _build(inbox, tmp_path / "b", tmp_path / "a.json")
+    assert code == 1
+    assert "fail_secure" in err
+    kept = extract.extract_document("notat.md", _SOURCE.format(s=ZERO_WIDTH).encode("utf-8"))
+    assert ZERO_WIDTH in kept.text
+    assert kept.soft_hyphens == 0
+
+
+def test_the_pdf_page_table_is_measured_over_the_normalised_text(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A PDF's `source_pages` locator is a table of CHARACTER offsets built
+    from the pages, while the text it indexes comes back through the door. Two
+    readings of one document, and a locator built against the other rendering
+    points a consumer at the wrong page with full confidence."""
+    pages = (
+        extract._PdfPage(1, f"Ar{SHY}beider med bitu{SHY}men."),
+        extract._PdfPage(2, f"Asfalt{SHY}betong."),
+    )
+    monkeypatch.setattr(extract, "_pdf_pages", lambda *a, **k: iter(pages))
+    table = extract._pdf_units(b"", False, False)
+    joined = extract._PDF_PAGE_SEPARATOR.join(extract._pdf_page_text(p) for p in pages)
+    text, removed = extract.normalise_extracted(joined)
+    assert removed == 3
+    assert table.starts[1] == text.index("Asfaltbetong")
+
+
+@pytest.mark.skipif(
+    not (Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json").is_file(),
+    reason="R761 is not on this machine",
+)
+def test_r761s_own_seventy_one_soft_hyphens_are_the_number_the_door_removes() -> None:
+    """PM's count, on the delivery the decision was taken for."""
+    source = Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json"
+    document = extract.extract_document(source.name, source.read_bytes())
+    assert document.soft_hyphens == 71
+    assert SHY not in document.text