feat(extract): one normalisation door removes U+00AD before the guard

Operator decision 2026-09-18, and the whole of it: okf removes the soft
hyphen, the guard is untouched, and the number is BOOKED.

`normalise_extracted(text) -> (text, removed)` in `extract`, applied
once at the end of `extract_document`, so both entry points and both
sides of a plan's `text_sha256` see one string. `_pdf_units` measures
its page offsets through the same door, because that table is
CHARACTER offsets rebuilt from the pages while the text it indexes
comes back normalised -- two readings of one document, and a table
built against the other names the wrong page with full confidence.
Removing a character removes no newline, so every line-indexed rule
downstream is unmoved by construction.

`ExtractedDocument.soft_hyphens` carries the count out;
`InboxResult.normalised` carries it per document; `account_run` reads
it off the RUN rather than recounting the source, because a second
count would be a second reader. It reaches the accounting JSON as
`normalised_soft_hyphen` at both levels and `log.md` as a
`**Normalisation**` bullet naming the count, the documents touched and
that no other character is moved.

EXPOSURE, measured and not assumed: 0 of the 78 readable documents of
the K2 reference corpus carry U+00AD or any of the four real
zero-width characters (the 8 unreadable ones raise before extraction),
0 in the pinned K2 bundle's concept bodies, and 0 across
`tests/fixtures`, `examples`, `skills`, `docs`, `src`, README and
CHANGELOG. The door cannot have moved a byte anyone here has measured.

Suite 2179 passed, 1 skipped; ruff and `mypy --strict src/` clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/llm_ingestion_okf/accounting.py

@@ -732,6 +732,12 @@ class DocumentAccount:
     counts: dict[str, int]
     fates: dict[str, Fate]
     error: str | None = None
+    #: How many U+00AD the normalisation door removed from this document's
+    #: text before the persist gate saw it (operator decision 2026-09-18).
+    #: Booked rather than silently applied: a door that changes a source's
+    #: bytes and says nothing is the same class of absence this whole module
+    #: exists to close.
+    normalised_soft_hyphen: int = 0
 
     @property
     def unaccounted(self) -> dict[str, int]:
@@ -758,6 +764,7 @@ class DocumentAccount:
             "source_file": self.source_file,
             "status": self.status,
             "code": self.code,
+            "normalised_soft_hyphen": self.normalised_soft_hyphen,
             "inventory": dict(self.counts),
             "fates": {kind: self.fates[kind].to_json() for kind in self.counts},
             "unaccounted": self.unaccounted,
@@ -806,6 +813,11 @@ class Accounting:
         """
         return sum(1 for d in self.documents if d.status == REJECTED)
 
+    @property
+    def normalised_soft_hyphen(self) -> int:
+        """Soft hyphens the normalisation door removed across the whole run."""
+        return sum(d.normalised_soft_hyphen for d in self.documents)
+
     @property
     def images_found(self) -> int:
         return sum(d.counts.get("image", 0) for d in self.documents)
@@ -820,6 +832,7 @@ class Accounting:
             "unaccounted": self.unaccounted,
             "double_booked": self.double_booked,
             "refused": self.refused,
+            "normalised_soft_hyphen": self.normalised_soft_hyphen,
             "documents": [d.to_json() for d in self.documents],
             "files": [f.to_json() for f in self.files],
         }
@@ -831,6 +844,12 @@ class Accounting:
             f"{self.unaccounted} unaccounted, {self.double_booked} double-booked; "
             f"{self.refused} of {len(self.documents)} document(s) refused whole."
         ]
+        touched = sum(1 for d in self.documents if d.normalised_soft_hyphen)
+        lines.append(
+            f"* **Normalisation**: {self.normalised_soft_hyphen} soft hyphen(s) (U+00AD) "
+            f"removed from {touched} of {len(self.documents)} document(s) before the persist "
+            "gate. No other character is touched."
+        )
         for doc in self.documents:
             if doc.status == REJECTED:
                 lines.append(
@@ -986,6 +1005,12 @@ def account_run(inbox: Path, walked: Sequence[Path], result: InboxResult) -> Acc
             documents.append(
                 DocumentAccount(name, "", None, inv.counts(), {k: Fate() for k in inv.counts()})
             )
+    # Read off the run, never recounted from the source: a second count would
+    # be a second reader, and the number this publishes has to be the number
+    # the normalisation door acted on.
+    removed = {item.source_file: item.soft_hyphens for item in result.normalised}
+    for document in documents:
+        document.normalised_soft_hyphen = removed.get(document.source_file, 0)
     return Accounting(documents, files)
 
 

src/llm_ingestion_okf/extract.py

@@ -381,6 +381,46 @@ _REMOTE_SOURCE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|//)")
 _DATA_URI = re.compile(r"^data:(?P<media>[^;,]*)(?P<base64>;base64)?,(?P<payload>.*)$", re.DOTALL)
 
 
+#: The ONE character the normalisation door removes, and the reason it is one
+#: character and not a class. `llm-ingestion-guard` 1.4.0 puts U+00AD in
+#: `_ZERO_WIDTH_CPS` beside U+200B, U+200C, U+200D and U+FEFF, and
+#: `output:zero-width-present` is an any-tier carrier: `fail_secure` at every
+#: trust level, with no sanitisation and no exception. Measured by PM
+#: 2026-09-18, R761 Prosesskoden:2025 carries 71 U+00AD and 0 of the other
+#: four, so a 701-page process code is unreadable for the whole chain over
+#: Norwegian hyphenation points inside words -- `ar[SHY]beider`,
+#: `bitu[SHY]men`, `asfalt[SHY]betong`. The verdict is formally right and
+#: materially a false positive, and the operator's answer (2026-09-18) is that
+#: okf removes this character before the guard and SAYS SO in the accounting.
+#:
+#: The other four stay. They carry no typographic job in running text, so
+#: removing one would be a decision about what the guard screens for, taken in
+#: the wrong repository. U+00A0 NBSP is not in the guard's set at all and is
+#: not touched either -- R761 ships 6 633 of them.
+SOFT_HYPHEN = "\u00ad"
+
+
+def normalise_extracted(text: str) -> tuple[str, int]:
+    """The normalisation door: the text without U+00AD, and how many were removed.
+
+    Applied ONCE, at the end of :func:`extract_document`, so every caller of
+    either entry point gets the same string and `propose` and Door B cannot
+    disagree about the text a plan's `text_sha256` indexes. The only other
+    place that has to know about it is :func:`_pdf_units`, which rebuilds a
+    table of CHARACTER offsets from the pages rather than from the returned
+    text: two readings of one document, and a table built against the other
+    one would name the wrong page with full confidence.
+
+    Removing a character never removes a newline, so every LINE-indexed rule
+    downstream -- the proposer's grammars, `xml_outline`, `pdf_outline`'s
+    per-page line check -- is unmoved by construction.
+    """
+    removed = text.count(SOFT_HYPHEN)
+    if not removed:
+        return text, 0
+    return text.replace(SOFT_HYPHEN, ""), removed
+
+
 @dataclass(frozen=True)
 class ExtractedDocument:
     """One dropped file's text, and the images that stand inside that text.
@@ -400,6 +440,10 @@ class ExtractedDocument:
     #: unpointed file with the same bytes as a carried one was carried through
     #: nothing (R761 ships eight such duplicates).
     files: tuple[str, ...] = ()
+    #: How many U+00AD the normalisation door removed from this text. Zero for
+    #: every document that carried none, which is 0 of the 78 readable
+    #: documents of the K2 reference corpus (measured 2026-09-19).
+    soft_hyphens: int = 0
 
 
 class _AssetCollector:
@@ -2586,7 +2630,11 @@ def _pdf_units(data: bytes, headings: bool, ocr: bool, assets: bool = False) ->
         # The page as it reaches the text, pointers included: a locator built
         # from the body alone would drift by two lines per carried image and
         # would name the wrong page from the first one onwards.
-        offset += len(_pdf_page_text(page)) + len(_PDF_PAGE_SEPARATOR)
+        # Through the normalisation door for the same reason one step smaller:
+        # the text this table indexes has had its soft hyphens removed, so
+        # measuring the page before the door would drift by one character per
+        # hyphen and name the wrong page.
+        offset += len(normalise_extracted(_pdf_page_text(page))[0]) + len(_PDF_PAGE_SEPARATOR)
     return SourceUnits("pages", tuple(starts), tuple(numbers))
 
 
@@ -2776,9 +2824,11 @@ def extract_document(
         text = _ASSET_READERS[suffix](data, collector)
     else:
         text = extractor(data)
+    rendered, soft_hyphens = normalise_extracted(renderer(text) if renderer is not None else text)
     return ExtractedDocument(
-        text=renderer(text) if renderer is not None else text,
+        text=rendered,
         images=tuple(collector.images) if collector is not None else (),
         rejected=tuple(collector.rejected) if collector is not None else (),
         files=tuple(collector.files) if collector is not None else (),
+        soft_hyphens=soft_hyphens,
     )

src/llm_ingestion_okf/inbox.py

@@ -643,6 +643,11 @@ class InboxResult:
     # ones that were found and not carried, with their codes. The content
     # accounting books a document's images from this, never from the bundle.
     document_assets: tuple[DocumentAssets, ...] = ()
+    # Per document the run READ: how many U+00AD the normalisation door
+    # removed before the persist gate saw the text. One entry per document
+    # that carried at least one, so a run over a corpus with none of them
+    # carries an empty tuple and says `0` rather than nothing.
+    normalised: tuple[DocumentNormalisation, ...] = ()
 
 
 @dataclass(frozen=True)
@@ -654,6 +659,19 @@ class DocumentAssets:
     rejected: tuple[AssetRejection, ...]
 
 
+@dataclass(frozen=True)
+class DocumentNormalisation:
+    """What the normalisation door removed from one document.
+
+    Recorded where the removal HAPPENED rather than counted again off the
+    source afterwards: a second count would be a second reader, and the number
+    the accounting publishes has to be the number the run acted on.
+    """
+
+    source_file: str
+    soft_hyphens: int
+
+
 def relative_source(path: Path, inbox: Path) -> str:
     """A dropped file's name as the provenance layer records it.
 
@@ -1108,6 +1126,7 @@ def process_inbox(
     refused_assets: list[AssetRejection] = []
     carried_files: set[str] = set()
     document_assets: list[DocumentAssets] = []
+    normalised: list[DocumentNormalisation] = []
 
     # Phase 1: name every file BEFORE any gate call or write, so an intra-run
     # collision is caught while both files can still be refused together. Under
@@ -1417,6 +1436,12 @@ def process_inbox(
                 posixpath.normpath((directory / reference).as_posix())
                 for reference in document.files
             )
+        if document.soft_hyphens:
+            normalised.append(
+                DocumentNormalisation(
+                    source_file=source_name(path), soft_hyphens=document.soft_hyphens
+                )
+            )
         if outputs:
             document_assets.append(
                 DocumentAssets(
@@ -1485,6 +1510,7 @@ def process_inbox(
         assets_rejected=tuple(refused_assets),
         carried_files=tuple(sorted(carried_files)),
         document_assets=tuple(document_assets),
+        normalised=tuple(normalised),
     )