llm-ingestion-okf/tests/test_soft_hyphen_door.py

"""The soft-hyphen normalisation door, in front of the persist gate.

Operator decision 2026-09-18. A long reference standard was refused WHOLE
by `llm-ingestion-guard` 1.4.0 -- `output:zero-width-present`, HIGH, an any-tier
carrier and therefore `fail_secure` at every trust level -- because the
publisher's source carries 71 U+00AD SOFT HYPHEN and 0 real zero-width
characters (U+200B, U+200C, U+200D, U+FEFF, U+2060 all 0, measured by PM on
both deliveries). Those 71 are Norwegian hyphenation points inside words, of
the kind `ar[SHY]beider` and `surdeigs[SHY]bakst`. The verdict is formally
right and materially a false positive.

Of the three ways out, the operator chose this one: okf removes U+00AD BEFORE
the guard sees the text and BOOKS the number in the content accounting. The
guard is not touched, and a real zero-width character is refused exactly as
before -- which is the known-negative every test here carries.

The three alternatives and why they are not here: weakening the guard is the
security repo's call and weakens every consumer's screen; delivering a cleaned
corpus fixes one document and no other; and doing nothing leaves a long
reference standard unreadable for the whole chain.

U+00A0 NBSP is NOT touched (6 633 of them in that document). The guard has no rule
about it -- `_ZERO_WIDTH_CPS` is exactly {200B, 200C, 200D, FEFF, 00AD} on
1.4.0, measured -- so nothing here has to.
"""

from __future__ import annotations

import contextlib
import io
import json
from pathlib import Path

import pytest

from llm_ingestion_okf import cli, extract

SHY = "­"
ZERO_WIDTH = "​"

#: Every character the door must leave alone, each one a way for it to reach
#: further than its own name. The four real zero-width carriers (the guard's
#: set minus U+00AD) and U+2060, which the guard does not screen for at all;
#: U+00A0 NBSP, which that document ships 6 633 of; the three hyphens a reader would
#: confuse with a soft one -- U+002D HYPHEN-MINUS, U+2010 HYPHEN, U+2011
#: NON-BREAKING HYPHEN; and a combining accent, because the door is the one
#: place in the chain that does NOT normalise to NFC.
UNTOUCHED = (
    "\u00a0",
    "\u200b",
    "\u200c",
    "\u200d",
    "\ufeff",
    "\u2060",
    "-",
    "\u2010",
    "\u2011",
    "\u0301",
)

_MIXED = (
    "".join(f"ar{{s}}beider {mark}{{s}}{mark} paa linje {n}\n" for n, mark in enumerate(UNTOUCHED))
    + "surdeigs{s}bakst\n"
)


def test_the_door_removes_one_character_and_leaves_every_other_one_where_it_was() -> None:
    """`log.md` says "No other character is touched" on every single run, and
    README and CLAUDE.md repeat it. Until this test the sentence was held by
    nothing: PM's mutant P6 -- a door that ALSO eats U+00A0 -- passed the whole
    suite green (2 171 passed, RC 0), and it would have eaten all 6 633 NBSP
    in that document while the log went on claiming otherwise.

    The invariant is exact rather than a share, and it is stated as the
    stronger of the two equivalent forms: instead of putting the removed
    characters back at their booked positions and comparing, the test builds
    the expected string with its OWN filter over the source. That fixes not
    only the multiset of surviving characters but their ORDER, so a door that
    removed and re-inserted elsewhere is caught too -- and it needs no
    positions, which the door does not return.

    All three counts come from the test, never from the door.
    """
    source = _MIXED.format(s=SHY)
    for mark in UNTOUCHED:
        assert source.count(mark) >= 2, mark
    assert len(UNTOUCHED) == 10

    removed_here = sum(1 for character in source if character == SHY)
    expected = "".join(character for character in source if character != SHY)
    assert removed_here == 21

    text, removed = extract.normalise_extracted(source)
    assert removed == removed_here
    assert text == expected
    assert SHY not in text
    for mark in UNTOUCHED:
        assert text.count(mark) == source.count(mark), mark


def test_the_promise_holds_through_the_one_door_and_not_only_in_it() -> None:
    """The same invariant one layer out, where the sentence is actually
    published: `extract_document` is the single place the door is applied, and
    a second remover sitting beside it would leave this test red while the
    function above stayed green. `.txt` is the extractor that returns the
    source verbatim, so the comparison is against the bytes that went in."""
    source = _MIXED.format(s=SHY)
    expected = "".join(character for character in source if character != SHY)
    document = extract.extract_document("notat.txt", source.encode("utf-8"))
    assert document.text == expected
    assert document.soft_hyphens == sum(1 for character in source if character == SHY)
    for mark in UNTOUCHED:
        assert document.text.count(mark) == source.count(mark), mark


def _build(inbox: Path, out: Path, accounting: Path) -> tuple[int, str]:
    err = io.StringIO()
    with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
        code = cli.main(
            [
                "build",
                str(inbox),
                "--bundle",
                str(out),
                "--bundle-id",
                "shy",
                "--okf-version",
                "0.2",
                "--accounting",
                str(accounting),
            ]
        )
    return code, err.getvalue()


_SOURCE = (
    "# Kapit{s}tel 7\n\nAr{s}beider med sjoko{s}lade og surdeigs{s}bakst.\n\n"
    "# Krav\n\nTilsvar{s}ende krav gjelder.\n"
)


def test_a_source_with_soft_hyphens_passes_the_default_gate(tmp_path: Path) -> None:
    """Reproduced on this tree 2026-09-19 before the door existed: three soft
    hyphens in one markdown file gave `0 of 1 extracted document(s) persisted;
    rejection codes: fail_secure 1` and exit 1."""
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    (inbox / "notat.md").write_text(_SOURCE.format(s=SHY), encoding="utf-8")
    accounting = tmp_path / "a.json"
    code, err = _build(inbox, tmp_path / "b", accounting)
    assert code == 0, err
    data = json.loads(accounting.read_text(encoding="utf-8"))
    assert data["normalised_soft_hyphen"] == 5
    assert data["documents"][0]["normalised_soft_hyphen"] == 5
    log = (tmp_path / "b" / "log.md").read_text(encoding="utf-8")
    assert "5 soft hyphen(s) (U+00AD) removed" in log


def test_the_log_says_where_the_soft_hyphen_count_comes_from(tmp_path: Path) -> None:
    """`normalised_soft_hyphen` is read off the RUN (`result.normalised`) and
    never recounted from the source, so it is the one number in the accounting
    with no independent denominator behind it. PM 2026-09-19 named the choice:
    a second counter, or a stated provenance. The second was taken, and this
    holds it -- an unstated one is the same class of defect as the door's own
    unheld promise beside it."""
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    (inbox / "notat.md").write_text(_SOURCE.format(s=SHY), encoding="utf-8")
    code, err = _build(inbox, tmp_path / "b", tmp_path / "a.json")
    assert code == 0, err
    log = (tmp_path / "b" / "log.md").read_text(encoding="utf-8")
    assert "The count is the door's own, read off the run and not recounted" in log
    assert "from the source." in log


def test_the_text_is_byte_identical_apart_from_the_removed_characters(tmp_path: Path) -> None:
    """The door removes the N characters and nothing else. Stated as an EXACT
    invariant rather than a share: the extracted text with the soft hyphens put
    back is the extracted text of the source that never had them."""
    with_shy = extract.extract_document("notat.md", _SOURCE.format(s=SHY).encode("utf-8"))
    without = extract.extract_document("notat.md", _SOURCE.format(s="").encode("utf-8"))
    assert with_shy.text == without.text
    assert with_shy.soft_hyphens == 5
    assert without.soft_hyphens == 0
    assert SHY not in with_shy.text


def test_a_real_zero_width_character_is_still_refused(tmp_path: Path) -> None:
    """The known-negative, and the whole reason the door is one character wide.
    U+200B is a carrier with no typographic job in Norwegian; U+00AD is a
    hyphenation point. Removing the first would be the guard's decision and
    would take a screen away from every consumer."""
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    (inbox / "notat.md").write_text(_SOURCE.format(s=ZERO_WIDTH), encoding="utf-8")
    code, err = _build(inbox, tmp_path / "b", tmp_path / "a.json")
    assert code == 1
    assert "fail_secure" in err
    kept = extract.extract_document("notat.md", _SOURCE.format(s=ZERO_WIDTH).encode("utf-8"))
    assert ZERO_WIDTH in kept.text
    assert kept.soft_hyphens == 0


def test_the_pdf_page_table_is_measured_over_the_normalised_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A PDF's `source_pages` locator is a table of CHARACTER offsets built
    from the pages, while the text it indexes comes back through the door. Two
    readings of one document, and a locator built against the other rendering
    points a consumer at the wrong page with full confidence."""
    pages = (
        extract._PdfPage(1, f"Ar{SHY}beider med sjoko{SHY}lade."),
        extract._PdfPage(2, f"Surdeigs{SHY}bakst."),
    )
    monkeypatch.setattr(extract, "_pdf_pages", lambda *a, **k: iter(pages))
    table = extract._pdf_units(b"", False, False)
    joined = extract._PDF_PAGE_SEPARATOR.join(extract._pdf_page_text(p) for p in pages)
    text, removed = extract.normalise_extracted(joined)
    assert removed == 3
    assert table.starts[1] == text.index("Surdeigsbakst")