llm-ingestion-okf/tests/test_soft_hyphen_door.py

"""The soft-hyphen normalisation door, in front of the persist gate.

Operator decision 2026-09-18. R761 Prosesskoden:2025 is refused WHOLE by
`llm-ingestion-guard` 1.4.0 -- `output:zero-width-present`, HIGH, an any-tier
carrier and therefore `fail_secure` at every trust level -- because the
publisher's source carries 71 U+00AD SOFT HYPHEN and 0 real zero-width
characters (U+200B, U+200C, U+200D, U+FEFF, U+2060 all 0, measured by PM on
both deliveries). Those 71 are Norwegian hyphenation points inside words:
`ar[SHY]beider`, `bitu[SHY]men`, `asfalt[SHY]betong`. The verdict is formally
right and materially a false positive.

Of the three ways out, the operator chose this one: okf removes U+00AD BEFORE
the guard sees the text and BOOKS the number in the content accounting. The
guard is not touched, and a real zero-width character is refused exactly as
before -- which is the known-negative every test here carries.

The three alternatives and why they are not here: weakening the guard is the
security repo's call and weakens every consumer's screen; delivering a cleaned
corpus fixes one document and no other; and doing nothing leaves a 701-page
process code unreadable for the whole chain.

U+00A0 NBSP is NOT touched (6 633 of them in R761). The guard has no rule
about it -- `_ZERO_WIDTH_CPS` is exactly {200B, 200C, 200D, FEFF, 00AD} on
1.4.0, measured -- so nothing here has to.
"""

from __future__ import annotations

import contextlib
import io
import json
from pathlib import Path

import pytest

from llm_ingestion_okf import cli, extract

SHY = "­"
ZERO_WIDTH = "​"


def _build(inbox: Path, out: Path, accounting: Path) -> tuple[int, str]:
    err = io.StringIO()
    with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
        code = cli.main(
            [
                "build",
                str(inbox),
                "--bundle",
                str(out),
                "--bundle-id",
                "shy",
                "--okf-version",
                "0.2",
                "--accounting",
                str(accounting),
            ]
        )
    return code, err.getvalue()


_SOURCE = (
    "# Pro{s}sess 84\n\nAr{s}beider med bitu{s}men og asfalt{s}betong.\n\n"
    "# Krav\n\nTilsvar{s}ende krav gjelder.\n"
)


def test_a_source_with_soft_hyphens_passes_the_default_gate(tmp_path: Path) -> None:
    """Reproduced on this tree 2026-09-19 before the door existed: three soft
    hyphens in one markdown file gave `0 of 1 extracted document(s) persisted;
    rejection codes: fail_secure 1` and exit 1."""
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    (inbox / "notat.md").write_text(_SOURCE.format(s=SHY), encoding="utf-8")
    accounting = tmp_path / "a.json"
    code, err = _build(inbox, tmp_path / "b", accounting)
    assert code == 0, err
    data = json.loads(accounting.read_text(encoding="utf-8"))
    assert data["normalised_soft_hyphen"] == 5
    assert data["documents"][0]["normalised_soft_hyphen"] == 5
    log = (tmp_path / "b" / "log.md").read_text(encoding="utf-8")
    assert "5 soft hyphen(s) (U+00AD) removed" in log


def test_the_text_is_byte_identical_apart_from_the_removed_characters(tmp_path: Path) -> None:
    """The door removes the N characters and nothing else. Stated as an EXACT
    invariant rather than a share: the extracted text with the soft hyphens put
    back is the extracted text of the source that never had them."""
    with_shy = extract.extract_document("notat.md", _SOURCE.format(s=SHY).encode("utf-8"))
    without = extract.extract_document("notat.md", _SOURCE.format(s="").encode("utf-8"))
    assert with_shy.text == without.text
    assert with_shy.soft_hyphens == 5
    assert without.soft_hyphens == 0
    assert SHY not in with_shy.text


def test_a_real_zero_width_character_is_still_refused(tmp_path: Path) -> None:
    """The known-negative, and the whole reason the door is one character wide.
    U+200B is a carrier with no typographic job in Norwegian; U+00AD is a
    hyphenation point. Removing the first would be the guard's decision and
    would take a screen away from every consumer."""
    inbox = tmp_path / "inbox"
    inbox.mkdir()
    (inbox / "notat.md").write_text(_SOURCE.format(s=ZERO_WIDTH), encoding="utf-8")
    code, err = _build(inbox, tmp_path / "b", tmp_path / "a.json")
    assert code == 1
    assert "fail_secure" in err
    kept = extract.extract_document("notat.md", _SOURCE.format(s=ZERO_WIDTH).encode("utf-8"))
    assert ZERO_WIDTH in kept.text
    assert kept.soft_hyphens == 0


def test_the_pdf_page_table_is_measured_over_the_normalised_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A PDF's `source_pages` locator is a table of CHARACTER offsets built
    from the pages, while the text it indexes comes back through the door. Two
    readings of one document, and a locator built against the other rendering
    points a consumer at the wrong page with full confidence."""
    pages = (
        extract._PdfPage(1, f"Ar{SHY}beider med bitu{SHY}men."),
        extract._PdfPage(2, f"Asfalt{SHY}betong."),
    )
    monkeypatch.setattr(extract, "_pdf_pages", lambda *a, **k: iter(pages))
    table = extract._pdf_units(b"", False, False)
    joined = extract._PDF_PAGE_SEPARATOR.join(extract._pdf_page_text(p) for p in pages)
    text, removed = extract.normalise_extracted(joined)
    assert removed == 3
    assert table.starts[1] == text.index("Asfaltbetong")


@pytest.mark.skipif(
    not (Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json").is_file(),
    reason="R761 is not on this machine",
)
def test_r761s_own_seventy_one_soft_hyphens_are_the_number_the_door_removes() -> None:
    """PM's count, on the delivery the decision was taken for."""
    source = Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json"
    document = extract.extract_document(source.name, source.read_bytes())
    assert document.soft_hyphens == 71
    assert SHY not in document.text