llm-ingestion-okf/tests/test_xml_sts.py

"""`.xml`: a NISO-STS document carries its own structure, and it was unreadable.

MEASURED OUTSIDE THIS REPOSITORY AND REPRODUCED HERE: the zip a publisher's
own viewer delivers as "Html" holds 0 html, 1 xml and 109 images, and `okf
build` on it gave **110 of 110 unreadable, 0 plans, exit 2** -- `.xml` was in
neither extractor registry. The one xml file IS the whole product: a large
regulatory reference document, the same one round 12 met as a PDF,
in NISO-STS form. It carries several thousand `<sec>`, **N (36 %) of them with a
`<title>`**, and a `<sec>`-nesting depth distribution that is row for row the
fasit's own, the fasit being built from that same file.

THE CEILING IS THEREFORE STRUCTURAL AND NOT COMPUTED: N of N. The PDF
arm reached N - 2 of N (99.9 %) by bridging from (page, y) to a line index; here the
publisher states the structure in elements, so nothing is recovered and nothing
is guessed.

WHAT THESE TESTS PIN, each because the measurement said it could go wrong:

- `<label>` carries the number and `<title>` carries the text. Emitting only
  `<title>` scores 0 of N while every line of the code looks right, because
  the number is what okf reduces to a directory name.
- a `<sec>` with a `<label>` and NO `<title>` is never a heading. Most (64 %) of
  the `<sec>` are lettered points (`a)`, `c)`) inside a process description, and one
  heading each would bury the document's own N.
- the ONE titled section with no label -- `Forord` -- emitted without a
  numbering token, because the corrected instrument key looks it up by title.
- a `<table-wrap>` as ONE markdown table block. The PDF path delivered 0 of 10.
- XML that is NOT STS producing text and ONE plan, never zero and never an
  invented structure from element names.
- a DTD REFUSED rather than parsed. Measured on this machine's interpreter
  (3.14.0, pyexpat 2.7.3): an external entity is refused by the stdlib, but the
  billion-laughs limit comes from libexpat >= 2.4.0 and NOT from Python, while
  `pyproject.toml` requires only `>=3.10`. Refusing every DTD is a guarantee
  about this code; relying on the parser is a guarantee about the machine.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from llm_ingestion_okf import extract, propose
from llm_ingestion_okf.errors import IngestError

FIXTURES = Path(__file__).parent / "fixtures"
STS = FIXTURES / "sts-mini.xml"
GENERIC = FIXTURES / "generic-feed.xml"
BOMB = FIXTURES / "xml-doctype-bomb.xml"
MALFORMED = FIXTURES / "xml-malformed.xml"
EMPTY_LABEL = FIXTURES / "sts-empty-label.xml"

STS_EXPECTED = """\
Testnormal for fiksturbruk
# Forord
Dette forordet baerer ingen nummerering.
# 1 Bruksomraade
Normalen gjelder for testing av ekstraktoren.
## 1.1 Omfang
Omfanget dekker hele arbeidet.
### 1.1.1 Materialer
Materialene skal vaere godkjente.
a) Betong skal ha fasthetsklasse B35.
b) Armering skal vaere av klasse B500NC.
# 2 Tabeller
Tabellen under er en enkelt tabellblokk.
Tabell 2-1
| Prosess | Enhet |
| --- | --- |
| Sprengning | kubikkmeter |
Foerste punkt i lista.
Andre punkt i lista."""


def test_an_sts_document_becomes_the_markdown_the_office_rows_produce() -> None:
    """The whole emission, asserted as one string rather than in pieces.

    A per-property test would pass on an output whose LINES were right and
    whose order was not, and order is what every boundary grammar reads.
    """
    assert extract.extract_text(STS.name, STS.read_bytes()) == STS_EXPECTED


def test_the_label_and_the_title_reach_the_proposer_as_one_heading() -> None:
    """Title, LEVEL and OFFSET -- not "it found more than one"."""
    text = extract.extract_text(STS.name, STS.read_bytes())
    candidates = propose.find_candidates(text)

    # The TITLE keeps its numbering token and the `number` field carries it as
    # well -- that is `propose`'s own documented ATX behaviour and this reader
    # does not reach into it. `1 Bruksomraade` gets no `number` because
    # `_NUMBERED` requires at least one dot, which is why both shapes are here.
    assert [(c.rule, c.number, c.title, c.level) for c in candidates] == [
        ("rule:heading", None, "Forord", 1),
        ("rule:heading", None, "1 Bruksomraade", 1),
        ("rule:heading", "1.1", "1.1 Omfang", 2),
        ("rule:heading", "1.1.1", "1.1.1 Materialer", 3),
        ("rule:heading", None, "2 Tabeller", 1),
        # The table-wrap, as ONE candidate rather than one per row.
        ("rule:table-block", None, "Tabell linje 15", 9),
    ]
    candidates = [c for c in candidates if c.rule == "rule:heading"]
    # The offsets, stated against the text itself: a candidate naming the right
    # title at the wrong offset would pass the list above.
    for candidate, line in zip(
        candidates,
        ["# Forord", "# 1 Bruksomraade", "## 1.1 Omfang", "### 1.1.1 Materialer", "# 2 Tabeller"],
        strict=True,
    ):
        assert text[candidate.start : candidate.start + len(line)] == line


def test_a_lettered_point_is_a_body_line_and_never_a_heading() -> None:
    """64 % of the reference standard's `<sec>` are these. One heading each buries its N."""
    text = extract.extract_text(STS.name, STS.read_bytes())

    assert "a) Betong skal ha fasthetsklasse B35." in text.split("\n")
    assert not any(line.startswith("#") and "Betong" in line for line in text.split("\n"))
    assert [c.title for c in propose.find_candidates(text) if "Betong" in c.title] == []


def test_a_table_wrap_is_one_table_block_with_its_label_above_it() -> None:
    """The separator line is what makes it a BLOCK and not two pipe lines."""
    lines = extract.extract_text(STS.name, STS.read_bytes()).split("\n")
    at = lines.index("Tabell 2-1")

    assert lines[at + 1 : at + 4] == [
        "| Prosess | Enhet |",
        "| --- | --- |",
        "| Sprengning | kubikkmeter |",
    ]


def test_the_text_survives_the_markup_exactly() -> None:
    """EXACT, not a percentage: strip what the emission added and compare."""
    import xml.etree.ElementTree as ET

    text = extract.extract_text(STS.name, STS.read_bytes())
    source = ET.fromstring(STS.read_text(encoding="utf-8"))

    kept = []
    for line in text.split("\n"):
        if set(line) <= set("| -:"):
            continue  # the table separator, which no source character produced
        kept.append(line.lstrip("#").replace("|", ""))
    emitted = "".join("".join(kept).split())
    original = "".join("".join(source.itertext()).split())
    assert emitted == original


def test_generic_xml_keeps_its_text_and_invents_no_structure() -> None:
    """One plan with the content preserved -- never zero, never element names
    promoted to headings. The schema test is NAMED (`<standard>` root, or any
    `<sec>`), so a document that is not STS is not guessed at."""
    text = extract.extract_text(GENERIC.name, GENERIC.read_bytes())

    assert "Kantinen er stengt for vedlikehold natt til fredag." in text
    assert "Redusert åpningstid" in text
    assert not any(line.startswith("#") for line in text.split("\n"))
    assert propose.find_candidates(text) == []


def test_a_document_type_declaration_is_refused_and_never_expanded() -> None:
    """A guarantee about this code, not about the machine's libexpat."""
    data = BOMB.read_bytes()
    assert b"<!DOCTYPE" in data  # the fixture really carries one

    with pytest.raises(IngestError) as caught:
        extract.extract_text(BOMB.name, data)
    assert caught.value.code == "extractor_xml_doctype"
    # The expansion must not appear ANYWHERE, including in the error text.
    assert "SPRENGSTOFF" not in str(caught.value)


def test_malformed_xml_raises_a_typed_error_and_not_a_parse_error() -> None:
    """Not a leaked `ParseError`, and not zero concepts in silence."""
    with pytest.raises(IngestError) as caught:
        extract.extract_text(MALFORMED.name, MALFORMED.read_bytes())
    assert caught.value.code == "extractor_xml_parse_error"


def test_xml_is_a_core_type_and_never_reaches_the_converter() -> None:
    """Stdlib parsing, so `[extract]` would make a stdlib type binary-dependent
    -- and the converter is a second parser this hardening never sees."""
    from llm_ingestion_okf.extract import (
        _CORE_EXTRACTORS,
        _OPTIONAL_EXTRACTORS,
        _PANDOC_FORMATS,
    )

    assert ".xml" in _CORE_EXTRACTORS
    assert ".xml" not in _OPTIONAL_EXTRACTORS
    assert ".xml" not in _PANDOC_FORMATS


def test_a_bundle_is_built_end_to_end_from_an_sts_document(tmp_path: Path) -> None:
    """The registries are COUPLED, and only a run through the whole chain says so.

    `segmentation._STDLIB_EXTRACTOR_IDS` names the ids whose extracted text is
    versioned by this package's own literal, and its own comment says a row
    added to the extraction registry and not there "fails loudly on the first
    proposal for that type". It does -- and no unit test of the extractor can
    see it, because the failure is two layers away in the version the plan is
    keyed to. This was found by running `okf build`, not by a test, which is
    why the test exists now.
    """
    from llm_ingestion_okf import cli
    from llm_ingestion_okf.segmentation import _STDLIB_EXTRACTOR_IDS, observed_extractor_version

    assert "xml" in _STDLIB_EXTRACTOR_IDS
    assert observed_extractor_version("xml")

    folder = tmp_path / "docs"
    folder.mkdir()
    (folder / "sts-mini.xml").write_bytes(STS.read_bytes())
    bundle = tmp_path / "bundle"
    report = cli.build(folder, bundle=bundle, bundle_id="mini", okf_version="0.2")
    assert report.codes == ()
    assert report.unaccounted == ()

    titles = sorted(
        line.split(":", 1)[1].strip()
        for concept in bundle.rglob("*.md")
        if concept.name not in ("index.md", "log.md") and concept.parent != bundle
        for line in concept.read_text(encoding="utf-8").splitlines()
        if line.startswith("title:")
    )
    assert titles, "an STS document must produce concepts"
    assert "Forord" in titles


def test_a_label_with_no_body_under_it_still_reaches_the_text() -> None:
    """A pending label must not be overwritten by the next one, or by a heading.

    MEASURED ON THE REFERENCE STANDARD, and it is exactly two characters: one `x)` marks a
    `<sec>` that carries a label and nothing else, and the label was held as a
    prefix for a body line that never came -- the next thing emitted was the
    following section's heading. Non-whitespace preservation was 1 283 393
    against 1 283 395, ratio 0.999998. An EXACT invariant does not get to be
    0.999998, and a percentage would have hidden which two characters they
    were.
    """
    import xml.etree.ElementTree as ET

    text = extract.extract_text(EMPTY_LABEL.name, EMPTY_LABEL.read_bytes())
    source = ET.fromstring(EMPTY_LABEL.read_text(encoding="utf-8"))

    assert "x)" in text
    emitted = "".join("".join(line.lstrip("#") for line in text.split("\n")).split())
    original = "".join("".join(source.itertext()).split())
    assert emitted == original