llm-ingestion-okf/tests/test_document_identity.py

"""A document that states who it is names its own directory (K3-19 a).

MEASURED OUTSIDE THIS REPOSITORY: a NISO-STS delivery built with `okf build`
put all 2 761 concepts under a directory named for the delivery path's file
name, a UUID that occurs **0 times** in the document itself. The document
carries exactly one `<std-ident>` whose `<doc-number>` says what it is, and one
`<title-wrap>` -- and nothing in this package read either. Every concept's
`sources` entry then named the file name twice, as the address and as the
title.

THE PRECEDENCE, and each layer is tested here: an explicit per-run value beats
what the document declares, and what the document declares beats the file
name. The explicit layer is `--frontmatter`, tested in
`tests/test_run_frontmatter.py`; this file holds the two layers below it.

WHAT THESE TESTS PIN:

- the directory comes from `<doc-number>` through the id grammar, and ONLY the
  file's own stem is replaced -- the folders above it are the operator's;
- `sources[0].title` is `<doc-number>` + `<year>` when the document states
  them, `<title-wrap>` when it states only a title, the file name otherwise;
- a declared value that would end the `sources` flow mapping early falls to
  the next layer instead of being written or mangled -- the measured document's
  own `<full>` title carries a comma;
- two documents in one run that declare the SAME name both keep their file
  name. The collision gate that already exists refuses both and tells the
  operator to rename one, and a name read from the document is not one a
  rename can change;
- more than one `<doc-number>` is no doc number: picking one would be a guess.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from llm_ingestion_okf import cli, extract
from llm_ingestion_okf.materialize import parse_frontmatter

FIXTURES = Path(__file__).parent / "fixtures"
IDENTITY = FIXTURES / "sts-identity.xml"
TITLE_ONLY = FIXTURES / "sts-mini.xml"
NEITHER = FIXTURES / "sts-empty-label.xml"
GENERIC = FIXTURES / "generic-feed.xml"

# A delivery path's name. Like the one measured, it says nothing about the
# document inside it.
OPAQUE = "0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0.xml"

FULL_TITLE = "R900 Testnormalen Standard for fiksturer, tester og kontroll"


def _build(tmp_path: Path, files: dict[str, bytes]) -> Path:
    folder = tmp_path / "docs"
    for name, data in files.items():
        (folder / name).parent.mkdir(parents=True, exist_ok=True)
        (folder / name).write_bytes(data)
    bundle = tmp_path / "bundle"
    report = cli.build(folder, bundle, bundle_id="identity", okf_version="0.2")
    assert report.codes == ()
    assert report.unaccounted == ()
    return bundle


def _concepts(bundle: Path) -> dict[str, dict[str, str]]:
    return {
        path.relative_to(bundle).as_posix(): parse_frontmatter(path)
        for path in sorted(bundle.rglob("*.md"))
        if path.name not in ("index.md", "log.md")
    }


def _sources(resource: str, title: str) -> str:
    return f"[{{ resource: {resource}, title: {title} }}]"


# --- what the document declares ---------------------------------------------


def test_the_identity_is_read_from_the_documents_own_elements() -> None:
    identity = extract.declared_identity(IDENTITY.name, IDENTITY.read_bytes())

    assert identity == extract.DeclaredIdentity(
        doc_number="R900 Testnormalen", year="2024", title=FULL_TITLE
    )


def test_a_title_wrap_without_a_doc_number_is_half_an_identity() -> None:
    identity = extract.declared_identity(TITLE_ONLY.name, TITLE_ONLY.read_bytes())

    assert identity == extract.DeclaredIdentity(
        doc_number=None, year=None, title="Testnormal for fiksturbruk"
    )


@pytest.mark.parametrize("fixture", [NEITHER, GENERIC], ids=["sts-no-front", "not-sts"])
def test_a_document_declaring_neither_has_no_identity(fixture: Path) -> None:
    assert extract.declared_identity(fixture.name, fixture.read_bytes()) is None


def test_only_the_row_that_can_read_a_declaration_is_asked() -> None:
    """A markdown file whose text LOOKS like an identity declares nothing."""
    data = b"<std-ident><doc-number>R900 Testnormalen</doc-number></std-ident>\n"
    assert extract.declared_identity("notat.md", data) is None


def test_two_doc_numbers_are_no_doc_number() -> None:
    """An adopted standard carries one `<std-ident>` per body that issued it.

    Choosing one of them would be a guess, so neither is used and the file
    name stays the directory.
    """
    data = IDENTITY.read_bytes().replace(
        b"</std-doc-meta>",
        b"<std-ident><doc-number>NS 9000</doc-number><year>2020</year></std-ident></std-doc-meta>",
    )
    identity = extract.declared_identity(IDENTITY.name, data)

    assert identity is not None
    assert identity.doc_number is None
    assert identity.year is None
    assert identity.title == FULL_TITLE


# --- the directory and the address, through the whole build -----------------


def test_the_doc_number_names_the_document_directory(tmp_path: Path) -> None:
    concepts = _concepts(_build(tmp_path, {OPAQUE: IDENTITY.read_bytes()}))

    assert concepts, "the fixture must produce concepts for this to mean anything"
    assert all(name.startswith("r900-testnormalen/") for name in concepts), sorted(concepts)
    assert not any(Path(OPAQUE).stem in name for name in concepts)


def test_the_address_keeps_the_file_and_the_title_is_the_documents(tmp_path: Path) -> None:
    """`resource` is where the bytes are; `title` is what the document calls itself.

    `<doc-number>` + `<year>` rather than `<full>`: the fixture's `<full>`, like
    the measured document's, carries a comma, which ends a flow mapping.
    """
    concepts = _concepts(_build(tmp_path, {OPAQUE: IDENTITY.read_bytes()}))

    assert {values["sources"] for values in concepts.values()} == {
        _sources(OPAQUE, "R900 Testnormalen 2024")
    }


def test_a_nested_document_keeps_the_folders_above_it(tmp_path: Path) -> None:
    concepts = _concepts(_build(tmp_path, {f"leveranse/{OPAQUE}": IDENTITY.read_bytes()}))

    assert all(name.startswith("leveranse/r900-testnormalen/") for name in concepts)
    assert {values["sources"] for values in concepts.values()} == {
        _sources(f"leveranse/{OPAQUE}", "R900 Testnormalen 2024")
    }


def test_a_title_without_a_doc_number_names_the_address_and_not_the_directory(
    tmp_path: Path,
) -> None:
    concepts = _concepts(_build(tmp_path, {"sts-mini.xml": TITLE_ONLY.read_bytes()}))

    assert all(name.startswith("sts-mini/") for name in concepts)
    assert {values["sources"] for values in concepts.values()} == {
        _sources("sts-mini.xml", "Testnormal for fiksturbruk")
    }


def test_no_declaration_keeps_the_file_name_in_both_places(tmp_path: Path) -> None:
    concepts = _concepts(_build(tmp_path, {"sts-empty-label.xml": NEITHER.read_bytes()}))

    assert all(name.startswith("sts-empty-label/") for name in concepts)
    assert {values["sources"] for values in concepts.values()} == {
        _sources("sts-empty-label.xml", "sts-empty-label.xml")
    }


def test_a_declared_title_that_would_close_the_flow_mapping_falls_to_the_file_name(
    tmp_path: Path,
) -> None:
    """Refused as a VALUE, never as a document, and never cleaned up.

    Removing the comma would write a title the document does not carry.
    """
    data = TITLE_ONLY.read_bytes().replace(
        b"Testnormal for fiksturbruk", b"Testnormal, for fiksturbruk"
    )
    concepts = _concepts(_build(tmp_path, {"sts-mini.xml": data}))

    assert {values["sources"] for values in concepts.values()} == {
        _sources("sts-mini.xml", "sts-mini.xml")
    }


# --- a declared name two documents claim --------------------------------------


def test_two_documents_declaring_one_doc_number_both_keep_their_file_name(
    tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    """Neither is overwritten, neither wins, and the fall is said out loud."""
    second = IDENTITY.read_bytes().replace(b"<year>2024</year>", b"<year>2023</year>")
    bundle = _build(tmp_path, {"a.xml": IDENTITY.read_bytes(), "b.xml": second})
    concepts = _concepts(bundle)

    assert not any(name.startswith("r900-testnormalen/") for name in concepts)
    assert {name.split("/", 1)[0] for name in concepts} == {"a", "b"}
    # Every section of both documents landed: same sections, two directories.
    assert sum(name.startswith("a/") for name in concepts) == sum(
        name.startswith("b/") for name in concepts
    )
    assert "R900 Testnormalen" in capsys.readouterr().err


def test_a_declared_name_another_documents_file_name_holds_is_not_taken(
    tmp_path: Path,
) -> None:
    markdown = b"# Innledning\n\nTekst her.\n\n# Omfang\n\nMer tekst her.\n"
    concepts = _concepts(
        _build(tmp_path, {"r900-testnormalen.md": markdown, OPAQUE: IDENTITY.read_bytes()})
    )

    stems = {name.split("/", 1)[0] for name in concepts}
    assert stems == {"r900-testnormalen", Path(OPAQUE).stem}
    assert all(
        values["source_file"] == "r900-testnormalen.md"
        for name, values in concepts.items()
        if name.startswith("r900-testnormalen/")
    )