llm-ingestion-okf/tests/test_quality.py

"""`okf quality` -- the per-file-type verdict, and what it refuses to say.

`okf check` is a CONTRACT check: it asks whether a payload carries what a claim
must rest on. Measured 2026-09-10 by `vegnormal-okf`, it returned 0 findings and
exit 0 on three arms over one corpus whose hit@k ranged from 6 of 6 to 0 of 6.
This module is the other question -- did the cut find anything worth reading --
and its whole discipline is that it answers it PER FILE TYPE, with the
denominator printed, and never answers PASS for a type it has no measurement
for.

Each rule below is exercised in both directions on purpose: a rule that has only
ever been run against the case it fires on has not been measured, it has been
asserted. The thresholds themselves, and the four evidence corpora they were
read off, live in `docs/2026-09-12-g37-terskler.md`.
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from llm_ingestion_okf import quality

_FRONTMATTER = """---
type: reference
title: {title}
source_file: {source_file}
ingested_at: 1970-01-01T00:00:00Z
---

{body}
"""


def _bundle(root: Path, concepts: list[tuple[str, str, str]]) -> Path:
    """A flat bundle: one root index, one file per concept.

    `concepts` is `(stem, source_file, body)`. `source_file` may be the empty
    string, which is the shape a bundle from another producer arrives in.
    """
    root.mkdir(parents=True, exist_ok=True)
    lines = ["---", "okf_version: 0.2", "bundle_id: quality-fixture", "---", ""]
    for stem, source_file, body in concepts:
        (root / f"{stem}.md").parent.mkdir(parents=True, exist_ok=True)
        (root / f"{stem}.md").write_text(
            _FRONTMATTER.format(title=stem, source_file=source_file, body=body),
            encoding="utf-8",
            newline="",
        )
        lines.append(f"- [{stem}]({stem}.md)")
    (root / "index.md").write_text("\n".join(lines) + "\n", encoding="utf-8", newline="")
    return root


def _pdf_documents(count: int, *, single: int) -> list[tuple[str, str, str]]:
    """`count` PDF documents, `single` of them yielding one concept only."""
    concepts: list[tuple[str, str, str]] = []
    for index in range(count):
        source = f"doc-{index}.pdf"
        segments = 1 if index < single else 3
        for segment in range(segments):
            concepts.append((f"doc-{index}-{segment}", source, "Body text enough to count."))
    return concepts


def test_a_type_at_its_measured_reference_passes(tmp_path: Path) -> None:
    """The known-positive: 8 of 32 is the reference, and 8 of 32 is not worse."""
    root = _bundle(tmp_path / "at-reference", _pdf_documents(32, single=8))
    report = quality.measure_bundle(root)
    row = report.row(".pdf")
    assert row.verdict == "PASS"
    assert (row.structure_null, row.documents) == (8, 32)


def test_a_type_worse_than_its_reference_fails(tmp_path: Path) -> None:
    """The known-negative for the same rule: 9 of 32 is worse than 8 of 32."""
    root = _bundle(tmp_path / "over-reference", _pdf_documents(32, single=9))
    report = quality.measure_bundle(root)
    assert report.row(".pdf").verdict == "FAIL"
    assert report.exit_code == 1


def test_an_empty_concept_fails_a_type_that_has_no_threshold(tmp_path: Path) -> None:
    """FAIL is reachable for every type; PASS is not. `.md` has no threshold."""
    root = _bundle(
        tmp_path / "empty-body",
        [("a", "note.md", "Body text."), ("b", "note.md", "   \n")],
    )
    report = quality.measure_bundle(root)
    row = report.row(".md")
    assert row.verdict == "FAIL"
    assert (row.empty, row.concepts) == (1, 2)


def test_a_healthy_type_without_a_threshold_is_unmeasured_and_never_pass(
    tmp_path: Path,
) -> None:
    """The other direction of the same rule: nothing wrong, and still not PASS."""
    root = _bundle(
        tmp_path / "no-threshold",
        [("a", "note.md", "Body text."), ("b", "note.md", "More body text.")],
    )
    report = quality.measure_bundle(root)
    assert report.row(".md").verdict == "UNMEASURED"
    assert report.exit_code == 3


def test_concepts_without_a_source_file_are_unmeasured(tmp_path: Path) -> None:
    """The shape three of the four evidence corpora arrive in.

    `n100-2023`, `n200-2024` and `n500-2024` carry `source_file` on 0 of 446,
    0 of 1 133 and 0 of 270 concepts, so a per-file-type gate has no type to
    speak about. Measured 2026-09-12; the gate says so instead of passing them.
    """
    root = _bundle(tmp_path / "foreign", [("a", "", "Body text."), ("b", "", "More text.")])
    report = quality.measure_bundle(root)
    row = report.row(quality.NO_SOURCE_FILE)
    assert row.verdict == "UNMEASURED"
    assert report.exit_code == 3


def test_a_failing_type_outweighs_a_passing_one(tmp_path: Path) -> None:
    concepts = _pdf_documents(32, single=9)
    for index in range(5):  # five documents: the floor a rate needs to be a rate
        concepts.append((f"note-{index}-0", f"note-{index}.docx", "Body text."))
        concepts.append((f"note-{index}-1", f"note-{index}.docx", "More body text."))
    report = quality.measure_bundle(_bundle(tmp_path / "mixed", concepts))
    assert report.row(".docx").verdict == "PASS"
    assert report.row(".pdf").verdict == "FAIL"
    assert report.exit_code == 1


def test_every_row_prints_its_denominator(tmp_path: Path) -> None:
    root = _bundle(tmp_path / "printed", _pdf_documents(32, single=8))
    rendered = quality.measure_bundle(root).render()
    assert "8/32" in rendered
    assert "0/80" in rendered  # empty concepts, over the concepts of the type
    assert "documents" in rendered


def test_the_bundle_run_log_is_reported_when_the_bundle_carries_one(tmp_path: Path) -> None:
    """A rejected document leaves NO row in the bundle: the log is the only trace.

    On the pinned K2 bundle the log reads `N = 43, merged 39, coded rejections
    4`, and the bundle itself shows 39 documents -- so the gate's own
    denominator is the bundle's, never the corpus's, and it says which one it
    is using.
    """
    root = _bundle(tmp_path / "logged", _pdf_documents(2, single=0))
    (root / "log.md").write_text(
        "* **Ingested**: /x - N = 3 (the corpus directory's file count, computed at "
        "run time), merged = 2 (2 substantive, 0 degenerate), coded rejections = 1.\n",
        encoding="utf-8",
        newline="",
    )
    report = quality.measure_bundle(root)
    assert report.run_log == "N = 3, merged = 2, coded rejections = 1"
    assert "coded rejections = 1" in report.render()


def test_a_bundle_without_a_log_says_so_rather_than_reporting_zero(tmp_path: Path) -> None:
    root = _bundle(tmp_path / "unlogged", _pdf_documents(2, single=0))
    report = quality.measure_bundle(root)
    assert report.run_log is None
    assert "no run log" in report.render()


def test_a_missing_bundle_exits_two(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
    assert quality.main([str(tmp_path / "nowhere")]) == 2


def test_the_cli_reaches_the_gate_through_okf_quality(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
    from llm_ingestion_okf import cli

    root = _bundle(tmp_path / "through-cli", _pdf_documents(32, single=9))
    assert cli.main(["quality", str(root)]) == 1
    assert ".pdf" in capsys.readouterr().out


def test_every_threshold_names_the_corpus_and_the_denominator_it_was_read_off() -> None:
    """A threshold without an N is the thing this whole gate exists to refuse."""
    assert quality.THRESHOLDS, "no thresholds at all would make every type UNMEASURED"
    for extension, threshold in quality.THRESHOLDS.items():
        assert extension.startswith("."), extension
        assert threshold.documents >= quality.MIN_DOCUMENTS_FOR_A_THRESHOLD
        assert threshold.source, f"{extension} threshold names no source"
        assert threshold.limit_documents == threshold.documents


def test_a_rate_over_too_few_documents_is_not_a_rate(tmp_path: Path) -> None:
    """FOUND BY RUNNING THE GATE, not by reading it.

    `~/repos/vegnormal-okf/build/sk2-bundle-default` is one PDF cut into 2 182
    concepts. Against the 32-document reference its one-concept share is 0 of 1,
    which the first version of this rule read as PASS -- a verdict resting on a
    denominator of one, which is the exact failure `MIN_DOCUMENTS_FOR_A_THRESHOLD`
    exists to refuse. The floor binds the BUNDLE's denominator too, not only the
    threshold's.
    """
    root = _bundle(tmp_path / "one-document", _pdf_documents(1, single=0))
    assert quality.measure_bundle(root).row(".pdf").verdict == "UNMEASURED"


def test_the_floor_admits_a_bundle_that_reaches_it(tmp_path: Path) -> None:
    """The other direction: five documents is the floor, and five is enough."""
    root = _bundle(tmp_path / "five-documents", _pdf_documents(5, single=1))
    assert quality.measure_bundle(root).row(".pdf").verdict == "PASS"


def test_an_empty_concept_fails_even_below_the_floor(tmp_path: Path) -> None:
    """The empty-body bar is per concept, not a rate, so the floor does not gate it."""
    root = _bundle(tmp_path / "one-document-empty", [("a", "only.pdf", "  \n")])
    assert quality.measure_bundle(root).row(".pdf").verdict == "FAIL"


def test_the_no_source_file_row_reports_no_document_count(tmp_path: Path) -> None:
    """A row that is not a file type has no documents either.

    Every concept without the key shares the same empty `source_file`, so a
    naive grouping reports `documents 1` and `one-concept documents 0/1` for a
    bundle of 446 concepts -- three numbers that look measured and mean nothing.
    """
    root = _bundle(tmp_path / "foreign-render", [("a", "", "Body."), ("b", "", "More.")])
    rendered = quality.measure_bundle(root).render()
    assert "documents 1" not in rendered
    assert "one-concept documents" not in rendered
    assert "empty 0/2" in rendered
    assert "no source_file on 2 of 2 concepts" in rendered


# --- G37b: `--fasit` and `boundary_share` -------------------------------------
#
# The metric that separates the two arms this gate could not tell apart. It is
# the ONLY one measured that orders them correctly and it needs the publisher's
# own declared structure, so it arrives as an input rather than as a constant:
# `okf quality <bundle> --fasit <json>`.
#
# Two things are pinned here that were measured before any of it was written
# (`docs/2026-09-12-g37-terskler.md` SS 7):
#
# 1. The fasit's `norm` key is reproduced from its own `title` by stripping all
#    whitespace and lowercasing -- 2 761 of 2 761 rows, so the normalisation is
#    not a guess.
# 2. That normalisation ALONE reaches 22 of 2 761 on the known-good arm, not
#    99 %, because okf's default route moves the numbering token STS glues into
#    `<title>` over into the concept id. The second match form -- the
#    (directory, residual title) pair -- is what takes it to 2 759 of 2 761.

_GOOD_ARM = Path.home() / "repos/vegnormal-okf/build/ferdig/r761-2025-generisk"
_BAD_ARM = Path.home() / "repos/vegnormal-okf/build/860019-mdb-100"
_OTHER_PRODUCT = Path.home() / "corpora/okf-telling-20260829/K2-bundle-default-20260912"
_FASIT = Path.home() / "repos/vegnormal-okf/build/sk2-fasit-2761.json"


def _fasit(path: Path, titles: list[str]) -> Path:
    """A fasit file in the shipped shape: a list of rows carrying title and norm."""
    path.write_text(
        json.dumps(
            [{"title": title, "norm": quality.normalise_title(title)} for title in titles],
            ensure_ascii=False,
        ),
        encoding="utf-8",
        newline="",
    )
    return path


def _declaring(titles: list[str]) -> list[tuple[str, str, str]]:
    """One concept per declared title, matching it literally."""
    return [(f"c{index}", "doc.xml", "Body text.") for index, _ in enumerate(titles)]


def test_the_normalisation_reproduces_the_fasit_key() -> None:
    """`norm` is whitespace-stripped, lowercased `title` -- not a guess.

    Measured over the shipped fasit before anything was written: 2 761 of 2 761
    rows reproduce, and the real file pins it below where it exists.
    """
    assert quality.normalise_title("  2.1 Hoved Prosesser\n") == "2.1hovedprosesser"
    assert quality.normalise_title("1Bruksområder for Prosesskoden") == (
        "1bruksområderforprosesskoden"
    )


def test_a_bundle_recovering_every_declared_boundary_passes(tmp_path: Path) -> None:
    """The known-positive, in miniature: every declared title became a concept."""
    titles = [f"{n} Kapittel {n}" for n in range(1, 7)]
    root = tmp_path / "recovered"
    root.mkdir()
    bundle = _bundle(root, [(quality.normalise_title(t)[:12], "doc.xml", "Body.") for t in titles])
    # The concept titles ARE the declared titles: rewrite the frontmatter.
    for index, title in enumerate(titles):
        stem = quality.normalise_title(title)[:12]
        (bundle / f"{stem}.md").write_text(
            _FRONTMATTER.format(title=title, source_file="doc.xml", body="Body."),
            encoding="utf-8",
            newline="",
        )
    report = quality.measure_bundle(
        bundle, fasit=quality.load_fasit(_fasit(tmp_path / "f.json", titles))
    )
    assert report.boundaries is not None
    assert (report.boundaries.recovered, report.boundaries.declared) == (6, 6)
    assert report.boundaries.verdict == "PASS"
    assert report.exit_code == 0


def test_a_bundle_recovering_few_declared_boundaries_fails(tmp_path: Path) -> None:
    """The known-negative for the same rule, and the arm it was built for.

    `860019-mdb-100` recovers 1 148 of 2 761. Here five of six is already worse
    than the bar of 2 759/2 761 -- the bar is that tight, which is a property of
    a regression bar set at its reference and is said out loud in the document.
    """
    titles = [f"{n} Kapittel {n}" for n in range(1, 7)]
    root = tmp_path / "lost"
    root.mkdir()
    bundle = _bundle(root, [(f"c{i}", "doc.md", "Body.") for i in range(5)])
    for index, title in enumerate(titles[:5]):
        (bundle / f"c{index}.md").write_text(
            _FRONTMATTER.format(title=title, source_file="doc.md", body="Body."),
            encoding="utf-8",
            newline="",
        )
    report = quality.measure_bundle(
        bundle, fasit=quality.load_fasit(_fasit(tmp_path / "f.json", titles))
    )
    assert report.boundaries is not None
    assert (report.boundaries.recovered, report.boundaries.declared) == (5, 6)
    assert report.boundaries.verdict == "FAIL"
    assert report.exit_code == 1


def test_the_numbering_token_is_matched_through_the_concept_directory(tmp_path: Path) -> None:
    """The second match form, and the reason P1's literal reading is not enough.

    STS glues the numbering into the title (`11.1Fastmerker`); okf's default
    route moves it into the concept id (`11-1/...`) and keeps the residual as
    the title. Measured on the known-good arm: the literal form alone reaches
    22 of 2 761, the pair form 2 737, either 2 759.
    """
    titles = ["11.1Fastmerker"]
    bundle = _bundle(tmp_path / "paired", [("11-1/p1", "doc.pdf", "Body.")])
    (bundle / "11-1/p1.md").write_text(
        _FRONTMATTER.format(title="Fastmerker", source_file="doc.pdf", body="Body."),
        encoding="utf-8",
        newline="",
    )
    fasit = quality.load_fasit(_fasit(tmp_path / "f.json", titles))
    report = quality.measure_bundle(bundle, fasit=fasit)
    assert report.boundaries is not None
    assert report.boundaries.literal == 0
    assert report.boundaries.paired == 1
    assert report.boundaries.recovered == 1


def test_without_a_fasit_the_gate_is_byte_for_byte_what_it_was(tmp_path: Path) -> None:
    """The regression the order asks for: no `--fasit`, no change of any kind."""
    root = _bundle(tmp_path / "unchanged", [("a", "note.md", "Body."), ("b", "note.md", "More.")])
    report = quality.measure_bundle(root)
    assert report.boundaries is None
    assert report.exit_code == 3
    assert "boundary" not in report.render().split("## What this verdict is not")[0].lower()


def test_a_fasit_that_is_not_a_list_exits_two_rather_than_unmeasured(tmp_path: Path) -> None:
    """An unreadable fasit is a run that did not happen, never a quiet verdict."""
    bad = tmp_path / "object.json"
    bad.write_text('{"title": "x", "norm": "x"}', encoding="utf-8")
    root = _bundle(tmp_path / "b", [("a", "note.md", "Body.")])
    assert quality.main([str(root), "--fasit", str(bad)]) == 2


def test_a_fasit_row_missing_a_key_exits_two(tmp_path: Path) -> None:
    bad = tmp_path / "rows.json"
    bad.write_text('[{"title": "x", "norm": "x"}, {"title": "y"}]', encoding="utf-8")
    root = _bundle(tmp_path / "b", [("a", "note.md", "Body.")])
    assert quality.main([str(root), "--fasit", str(bad)]) == 2


def test_a_fasit_that_is_not_json_exits_two(tmp_path: Path) -> None:
    bad = tmp_path / "broken.json"
    bad.write_text("not json at all", encoding="utf-8")
    root = _bundle(tmp_path / "b", [("a", "note.md", "Body.")])
    assert quality.main([str(root), "--fasit", str(bad)]) == 2


def test_a_missing_fasit_file_exits_two(tmp_path: Path) -> None:
    root = _bundle(tmp_path / "b", [("a", "note.md", "Body.")])
    assert quality.main([str(root), "--fasit", str(tmp_path / "nowhere.json")]) == 2


def test_a_valid_fasit_reaches_the_gate_through_the_cli(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
    titles = [f"{n} Kapittel {n}" for n in range(1, 7)]
    bundle = _bundle(tmp_path / "cli", [(f"c{i}", "doc.md", "Body.") for i in range(1)])
    (bundle / "c0.md").write_text(
        _FRONTMATTER.format(title=titles[0], source_file="doc.md", body="Body."),
        encoding="utf-8",
        newline="",
    )
    from llm_ingestion_okf import cli

    assert (
        cli.main(["quality", str(bundle), "--fasit", str(_fasit(tmp_path / "f.json", titles))]) == 1
    )
    out = capsys.readouterr().out
    assert "boundary_share" in out
    assert "1/6" in out


def test_a_fasit_below_the_floor_is_unmeasured_and_never_pass(tmp_path: Path) -> None:
    """A share over four declared boundaries is not a rate either."""
    titles = ["1 A", "2 B", "3 C", "4 D"]
    bundle = _bundle(tmp_path / "tiny", [(f"c{i}", "doc.md", "Body.") for i in range(4)])
    for index, title in enumerate(titles):
        (bundle / f"c{index}.md").write_text(
            _FRONTMATTER.format(title=title, source_file="doc.md", body="Body."),
            encoding="utf-8",
            newline="",
        )
    report = quality.measure_bundle(
        bundle, fasit=quality.load_fasit(_fasit(tmp_path / "f.json", titles))
    )
    assert report.boundaries is not None
    assert report.boundaries.verdict == "UNMEASURED"
    assert report.boundaries.recovered == 4


def test_the_boundary_row_says_its_threshold_rests_on_one_corpus(tmp_path: Path) -> None:
    """P2, in the printout rather than only in the document."""
    titles = [f"{n} Kapittel {n}" for n in range(1, 7)]
    bundle = _bundle(tmp_path / "caveat", [("c0", "doc.md", "Body.")])
    (bundle / "c0.md").write_text(
        _FRONTMATTER.format(title=titles[0], source_file="doc.md", body="Body."),
        encoding="utf-8",
        newline="",
    )
    rendered = quality.measure_bundle(
        bundle, fasit=quality.load_fasit(_fasit(tmp_path / "f.json", titles))
    ).render()
    assert "one product" in rendered
    assert "N = 1" in rendered


def test_the_boundary_threshold_names_its_corpus_and_denominator() -> None:
    bar = quality.BOUNDARY_THRESHOLD
    assert bar.limit_declared >= quality.MIN_DECLARED_FOR_A_THRESHOLD
    assert bar.corpora == 1
    assert bar.source


@pytest.mark.skipif(not (_GOOD_ARM.is_dir() and _FASIT.is_file()), reason="local gold set absent")
def test_the_known_good_arm_reproduces_its_published_recall() -> None:
    """2 759 of 2 761, and the decomposition P1 asked for: 22 literal, 2 737 paired."""
    report = quality.measure_bundle(_GOOD_ARM, fasit=quality.load_fasit(_FASIT))
    assert report.boundaries is not None
    assert (report.boundaries.recovered, report.boundaries.declared) == (2759, 2761)
    assert (report.boundaries.literal, report.boundaries.paired) == (22, 2737)
    assert report.boundaries.verdict == "PASS"


@pytest.mark.skipif(not (_BAD_ARM.is_dir() and _FASIT.is_file()), reason="local gold set absent")
def test_the_known_bad_arm_reproduces_its_published_recall() -> None:
    """1 148 of 2 761 -- the 41.6 % the whole gate could not previously see."""
    report = quality.measure_bundle(_BAD_ARM, fasit=quality.load_fasit(_FASIT))
    assert report.boundaries is not None
    assert (report.boundaries.recovered, report.boundaries.declared) == (1148, 2761)
    assert report.boundaries.verdict == "FAIL"
    assert report.exit_code == 1


@pytest.mark.skipif(
    not (_OTHER_PRODUCT.is_dir() and _FASIT.is_file()), reason="local gold set absent"
)
def test_a_bundle_of_another_product_scores_zero_and_that_is_the_assertion(tmp_path: Path) -> None:
    """The adverse case, measured rather than argued.

    The fasit describes ONE product. The pinned 43-document K2 bundle is a
    different corpus entirely and scores 0 of 2 761 -- a FAIL that says the
    caller's assertion was wrong, not that the bundle is bad. `--fasit` is an
    assertion by the caller, the way `okf consume --ref` is.
    """
    report = quality.measure_bundle(_OTHER_PRODUCT, fasit=quality.load_fasit(_FASIT))
    assert report.boundaries is not None
    assert report.boundaries.recovered == 0
    assert report.boundaries.verdict == "FAIL"


@pytest.mark.skipif(not _FASIT.is_file(), reason="local gold set absent")
def test_the_shipped_fasit_reproduces_its_own_key_under_this_normalisation() -> None:
    """2 761 of 2 761 rows: `norm` is exactly whitespace-stripped, lowercased title."""
    rows = quality.load_fasit(_FASIT)
    assert len(rows) == 2761
    assert sum(1 for row in rows if quality.normalise_title(row.title) == row.norm) == 2761