test(extract): the two registries are coupled, and only a whole run says so

RED, 12 failing. Two corrections to the previous commit's tests, both found by
running rather than by reading.

ONE, AND IT IS A REAL GAP THIS COMMIT CLOSES: `okf build` on an STS document
does not fail in the extractor at all -- it fails two layers away, in
`segmentation.observed_extractor_version`, with `no extractor version is known
for extractor_id 'xml'`. That coupling is deliberate and its own comment says
so: a row added to the extraction registry and not to `_STDLIB_EXTRACTOR_IDS`
"fails loudly on the first proposal for that type". No unit test of an
extractor can see it, because the missing thing is the version the plan is
KEYED to, not the text. The previous commit's tests would all have gone green
with the build still refusing every file. This test is the one that would not
have.

TWO: the expectation that the proposer strips a numbering token out of an ATX
title was wrong about `propose`, not about this reader. Measured: `_ATX` keeps
the whole title and fills `number` in addition, and `_NUMBERED` needs at least
one dot, so `1 Bruksomraade` gets no number while `1.1 Omfang` gets one and
keeps it. Both shapes are now asserted, along with the table-wrap arriving as
ONE `rule:table-block` candidate rather than one per row.

pytest -q: 12 failed, 1554 passed, 1 skipped.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-10 03:36:13 +02:00
commit 5bb3b045c2

View file

@ -85,13 +85,20 @@ def test_the_label_and_the_title_reach_the_proposer_as_one_heading() -> None:
text = extract.extract_text(STS.name, STS.read_bytes())
candidates = propose.find_candidates(text)
assert [(c.number, c.title, c.level) for c in candidates] == [
(None, "Forord", 1),
(None, "1 Bruksomraade", 1),
("1.1", "Omfang", 2),
("1.1.1", "Materialer", 3),
(None, "2 Tabeller", 1),
# The TITLE keeps its numbering token and the `number` field carries it as
# well -- that is `propose`'s own documented ATX behaviour and this reader
# does not reach into it. `1 Bruksomraade` gets no `number` because
# `_NUMBERED` requires at least one dot, which is why both shapes are here.
assert [(c.rule, c.number, c.title, c.level) for c in candidates] == [
("rule:heading", None, "Forord", 1),
("rule:heading", None, "1 Bruksomraade", 1),
("rule:heading", "1.1", "1.1 Omfang", 2),
("rule:heading", "1.1.1", "1.1.1 Materialer", 3),
("rule:heading", None, "2 Tabeller", 1),
# The table-wrap, as ONE candidate rather than one per row.
("rule:table-block", None, "Tabell linje 15", 9),
]
candidates = [c for c in candidates if c.rule == "rule:heading"]
# The offsets, stated against the text itself: a candidate naming the right
# title at the wrong offset would pass the list above.
for candidate, line in zip(
@ -183,3 +190,39 @@ def test_xml_is_a_core_type_and_never_reaches_the_converter() -> None:
assert ".xml" in _CORE_EXTRACTORS
assert ".xml" not in _OPTIONAL_EXTRACTORS
assert ".xml" not in _PANDOC_FORMATS
def test_a_bundle_is_built_end_to_end_from_an_sts_document(tmp_path: Path) -> None:
"""The registries are COUPLED, and only a run through the whole chain says so.
`segmentation._STDLIB_EXTRACTOR_IDS` names the ids whose extracted text is
versioned by this package's own literal, and its own comment says a row
added to the extraction registry and not there "fails loudly on the first
proposal for that type". It does -- and no unit test of the extractor can
see it, because the failure is two layers away in the version the plan is
keyed to. This was found by running `okf build`, not by a test, which is
why the test exists now.
"""
from llm_ingestion_okf import cli
from llm_ingestion_okf.segmentation import _STDLIB_EXTRACTOR_IDS, observed_extractor_version
assert "xml" in _STDLIB_EXTRACTOR_IDS
assert observed_extractor_version("xml")
folder = tmp_path / "docs"
folder.mkdir()
(folder / "sts-mini.xml").write_bytes(STS.read_bytes())
bundle = tmp_path / "bundle"
report = cli.build(folder, bundle=bundle, bundle_id="mini", okf_version="0.2")
assert report.codes == ()
assert report.unaccounted == ()
titles = sorted(
line.split(":", 1)[1].strip()
for concept in bundle.rglob("*.md")
if concept.name not in ("index.md", "log.md") and concept.parent != bundle
for line in concept.read_text(encoding="utf-8").splitlines()
if line.startswith("title:")
)
assert titles, "an STS document must produce concepts"
assert "Forord" in titles