test(extract): the two registries are coupled, and only a whole run says so
RED, 12 failing. Two corrections to the previous commit's tests, both found by running rather than by reading. ONE, AND IT IS A REAL GAP THIS COMMIT CLOSES: `okf build` on an STS document does not fail in the extractor at all -- it fails two layers away, in `segmentation.observed_extractor_version`, with `no extractor version is known for extractor_id 'xml'`. That coupling is deliberate and its own comment says so: a row added to the extraction registry and not to `_STDLIB_EXTRACTOR_IDS` "fails loudly on the first proposal for that type". No unit test of an extractor can see it, because the missing thing is the version the plan is KEYED to, not the text. The previous commit's tests would all have gone green with the build still refusing every file. This test is the one that would not have. TWO: the expectation that the proposer strips a numbering token out of an ATX title was wrong about `propose`, not about this reader. Measured: `_ATX` keeps the whole title and fills `number` in addition, and `_NUMBERED` needs at least one dot, so `1 Bruksomraade` gets no number while `1.1 Omfang` gets one and keeps it. Both shapes are now asserted, along with the table-wrap arriving as ONE `rule:table-block` candidate rather than one per row. pytest -q: 12 failed, 1554 passed, 1 skipped. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
324cd0b1e5
commit
5bb3b045c2
1 changed files with 49 additions and 6 deletions
|
|
@ -85,13 +85,20 @@ def test_the_label_and_the_title_reach_the_proposer_as_one_heading() -> None:
|
||||||
text = extract.extract_text(STS.name, STS.read_bytes())
|
text = extract.extract_text(STS.name, STS.read_bytes())
|
||||||
candidates = propose.find_candidates(text)
|
candidates = propose.find_candidates(text)
|
||||||
|
|
||||||
assert [(c.number, c.title, c.level) for c in candidates] == [
|
# The TITLE keeps its numbering token and the `number` field carries it as
|
||||||
(None, "Forord", 1),
|
# well -- that is `propose`'s own documented ATX behaviour and this reader
|
||||||
(None, "1 Bruksomraade", 1),
|
# does not reach into it. `1 Bruksomraade` gets no `number` because
|
||||||
("1.1", "Omfang", 2),
|
# `_NUMBERED` requires at least one dot, which is why both shapes are here.
|
||||||
("1.1.1", "Materialer", 3),
|
assert [(c.rule, c.number, c.title, c.level) for c in candidates] == [
|
||||||
(None, "2 Tabeller", 1),
|
("rule:heading", None, "Forord", 1),
|
||||||
|
("rule:heading", None, "1 Bruksomraade", 1),
|
||||||
|
("rule:heading", "1.1", "1.1 Omfang", 2),
|
||||||
|
("rule:heading", "1.1.1", "1.1.1 Materialer", 3),
|
||||||
|
("rule:heading", None, "2 Tabeller", 1),
|
||||||
|
# The table-wrap, as ONE candidate rather than one per row.
|
||||||
|
("rule:table-block", None, "Tabell linje 15", 9),
|
||||||
]
|
]
|
||||||
|
candidates = [c for c in candidates if c.rule == "rule:heading"]
|
||||||
# The offsets, stated against the text itself: a candidate naming the right
|
# The offsets, stated against the text itself: a candidate naming the right
|
||||||
# title at the wrong offset would pass the list above.
|
# title at the wrong offset would pass the list above.
|
||||||
for candidate, line in zip(
|
for candidate, line in zip(
|
||||||
|
|
@ -183,3 +190,39 @@ def test_xml_is_a_core_type_and_never_reaches_the_converter() -> None:
|
||||||
assert ".xml" in _CORE_EXTRACTORS
|
assert ".xml" in _CORE_EXTRACTORS
|
||||||
assert ".xml" not in _OPTIONAL_EXTRACTORS
|
assert ".xml" not in _OPTIONAL_EXTRACTORS
|
||||||
assert ".xml" not in _PANDOC_FORMATS
|
assert ".xml" not in _PANDOC_FORMATS
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bundle_is_built_end_to_end_from_an_sts_document(tmp_path: Path) -> None:
|
||||||
|
"""The registries are COUPLED, and only a run through the whole chain says so.
|
||||||
|
|
||||||
|
`segmentation._STDLIB_EXTRACTOR_IDS` names the ids whose extracted text is
|
||||||
|
versioned by this package's own literal, and its own comment says a row
|
||||||
|
added to the extraction registry and not there "fails loudly on the first
|
||||||
|
proposal for that type". It does -- and no unit test of the extractor can
|
||||||
|
see it, because the failure is two layers away in the version the plan is
|
||||||
|
keyed to. This was found by running `okf build`, not by a test, which is
|
||||||
|
why the test exists now.
|
||||||
|
"""
|
||||||
|
from llm_ingestion_okf import cli
|
||||||
|
from llm_ingestion_okf.segmentation import _STDLIB_EXTRACTOR_IDS, observed_extractor_version
|
||||||
|
|
||||||
|
assert "xml" in _STDLIB_EXTRACTOR_IDS
|
||||||
|
assert observed_extractor_version("xml")
|
||||||
|
|
||||||
|
folder = tmp_path / "docs"
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "sts-mini.xml").write_bytes(STS.read_bytes())
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
report = cli.build(folder, bundle=bundle, bundle_id="mini", okf_version="0.2")
|
||||||
|
assert report.codes == ()
|
||||||
|
assert report.unaccounted == ()
|
||||||
|
|
||||||
|
titles = sorted(
|
||||||
|
line.split(":", 1)[1].strip()
|
||||||
|
for concept in bundle.rglob("*.md")
|
||||||
|
if concept.name not in ("index.md", "log.md") and concept.parent != bundle
|
||||||
|
for line in concept.read_text(encoding="utf-8").splitlines()
|
||||||
|
if line.startswith("title:")
|
||||||
|
)
|
||||||
|
assert titles, "an STS document must produce concepts"
|
||||||
|
assert "Forord" in titles
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue