Kjell Tore Guttormsen
9d1f4b14ed
Every fixture, test document, tool example and document now uses an invented kitchen-and-baking handbook series, written in this repository. The package's behaviour is unchanged; src/ changes are comments and help text only. - Generated fixtures are regenerated from their generators. Their structural counts are identical before and after: elements, images, rows, cells, headings, bookmarks and the witness inventory's per-document totals. The image-inbox and accounting documents are renamed kapittel-84-*. - tools/okf_accounting_gate.py: the two options that named one real corpus each are replaced by a generic, repeatable --corpus PATH with no default. Row 5 compares the PDF pair alone. Gate verdict unchanged: RED rows 2, 3, 6. - tools/okf_witness.py: the STS JSON reader for one publisher's delivery is removed, along with its three twins and five tests. The mutation harness loses W09. - docs/: 13 dated reports that documented runs on a retired reference corpus are removed, and 40 are neutralized. Dead links are removed, and no new dangling path is introduced. - The synthetic MCP-gate corpus and the residual probe words are neutral. Valgt: keep the `okf quality --fasit` bar value (the measured fraction, one corpus) and rewrite only its provenance, because the verdict stays unchanged and the number names nothing. Term check with the local list: 0 of 411 tracked files, 0 file names, 0 of 27 binary fixtures. Suite after git add: 2457 passed, 1 skipped. The base tree had 2460 passed and 2 skipped; five tests went with the JSON reader and four were added by the term check. ruff, ruff format and mypy --strict src/ are clean. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
457 lines
18 KiB
Python
457 lines
18 KiB
Python
"""Structure derivation: what ingest can work out about a dropped document.
|
|
|
|
Pure functions over text and a filename — no I/O, no bundle, no door. The
|
|
whole point of the module is that a consumer can REASON over a bundle rather
|
|
than only look things up in it, so every fact here is either DECLARED by the
|
|
producer or INFERRED by us, and the difference is carried in the output. An
|
|
unmarked heuristic is worse than no heuristic: the consumer cannot know when
|
|
to doubt it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf.structure import (
|
|
BundleStructure,
|
|
DocumentStructure,
|
|
derive_document_structure,
|
|
resolve_structure,
|
|
)
|
|
|
|
|
|
def derive(text: str, source_file: str = "note.md") -> DocumentStructure:
|
|
return derive_document_structure(text, source_file=source_file)
|
|
|
|
|
|
# --- title ----------------------------------------------------------------
|
|
#
|
|
# The answer the previous session recorded for the Door B / Door A capability
|
|
# gap: Door B needs a title, and a profile cannot supply one because
|
|
# `BundleProfile.index` carries no title field at all. Three sources, in
|
|
# descending certainty.
|
|
|
|
|
|
def test_title_from_frontmatter_is_declared() -> None:
|
|
doc = derive("---\ntitle: Surdeigsbaking\n---\n\nbody\n", "q500-surdeigsbaking.md")
|
|
assert doc.title == "Surdeigsbaking"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_title_from_leading_heading_is_derived() -> None:
|
|
doc = derive("# Surdeigsbaking\n\nbody\n", "q500-surdeigsbaking.md")
|
|
assert doc.title == "Surdeigsbaking"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_falls_back_to_the_filename_stem_and_is_derived() -> None:
|
|
doc = derive("body with no heading\n", "q500-surdeigsbaking.md")
|
|
assert doc.title == "q500-surdeigsbaking"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_frontmatter_title_beats_a_heading() -> None:
|
|
doc = derive("---\ntitle: Declared\n---\n\n# Heading\n", "x.md")
|
|
assert doc.title == "Declared"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_a_heading_below_the_first_prose_line_is_not_the_title() -> None:
|
|
# Only a LEADING heading is the document's title; a heading further down is
|
|
# a section of it. Taking any heading would retitle every document whose
|
|
# body happens to start with prose.
|
|
doc = derive("intro prose\n\n# Section Two\n", "the-file.md")
|
|
assert doc.title == "the-file"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_is_nfc_normalised() -> None:
|
|
# macOS hands filenames over decomposed; the repo normalises before use so
|
|
# one visual name cannot reduce two ways.
|
|
doc = derive("body\n", "prøve.md")
|
|
assert doc.title == "prøve"
|
|
|
|
|
|
# --- document number ------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source_file", "expected"),
|
|
[
|
|
("q500-surdeigsbaking.md", "Q500"),
|
|
("Q500 Surdeigsbaking.md", "Q500"),
|
|
("w720-something.md", "W720"),
|
|
("4.2.1-details.md", "4.2.1"),
|
|
("P610.4 note.md", "P610.4"),
|
|
],
|
|
)
|
|
def test_number_is_read_off_the_filename_and_marked_derived(
|
|
source_file: str, expected: str
|
|
) -> None:
|
|
doc = derive("body\n", source_file)
|
|
assert doc.number == expected
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_declared_number_wins_and_is_not_marked_derived() -> None:
|
|
doc = derive("---\nnumber: Q200\n---\n\nbody\n", "q500-surdeigsbaking.md")
|
|
assert doc.number == "Q200"
|
|
assert "number" not in doc.derived
|
|
|
|
|
|
def test_number_falls_back_to_the_title_when_the_filename_has_none() -> None:
|
|
doc = derive("# Q500 Surdeigsbaking\n", "dropped-file.md")
|
|
assert doc.number == "Q500"
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_bare_integer_is_not_a_document_number() -> None:
|
|
# "2026-notes" and "12 things" are ordinary names, not numbering. A pure
|
|
# number needs a dotted form to count; anything looser would label most of
|
|
# a second brain with a document number it never had.
|
|
assert derive("body\n", "12-things.md").number is None
|
|
assert derive("body\n", "2026-notes.md").number is None
|
|
|
|
|
|
def test_no_number_anywhere_is_none_not_an_invention() -> None:
|
|
doc = derive("# Just A Title\n", "just-a-title.md")
|
|
assert doc.number is None
|
|
assert doc.parent_number is None
|
|
|
|
|
|
# --- hierarchy ------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("number_source", "expected_parent"),
|
|
[
|
|
("4.2.1-x.md", "4.2"),
|
|
("P610.4 x.md", "P610"),
|
|
("q500-x.md", None),
|
|
# `4.2` would drop to `4`, and a BARE integer is not a document number
|
|
# under this module's own grammar. A pointer nothing could ever satisfy
|
|
# is not a pointer: it would sit in the unresolved list forever, and an
|
|
# unresolved list that never clears trains a consumer to ignore it.
|
|
("4.2-x.md", None),
|
|
],
|
|
)
|
|
def test_parent_is_the_number_minus_its_last_component(
|
|
number_source: str, expected_parent: str | None
|
|
) -> None:
|
|
assert derive("body\n", number_source).parent_number == expected_parent
|
|
|
|
|
|
def test_parent_is_structural_and_carries_the_numbers_confidence() -> None:
|
|
# The parent is not an independent guess: given a number, dropping the last
|
|
# dotted component is arithmetic. So it is only as certain as the number.
|
|
declared = derive("---\nnumber: 4.2.1\n---\n\nbody\n", "whatever.md")
|
|
assert declared.parent_number == "4.2"
|
|
assert "parent" not in declared.derived
|
|
|
|
inferred = derive("body\n", "4.2.1-x.md")
|
|
assert inferred.parent_number == "4.2"
|
|
assert "parent" in inferred.derived
|
|
|
|
|
|
# --- cross references -----------------------------------------------------
|
|
|
|
|
|
def test_number_mentions_in_the_body_are_references() -> None:
|
|
doc = derive("See Q200 and jf. Q300 kap. 4 for details.\n", "q500-x.md")
|
|
assert doc.references == ("Q200", "Q300")
|
|
assert "references" in doc.derived
|
|
|
|
|
|
def test_a_document_never_references_itself() -> None:
|
|
doc = derive("Q500 says that Q500 applies, see Q200.\n", "q500-x.md")
|
|
assert doc.references == ("Q200",)
|
|
|
|
|
|
def test_references_are_deduplicated_and_ordered_by_first_appearance() -> None:
|
|
# Each mention carries a cue, because a number without one is no longer a
|
|
# reference at all. What this test pins is the ordering and the dedup.
|
|
doc = derive("se Q300 then se Q200 then se Q300 again.\n", "q500-x.md")
|
|
assert doc.references == ("Q300", "Q200")
|
|
|
|
|
|
def test_markdown_link_targets_are_references_too() -> None:
|
|
doc = derive("See [the other](other-doc.md) and [again](other-doc.md).\n", "q500-x.md")
|
|
assert "other-doc.md" in doc.references
|
|
|
|
|
|
def test_external_links_are_not_references() -> None:
|
|
# A cross-reference is inside the bundle. An http link is somebody else's
|
|
# document and resolving it is not this library's job.
|
|
doc = derive("See [upstream](https://example.test/a.md).\n", "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_declared_references_are_not_marked_derived() -> None:
|
|
doc = derive("---\nreferences: [Q200, Q300]\n---\n\nbody mentioning Q400\n", "q500-x.md")
|
|
assert doc.references == ("Q200", "Q300")
|
|
assert "references" not in doc.derived
|
|
|
|
|
|
# --- supersedes and version ----------------------------------------------
|
|
|
|
|
|
def test_supersedes_is_declared_only_never_inferred_per_document() -> None:
|
|
doc = derive("---\nsupersedes: [q500-2018]\n---\n\nbody\n", "q500-2021.md")
|
|
assert doc.supersedes == ("q500-2018",)
|
|
assert "supersedes" not in doc.derived
|
|
|
|
|
|
def test_no_declared_supersedes_means_none_at_document_level() -> None:
|
|
# Whether one document supersedes another is a fact about a PAIR, so a
|
|
# single document cannot answer it. The bundle-level resolver may propose
|
|
# it; this function must not.
|
|
doc = derive("This replaces the 2018 edition.\n", "q500-2021.md")
|
|
assert doc.supersedes == ()
|
|
|
|
|
|
def test_version_is_declared_only() -> None:
|
|
assert derive("---\nversion: '2021'\n---\n\nbody\n", "x.md").version == "2021"
|
|
assert derive("body\n", "q500-2021.md").version is None
|
|
|
|
|
|
# --- pass-through of the producer's own keys ------------------------------
|
|
|
|
|
|
def test_declared_frontmatter_is_carried_verbatim() -> None:
|
|
# The measured gap: documents carry status/date 55/55 while the index
|
|
# carries them 0/55. Derivation has to surface them for the index to be
|
|
# able to project them at all.
|
|
doc = derive("---\nstatus: gjeldende\ndate: 2026-01-01\n---\n\nbody\n", "x.md")
|
|
assert doc.declared["status"] == "gjeldende"
|
|
assert doc.declared["date"] == "2026-01-01"
|
|
|
|
|
|
def test_a_document_with_no_frontmatter_declares_nothing() -> None:
|
|
assert derive("# Title\n\nbody\n", "x.md").declared == {}
|
|
|
|
|
|
def test_derivation_is_deterministic() -> None:
|
|
text = "---\nstatus: gjeldende\n---\n\n# Q500 Surdeigsbaking\n\nSee Q200.\n"
|
|
assert derive(text, "q500-surdeigsbaking.md") == derive(text, "q500-surdeigsbaking.md")
|
|
|
|
|
|
# --- bundle-level resolution ----------------------------------------------
|
|
#
|
|
# The additive requirement in one design decision: resolution is a PURE
|
|
# function of the whole document set. Nothing is diffed and nothing is
|
|
# appended, so rebuild-from-scratch and incremental update cannot disagree —
|
|
# not because a diffing algorithm was proved correct, but because there is no
|
|
# diffing algorithm to prove.
|
|
|
|
|
|
def doc(name: str, text: str, source_file: str | None = None) -> tuple[str, DocumentStructure]:
|
|
return name, derive(text, source_file or f"{name}.md")
|
|
|
|
|
|
def resolve(*pairs: tuple[str, DocumentStructure]) -> BundleStructure:
|
|
return resolve_structure(dict(pairs))
|
|
|
|
|
|
def edges_of(bundle: BundleStructure, kind: str) -> list[tuple[str, str, str | None]]:
|
|
return [(e.source, e.subject, e.target) for e in bundle.edges if e.kind == kind]
|
|
|
|
|
|
def test_a_reference_resolves_to_the_document_carrying_that_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-q500", "See Q200 for details.\n", "q500-surdeigsbaking.md"),
|
|
doc("inbox-q200", "body\n", "q200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-q500", "Q200", "inbox-q200")]
|
|
assert bundle.unresolved == ()
|
|
|
|
|
|
def test_a_reference_to_a_document_not_yet_dropped_is_UNRESOLVED_not_dropped() -> None:
|
|
# Normal state while a bundle is still being built up. It must be visible
|
|
# as unfulfilled: an absence that does not scream is the most dangerous
|
|
# state this repo knows.
|
|
bundle = resolve(doc("inbox-q500", "See Q200.\n", "q500-x.md"))
|
|
assert edges_of(bundle, "references") == [("inbox-q500", "Q200", None)]
|
|
assert [e.subject for e in bundle.unresolved] == ["Q200"]
|
|
|
|
|
|
def test_a_reference_resolves_once_its_target_arrives_in_a_later_round() -> None:
|
|
first = doc("inbox-q500", "See Q200.\n", "q500-x.md")
|
|
assert resolve(first).unresolved != ()
|
|
later = resolve(first, doc("inbox-q200", "body\n", "q200-y.md"))
|
|
assert later.unresolved == ()
|
|
assert edges_of(later, "references") == [("inbox-q500", "Q200", "inbox-q200")]
|
|
|
|
|
|
def test_a_link_target_resolves_against_the_source_filename() -> None:
|
|
# Door B renames what it writes, so a producer's link points at the name
|
|
# the file arrived as, never at the concept name. Resolving only concept
|
|
# names would report every intra-bundle link as dangling.
|
|
bundle = resolve(
|
|
doc("inbox-a", "See [other](q200-grunnlag.md).\n", "a.md"),
|
|
doc("inbox-q200", "body\n", "q200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-a", "q200-grunnlag.md", "inbox-q200")]
|
|
|
|
|
|
def test_parent_resolves_to_the_document_carrying_the_parent_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-421", "body\n", "4.2.1-details.md"),
|
|
doc("inbox-42", "body\n", "4.2-section.md"),
|
|
)
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", "inbox-42")]
|
|
|
|
|
|
def test_a_missing_parent_is_unresolved_rather_than_absent() -> None:
|
|
bundle = resolve(doc("inbox-421", "body\n", "4.2.1-details.md"))
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", None)]
|
|
|
|
|
|
def test_declared_supersedes_resolves_and_is_not_derived() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-new", "---\nsupersedes: [q500-2018]\n---\nbody\n", "q500-2021.md"),
|
|
doc("inbox-old", "body\n", "q500-2018.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == [("inbox-new", "q500-2018", "inbox-old")]
|
|
assert [e.derived for e in bundle.edges if e.kind == "supersedes"] == [False]
|
|
|
|
|
|
def test_same_number_with_ordered_versions_yields_a_DERIVED_supersedes_chain() -> None:
|
|
# The one pair-level heuristic in the module, and it is marked as such.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: Q500\nversion: '2018'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: Q500\nversion: '2021'\n---\nbody\n", "b.md"),
|
|
doc("inbox-c", "---\nnumber: Q500\nversion: '2026'\n---\nbody\n", "c.md"),
|
|
)
|
|
chain = [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"]
|
|
assert chain == [("inbox-b", "inbox-a"), ("inbox-c", "inbox-b")]
|
|
assert all(e.derived for e in bundle.edges if e.kind == "supersedes")
|
|
|
|
|
|
def test_same_number_without_versions_proposes_nothing() -> None:
|
|
# Two documents sharing a number and no way to order them is exactly the
|
|
# case where a guess would be indistinguishable from a fact.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: Q500\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: Q500\n---\nbody\n", "b.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == []
|
|
|
|
|
|
def test_version_ordering_is_numeric_not_lexicographic() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: Q500\nversion: '9'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: Q500\nversion: '10'\n---\nbody\n", "b.md"),
|
|
)
|
|
assert [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"] == [
|
|
("inbox-b", "inbox-a")
|
|
]
|
|
|
|
|
|
# --- the invariants the additive requirement turns on ---------------------
|
|
|
|
|
|
def test_resolution_does_not_depend_on_the_order_documents_were_added() -> None:
|
|
a = doc("inbox-a", "See Q200.\n", "q500-x.md")
|
|
b = doc("inbox-q200", "See Q500.\n", "q200-y.md")
|
|
assert resolve(a, b) == resolve(b, a)
|
|
|
|
|
|
def test_resolving_the_same_document_twice_yields_one_edge_not_two() -> None:
|
|
a = doc("inbox-a", "See Q200.\n", "q500-x.md")
|
|
b = doc("inbox-q200", "body\n", "q200-y.md")
|
|
once = resolve(a, b)
|
|
# A mapping cannot hold the same key twice, which is the point: identity is
|
|
# the concept name, so re-dropping a file replaces rather than accumulates.
|
|
assert resolve(a, b, a) == once
|
|
assert len(edges_of(once, "references")) == 1
|
|
|
|
|
|
def test_edges_are_deterministically_ordered() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-b", "See Q100 and Q300.\n", "q200-b.md"),
|
|
doc("inbox-a", "See Q300.\n", "q100-a.md"),
|
|
)
|
|
assert list(bundle.edges) == sorted(bundle.edges, key=lambda e: (e.source, e.kind, e.subject))
|
|
|
|
|
|
# --- what a derived reference IS ------------------------------------------
|
|
#
|
|
# Measured 2026-08-29 against two real corpora, because the previous rule --
|
|
# "every number-shaped token in the body" -- was reported producing 12 false
|
|
# references out of 12 on a third. Denominators, before the rule below:
|
|
#
|
|
# corpus A 2 561 unique normative concept documents -> 2 838 subjects, of
|
|
# which 702 (24.7 %) were hexadecimal fragments of UUIDs lifted out
|
|
# of HTML/MathML attribute values, plus unit symbols (`kN/m2` ->
|
|
# `M2`), table row labels and formula numbers;
|
|
# corpus B 28 of this repository's own documents -> 559 subjects, dominated
|
|
# by software versions (`v0.3.0`, `3.14.0`, `0.11.10`), a licence
|
|
# id (`AGPL-3.0` -> `3.0`), a container version (`%PDF-1.6` ->
|
|
# `1.6`), a hash algorithm name (`SHA256`), and tokens lifted out
|
|
# of escape sequences in quoted source (`\n60` -> `N60`).
|
|
#
|
|
# The version class cannot be excluded structurally: corpus A proves `W221` and
|
|
# `W240` are genuine document numbers and corpus B proves `V0.3.0` is a
|
|
# software version, and they are the same token shape. So no blacklist can
|
|
# work, and a positive cue is the only rule that can be defended against data.
|
|
# A missing reference is visible to the reader; a false one is not.
|
|
|
|
|
|
def test_a_package_version_is_not_a_reference() -> None:
|
|
doc = derive("Built on pdfplumber 0.11.10 and Python 3.14.0, licence AGPL-3.0.\n", "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_v_prefixed_version_is_not_a_reference() -> None:
|
|
# The reported failure class: a version string carried in a title, read as
|
|
# a document number. `W221` in corpus A is a real document; `v0.5.0` here
|
|
# is not, and only the cue in front of it can tell them apart.
|
|
doc = derive("The classifier lags v0.3.0 and the gap is wider now (v0.5.0a2).\n", "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_number_inside_markup_is_not_a_reference() -> None:
|
|
doc = derive('<math id="id-647a093b-a70f-4e38-d277-b139040226a4">x</math>\n', "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_measurement_is_not_a_reference() -> None:
|
|
doc = derive("Compressive strength at least 500 kN/m2, used only below 2.5 m.\n", "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_cue_word_makes_a_number_a_reference() -> None:
|
|
# KNOWN-POSITIVE. Without it the two tests above pass for the wrong reason:
|
|
# a rule that finds nothing at all also finds nothing false.
|
|
doc = derive("Laid to depth as se kapittel 2.6.2.2 and tabell 4.8.4-3 give.\n", "q500-x.md")
|
|
assert doc.references == ("2.6.2.2", "4.8.4")
|
|
assert "references" in doc.derived
|
|
|
|
|
|
def test_a_section_sign_is_a_cue() -> None:
|
|
doc = derive("Emitted as one block, in §10.2's own listing order.\n", "q500-x.md")
|
|
assert doc.references == ("10.2",)
|
|
|
|
|
|
def test_a_cue_only_counts_at_a_word_boundary() -> None:
|
|
# A Norwegian compound ending in `-klasse` ends in the cue `se`. Matching
|
|
# an unanchored cue admitted 86 such class designations on corpus A, none
|
|
# of which name a document.
|
|
doc = derive("Se også P210; ovnsklasse C5 gjelder.\n", "q500-x.md")
|
|
assert doc.references == ("P210",)
|
|
|
|
|
|
def test_a_fragment_only_link_target_is_not_a_reference() -> None:
|
|
# An anchor points inside THIS document. It can never resolve to another
|
|
# concept, so emitting it states a relation that cannot exist.
|
|
doc = derive("See [table pairing](#table-pairing).\n", "q500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_template_placeholder_link_target_is_not_a_reference() -> None:
|
|
# Documentation about link SYNTAX carries example targets. A brace cannot
|
|
# occur in a concept name (`reduce_to_id_grammar` emits lowercase
|
|
# alphanumerics and hyphens), so such a target can never resolve — and an
|
|
# edge that can never resolve is a claim, not a pending pointer.
|
|
doc = derive("Managed lines read `- [title](ingest-{id}.md)`.\n", "q500-x.md")
|
|
assert doc.references == ()
|