Kjell Tore Guttormsen
05cda5ded5
A bundle a consumer can only look things up in is a filing cabinet. This adds the derivation half of what lets one REASON over it: per-document title, number, parent and references, plus bundle-level resolution of every pointer. Two rules carry the design. Every fact is marked DECLARED or DERIVED, because an unmarked heuristic is worse than no heuristic -- a consumer cannot know when to doubt it. And resolution is a PURE function of the whole document set rather than a diff, which is what makes rebuild-from-scratch equal an incremental update by construction: there is no diffing algorithm to prove correct. An unresolved pointer is kept and reported, never dropped: while a bundle is built up over several rounds, pointing at something not dropped yet is the normal state, and the dangerous version of it is the one that leaves no trace. Symmetrically, a parent our own grammar could never admit (4.2 -> 4, a bare integer) is not emitted at all -- an unresolved list that never clears is one a consumer learns to ignore. 45 new tests; suite 615 -> 660.
372 lines
14 KiB
Python
372 lines
14 KiB
Python
"""Structure derivation: what ingest can work out about a dropped document.
|
|
|
|
Pure functions over text and a filename — no I/O, no bundle, no door. The
|
|
whole point of the module is that a consumer can REASON over a bundle rather
|
|
than only look things up in it, so every fact here is either DECLARED by the
|
|
producer or INFERRED by us, and the difference is carried in the output. An
|
|
unmarked heuristic is worse than no heuristic: the consumer cannot know when
|
|
to doubt it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf.structure import (
|
|
BundleStructure,
|
|
DocumentStructure,
|
|
derive_document_structure,
|
|
resolve_structure,
|
|
)
|
|
|
|
|
|
def derive(text: str, source_file: str = "note.md") -> DocumentStructure:
|
|
return derive_document_structure(text, source_file=source_file)
|
|
|
|
|
|
# --- title ----------------------------------------------------------------
|
|
#
|
|
# The answer the previous session recorded for the Door B / Door A capability
|
|
# gap: Door B needs a title, and a profile cannot supply one because
|
|
# `BundleProfile.index` carries no title field at all. Three sources, in
|
|
# descending certainty.
|
|
|
|
|
|
def test_title_from_frontmatter_is_declared() -> None:
|
|
doc = derive("---\ntitle: Vegbygging\n---\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.title == "Vegbygging"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_title_from_leading_heading_is_derived() -> None:
|
|
doc = derive("# Vegbygging\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.title == "Vegbygging"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_falls_back_to_the_filename_stem_and_is_derived() -> None:
|
|
doc = derive("body with no heading\n", "n500-vegbygging.md")
|
|
assert doc.title == "n500-vegbygging"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_frontmatter_title_beats_a_heading() -> None:
|
|
doc = derive("---\ntitle: Declared\n---\n\n# Heading\n", "x.md")
|
|
assert doc.title == "Declared"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_a_heading_below_the_first_prose_line_is_not_the_title() -> None:
|
|
# Only a LEADING heading is the document's title; a heading further down is
|
|
# a section of it. Taking any heading would retitle every document whose
|
|
# body happens to start with prose.
|
|
doc = derive("intro prose\n\n# Section Two\n", "the-file.md")
|
|
assert doc.title == "the-file"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_is_nfc_normalised() -> None:
|
|
# macOS hands filenames over decomposed; the repo normalises before use so
|
|
# one visual name cannot reduce two ways.
|
|
doc = derive("body\n", "prøve.md")
|
|
assert doc.title == "prøve"
|
|
|
|
|
|
# --- document number ------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source_file", "expected"),
|
|
[
|
|
("n500-vegbygging.md", "N500"),
|
|
("N500 Vegbygging.md", "N500"),
|
|
("v720-something.md", "V720"),
|
|
("4.2.1-details.md", "4.2.1"),
|
|
("R610.4 note.md", "R610.4"),
|
|
],
|
|
)
|
|
def test_number_is_read_off_the_filename_and_marked_derived(
|
|
source_file: str, expected: str
|
|
) -> None:
|
|
doc = derive("body\n", source_file)
|
|
assert doc.number == expected
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_declared_number_wins_and_is_not_marked_derived() -> None:
|
|
doc = derive("---\nnumber: N200\n---\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.number == "N200"
|
|
assert "number" not in doc.derived
|
|
|
|
|
|
def test_number_falls_back_to_the_title_when_the_filename_has_none() -> None:
|
|
doc = derive("# N500 Vegbygging\n", "dropped-file.md")
|
|
assert doc.number == "N500"
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_bare_integer_is_not_a_document_number() -> None:
|
|
# "2026-notes" and "12 things" are ordinary names, not numbering. A pure
|
|
# number needs a dotted form to count; anything looser would label most of
|
|
# a second brain with a document number it never had.
|
|
assert derive("body\n", "12-things.md").number is None
|
|
assert derive("body\n", "2026-notes.md").number is None
|
|
|
|
|
|
def test_no_number_anywhere_is_none_not_an_invention() -> None:
|
|
doc = derive("# Just A Title\n", "just-a-title.md")
|
|
assert doc.number is None
|
|
assert doc.parent_number is None
|
|
|
|
|
|
# --- hierarchy ------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("number_source", "expected_parent"),
|
|
[
|
|
("4.2.1-x.md", "4.2"),
|
|
("R610.4 x.md", "R610"),
|
|
("n500-x.md", None),
|
|
# `4.2` would drop to `4`, and a BARE integer is not a document number
|
|
# under this module's own grammar. A pointer nothing could ever satisfy
|
|
# is not a pointer: it would sit in the unresolved list forever, and an
|
|
# unresolved list that never clears trains a consumer to ignore it.
|
|
("4.2-x.md", None),
|
|
],
|
|
)
|
|
def test_parent_is_the_number_minus_its_last_component(
|
|
number_source: str, expected_parent: str | None
|
|
) -> None:
|
|
assert derive("body\n", number_source).parent_number == expected_parent
|
|
|
|
|
|
def test_parent_is_structural_and_carries_the_numbers_confidence() -> None:
|
|
# The parent is not an independent guess: given a number, dropping the last
|
|
# dotted component is arithmetic. So it is only as certain as the number.
|
|
declared = derive("---\nnumber: 4.2.1\n---\n\nbody\n", "whatever.md")
|
|
assert declared.parent_number == "4.2"
|
|
assert "parent" not in declared.derived
|
|
|
|
inferred = derive("body\n", "4.2.1-x.md")
|
|
assert inferred.parent_number == "4.2"
|
|
assert "parent" in inferred.derived
|
|
|
|
|
|
# --- cross references -----------------------------------------------------
|
|
|
|
|
|
def test_number_mentions_in_the_body_are_references() -> None:
|
|
doc = derive("See N200 and jf. N300 kap. 4 for details.\n", "n500-x.md")
|
|
assert doc.references == ("N200", "N300")
|
|
assert "references" in doc.derived
|
|
|
|
|
|
def test_a_document_never_references_itself() -> None:
|
|
doc = derive("N500 says that N500 applies, see N200.\n", "n500-x.md")
|
|
assert doc.references == ("N200",)
|
|
|
|
|
|
def test_references_are_deduplicated_and_ordered_by_first_appearance() -> None:
|
|
doc = derive("N300 then N200 then N300 again.\n", "n500-x.md")
|
|
assert doc.references == ("N300", "N200")
|
|
|
|
|
|
def test_markdown_link_targets_are_references_too() -> None:
|
|
doc = derive("See [the other](other-doc.md) and [again](other-doc.md).\n", "n500-x.md")
|
|
assert "other-doc.md" in doc.references
|
|
|
|
|
|
def test_external_links_are_not_references() -> None:
|
|
# A cross-reference is inside the bundle. An http link is somebody else's
|
|
# document and resolving it is not this library's job.
|
|
doc = derive("See [upstream](https://example.test/a.md).\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_declared_references_are_not_marked_derived() -> None:
|
|
doc = derive("---\nreferences: [N200, N300]\n---\n\nbody mentioning N400\n", "n500-x.md")
|
|
assert doc.references == ("N200", "N300")
|
|
assert "references" not in doc.derived
|
|
|
|
|
|
# --- supersedes and version ----------------------------------------------
|
|
|
|
|
|
def test_supersedes_is_declared_only_never_inferred_per_document() -> None:
|
|
doc = derive("---\nsupersedes: [n500-2018]\n---\n\nbody\n", "n500-2021.md")
|
|
assert doc.supersedes == ("n500-2018",)
|
|
assert "supersedes" not in doc.derived
|
|
|
|
|
|
def test_no_declared_supersedes_means_none_at_document_level() -> None:
|
|
# Whether one document supersedes another is a fact about a PAIR, so a
|
|
# single document cannot answer it. The bundle-level resolver may propose
|
|
# it; this function must not.
|
|
doc = derive("This replaces the 2018 edition.\n", "n500-2021.md")
|
|
assert doc.supersedes == ()
|
|
|
|
|
|
def test_version_is_declared_only() -> None:
|
|
assert derive("---\nversion: '2021'\n---\n\nbody\n", "x.md").version == "2021"
|
|
assert derive("body\n", "n500-2021.md").version is None
|
|
|
|
|
|
# --- pass-through of the producer's own keys ------------------------------
|
|
|
|
|
|
def test_declared_frontmatter_is_carried_verbatim() -> None:
|
|
# The measured gap: documents carry status/date 55/55 while the index
|
|
# carries them 0/55. Derivation has to surface them for the index to be
|
|
# able to project them at all.
|
|
doc = derive("---\nstatus: gjeldende\ndate: 2026-01-01\n---\n\nbody\n", "x.md")
|
|
assert doc.declared["status"] == "gjeldende"
|
|
assert doc.declared["date"] == "2026-01-01"
|
|
|
|
|
|
def test_a_document_with_no_frontmatter_declares_nothing() -> None:
|
|
assert derive("# Title\n\nbody\n", "x.md").declared == {}
|
|
|
|
|
|
def test_derivation_is_deterministic() -> None:
|
|
text = "---\nstatus: gjeldende\n---\n\n# N500 Vegbygging\n\nSee N200.\n"
|
|
assert derive(text, "n500-vegbygging.md") == derive(text, "n500-vegbygging.md")
|
|
|
|
|
|
# --- bundle-level resolution ----------------------------------------------
|
|
#
|
|
# The additive requirement in one design decision: resolution is a PURE
|
|
# function of the whole document set. Nothing is diffed and nothing is
|
|
# appended, so rebuild-from-scratch and incremental update cannot disagree —
|
|
# not because a diffing algorithm was proved correct, but because there is no
|
|
# diffing algorithm to prove.
|
|
|
|
|
|
def doc(name: str, text: str, source_file: str | None = None) -> tuple[str, DocumentStructure]:
|
|
return name, derive(text, source_file or f"{name}.md")
|
|
|
|
|
|
def resolve(*pairs: tuple[str, DocumentStructure]) -> BundleStructure:
|
|
return resolve_structure(dict(pairs))
|
|
|
|
|
|
def edges_of(bundle: BundleStructure, kind: str) -> list[tuple[str, str, str | None]]:
|
|
return [(e.source, e.subject, e.target) for e in bundle.edges if e.kind == kind]
|
|
|
|
|
|
def test_a_reference_resolves_to_the_document_carrying_that_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-n500", "See N200 for details.\n", "n500-vegbygging.md"),
|
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
|
assert bundle.unresolved == ()
|
|
|
|
|
|
def test_a_reference_to_a_document_not_yet_dropped_is_UNRESOLVED_not_dropped() -> None:
|
|
# Normal state while a bundle is still being built up. It must be visible
|
|
# as unfulfilled: an absence that does not scream is the most dangerous
|
|
# state this repo knows.
|
|
bundle = resolve(doc("inbox-n500", "See N200.\n", "n500-x.md"))
|
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", None)]
|
|
assert [e.subject for e in bundle.unresolved] == ["N200"]
|
|
|
|
|
|
def test_a_reference_resolves_once_its_target_arrives_in_a_later_round() -> None:
|
|
first = doc("inbox-n500", "See N200.\n", "n500-x.md")
|
|
assert resolve(first).unresolved != ()
|
|
later = resolve(first, doc("inbox-n200", "body\n", "n200-y.md"))
|
|
assert later.unresolved == ()
|
|
assert edges_of(later, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
|
|
|
|
|
def test_a_link_target_resolves_against_the_source_filename() -> None:
|
|
# Door B renames what it writes, so a producer's link points at the name
|
|
# the file arrived as, never at the concept name. Resolving only concept
|
|
# names would report every intra-bundle link as dangling.
|
|
bundle = resolve(
|
|
doc("inbox-a", "See [other](n200-grunnlag.md).\n", "a.md"),
|
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-a", "n200-grunnlag.md", "inbox-n200")]
|
|
|
|
|
|
def test_parent_resolves_to_the_document_carrying_the_parent_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-421", "body\n", "4.2.1-details.md"),
|
|
doc("inbox-42", "body\n", "4.2-section.md"),
|
|
)
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", "inbox-42")]
|
|
|
|
|
|
def test_a_missing_parent_is_unresolved_rather_than_absent() -> None:
|
|
bundle = resolve(doc("inbox-421", "body\n", "4.2.1-details.md"))
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", None)]
|
|
|
|
|
|
def test_declared_supersedes_resolves_and_is_not_derived() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-new", "---\nsupersedes: [n500-2018]\n---\nbody\n", "n500-2021.md"),
|
|
doc("inbox-old", "body\n", "n500-2018.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == [("inbox-new", "n500-2018", "inbox-old")]
|
|
assert [e.derived for e in bundle.edges if e.kind == "supersedes"] == [False]
|
|
|
|
|
|
def test_same_number_with_ordered_versions_yields_a_DERIVED_supersedes_chain() -> None:
|
|
# The one pair-level heuristic in the module, and it is marked as such.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\nversion: '2018'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\nversion: '2021'\n---\nbody\n", "b.md"),
|
|
doc("inbox-c", "---\nnumber: N500\nversion: '2026'\n---\nbody\n", "c.md"),
|
|
)
|
|
chain = [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"]
|
|
assert chain == [("inbox-b", "inbox-a"), ("inbox-c", "inbox-b")]
|
|
assert all(e.derived for e in bundle.edges if e.kind == "supersedes")
|
|
|
|
|
|
def test_same_number_without_versions_proposes_nothing() -> None:
|
|
# Two documents sharing a number and no way to order them is exactly the
|
|
# case where a guess would be indistinguishable from a fact.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\n---\nbody\n", "b.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == []
|
|
|
|
|
|
def test_version_ordering_is_numeric_not_lexicographic() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\nversion: '9'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\nversion: '10'\n---\nbody\n", "b.md"),
|
|
)
|
|
assert [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"] == [
|
|
("inbox-b", "inbox-a")
|
|
]
|
|
|
|
|
|
# --- the invariants the additive requirement turns on ---------------------
|
|
|
|
|
|
def test_resolution_does_not_depend_on_the_order_documents_were_added() -> None:
|
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
|
b = doc("inbox-n200", "See N500.\n", "n200-y.md")
|
|
assert resolve(a, b) == resolve(b, a)
|
|
|
|
|
|
def test_resolving_the_same_document_twice_yields_one_edge_not_two() -> None:
|
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
|
b = doc("inbox-n200", "body\n", "n200-y.md")
|
|
once = resolve(a, b)
|
|
# A mapping cannot hold the same key twice, which is the point: identity is
|
|
# the concept name, so re-dropping a file replaces rather than accumulates.
|
|
assert resolve(a, b, a) == once
|
|
assert len(edges_of(once, "references")) == 1
|
|
|
|
|
|
def test_edges_are_deterministically_ordered() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-b", "See N100 and N300.\n", "n200-b.md"),
|
|
doc("inbox-a", "See N300.\n", "n100-a.md"),
|
|
)
|
|
assert list(bundle.edges) == sorted(bundle.edges, key=lambda e: (e.source, e.kind, e.subject))
|