Kjell Tore Guttormsen
2827be0ece
STRUCTURED_V1 derived `references` from every number-shaped token in a body. A consumer measured 12 false references out of 12 on their corpus, ten of them version numbers in titles read as document numbers. Measured again here before changing anything, on two corpora, through `derive_document_structure` itself: corpus A a consumer's normative bundles 2 561 docs -> 2 838 subjects corpus B this repository's own docs/ 28 docs -> 559 subjects The reported class reproduces, and two larger ones they did not report turn up: 702 of corpus A's subjects (24.7 %) are hexadecimal fragments of UUIDs read out of `id="..."` attributes in embedded MathML, and corpus B lifts tokens out of escape sequences in quoted source (`\n60` -> `N60`), licence identifiers (`AGPL-3.0` -> `3.0`) and package pins. A derived subject is now a bundle-local link target, or a number immediately preceded by a cue from a closed set. Positive rather than a blacklist because the data forces it: `V221` is a genuine document number in corpus A and `V0.3.0` is a software version in corpus B, and they are the same token shape, so only the words in front of them can tell them apart. The cue matches at a word boundary (a Norwegian compound ending in `-klasse` otherwise satisfies the cue `se`, which admitted 86 class designations) and the window is NFC-normalised so a cue survives a decomposed filesystem. Fragment-only and brace-carrying link targets go too: neither can name a concept, so neither is a pending pointer. After: 2 838 -> 1 279 (A) and 559 -> 72 (B). Hand-classified against the occurrence that actually passed the gate: 30 of 30 sampled genuine on A, 60 of 60 on B. Residual known falses: 9 of B's 72, all illustrative link targets in prose about link syntax. The prefix-resolution rule stays open, per the order's condition: of 2 589 documents, 2 562 carry a number and 0 of those are dotted, so unique-prefix match has no data here to be defended against. Cost stated rather than hidden: a corpus phrasing cross-references outside this vocabulary derives nothing and must declare `references`. A missing reference is visible to the reader; a false one is not. Record: docs/plan/references-cue-rule.md Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01ATyA5Lx53N1rKzwMnuMoem
457 lines
18 KiB
Python
457 lines
18 KiB
Python
"""Structure derivation: what ingest can work out about a dropped document.
|
|
|
|
Pure functions over text and a filename — no I/O, no bundle, no door. The
|
|
whole point of the module is that a consumer can REASON over a bundle rather
|
|
than only look things up in it, so every fact here is either DECLARED by the
|
|
producer or INFERRED by us, and the difference is carried in the output. An
|
|
unmarked heuristic is worse than no heuristic: the consumer cannot know when
|
|
to doubt it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf.structure import (
|
|
BundleStructure,
|
|
DocumentStructure,
|
|
derive_document_structure,
|
|
resolve_structure,
|
|
)
|
|
|
|
|
|
def derive(text: str, source_file: str = "note.md") -> DocumentStructure:
|
|
return derive_document_structure(text, source_file=source_file)
|
|
|
|
|
|
# --- title ----------------------------------------------------------------
|
|
#
|
|
# The answer the previous session recorded for the Door B / Door A capability
|
|
# gap: Door B needs a title, and a profile cannot supply one because
|
|
# `BundleProfile.index` carries no title field at all. Three sources, in
|
|
# descending certainty.
|
|
|
|
|
|
def test_title_from_frontmatter_is_declared() -> None:
|
|
doc = derive("---\ntitle: Vegbygging\n---\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.title == "Vegbygging"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_title_from_leading_heading_is_derived() -> None:
|
|
doc = derive("# Vegbygging\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.title == "Vegbygging"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_falls_back_to_the_filename_stem_and_is_derived() -> None:
|
|
doc = derive("body with no heading\n", "n500-vegbygging.md")
|
|
assert doc.title == "n500-vegbygging"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_frontmatter_title_beats_a_heading() -> None:
|
|
doc = derive("---\ntitle: Declared\n---\n\n# Heading\n", "x.md")
|
|
assert doc.title == "Declared"
|
|
assert "title" not in doc.derived
|
|
|
|
|
|
def test_a_heading_below_the_first_prose_line_is_not_the_title() -> None:
|
|
# Only a LEADING heading is the document's title; a heading further down is
|
|
# a section of it. Taking any heading would retitle every document whose
|
|
# body happens to start with prose.
|
|
doc = derive("intro prose\n\n# Section Two\n", "the-file.md")
|
|
assert doc.title == "the-file"
|
|
assert "title" in doc.derived
|
|
|
|
|
|
def test_title_is_nfc_normalised() -> None:
|
|
# macOS hands filenames over decomposed; the repo normalises before use so
|
|
# one visual name cannot reduce two ways.
|
|
doc = derive("body\n", "prøve.md")
|
|
assert doc.title == "prøve"
|
|
|
|
|
|
# --- document number ------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source_file", "expected"),
|
|
[
|
|
("n500-vegbygging.md", "N500"),
|
|
("N500 Vegbygging.md", "N500"),
|
|
("v720-something.md", "V720"),
|
|
("4.2.1-details.md", "4.2.1"),
|
|
("R610.4 note.md", "R610.4"),
|
|
],
|
|
)
|
|
def test_number_is_read_off_the_filename_and_marked_derived(
|
|
source_file: str, expected: str
|
|
) -> None:
|
|
doc = derive("body\n", source_file)
|
|
assert doc.number == expected
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_declared_number_wins_and_is_not_marked_derived() -> None:
|
|
doc = derive("---\nnumber: N200\n---\n\nbody\n", "n500-vegbygging.md")
|
|
assert doc.number == "N200"
|
|
assert "number" not in doc.derived
|
|
|
|
|
|
def test_number_falls_back_to_the_title_when_the_filename_has_none() -> None:
|
|
doc = derive("# N500 Vegbygging\n", "dropped-file.md")
|
|
assert doc.number == "N500"
|
|
assert "number" in doc.derived
|
|
|
|
|
|
def test_a_bare_integer_is_not_a_document_number() -> None:
|
|
# "2026-notes" and "12 things" are ordinary names, not numbering. A pure
|
|
# number needs a dotted form to count; anything looser would label most of
|
|
# a second brain with a document number it never had.
|
|
assert derive("body\n", "12-things.md").number is None
|
|
assert derive("body\n", "2026-notes.md").number is None
|
|
|
|
|
|
def test_no_number_anywhere_is_none_not_an_invention() -> None:
|
|
doc = derive("# Just A Title\n", "just-a-title.md")
|
|
assert doc.number is None
|
|
assert doc.parent_number is None
|
|
|
|
|
|
# --- hierarchy ------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("number_source", "expected_parent"),
|
|
[
|
|
("4.2.1-x.md", "4.2"),
|
|
("R610.4 x.md", "R610"),
|
|
("n500-x.md", None),
|
|
# `4.2` would drop to `4`, and a BARE integer is not a document number
|
|
# under this module's own grammar. A pointer nothing could ever satisfy
|
|
# is not a pointer: it would sit in the unresolved list forever, and an
|
|
# unresolved list that never clears trains a consumer to ignore it.
|
|
("4.2-x.md", None),
|
|
],
|
|
)
|
|
def test_parent_is_the_number_minus_its_last_component(
|
|
number_source: str, expected_parent: str | None
|
|
) -> None:
|
|
assert derive("body\n", number_source).parent_number == expected_parent
|
|
|
|
|
|
def test_parent_is_structural_and_carries_the_numbers_confidence() -> None:
|
|
# The parent is not an independent guess: given a number, dropping the last
|
|
# dotted component is arithmetic. So it is only as certain as the number.
|
|
declared = derive("---\nnumber: 4.2.1\n---\n\nbody\n", "whatever.md")
|
|
assert declared.parent_number == "4.2"
|
|
assert "parent" not in declared.derived
|
|
|
|
inferred = derive("body\n", "4.2.1-x.md")
|
|
assert inferred.parent_number == "4.2"
|
|
assert "parent" in inferred.derived
|
|
|
|
|
|
# --- cross references -----------------------------------------------------
|
|
|
|
|
|
def test_number_mentions_in_the_body_are_references() -> None:
|
|
doc = derive("See N200 and jf. N300 kap. 4 for details.\n", "n500-x.md")
|
|
assert doc.references == ("N200", "N300")
|
|
assert "references" in doc.derived
|
|
|
|
|
|
def test_a_document_never_references_itself() -> None:
|
|
doc = derive("N500 says that N500 applies, see N200.\n", "n500-x.md")
|
|
assert doc.references == ("N200",)
|
|
|
|
|
|
def test_references_are_deduplicated_and_ordered_by_first_appearance() -> None:
|
|
# Each mention carries a cue, because a number without one is no longer a
|
|
# reference at all. What this test pins is the ordering and the dedup.
|
|
doc = derive("se N300 then se N200 then se N300 again.\n", "n500-x.md")
|
|
assert doc.references == ("N300", "N200")
|
|
|
|
|
|
def test_markdown_link_targets_are_references_too() -> None:
|
|
doc = derive("See [the other](other-doc.md) and [again](other-doc.md).\n", "n500-x.md")
|
|
assert "other-doc.md" in doc.references
|
|
|
|
|
|
def test_external_links_are_not_references() -> None:
|
|
# A cross-reference is inside the bundle. An http link is somebody else's
|
|
# document and resolving it is not this library's job.
|
|
doc = derive("See [upstream](https://example.test/a.md).\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_declared_references_are_not_marked_derived() -> None:
|
|
doc = derive("---\nreferences: [N200, N300]\n---\n\nbody mentioning N400\n", "n500-x.md")
|
|
assert doc.references == ("N200", "N300")
|
|
assert "references" not in doc.derived
|
|
|
|
|
|
# --- supersedes and version ----------------------------------------------
|
|
|
|
|
|
def test_supersedes_is_declared_only_never_inferred_per_document() -> None:
|
|
doc = derive("---\nsupersedes: [n500-2018]\n---\n\nbody\n", "n500-2021.md")
|
|
assert doc.supersedes == ("n500-2018",)
|
|
assert "supersedes" not in doc.derived
|
|
|
|
|
|
def test_no_declared_supersedes_means_none_at_document_level() -> None:
|
|
# Whether one document supersedes another is a fact about a PAIR, so a
|
|
# single document cannot answer it. The bundle-level resolver may propose
|
|
# it; this function must not.
|
|
doc = derive("This replaces the 2018 edition.\n", "n500-2021.md")
|
|
assert doc.supersedes == ()
|
|
|
|
|
|
def test_version_is_declared_only() -> None:
|
|
assert derive("---\nversion: '2021'\n---\n\nbody\n", "x.md").version == "2021"
|
|
assert derive("body\n", "n500-2021.md").version is None
|
|
|
|
|
|
# --- pass-through of the producer's own keys ------------------------------
|
|
|
|
|
|
def test_declared_frontmatter_is_carried_verbatim() -> None:
|
|
# The measured gap: documents carry status/date 55/55 while the index
|
|
# carries them 0/55. Derivation has to surface them for the index to be
|
|
# able to project them at all.
|
|
doc = derive("---\nstatus: gjeldende\ndate: 2026-01-01\n---\n\nbody\n", "x.md")
|
|
assert doc.declared["status"] == "gjeldende"
|
|
assert doc.declared["date"] == "2026-01-01"
|
|
|
|
|
|
def test_a_document_with_no_frontmatter_declares_nothing() -> None:
|
|
assert derive("# Title\n\nbody\n", "x.md").declared == {}
|
|
|
|
|
|
def test_derivation_is_deterministic() -> None:
|
|
text = "---\nstatus: gjeldende\n---\n\n# N500 Vegbygging\n\nSee N200.\n"
|
|
assert derive(text, "n500-vegbygging.md") == derive(text, "n500-vegbygging.md")
|
|
|
|
|
|
# --- bundle-level resolution ----------------------------------------------
|
|
#
|
|
# The additive requirement in one design decision: resolution is a PURE
|
|
# function of the whole document set. Nothing is diffed and nothing is
|
|
# appended, so rebuild-from-scratch and incremental update cannot disagree —
|
|
# not because a diffing algorithm was proved correct, but because there is no
|
|
# diffing algorithm to prove.
|
|
|
|
|
|
def doc(name: str, text: str, source_file: str | None = None) -> tuple[str, DocumentStructure]:
|
|
return name, derive(text, source_file or f"{name}.md")
|
|
|
|
|
|
def resolve(*pairs: tuple[str, DocumentStructure]) -> BundleStructure:
|
|
return resolve_structure(dict(pairs))
|
|
|
|
|
|
def edges_of(bundle: BundleStructure, kind: str) -> list[tuple[str, str, str | None]]:
|
|
return [(e.source, e.subject, e.target) for e in bundle.edges if e.kind == kind]
|
|
|
|
|
|
def test_a_reference_resolves_to_the_document_carrying_that_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-n500", "See N200 for details.\n", "n500-vegbygging.md"),
|
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
|
assert bundle.unresolved == ()
|
|
|
|
|
|
def test_a_reference_to_a_document_not_yet_dropped_is_UNRESOLVED_not_dropped() -> None:
|
|
# Normal state while a bundle is still being built up. It must be visible
|
|
# as unfulfilled: an absence that does not scream is the most dangerous
|
|
# state this repo knows.
|
|
bundle = resolve(doc("inbox-n500", "See N200.\n", "n500-x.md"))
|
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", None)]
|
|
assert [e.subject for e in bundle.unresolved] == ["N200"]
|
|
|
|
|
|
def test_a_reference_resolves_once_its_target_arrives_in_a_later_round() -> None:
|
|
first = doc("inbox-n500", "See N200.\n", "n500-x.md")
|
|
assert resolve(first).unresolved != ()
|
|
later = resolve(first, doc("inbox-n200", "body\n", "n200-y.md"))
|
|
assert later.unresolved == ()
|
|
assert edges_of(later, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
|
|
|
|
|
def test_a_link_target_resolves_against_the_source_filename() -> None:
|
|
# Door B renames what it writes, so a producer's link points at the name
|
|
# the file arrived as, never at the concept name. Resolving only concept
|
|
# names would report every intra-bundle link as dangling.
|
|
bundle = resolve(
|
|
doc("inbox-a", "See [other](n200-grunnlag.md).\n", "a.md"),
|
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
|
)
|
|
assert edges_of(bundle, "references") == [("inbox-a", "n200-grunnlag.md", "inbox-n200")]
|
|
|
|
|
|
def test_parent_resolves_to_the_document_carrying_the_parent_number() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-421", "body\n", "4.2.1-details.md"),
|
|
doc("inbox-42", "body\n", "4.2-section.md"),
|
|
)
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", "inbox-42")]
|
|
|
|
|
|
def test_a_missing_parent_is_unresolved_rather_than_absent() -> None:
|
|
bundle = resolve(doc("inbox-421", "body\n", "4.2.1-details.md"))
|
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", None)]
|
|
|
|
|
|
def test_declared_supersedes_resolves_and_is_not_derived() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-new", "---\nsupersedes: [n500-2018]\n---\nbody\n", "n500-2021.md"),
|
|
doc("inbox-old", "body\n", "n500-2018.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == [("inbox-new", "n500-2018", "inbox-old")]
|
|
assert [e.derived for e in bundle.edges if e.kind == "supersedes"] == [False]
|
|
|
|
|
|
def test_same_number_with_ordered_versions_yields_a_DERIVED_supersedes_chain() -> None:
|
|
# The one pair-level heuristic in the module, and it is marked as such.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\nversion: '2018'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\nversion: '2021'\n---\nbody\n", "b.md"),
|
|
doc("inbox-c", "---\nnumber: N500\nversion: '2026'\n---\nbody\n", "c.md"),
|
|
)
|
|
chain = [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"]
|
|
assert chain == [("inbox-b", "inbox-a"), ("inbox-c", "inbox-b")]
|
|
assert all(e.derived for e in bundle.edges if e.kind == "supersedes")
|
|
|
|
|
|
def test_same_number_without_versions_proposes_nothing() -> None:
|
|
# Two documents sharing a number and no way to order them is exactly the
|
|
# case where a guess would be indistinguishable from a fact.
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\n---\nbody\n", "b.md"),
|
|
)
|
|
assert edges_of(bundle, "supersedes") == []
|
|
|
|
|
|
def test_version_ordering_is_numeric_not_lexicographic() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-a", "---\nnumber: N500\nversion: '9'\n---\nbody\n", "a.md"),
|
|
doc("inbox-b", "---\nnumber: N500\nversion: '10'\n---\nbody\n", "b.md"),
|
|
)
|
|
assert [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"] == [
|
|
("inbox-b", "inbox-a")
|
|
]
|
|
|
|
|
|
# --- the invariants the additive requirement turns on ---------------------
|
|
|
|
|
|
def test_resolution_does_not_depend_on_the_order_documents_were_added() -> None:
|
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
|
b = doc("inbox-n200", "See N500.\n", "n200-y.md")
|
|
assert resolve(a, b) == resolve(b, a)
|
|
|
|
|
|
def test_resolving_the_same_document_twice_yields_one_edge_not_two() -> None:
|
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
|
b = doc("inbox-n200", "body\n", "n200-y.md")
|
|
once = resolve(a, b)
|
|
# A mapping cannot hold the same key twice, which is the point: identity is
|
|
# the concept name, so re-dropping a file replaces rather than accumulates.
|
|
assert resolve(a, b, a) == once
|
|
assert len(edges_of(once, "references")) == 1
|
|
|
|
|
|
def test_edges_are_deterministically_ordered() -> None:
|
|
bundle = resolve(
|
|
doc("inbox-b", "See N100 and N300.\n", "n200-b.md"),
|
|
doc("inbox-a", "See N300.\n", "n100-a.md"),
|
|
)
|
|
assert list(bundle.edges) == sorted(bundle.edges, key=lambda e: (e.source, e.kind, e.subject))
|
|
|
|
|
|
# --- what a derived reference IS ------------------------------------------
|
|
#
|
|
# Measured 2026-08-29 against two real corpora, because the previous rule --
|
|
# "every number-shaped token in the body" -- was reported producing 12 false
|
|
# references out of 12 on a third. Denominators, before the rule below:
|
|
#
|
|
# corpus A 2 561 unique normative concept documents -> 2 838 subjects, of
|
|
# which 702 (24.7 %) were hexadecimal fragments of UUIDs lifted out
|
|
# of HTML/MathML attribute values, plus unit symbols (`kN/m2` ->
|
|
# `M2`), table row labels and formula numbers;
|
|
# corpus B 28 of this repository's own documents -> 559 subjects, dominated
|
|
# by software versions (`v0.3.0`, `3.14.0`, `0.11.10`), a licence
|
|
# id (`AGPL-3.0` -> `3.0`), a container version (`%PDF-1.6` ->
|
|
# `1.6`), a hash algorithm name (`SHA256`), and tokens lifted out
|
|
# of escape sequences in quoted source (`\n60` -> `N60`).
|
|
#
|
|
# The version class cannot be excluded structurally: corpus A proves `V221` and
|
|
# `V240` are genuine document numbers and corpus B proves `V0.3.0` is a
|
|
# software version, and they are the same token shape. So no blacklist can
|
|
# work, and a positive cue is the only rule that can be defended against data.
|
|
# A missing reference is visible to the reader; a false one is not.
|
|
|
|
|
|
def test_a_package_version_is_not_a_reference() -> None:
|
|
doc = derive("Built on pdfplumber 0.11.10 and Python 3.14.0, licence AGPL-3.0.\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_v_prefixed_version_is_not_a_reference() -> None:
|
|
# The reported failure class: a version string carried in a title, read as
|
|
# a document number. `V221` in corpus A is a real document; `v0.5.0` here
|
|
# is not, and only the cue in front of it can tell them apart.
|
|
doc = derive("The classifier lags v0.3.0 and the gap is wider now (v0.5.0a2).\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_number_inside_markup_is_not_a_reference() -> None:
|
|
doc = derive('<math id="id-647a093b-a70f-4e38-d277-b139040226a4">x</math>\n', "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_measurement_is_not_a_reference() -> None:
|
|
doc = derive("Compressive strength at least 500 kN/m2, used only below 2.5 m.\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_cue_word_makes_a_number_a_reference() -> None:
|
|
# KNOWN-POSITIVE. Without it the two tests above pass for the wrong reason:
|
|
# a rule that finds nothing at all also finds nothing false.
|
|
doc = derive("Laid to depth as se kapittel 2.6.2.2 and tabell 4.8.4-3 give.\n", "n500-x.md")
|
|
assert doc.references == ("2.6.2.2", "4.8.4")
|
|
assert "references" in doc.derived
|
|
|
|
|
|
def test_a_section_sign_is_a_cue() -> None:
|
|
doc = derive("Emitted as one block, in §10.2's own listing order.\n", "n500-x.md")
|
|
assert doc.references == ("10.2",)
|
|
|
|
|
|
def test_a_cue_only_counts_at_a_word_boundary() -> None:
|
|
# A Norwegian compound ending in `-klasse` ends in the cue `se`. Matching
|
|
# an unanchored cue admitted 86 such class designations on corpus A, none
|
|
# of which name a document.
|
|
doc = derive("Se også R210; lysklasse C5 gjelder.\n", "n500-x.md")
|
|
assert doc.references == ("R210",)
|
|
|
|
|
|
def test_a_fragment_only_link_target_is_not_a_reference() -> None:
|
|
# An anchor points inside THIS document. It can never resolve to another
|
|
# concept, so emitting it states a relation that cannot exist.
|
|
doc = derive("See [table pairing](#table-pairing).\n", "n500-x.md")
|
|
assert doc.references == ()
|
|
|
|
|
|
def test_a_template_placeholder_link_target_is_not_a_reference() -> None:
|
|
# Documentation about link SYNTAX carries example targets. A brace cannot
|
|
# occur in a concept name (`reduce_to_id_grammar` emits lowercase
|
|
# alphanumerics and hyphens), so such a target can never resolve — and an
|
|
# edge that can never resolve is a claim, not a pending pointer.
|
|
doc = derive("Managed lines read `- [title](ingest-{id}.md)`.\n", "n500-x.md")
|
|
assert doc.references == ()
|