"""Structure derivation: numbering, hierarchy, cross-references, supersession. A bundle a consumer can only look things up in is a filing cabinet. A bundle a consumer can REASON over needs the relations between its documents carried in the bundle itself — which document is a child of which, what supersedes what, what points at what. Producers rarely write that down, so ingest derives it. Two rules govern everything here, and both exist because the alternative was measured to be worse: 1. **Every fact is marked DECLARED or DERIVED.** `derived` names exactly the fields this module inferred; a field present and absent from `derived` was stated by the producer. An unmarked heuristic is worse than no heuristic because the consumer cannot know when to doubt it — so a consumer that trusts nothing derived can still use everything declared, and one that accepts both knows which half it is betting on. 2. **Nothing here is a fact about a PAIR of documents.** Supersession and reference resolution need the whole bundle, so this module records only the SUBJECT that was pointed at (`references`, `supersedes`), never a resolved target. Resolution is :mod:`llm_ingestion_okf.structure`'s bundle half (:func:`resolve_structure`), which is a pure function of the whole document set — which is what makes rebuild-from-scratch and incremental update agree by construction rather than by a diffing algorithm we would have to prove. Pure: no filesystem, no bundle, no door, no model call. LF assumptions are the caller's; this module reads whatever text it is handed. """ from __future__ import annotations import re import unicodedata from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from pathlib import Path # A document number is either an alpha-prefixed identifier (`N500`, `V720`, # `R610.4`) or a dotted numeric section (`4.2.1`). A BARE integer is # deliberately not a number: `12-things.md` and `2026-notes.md` are ordinary # names, and admitting them would stamp a document number on most of a second # brain that never had one. The trailing guard keeps `n500x` from reducing to # `N500` — a partial match of a longer word is not an identifier. The guard is # `\w` and NOT `[\w.]`: a number at the end of a sentence ("see N200.") is # followed by a full stop, and forbidding one there silently dropped every # reference that happened to close a sentence. `R610.4` is unaffected because # the dotted tail is greedy and consumes it first. _NUMBER = r"(?:[A-Za-z]{1,3}\d{1,5}(?:\.\d{1,4})*|\d{1,4}(?:\.\d{1,4})+)(?!\w)" _NUMBER_AT_START = re.compile(rf"^({_NUMBER})") _NUMBER_ANYWHERE = re.compile(rf"(? `M2`), table row labels, # formula numbers, licence identifiers (`AGPL-3.0` -> `3.0`) and package pins. # Each of those stands in the index as a relation this library asserted, under # the producer's name. A reference the reader never sees is a smaller harm # than one the reader cannot tell from a fact. # # The consequence is stated rather than hidden: a corpus whose cross-references # are phrased outside this vocabulary — another language, a house style — gets # NOTHING derived, and must declare `references` itself. Silence is the honest # failure; a guess dressed as a relation is not. # # The leading guard is a word boundary that also covers the Norwegian letters # `\w` handles but `[a-z]` does not. Without it `belysningsklasse C5` ends in # the cue `se`, which admitted 86 class designations on the corpus measured. _CUE = re.compile( r"(?:(? str: # A producer quotes a scalar to keep YAML from retyping it (`version: # '2021'` is a string, not an integer). The quotes are the encoding, not # the value, and carrying them through would put them in the index. if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": return value[1:-1] return value def _parse_flow_list(value: str) -> tuple[str, ...]: """A YAML *flow* sequence (`[a, b]`) or a bare scalar, as a tuple. Flow form only, matching this library's standing emission rule: the line-oriented parser cannot read a block list at all, so a value it can write is a value it can read back. """ stripped = value.strip() if not (stripped.startswith("[") and stripped.endswith("]")): return (_unquote(stripped),) if stripped else () items = (_unquote(item.strip()) for item in stripped[1:-1].split(",")) return tuple(item for item in items if item) def _split_frontmatter(text: str) -> tuple[dict[str, str], int]: """The leading `---` block as keys, and the offset where the body starts. A third copy of this library's line-oriented grammar (`materialize` reads a path, `profiles` returns body LINES) because this one needs a character OFFSET: the reference scan masks the frontmatter region rather than re-joining the body, so that every match position stays comparable against the original text and first-appearance order survives. """ if not text.startswith("---"): return {}, 0 lines = text.splitlines(keepends=True) if lines[0].strip() != "---": return {}, 0 declared: dict[str, str] = {} offset = len(lines[0]) for line in lines[1:]: offset += len(line) if line.strip() == "---": return declared, offset # An INDENTED key belongs to the block above it, not to the document. # Without this, `key.strip()` would flatten it into the same namespace # as the top-level keys and, arriving later, SUBSTITUTE for one of them # -- a `sources:` entry's own `title:` silently becoming the document's, # carrying `number` and `parent` with it. Skipping is deliberately not # parsing: the nested value is not read, only refused. The structured # reader is D1b. if line[:1] in (" ", "\t"): continue key, sep, value = line.partition(":") if sep: declared[key.strip()] = _unquote(value.strip()) # An unterminated block is not frontmatter; the whole text is body. return {}, 0 def _normalize_number(token: str) -> str: # `n500` and `N500` are the same identifier written twice. Uppercasing the # alpha prefix is what lets a reference find its target without every # consumer having to case-fold for itself. return token.upper() def _leading_heading(body: str) -> str | None: for line in body.splitlines(): if not line.strip(): continue # Only a LEADING heading is the document's title. A heading further # down is a section of the document, and taking it would retitle every # document whose body happens to open with prose. return line[2:].strip() if line.startswith("# ") else None return None def _number_in(text: str) -> str | None: match = _NUMBER_AT_START.match(text.strip()) return _normalize_number(match.group(1)) if match else None def _parent_of(number: str) -> str | None: head, sep, _ = number.rpartition(".") if not sep: return None # The parent must itself be something this grammar would recognise as a # document number. `4.2` drops to `4`, and a bare integer is not a number # here — emitting it would create an unresolved pointer that no document # could ever satisfy, and an unresolved list that never clears is one a # consumer learns to ignore. return head if _NUMBER_AT_START.fullmatch(head) else None def _scan_references(body: str, offset: int, own_number: str | None) -> tuple[str, ...]: """Bundle-local reference subjects, ordered by first appearance. Two kinds of pointer, and nothing else. A markdown link is an explicit one the producer authored, so it is taken as written. A number in running prose is taken only when a :data:`_CUE` stands immediately in front of it — everything else number-shaped is a version, a measurement, a label or an identifier fragment, and emitting it puts a relation this library invented into the index under the producer's name. Link targets are collected first and their spans masked with spaces before the number scan runs, so a link to `n500.md` yields the link target once rather than the target plus a phantom `N500` read out of the URL. Masking with spaces rather than deleting keeps every later offset aligned, which is what makes "first appearance" a property of the original text — and it is also what keeps a cue from being read across a link it does not precede. """ found: list[tuple[int, str]] = [] masked = list(body) for match in _LINK.finditer(body): target = match.group(1) start, end = match.span(1) for position in range(start, end): masked[position] = " " # A fragment-only target points inside THIS document, and a target # carrying a brace is a template placeholder from prose ABOUT links # (`reduce_to_id_grammar` cannot emit a brace). Neither can ever # resolve to another concept, so carrying them states a relation that # cannot exist rather than one not dropped yet. if target.startswith(("http://", "https://", "//", "mailto:", "#")): continue if "{" in target or "}" in target: continue found.append((offset + start, target)) scanned = "".join(masked) for match in _NUMBER_ANYWHERE.finditer(scanned): number = _normalize_number(match.group(1)) if number == own_number: continue # NFC first: a cue carrying a Norwegian letter arrives decomposed from # a macOS filesystem, and a cue that matches only one of the two forms # is a rule that holds or not depending on where the file was written. window = unicodedata.normalize( "NFC", scanned[max(0, match.start() - _CUE_WINDOW) : match.start()] ) if _CUE.search(window): found.append((offset + match.start(), number)) ordered: list[str] = [] for _, subject in sorted(found, key=lambda pair: pair[0]): if subject not in ordered: ordered.append(subject) return tuple(ordered) @dataclass(frozen=True) class DocumentStructure: """What ingest worked out about ONE document, and how sure it is. `references` and `supersedes` hold SUBJECTS — the number or filename that was pointed at — never resolved targets: whether the thing pointed at exists is a fact about the bundle, and a document does not know its bundle. `declared` is the producer's own frontmatter, verbatim and unfiltered. It is here because the measured failure was an index that carried none of the metadata its documents carried; a projection cannot surface what derivation never returned. """ title: str source_file: str number: str | None = None parent_number: str | None = None version: str | None = None references: tuple[str, ...] = () supersedes: tuple[str, ...] = () declared: Mapping[str, str] = field(default_factory=dict) derived: frozenset[str] = frozenset() def derive_document_structure(text: str, *, source_file: str) -> DocumentStructure: """Derive one document's structure from its text and the name it arrived as. Certainty, highest first, per field: - `title` — the producer's `title` key (DECLARED); else a leading `# ` heading; else the filename stem. The last two are DERIVED. - `number` — the producer's `number` key (DECLARED); else the leading number token of the filename, else of the title (DERIVED). - `parent_number` — arithmetic on `number`, so it is exactly as certain as the number it came from and is never an independent guess. - `references` — the producer's `references` key (DECLARED); else every bundle-local link target and number mention in the body (DERIVED). - `supersedes`, `version` — DECLARED or absent. Supersession is a fact about a pair of documents, so one document cannot answer it; the bundle-level resolver may propose it, this function must not. """ declared, offset = _split_frontmatter(text) body = text[offset:] derived: set[str] = set() stem = unicodedata.normalize("NFC", Path(source_file).stem) title = declared.get("title") if title is None: derived.add("title") heading = _leading_heading(body) title = heading if heading is not None else stem title = unicodedata.normalize("NFC", title) number = declared.get("number") if number is None: candidate = _number_in(stem) or _number_in(title) if candidate is not None: derived.add("number") number = candidate parent_number = _parent_of(number) if number else None if parent_number is not None and "number" in derived: derived.add("parent") if "references" in declared: references = _parse_flow_list(declared["references"]) else: references = _scan_references(body, offset, number) if references: derived.add("references") return DocumentStructure( title=title, source_file=source_file, number=number, parent_number=parent_number, version=declared.get("version"), references=references, supersedes=_parse_flow_list(declared["supersedes"]) if "supersedes" in declared else (), declared=declared, derived=frozenset(derived), ) # --- bundle-level resolution --------------------------------------------- @dataclass(frozen=True) class StructureEdge: """One relation between two documents, resolved or not. `subject` is what the source document pointed AT — a number, a filename — and `target` is the concept it turned out to be, or `None`. An unresolved edge is kept rather than dropped: while a bundle is being built up, a pointer to something not dropped yet is the NORMAL state, and the dangerous version of it is the one that leaves no trace. `derived` marks an edge this library proposed rather than one the producer declared. """ source: str kind: str subject: str target: str | None derived: bool @dataclass(frozen=True) class BundleStructure: """Every document's structure plus every relation between them. Produced by :func:`resolve_structure` from the WHOLE document set, which is the design answer to the additive requirement: an incremental update is just resolution over a larger set, so it cannot disagree with a rebuild from scratch, and re-dropping a file cannot double an edge because the concept name is the identity. """ documents: Mapping[str, DocumentStructure] edges: tuple[StructureEdge, ...] @property def unresolved(self) -> tuple[StructureEdge, ...]: """Every edge whose target is not (yet) in the bundle.""" return tuple(edge for edge in self.edges if edge.target is None) def _version_key(version: str) -> tuple[object, ...]: # Natural order: digit runs compare as integers so `10` follows `9`, and # everything else compares as text. Lexicographic order would put the 2026 # edition of a document before its 9th revision. parts = re.split(r"(\d+)", version) return tuple((1, int(part)) if part.isdigit() else (0, part) for part in parts if part) def _lookup(documents: Mapping[str, DocumentStructure]) -> dict[str, str]: """Every name a document can be pointed at by, mapped to its concept name. Numbers are folded to upper case (the form :func:`_normalize_number` writes) and filenames are matched both with and without their extension, because a producer's link points at the name the file ARRIVED as while the bundle holds the name Door B gave it. A key claimed by two documents is dropped rather than resolved to whichever came first: an ambiguous pointer that silently picks a winner is worse than one reported unresolved. """ claims: dict[str, set[str]] = {} for name, document in documents.items(): keys = {name, document.source_file, Path(document.source_file).stem} if document.number: keys.add(document.number) for key in keys: claims.setdefault(key.upper(), set()).add(name) return {key: next(iter(owners)) for key, owners in claims.items() if len(owners) == 1} def resolve_structure(documents: Mapping[str, DocumentStructure]) -> BundleStructure: """Resolve every pointer in `documents` against the bundle as a whole. A pure function of the whole set — no diff, no append, no memory of earlier rounds. That is deliberate and it is what makes the three additive invariants hold by construction rather than by argument: rebuild equals incremental, the input order does not matter, and re-dropping a document replaces its edges instead of duplicating them. Three kinds of edge, and the confidence of each comes from where it came from: `parent` and `references` inherit the source document's `derived` marking, declared `supersedes` is never derived, and the one relation this function proposes on its own — same number, ordered versions — always is. """ by_key = _lookup(documents) edges: list[StructureEdge] = [] for name in sorted(documents): document = documents[name] if document.parent_number is not None: edges.append( StructureEdge( source=name, kind="parent", subject=document.parent_number, target=by_key.get(document.parent_number.upper()), derived="parent" in document.derived, ) ) for subject in document.references: edges.append( StructureEdge( source=name, kind="references", subject=subject, target=by_key.get(subject.upper()), derived="references" in document.derived, ) ) for subject in document.supersedes: edges.append( StructureEdge( source=name, kind="supersedes", subject=subject, target=by_key.get(subject.upper()), derived=False, ) ) edges.extend(_derived_supersession(documents)) return BundleStructure( documents=dict(documents), edges=tuple(sorted(edges, key=lambda edge: (edge.source, edge.kind, edge.subject))), ) def _derived_supersession( documents: Mapping[str, DocumentStructure], ) -> list[StructureEdge]: """The one relation this library proposes without being told: a version chain. Documents sharing a number and each carrying a `version` are ordered by that version, and each supersedes its immediate predecessor. Every edge is marked derived. A group whose members do NOT all carry a version proposes nothing. Two documents with the same number and no way to order them is exactly the case where a guess would be indistinguishable from a fact — and supersession is the relation a consumer is most likely to act on, so a wrong one here costs more than a missing one. Documents that declare their own `supersedes` are left out of the chain entirely: the producer has answered the question. """ groups: dict[str, list[str]] = {} for name, document in documents.items(): if document.number and document.version and not document.supersedes: groups.setdefault(document.number, []).append(name) proposed: list[StructureEdge] = [] for number in sorted(groups): members = groups[number] if len(members) < 2: continue ordered = sorted( members, key=lambda name: (_version_key(documents[name].version or ""), name) ) for older, newer in zip(ordered, ordered[1:]): proposed.append( StructureEdge( source=newer, kind="supersedes", subject=documents[older].number or older, target=older, derived=True, ) ) return proposed # --- projection to and from frontmatter ----------------------------------- # The order `derived` lists its members in. Fixed rather than sorted so the # marker reads in the order the fields are established (a title before the # number read out of it), and so two runs over the same document emit the same # bytes. _DERIVED_ORDER = ("title", "number", "parent", "references", "supersedes") def _render_flow_list(items: Sequence[str]) -> str: # Flow form, never block: this library's parser round-trips a flow sequence # as an opaque value and cannot read a block one at all, so a value we can # write is a value we can read back. return f"[{', '.join(items)}]" def structure_frontmatter(document: DocumentStructure, keys: Sequence[str]) -> dict[str, str]: """The structure keys a concept file carries, for the keys a profile names. Written into the CONCEPT rather than only into the index, because the index is a projection: a later round rebuilds it by reading these keys back, and a fact that lived only in the index would be lost the moment the index was reprojected. """ values: dict[str, str] = {} for key in keys: if key == "derived": continue if key == "number" and document.number: values[key] = document.number elif key == "parent" and document.parent_number: values[key] = document.parent_number elif key == "version" and document.version: values[key] = document.version elif key == "references" and document.references: values[key] = _render_flow_list(document.references) elif key == "supersedes" and document.supersedes: values[key] = _render_flow_list(document.supersedes) elif key in document.declared: values[key] = document.declared[key] marked = [field_name for field_name in _DERIVED_ORDER if field_name in document.derived] if marked and "derived" in keys: values["derived"] = _render_flow_list(marked) return values def structure_from_frontmatter(values: Mapping[str, str]) -> DocumentStructure: """A concept file's stored frontmatter read back as its structure. The inverse of :func:`structure_frontmatter` over the keys it writes, plus the door's own `title` and `source_file`. Reading the STORED keys rather than re-deriving from the body is what makes a reprojection cheap and, more importantly, stable: re-derivation would make a bundle's index depend on the version of this library that last touched it. """ return DocumentStructure( title=values.get("title", ""), source_file=values.get("source_file", ""), number=values.get("number"), parent_number=values.get("parent"), version=values.get("version"), references=_parse_flow_list(values["references"]) if "references" in values else (), supersedes=_parse_flow_list(values["supersedes"]) if "supersedes" in values else (), declared=dict(values), derived=frozenset(_parse_flow_list(values["derived"]) if "derived" in values else ()), ) # The suffix a subject wears when nothing in the bundle answers to it. A # pointer that is merely absent from the index is indistinguishable from one # that was never made -- and an absence that does not scream is the most # dangerous state this repo knows. UNRESOLVED_MARKER = "?" def facet_values(name: str, bundle: BundleStructure, keys: Sequence[str]) -> dict[str, str]: """One document's facets, as an index entry carries them. Relations are rendered as their SUBJECTS, each suffixed with :data:`UNRESOLVED_MARKER` when the bundle holds nothing answering to it, so an index reader sees the difference between "points at N200" and "points at an N200 that is not here". `derived` gathers the document's own inferred fields plus any relation this library proposed rather than read. """ document = bundle.documents[name] values = structure_frontmatter(document, [key for key in keys if key != "derived"]) marked = {field_name for field_name in document.derived} edges = [edge for edge in bundle.edges if edge.source == name] for kind, key in ( ("parent", "parent"), ("references", "references"), ("supersedes", "supersedes"), ): if key not in keys: continue subjects = [ edge.subject + (UNRESOLVED_MARKER if edge.target is None else "") for edge in edges if edge.kind == kind ] if any(edge.kind == kind and edge.derived for edge in edges): marked.add(key) if not subjects: values.pop(key, None) elif kind == "parent": values[key] = subjects[0] else: values[key] = _render_flow_list(subjects) listed = [field_name for field_name in _DERIVED_ORDER if field_name in marked] if listed and "derived" in keys: values["derived"] = _render_flow_list(listed) return values