feat(structure): derive numbering, hierarchy and cross-references, marked declared vs derived
A bundle a consumer can only look things up in is a filing cabinet. This adds the derivation half of what lets one REASON over it: per-document title, number, parent and references, plus bundle-level resolution of every pointer. Two rules carry the design. Every fact is marked DECLARED or DERIVED, because an unmarked heuristic is worse than no heuristic -- a consumer cannot know when to doubt it. And resolution is a PURE function of the whole document set rather than a diff, which is what makes rebuild-from-scratch equal an incremental update by construction: there is no diffing algorithm to prove correct. An unresolved pointer is kept and reported, never dropped: while a bundle is built up over several rounds, pointing at something not dropped yet is the normal state, and the dangerous version of it is the one that leaves no trace. Symmetrically, a parent our own grammar could never admit (4.2 -> 4, a bare integer) is not emitted at all -- an unresolved list that never clears is one a consumer learns to ignore. 45 new tests; suite 615 -> 660.
This commit is contained in:
parent
dc9ea599c5
commit
05cda5ded5
2 changed files with 801 additions and 0 deletions
429
src/llm_ingestion_okf/structure.py
Normal file
429
src/llm_ingestion_okf/structure.py
Normal file
|
|
@ -0,0 +1,429 @@
|
||||||
|
"""Structure derivation: numbering, hierarchy, cross-references, supersession.
|
||||||
|
|
||||||
|
A bundle a consumer can only look things up in is a filing cabinet. A bundle a
|
||||||
|
consumer can REASON over needs the relations between its documents carried in
|
||||||
|
the bundle itself — which document is a child of which, what supersedes what,
|
||||||
|
what points at what. Producers rarely write that down, so ingest derives it.
|
||||||
|
|
||||||
|
Two rules govern everything here, and both exist because the alternative was
|
||||||
|
measured to be worse:
|
||||||
|
|
||||||
|
1. **Every fact is marked DECLARED or DERIVED.** `derived` names exactly the
|
||||||
|
fields this module inferred; a field present and absent from `derived` was
|
||||||
|
stated by the producer. An unmarked heuristic is worse than no heuristic
|
||||||
|
because the consumer cannot know when to doubt it — so a consumer that
|
||||||
|
trusts nothing derived can still use everything declared, and one that
|
||||||
|
accepts both knows which half it is betting on.
|
||||||
|
2. **Nothing here is a fact about a PAIR of documents.** Supersession and
|
||||||
|
reference resolution need the whole bundle, so this module records only the
|
||||||
|
SUBJECT that was pointed at (`references`, `supersedes`), never a resolved
|
||||||
|
target. Resolution is :mod:`llm_ingestion_okf.structure`'s bundle half
|
||||||
|
(:func:`resolve_structure`), which is a pure function of the whole document
|
||||||
|
set — which is what makes rebuild-from-scratch and incremental update agree
|
||||||
|
by construction rather than by a diffing algorithm we would have to prove.
|
||||||
|
|
||||||
|
Pure: no filesystem, no bundle, no door, no model call. LF assumptions are the
|
||||||
|
caller's; this module reads whatever text it is handed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# A document number is either an alpha-prefixed identifier (`N500`, `V720`,
|
||||||
|
# `R610.4`) or a dotted numeric section (`4.2.1`). A BARE integer is
|
||||||
|
# deliberately not a number: `12-things.md` and `2026-notes.md` are ordinary
|
||||||
|
# names, and admitting them would stamp a document number on most of a second
|
||||||
|
# brain that never had one. The trailing guard keeps `n500x` from reducing to
|
||||||
|
# `N500` — a partial match of a longer word is not an identifier. The guard is
|
||||||
|
# `\w` and NOT `[\w.]`: a number at the end of a sentence ("see N200.") is
|
||||||
|
# followed by a full stop, and forbidding one there silently dropped every
|
||||||
|
# reference that happened to close a sentence. `R610.4` is unaffected because
|
||||||
|
# the dotted tail is greedy and consumes it first.
|
||||||
|
_NUMBER = r"(?:[A-Za-z]{1,3}\d{1,5}(?:\.\d{1,4})*|\d{1,4}(?:\.\d{1,4})+)(?!\w)"
|
||||||
|
_NUMBER_AT_START = re.compile(rf"^({_NUMBER})")
|
||||||
|
_NUMBER_ANYWHERE = re.compile(rf"(?<![\w.])({_NUMBER})")
|
||||||
|
|
||||||
|
# Markdown inline links. Only the target matters here, and only a bundle-local
|
||||||
|
# one: an `http(s)` target is somebody else's document, and resolving it is not
|
||||||
|
# this library's job.
|
||||||
|
_LINK = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)")
|
||||||
|
|
||||||
|
# The fields this module can infer. Named as a constant because `derived` is a
|
||||||
|
# contract with the consumer, not an implementation detail.
|
||||||
|
DERIVABLE_FIELDS = frozenset({"title", "number", "parent", "references"})
|
||||||
|
|
||||||
|
|
||||||
|
def _unquote(value: str) -> str:
|
||||||
|
# A producer quotes a scalar to keep YAML from retyping it (`version:
|
||||||
|
# '2021'` is a string, not an integer). The quotes are the encoding, not
|
||||||
|
# the value, and carrying them through would put them in the index.
|
||||||
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
||||||
|
return value[1:-1]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_flow_list(value: str) -> tuple[str, ...]:
|
||||||
|
"""A YAML *flow* sequence (`[a, b]`) or a bare scalar, as a tuple.
|
||||||
|
|
||||||
|
Flow form only, matching this library's standing emission rule: the
|
||||||
|
line-oriented parser cannot read a block list at all, so a value it can
|
||||||
|
write is a value it can read back.
|
||||||
|
"""
|
||||||
|
stripped = value.strip()
|
||||||
|
if not (stripped.startswith("[") and stripped.endswith("]")):
|
||||||
|
return (_unquote(stripped),) if stripped else ()
|
||||||
|
items = (_unquote(item.strip()) for item in stripped[1:-1].split(","))
|
||||||
|
return tuple(item for item in items if item)
|
||||||
|
|
||||||
|
|
||||||
|
def _split_frontmatter(text: str) -> tuple[dict[str, str], int]:
|
||||||
|
"""The leading `---` block as keys, and the offset where the body starts.
|
||||||
|
|
||||||
|
A third copy of this library's line-oriented grammar (`materialize` reads a
|
||||||
|
path, `profiles` returns body LINES) because this one needs a character
|
||||||
|
OFFSET: the reference scan masks the frontmatter region rather than
|
||||||
|
re-joining the body, so that every match position stays comparable against
|
||||||
|
the original text and first-appearance order survives.
|
||||||
|
"""
|
||||||
|
if not text.startswith("---"):
|
||||||
|
return {}, 0
|
||||||
|
lines = text.splitlines(keepends=True)
|
||||||
|
if lines[0].strip() != "---":
|
||||||
|
return {}, 0
|
||||||
|
declared: dict[str, str] = {}
|
||||||
|
offset = len(lines[0])
|
||||||
|
for line in lines[1:]:
|
||||||
|
offset += len(line)
|
||||||
|
if line.strip() == "---":
|
||||||
|
return declared, offset
|
||||||
|
key, sep, value = line.partition(":")
|
||||||
|
if sep:
|
||||||
|
declared[key.strip()] = _unquote(value.strip())
|
||||||
|
# An unterminated block is not frontmatter; the whole text is body.
|
||||||
|
return {}, 0
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_number(token: str) -> str:
|
||||||
|
# `n500` and `N500` are the same identifier written twice. Uppercasing the
|
||||||
|
# alpha prefix is what lets a reference find its target without every
|
||||||
|
# consumer having to case-fold for itself.
|
||||||
|
return token.upper()
|
||||||
|
|
||||||
|
|
||||||
|
def _leading_heading(body: str) -> str | None:
|
||||||
|
for line in body.splitlines():
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
# Only a LEADING heading is the document's title. A heading further
|
||||||
|
# down is a section of the document, and taking it would retitle every
|
||||||
|
# document whose body happens to open with prose.
|
||||||
|
return line[2:].strip() if line.startswith("# ") else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _number_in(text: str) -> str | None:
|
||||||
|
match = _NUMBER_AT_START.match(text.strip())
|
||||||
|
return _normalize_number(match.group(1)) if match else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parent_of(number: str) -> str | None:
|
||||||
|
head, sep, _ = number.rpartition(".")
|
||||||
|
if not sep:
|
||||||
|
return None
|
||||||
|
# The parent must itself be something this grammar would recognise as a
|
||||||
|
# document number. `4.2` drops to `4`, and a bare integer is not a number
|
||||||
|
# here — emitting it would create an unresolved pointer that no document
|
||||||
|
# could ever satisfy, and an unresolved list that never clears is one a
|
||||||
|
# consumer learns to ignore.
|
||||||
|
return head if _NUMBER_AT_START.fullmatch(head) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_references(body: str, offset: int, own_number: str | None) -> tuple[str, ...]:
|
||||||
|
"""Bundle-local reference subjects, ordered by first appearance.
|
||||||
|
|
||||||
|
Link targets are collected first and their spans masked with spaces before
|
||||||
|
the number scan runs, so a link to `n500.md` yields the link target once
|
||||||
|
rather than the target plus a phantom `N500` read out of the URL. Masking
|
||||||
|
with spaces rather than deleting keeps every later offset aligned, which is
|
||||||
|
what makes "first appearance" a property of the original text.
|
||||||
|
"""
|
||||||
|
found: list[tuple[int, str]] = []
|
||||||
|
masked = list(body)
|
||||||
|
for match in _LINK.finditer(body):
|
||||||
|
target = match.group(1)
|
||||||
|
start, end = match.span(1)
|
||||||
|
for position in range(start, end):
|
||||||
|
masked[position] = " "
|
||||||
|
if target.startswith(("http://", "https://", "//", "mailto:")):
|
||||||
|
continue
|
||||||
|
found.append((offset + start, target))
|
||||||
|
for match in _NUMBER_ANYWHERE.finditer("".join(masked)):
|
||||||
|
number = _normalize_number(match.group(1))
|
||||||
|
if number != own_number:
|
||||||
|
found.append((offset + match.start(), number))
|
||||||
|
|
||||||
|
ordered: list[str] = []
|
||||||
|
for _, subject in sorted(found, key=lambda pair: pair[0]):
|
||||||
|
if subject not in ordered:
|
||||||
|
ordered.append(subject)
|
||||||
|
return tuple(ordered)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DocumentStructure:
|
||||||
|
"""What ingest worked out about ONE document, and how sure it is.
|
||||||
|
|
||||||
|
`references` and `supersedes` hold SUBJECTS — the number or filename that
|
||||||
|
was pointed at — never resolved targets: whether the thing pointed at
|
||||||
|
exists is a fact about the bundle, and a document does not know its bundle.
|
||||||
|
|
||||||
|
`declared` is the producer's own frontmatter, verbatim and unfiltered. It
|
||||||
|
is here because the measured failure was an index that carried none of the
|
||||||
|
metadata its documents carried; a projection cannot surface what derivation
|
||||||
|
never returned.
|
||||||
|
"""
|
||||||
|
|
||||||
|
title: str
|
||||||
|
source_file: str
|
||||||
|
number: str | None = None
|
||||||
|
parent_number: str | None = None
|
||||||
|
version: str | None = None
|
||||||
|
references: tuple[str, ...] = ()
|
||||||
|
supersedes: tuple[str, ...] = ()
|
||||||
|
declared: Mapping[str, str] = field(default_factory=dict)
|
||||||
|
derived: frozenset[str] = frozenset()
|
||||||
|
|
||||||
|
|
||||||
|
def derive_document_structure(text: str, *, source_file: str) -> DocumentStructure:
|
||||||
|
"""Derive one document's structure from its text and the name it arrived as.
|
||||||
|
|
||||||
|
Certainty, highest first, per field:
|
||||||
|
|
||||||
|
- `title` — the producer's `title` key (DECLARED); else a leading `# `
|
||||||
|
heading; else the filename stem. The last two are DERIVED.
|
||||||
|
- `number` — the producer's `number` key (DECLARED); else the leading
|
||||||
|
number token of the filename, else of the title (DERIVED).
|
||||||
|
- `parent_number` — arithmetic on `number`, so it is exactly as certain as
|
||||||
|
the number it came from and is never an independent guess.
|
||||||
|
- `references` — the producer's `references` key (DECLARED); else every
|
||||||
|
bundle-local link target and number mention in the body (DERIVED).
|
||||||
|
- `supersedes`, `version` — DECLARED or absent. Supersession is a fact
|
||||||
|
about a pair of documents, so one document cannot answer it; the
|
||||||
|
bundle-level resolver may propose it, this function must not.
|
||||||
|
"""
|
||||||
|
declared, offset = _split_frontmatter(text)
|
||||||
|
body = text[offset:]
|
||||||
|
derived: set[str] = set()
|
||||||
|
|
||||||
|
stem = unicodedata.normalize("NFC", Path(source_file).stem)
|
||||||
|
|
||||||
|
title = declared.get("title")
|
||||||
|
if title is None:
|
||||||
|
derived.add("title")
|
||||||
|
heading = _leading_heading(body)
|
||||||
|
title = heading if heading is not None else stem
|
||||||
|
title = unicodedata.normalize("NFC", title)
|
||||||
|
|
||||||
|
number = declared.get("number")
|
||||||
|
if number is None:
|
||||||
|
candidate = _number_in(stem) or _number_in(title)
|
||||||
|
if candidate is not None:
|
||||||
|
derived.add("number")
|
||||||
|
number = candidate
|
||||||
|
|
||||||
|
parent_number = _parent_of(number) if number else None
|
||||||
|
if parent_number is not None and "number" in derived:
|
||||||
|
derived.add("parent")
|
||||||
|
|
||||||
|
if "references" in declared:
|
||||||
|
references = _parse_flow_list(declared["references"])
|
||||||
|
else:
|
||||||
|
references = _scan_references(body, offset, number)
|
||||||
|
if references:
|
||||||
|
derived.add("references")
|
||||||
|
|
||||||
|
return DocumentStructure(
|
||||||
|
title=title,
|
||||||
|
source_file=source_file,
|
||||||
|
number=number,
|
||||||
|
parent_number=parent_number,
|
||||||
|
version=declared.get("version"),
|
||||||
|
references=references,
|
||||||
|
supersedes=_parse_flow_list(declared["supersedes"]) if "supersedes" in declared else (),
|
||||||
|
declared=declared,
|
||||||
|
derived=frozenset(derived),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- bundle-level resolution ---------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class StructureEdge:
|
||||||
|
"""One relation between two documents, resolved or not.
|
||||||
|
|
||||||
|
`subject` is what the source document pointed AT — a number, a filename —
|
||||||
|
and `target` is the concept it turned out to be, or `None`. An unresolved
|
||||||
|
edge is kept rather than dropped: while a bundle is being built up, a
|
||||||
|
pointer to something not dropped yet is the NORMAL state, and the dangerous
|
||||||
|
version of it is the one that leaves no trace. `derived` marks an edge this
|
||||||
|
library proposed rather than one the producer declared.
|
||||||
|
"""
|
||||||
|
|
||||||
|
source: str
|
||||||
|
kind: str
|
||||||
|
subject: str
|
||||||
|
target: str | None
|
||||||
|
derived: bool
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BundleStructure:
|
||||||
|
"""Every document's structure plus every relation between them.
|
||||||
|
|
||||||
|
Produced by :func:`resolve_structure` from the WHOLE document set, which is
|
||||||
|
the design answer to the additive requirement: an incremental update is
|
||||||
|
just resolution over a larger set, so it cannot disagree with a rebuild
|
||||||
|
from scratch, and re-dropping a file cannot double an edge because the
|
||||||
|
concept name is the identity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
documents: Mapping[str, DocumentStructure]
|
||||||
|
edges: tuple[StructureEdge, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def unresolved(self) -> tuple[StructureEdge, ...]:
|
||||||
|
"""Every edge whose target is not (yet) in the bundle."""
|
||||||
|
return tuple(edge for edge in self.edges if edge.target is None)
|
||||||
|
|
||||||
|
|
||||||
|
def _version_key(version: str) -> tuple[object, ...]:
|
||||||
|
# Natural order: digit runs compare as integers so `10` follows `9`, and
|
||||||
|
# everything else compares as text. Lexicographic order would put the 2026
|
||||||
|
# edition of a document before its 9th revision.
|
||||||
|
parts = re.split(r"(\d+)", version)
|
||||||
|
return tuple((1, int(part)) if part.isdigit() else (0, part) for part in parts if part)
|
||||||
|
|
||||||
|
|
||||||
|
def _lookup(documents: Mapping[str, DocumentStructure]) -> dict[str, str]:
|
||||||
|
"""Every name a document can be pointed at by, mapped to its concept name.
|
||||||
|
|
||||||
|
Numbers are folded to upper case (the form :func:`_normalize_number` writes)
|
||||||
|
and filenames are matched both with and without their extension, because a
|
||||||
|
producer's link points at the name the file ARRIVED as while the bundle
|
||||||
|
holds the name Door B gave it. A key claimed by two documents is dropped
|
||||||
|
rather than resolved to whichever came first: an ambiguous pointer that
|
||||||
|
silently picks a winner is worse than one reported unresolved.
|
||||||
|
"""
|
||||||
|
claims: dict[str, set[str]] = {}
|
||||||
|
for name, document in documents.items():
|
||||||
|
keys = {name, document.source_file, Path(document.source_file).stem}
|
||||||
|
if document.number:
|
||||||
|
keys.add(document.number)
|
||||||
|
for key in keys:
|
||||||
|
claims.setdefault(key.upper(), set()).add(name)
|
||||||
|
return {key: next(iter(owners)) for key, owners in claims.items() if len(owners) == 1}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_structure(documents: Mapping[str, DocumentStructure]) -> BundleStructure:
|
||||||
|
"""Resolve every pointer in `documents` against the bundle as a whole.
|
||||||
|
|
||||||
|
A pure function of the whole set — no diff, no append, no memory of earlier
|
||||||
|
rounds. That is deliberate and it is what makes the three additive
|
||||||
|
invariants hold by construction rather than by argument: rebuild equals
|
||||||
|
incremental, the input order does not matter, and re-dropping a document
|
||||||
|
replaces its edges instead of duplicating them.
|
||||||
|
|
||||||
|
Three kinds of edge, and the confidence of each comes from where it came
|
||||||
|
from: `parent` and `references` inherit the source document's `derived`
|
||||||
|
marking, declared `supersedes` is never derived, and the one relation this
|
||||||
|
function proposes on its own — same number, ordered versions — always is.
|
||||||
|
"""
|
||||||
|
by_key = _lookup(documents)
|
||||||
|
edges: list[StructureEdge] = []
|
||||||
|
|
||||||
|
for name in sorted(documents):
|
||||||
|
document = documents[name]
|
||||||
|
if document.parent_number is not None:
|
||||||
|
edges.append(
|
||||||
|
StructureEdge(
|
||||||
|
source=name,
|
||||||
|
kind="parent",
|
||||||
|
subject=document.parent_number,
|
||||||
|
target=by_key.get(document.parent_number.upper()),
|
||||||
|
derived="parent" in document.derived,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for subject in document.references:
|
||||||
|
edges.append(
|
||||||
|
StructureEdge(
|
||||||
|
source=name,
|
||||||
|
kind="references",
|
||||||
|
subject=subject,
|
||||||
|
target=by_key.get(subject.upper()),
|
||||||
|
derived="references" in document.derived,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for subject in document.supersedes:
|
||||||
|
edges.append(
|
||||||
|
StructureEdge(
|
||||||
|
source=name,
|
||||||
|
kind="supersedes",
|
||||||
|
subject=subject,
|
||||||
|
target=by_key.get(subject.upper()),
|
||||||
|
derived=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
edges.extend(_derived_supersession(documents))
|
||||||
|
return BundleStructure(
|
||||||
|
documents=dict(documents),
|
||||||
|
edges=tuple(sorted(edges, key=lambda edge: (edge.source, edge.kind, edge.subject))),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _derived_supersession(
|
||||||
|
documents: Mapping[str, DocumentStructure],
|
||||||
|
) -> list[StructureEdge]:
|
||||||
|
"""The one relation this library proposes without being told: a version chain.
|
||||||
|
|
||||||
|
Documents sharing a number and each carrying a `version` are ordered by
|
||||||
|
that version, and each supersedes its immediate predecessor. Every edge is
|
||||||
|
marked derived.
|
||||||
|
|
||||||
|
A group whose members do NOT all carry a version proposes nothing. Two
|
||||||
|
documents with the same number and no way to order them is exactly the case
|
||||||
|
where a guess would be indistinguishable from a fact — and supersession is
|
||||||
|
the relation a consumer is most likely to act on, so a wrong one here costs
|
||||||
|
more than a missing one. Documents that declare their own `supersedes` are
|
||||||
|
left out of the chain entirely: the producer has answered the question.
|
||||||
|
"""
|
||||||
|
groups: dict[str, list[str]] = {}
|
||||||
|
for name, document in documents.items():
|
||||||
|
if document.number and document.version and not document.supersedes:
|
||||||
|
groups.setdefault(document.number, []).append(name)
|
||||||
|
|
||||||
|
proposed: list[StructureEdge] = []
|
||||||
|
for number in sorted(groups):
|
||||||
|
members = groups[number]
|
||||||
|
if len(members) < 2:
|
||||||
|
continue
|
||||||
|
ordered = sorted(
|
||||||
|
members, key=lambda name: (_version_key(documents[name].version or ""), name)
|
||||||
|
)
|
||||||
|
for older, newer in zip(ordered, ordered[1:]):
|
||||||
|
proposed.append(
|
||||||
|
StructureEdge(
|
||||||
|
source=newer,
|
||||||
|
kind="supersedes",
|
||||||
|
subject=documents[older].number or older,
|
||||||
|
target=older,
|
||||||
|
derived=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return proposed
|
||||||
372
tests/test_structure.py
Normal file
372
tests/test_structure.py
Normal file
|
|
@ -0,0 +1,372 @@
|
||||||
|
"""Structure derivation: what ingest can work out about a dropped document.
|
||||||
|
|
||||||
|
Pure functions over text and a filename — no I/O, no bundle, no door. The
|
||||||
|
whole point of the module is that a consumer can REASON over a bundle rather
|
||||||
|
than only look things up in it, so every fact here is either DECLARED by the
|
||||||
|
producer or INFERRED by us, and the difference is carried in the output. An
|
||||||
|
unmarked heuristic is worse than no heuristic: the consumer cannot know when
|
||||||
|
to doubt it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from llm_ingestion_okf.structure import (
|
||||||
|
BundleStructure,
|
||||||
|
DocumentStructure,
|
||||||
|
derive_document_structure,
|
||||||
|
resolve_structure,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def derive(text: str, source_file: str = "note.md") -> DocumentStructure:
|
||||||
|
return derive_document_structure(text, source_file=source_file)
|
||||||
|
|
||||||
|
|
||||||
|
# --- title ----------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# The answer the previous session recorded for the Door B / Door A capability
|
||||||
|
# gap: Door B needs a title, and a profile cannot supply one because
|
||||||
|
# `BundleProfile.index` carries no title field at all. Three sources, in
|
||||||
|
# descending certainty.
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_from_frontmatter_is_declared() -> None:
|
||||||
|
doc = derive("---\ntitle: Vegbygging\n---\n\nbody\n", "n500-vegbygging.md")
|
||||||
|
assert doc.title == "Vegbygging"
|
||||||
|
assert "title" not in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_from_leading_heading_is_derived() -> None:
|
||||||
|
doc = derive("# Vegbygging\n\nbody\n", "n500-vegbygging.md")
|
||||||
|
assert doc.title == "Vegbygging"
|
||||||
|
assert "title" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_falls_back_to_the_filename_stem_and_is_derived() -> None:
|
||||||
|
doc = derive("body with no heading\n", "n500-vegbygging.md")
|
||||||
|
assert doc.title == "n500-vegbygging"
|
||||||
|
assert "title" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_frontmatter_title_beats_a_heading() -> None:
|
||||||
|
doc = derive("---\ntitle: Declared\n---\n\n# Heading\n", "x.md")
|
||||||
|
assert doc.title == "Declared"
|
||||||
|
assert "title" not in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_heading_below_the_first_prose_line_is_not_the_title() -> None:
|
||||||
|
# Only a LEADING heading is the document's title; a heading further down is
|
||||||
|
# a section of it. Taking any heading would retitle every document whose
|
||||||
|
# body happens to start with prose.
|
||||||
|
doc = derive("intro prose\n\n# Section Two\n", "the-file.md")
|
||||||
|
assert doc.title == "the-file"
|
||||||
|
assert "title" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_title_is_nfc_normalised() -> None:
|
||||||
|
# macOS hands filenames over decomposed; the repo normalises before use so
|
||||||
|
# one visual name cannot reduce two ways.
|
||||||
|
doc = derive("body\n", "prøve.md")
|
||||||
|
assert doc.title == "prøve"
|
||||||
|
|
||||||
|
|
||||||
|
# --- document number ------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("source_file", "expected"),
|
||||||
|
[
|
||||||
|
("n500-vegbygging.md", "N500"),
|
||||||
|
("N500 Vegbygging.md", "N500"),
|
||||||
|
("v720-something.md", "V720"),
|
||||||
|
("4.2.1-details.md", "4.2.1"),
|
||||||
|
("R610.4 note.md", "R610.4"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_number_is_read_off_the_filename_and_marked_derived(
|
||||||
|
source_file: str, expected: str
|
||||||
|
) -> None:
|
||||||
|
doc = derive("body\n", source_file)
|
||||||
|
assert doc.number == expected
|
||||||
|
assert "number" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_declared_number_wins_and_is_not_marked_derived() -> None:
|
||||||
|
doc = derive("---\nnumber: N200\n---\n\nbody\n", "n500-vegbygging.md")
|
||||||
|
assert doc.number == "N200"
|
||||||
|
assert "number" not in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_number_falls_back_to_the_title_when_the_filename_has_none() -> None:
|
||||||
|
doc = derive("# N500 Vegbygging\n", "dropped-file.md")
|
||||||
|
assert doc.number == "N500"
|
||||||
|
assert "number" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bare_integer_is_not_a_document_number() -> None:
|
||||||
|
# "2026-notes" and "12 things" are ordinary names, not numbering. A pure
|
||||||
|
# number needs a dotted form to count; anything looser would label most of
|
||||||
|
# a second brain with a document number it never had.
|
||||||
|
assert derive("body\n", "12-things.md").number is None
|
||||||
|
assert derive("body\n", "2026-notes.md").number is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_number_anywhere_is_none_not_an_invention() -> None:
|
||||||
|
doc = derive("# Just A Title\n", "just-a-title.md")
|
||||||
|
assert doc.number is None
|
||||||
|
assert doc.parent_number is None
|
||||||
|
|
||||||
|
|
||||||
|
# --- hierarchy ------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("number_source", "expected_parent"),
|
||||||
|
[
|
||||||
|
("4.2.1-x.md", "4.2"),
|
||||||
|
("R610.4 x.md", "R610"),
|
||||||
|
("n500-x.md", None),
|
||||||
|
# `4.2` would drop to `4`, and a BARE integer is not a document number
|
||||||
|
# under this module's own grammar. A pointer nothing could ever satisfy
|
||||||
|
# is not a pointer: it would sit in the unresolved list forever, and an
|
||||||
|
# unresolved list that never clears trains a consumer to ignore it.
|
||||||
|
("4.2-x.md", None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_parent_is_the_number_minus_its_last_component(
|
||||||
|
number_source: str, expected_parent: str | None
|
||||||
|
) -> None:
|
||||||
|
assert derive("body\n", number_source).parent_number == expected_parent
|
||||||
|
|
||||||
|
|
||||||
|
def test_parent_is_structural_and_carries_the_numbers_confidence() -> None:
|
||||||
|
# The parent is not an independent guess: given a number, dropping the last
|
||||||
|
# dotted component is arithmetic. So it is only as certain as the number.
|
||||||
|
declared = derive("---\nnumber: 4.2.1\n---\n\nbody\n", "whatever.md")
|
||||||
|
assert declared.parent_number == "4.2"
|
||||||
|
assert "parent" not in declared.derived
|
||||||
|
|
||||||
|
inferred = derive("body\n", "4.2.1-x.md")
|
||||||
|
assert inferred.parent_number == "4.2"
|
||||||
|
assert "parent" in inferred.derived
|
||||||
|
|
||||||
|
|
||||||
|
# --- cross references -----------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_number_mentions_in_the_body_are_references() -> None:
|
||||||
|
doc = derive("See N200 and jf. N300 kap. 4 for details.\n", "n500-x.md")
|
||||||
|
assert doc.references == ("N200", "N300")
|
||||||
|
assert "references" in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_document_never_references_itself() -> None:
|
||||||
|
doc = derive("N500 says that N500 applies, see N200.\n", "n500-x.md")
|
||||||
|
assert doc.references == ("N200",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_references_are_deduplicated_and_ordered_by_first_appearance() -> None:
|
||||||
|
doc = derive("N300 then N200 then N300 again.\n", "n500-x.md")
|
||||||
|
assert doc.references == ("N300", "N200")
|
||||||
|
|
||||||
|
|
||||||
|
def test_markdown_link_targets_are_references_too() -> None:
|
||||||
|
doc = derive("See [the other](other-doc.md) and [again](other-doc.md).\n", "n500-x.md")
|
||||||
|
assert "other-doc.md" in doc.references
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_links_are_not_references() -> None:
|
||||||
|
# A cross-reference is inside the bundle. An http link is somebody else's
|
||||||
|
# document and resolving it is not this library's job.
|
||||||
|
doc = derive("See [upstream](https://example.test/a.md).\n", "n500-x.md")
|
||||||
|
assert doc.references == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_declared_references_are_not_marked_derived() -> None:
|
||||||
|
doc = derive("---\nreferences: [N200, N300]\n---\n\nbody mentioning N400\n", "n500-x.md")
|
||||||
|
assert doc.references == ("N200", "N300")
|
||||||
|
assert "references" not in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
# --- supersedes and version ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_supersedes_is_declared_only_never_inferred_per_document() -> None:
|
||||||
|
doc = derive("---\nsupersedes: [n500-2018]\n---\n\nbody\n", "n500-2021.md")
|
||||||
|
assert doc.supersedes == ("n500-2018",)
|
||||||
|
assert "supersedes" not in doc.derived
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_declared_supersedes_means_none_at_document_level() -> None:
|
||||||
|
# Whether one document supersedes another is a fact about a PAIR, so a
|
||||||
|
# single document cannot answer it. The bundle-level resolver may propose
|
||||||
|
# it; this function must not.
|
||||||
|
doc = derive("This replaces the 2018 edition.\n", "n500-2021.md")
|
||||||
|
assert doc.supersedes == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_version_is_declared_only() -> None:
|
||||||
|
assert derive("---\nversion: '2021'\n---\n\nbody\n", "x.md").version == "2021"
|
||||||
|
assert derive("body\n", "n500-2021.md").version is None
|
||||||
|
|
||||||
|
|
||||||
|
# --- pass-through of the producer's own keys ------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_declared_frontmatter_is_carried_verbatim() -> None:
|
||||||
|
# The measured gap: documents carry status/date 55/55 while the index
|
||||||
|
# carries them 0/55. Derivation has to surface them for the index to be
|
||||||
|
# able to project them at all.
|
||||||
|
doc = derive("---\nstatus: gjeldende\ndate: 2026-01-01\n---\n\nbody\n", "x.md")
|
||||||
|
assert doc.declared["status"] == "gjeldende"
|
||||||
|
assert doc.declared["date"] == "2026-01-01"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_document_with_no_frontmatter_declares_nothing() -> None:
|
||||||
|
assert derive("# Title\n\nbody\n", "x.md").declared == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_derivation_is_deterministic() -> None:
|
||||||
|
text = "---\nstatus: gjeldende\n---\n\n# N500 Vegbygging\n\nSee N200.\n"
|
||||||
|
assert derive(text, "n500-vegbygging.md") == derive(text, "n500-vegbygging.md")
|
||||||
|
|
||||||
|
|
||||||
|
# --- bundle-level resolution ----------------------------------------------
|
||||||
|
#
|
||||||
|
# The additive requirement in one design decision: resolution is a PURE
|
||||||
|
# function of the whole document set. Nothing is diffed and nothing is
|
||||||
|
# appended, so rebuild-from-scratch and incremental update cannot disagree —
|
||||||
|
# not because a diffing algorithm was proved correct, but because there is no
|
||||||
|
# diffing algorithm to prove.
|
||||||
|
|
||||||
|
|
||||||
|
def doc(name: str, text: str, source_file: str | None = None) -> tuple[str, DocumentStructure]:
|
||||||
|
return name, derive(text, source_file or f"{name}.md")
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(*pairs: tuple[str, DocumentStructure]) -> BundleStructure:
|
||||||
|
return resolve_structure(dict(pairs))
|
||||||
|
|
||||||
|
|
||||||
|
def edges_of(bundle: BundleStructure, kind: str) -> list[tuple[str, str, str | None]]:
|
||||||
|
return [(e.source, e.subject, e.target) for e in bundle.edges if e.kind == kind]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_reference_resolves_to_the_document_carrying_that_number() -> None:
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-n500", "See N200 for details.\n", "n500-vegbygging.md"),
|
||||||
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
||||||
|
)
|
||||||
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
||||||
|
assert bundle.unresolved == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_reference_to_a_document_not_yet_dropped_is_UNRESOLVED_not_dropped() -> None:
|
||||||
|
# Normal state while a bundle is still being built up. It must be visible
|
||||||
|
# as unfulfilled: an absence that does not scream is the most dangerous
|
||||||
|
# state this repo knows.
|
||||||
|
bundle = resolve(doc("inbox-n500", "See N200.\n", "n500-x.md"))
|
||||||
|
assert edges_of(bundle, "references") == [("inbox-n500", "N200", None)]
|
||||||
|
assert [e.subject for e in bundle.unresolved] == ["N200"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_reference_resolves_once_its_target_arrives_in_a_later_round() -> None:
|
||||||
|
first = doc("inbox-n500", "See N200.\n", "n500-x.md")
|
||||||
|
assert resolve(first).unresolved != ()
|
||||||
|
later = resolve(first, doc("inbox-n200", "body\n", "n200-y.md"))
|
||||||
|
assert later.unresolved == ()
|
||||||
|
assert edges_of(later, "references") == [("inbox-n500", "N200", "inbox-n200")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_link_target_resolves_against_the_source_filename() -> None:
|
||||||
|
# Door B renames what it writes, so a producer's link points at the name
|
||||||
|
# the file arrived as, never at the concept name. Resolving only concept
|
||||||
|
# names would report every intra-bundle link as dangling.
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-a", "See [other](n200-grunnlag.md).\n", "a.md"),
|
||||||
|
doc("inbox-n200", "body\n", "n200-grunnlag.md"),
|
||||||
|
)
|
||||||
|
assert edges_of(bundle, "references") == [("inbox-a", "n200-grunnlag.md", "inbox-n200")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_parent_resolves_to_the_document_carrying_the_parent_number() -> None:
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-421", "body\n", "4.2.1-details.md"),
|
||||||
|
doc("inbox-42", "body\n", "4.2-section.md"),
|
||||||
|
)
|
||||||
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", "inbox-42")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_missing_parent_is_unresolved_rather_than_absent() -> None:
|
||||||
|
bundle = resolve(doc("inbox-421", "body\n", "4.2.1-details.md"))
|
||||||
|
assert edges_of(bundle, "parent") == [("inbox-421", "4.2", None)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_declared_supersedes_resolves_and_is_not_derived() -> None:
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-new", "---\nsupersedes: [n500-2018]\n---\nbody\n", "n500-2021.md"),
|
||||||
|
doc("inbox-old", "body\n", "n500-2018.md"),
|
||||||
|
)
|
||||||
|
assert edges_of(bundle, "supersedes") == [("inbox-new", "n500-2018", "inbox-old")]
|
||||||
|
assert [e.derived for e in bundle.edges if e.kind == "supersedes"] == [False]
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_number_with_ordered_versions_yields_a_DERIVED_supersedes_chain() -> None:
|
||||||
|
# The one pair-level heuristic in the module, and it is marked as such.
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-a", "---\nnumber: N500\nversion: '2018'\n---\nbody\n", "a.md"),
|
||||||
|
doc("inbox-b", "---\nnumber: N500\nversion: '2021'\n---\nbody\n", "b.md"),
|
||||||
|
doc("inbox-c", "---\nnumber: N500\nversion: '2026'\n---\nbody\n", "c.md"),
|
||||||
|
)
|
||||||
|
chain = [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"]
|
||||||
|
assert chain == [("inbox-b", "inbox-a"), ("inbox-c", "inbox-b")]
|
||||||
|
assert all(e.derived for e in bundle.edges if e.kind == "supersedes")
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_number_without_versions_proposes_nothing() -> None:
|
||||||
|
# Two documents sharing a number and no way to order them is exactly the
|
||||||
|
# case where a guess would be indistinguishable from a fact.
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-a", "---\nnumber: N500\n---\nbody\n", "a.md"),
|
||||||
|
doc("inbox-b", "---\nnumber: N500\n---\nbody\n", "b.md"),
|
||||||
|
)
|
||||||
|
assert edges_of(bundle, "supersedes") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_version_ordering_is_numeric_not_lexicographic() -> None:
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-a", "---\nnumber: N500\nversion: '9'\n---\nbody\n", "a.md"),
|
||||||
|
doc("inbox-b", "---\nnumber: N500\nversion: '10'\n---\nbody\n", "b.md"),
|
||||||
|
)
|
||||||
|
assert [(e.source, e.target) for e in bundle.edges if e.kind == "supersedes"] == [
|
||||||
|
("inbox-b", "inbox-a")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# --- the invariants the additive requirement turns on ---------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolution_does_not_depend_on_the_order_documents_were_added() -> None:
|
||||||
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
||||||
|
b = doc("inbox-n200", "See N500.\n", "n200-y.md")
|
||||||
|
assert resolve(a, b) == resolve(b, a)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolving_the_same_document_twice_yields_one_edge_not_two() -> None:
|
||||||
|
a = doc("inbox-a", "See N200.\n", "n500-x.md")
|
||||||
|
b = doc("inbox-n200", "body\n", "n200-y.md")
|
||||||
|
once = resolve(a, b)
|
||||||
|
# A mapping cannot hold the same key twice, which is the point: identity is
|
||||||
|
# the concept name, so re-dropping a file replaces rather than accumulates.
|
||||||
|
assert resolve(a, b, a) == once
|
||||||
|
assert len(edges_of(once, "references")) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_edges_are_deterministically_ordered() -> None:
|
||||||
|
bundle = resolve(
|
||||||
|
doc("inbox-b", "See N100 and N300.\n", "n200-b.md"),
|
||||||
|
doc("inbox-a", "See N300.\n", "n100-a.md"),
|
||||||
|
)
|
||||||
|
assert list(bundle.edges) == sorted(bundle.edges, key=lambda e: (e.source, e.kind, e.subject))
|
||||||
Loading…
Add table
Add a link
Reference in a new issue