open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(profiles): a faceted index policy and the additive STRUCTURED_V1 profile

Browse source 
The measured defect, as data: the 2026-08-26 bake-off had every arm retrieve
40/40, so quality could not separate them. The only axis that did was trap
exposure -- 18/20 for the OKF-index arm against 8/20 for a frontmatter
head-scan -- and both sides measured the reason independently: the flat index
carries title/date/status/supersedes 0 times while its own documents carry them
55/55/55/5. The metadata is in the bundle; the index throws it away.

FacetPolicy lets an index entry keep it. The grammar is thin on purpose (one
separator, then key: value joined by '; ') because index lines are read by
regex on both sides of this library, and a value carrying either delimiter is
REFUSED rather than escaped -- validation, not repair, as everywhere else here.

Additive by construction, not by caution. entry_pattern IS link_pattern when a
policy carries no facets, so DEFAULT and STRICT_V1 match the same lines and
emit the same bytes; the goldens are the proof. Facets arrive as STRUCTURED_V1,
a new profile, because DEFAULT states commons' ingest-spec index layer and
changing its bytes from here would be this repo editing a contract it does not
own.

17 new tests; suite 660 -> 677.
This commit is contained in:
Kjell Tore Guttormsen 2026-08-27 00:30:42 +02:00
commit 52c82bc3d1
3 changed files with 351 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/llm_ingestion_okf/materialize.py
View file

@ -271,7 +271,7 @@ def _update_index_lines(
for line in lines:
content = line.rstrip("\r\n")
ending = line[len(content) :]
match = profile.index.link_pattern.match(content)
match = profile.index.entry_pattern.match(content)
if match is not None:
target = match.group("target")
if target in removed_targets:

199
src/llm_ingestion_okf/profiles.py
View file

@ -375,6 +375,86 @@ def _split_frontmatter(text: str) -> tuple[dict[str, str], list[str]]:
return head, []
@dataclass(frozen=True)
class FacetPolicy:
"""The metadata an index ENTRY carries beside its label and target.
A flat index is a table of contents. A faceted one is a table a consumer
can reason over without opening a single document - which is the whole
difference the 2026-08-26 bake-off measured: every arm retrieved 40/40, and
the only axis that separated them was trap exposure, where the flat index
lost precisely because it carried title/date/status/supersedes 0 times
while its own documents carried them 55/55/55/5.
`keys` is the ordered, closed set. Ordering by the POLICY rather than by
the caller's mapping is what keeps two callers passing the same facts from
emitting different bytes, exactly as the root frontmatter does; closing the
set is what keeps a value out of a file no reader of this contract looks
at.
The grammar is deliberately thin - `separator` once, then `key: value`
joined by `joiner` - because an index line is read by regex on both sides
of this library. A value carrying either delimiter is REFUSED rather than
escaped or repaired: escaping would make the line unreadable to a consumer
that splits naively, and this library validates rather than repairs
everywhere else.
"""
keys: tuple[str, ...]
separator: str = " \u2014 "
joiner: str = "; "
def __post_init__(self) -> None:
if not self.keys:
raise ValueError(
"a facet policy must name at least one key - a policy naming "
"none would render a separator with nothing after it"
)
def render(self, values: Mapping[str, str]) -> str:
"""The facet tail for `values`, or "" when none of them are present."""
unknown = sorted(set(values) - set(self.keys))
if unknown:
raise ValueError(
f"facet key(s) {', '.join(repr(key) for key in unknown)} are not named "
f"by this index policy, which pins {self.keys} - rendering an unnamed "
"key would put a value in a file no reader of this contract looks at"
)
present = [(key, values[key]) for key in self.keys if values.get(key)]
for key, value in present:
if "\n" in value or "\r" in value:
raise ValueError(f"facet {key!r} must be single-line, got {value!r}")
for label, delimiter in (("separator", self.separator), ("joiner", self.joiner)):
if delimiter in value:
raise ValueError(
f"facet {key!r} contains this policy's {label} {delimiter!r} "
f"({value!r}) - refusing to escape or repair it, which would "
"make the line parse one way here and another way downstream"
)
if not present:
return ""
return self.separator + self.joiner.join(f"{key}: {value}" for key, value in present)
def parse(self, tail: str) -> dict[str, str]:
"""The facet tail read back. The inverse of `render` by construction."""
parsed: dict[str, str] = {}
for chunk in tail.split(self.joiner):
key, sep, value = chunk.partition(":")
if sep:
parsed[key.strip()] = value.strip()
return parsed
@dataclass(frozen=True)
class IndexEntry:
"""One managed index line, read back into its parts."""
label: str
target: str
description: str | None = None
facets: Mapping[str, str] = field(default_factory=dict)
@dataclass(frozen=True)
class IndexViolation:
"""One way an index file departs from the policy.
@ -443,8 +523,31 @@ class IndexPolicy:
entries_match_directory: bool = False
root_frontmatter: tuple[str, ...] = ()
root_frontmatter_required: frozenset[str] = field(default_factory=frozenset)
facets: FacetPolicy | None = None
_faceted_pattern: re.Pattern[str] | None = field(
init=False, repr=False, compare=False, default=None
)
def __post_init__(self) -> None:
if self.facets is not None:
# The faceted pattern is BUILT from the base one, so an unanchored
# base would silently produce an unanchored faceted pattern - and
# index maintenance keys on this pattern to decide which lines it
# may rewrite. A substring match there edits curated prose.
if not self.link_pattern.pattern.endswith("$"):
raise ValueError(
"a link pattern carrying facets must be anchored to the end "
"of the line ('$'), because the faceted pattern is derived "
"from it and an unanchored match would rewrite curated prose"
)
object.__setattr__(
self,
"_faceted_pattern",
re.compile(
self.link_pattern.pattern[:-1]
+ f"(?:{re.escape(self.facets.separator)}(?P<facets>.+))?$"
),
)
stray = sorted(self.root_frontmatter_required - set(self.root_frontmatter))
if stray:
raise ValueError(
@ -464,7 +567,24 @@ class IndexPolicy:
"""Whether an entry carries a description alongside label and target."""
return "{description}" in self.link_template
def render_link(self, label: str, target: str, description: str | None = None) -> str:
@property
def entry_pattern(self) -> re.Pattern[str]:
"""The pattern that recognises a managed line, facets included.
IS `link_pattern` when this policy carries no facets, which is what
makes the feature additive: DEFAULT and STRICT_V1 match exactly the
lines they always matched, byte for byte.
"""
return self._faceted_pattern if self._faceted_pattern is not None else self.link_pattern
def render_link(
self,
label: str,
target: str,
description: str | None = None,
*,
facets: Mapping[str, str] | None = None,
) -> str:
if self.requires_description and description is None:
raise ValueError(
"this index policy's entries carry a description; rendering "
@ -475,7 +595,33 @@ class IndexPolicy:
"this index policy's entries carry no description; the value "
"offered would be dropped silently"
)
return self.link_template.format(label=label, target=target, description=description)
if self.facets is None and facets:
raise ValueError(
"this index policy carries no facets; the values offered would be dropped silently"
)
line = self.link_template.format(label=label, target=target, description=description)
if self.facets is None or facets is None:
return line
return line + self.facets.render(facets)
def parse_entry(self, line: str) -> IndexEntry | None:
"""One managed line read back into its parts, or `None` for anything else.
Anything this returns `None` for is curated content and survives
verbatim: the index is the one file where this library writes beside
somebody else's prose.
"""
match = self.entry_pattern.match(line.rstrip("\r\n"))
if match is None:
return None
groups = match.groupdict()
tail = groups.get("facets")
return IndexEntry(
label=match.group("label"),
target=match.group("target"),
description=groups.get("description"),
facets=self.facets.parse(tail) if (self.facets is not None and tail) else {},
)
def required_indexes(self, directories: Sequence[str]) -> tuple[str, ...]:
"""The index paths this policy requires, given the caller's directories.
@ -526,7 +672,7 @@ class IndexPolicy:
if line.startswith("# "):
headings.append(line)
continue
match = self.link_pattern.match(line)
match = self.entry_pattern.match(line)
if match is not None:
listed.add(match.group("target"))
continue
@ -728,6 +874,53 @@ STRICT_V1 = BundleProfile(
)
# The ordered structure keys: what `structure.py` derives, plus the two
# producer-declared keys the 2026-08-26 bake-off measured missing from the
# index (`status` 55/55 in the documents, 0/55 in the index; `date` likewise).
# One tuple feeds both the frontmatter order and the facet set, because two
# lists of the same keys drift.
_STRUCTURE_KEYS = (
"number",
"parent",
"status",
"date",
"version",
"supersedes",
"references",
# LAST, and load-bearing: it names which of the keys before it this library
# INFERRED rather than read. A consumer that trusts nothing derived can
# still use everything else, and one that accepts both knows which half it
# is betting on. An unmarked heuristic is worse than no heuristic.
"derived",
)
# DEFAULT plus structure. Additive in the strict sense: the namespaces, the
# type policy and the ownership stamp are DEFAULT's own objects, so a bundle
# written under either profile stays re-runnable under the other, and DEFAULT's
# bytes do not move.
#
# Why a new profile rather than facets on DEFAULT: DEFAULT states commons'
# ingest-spec 6 index layer. Changing its rendered bytes from here would be
# this repo editing another repo's contract (O2), and it would churn every
# golden fixture that door has ever written. The measured defect is real, but
# the fix belongs beside the contract it changes, not inside one we do not own.
#
# The cost is deliberate and small. Facets are rendered only where a value
# exists, so a bundle whose documents declare nothing pays nothing, and the
# 2026-08-26 arm that lost on trap exposure was 6 031 characters against
# 21 879 for the head-scan it lost to - the headroom for carrying the metadata
# back into the index is most of that gap.
STRUCTURED_V1 = BundleProfile(
types=DEFAULT.types,
frontmatter=FrontmatterSchema(
order=(*DEFAULT.frontmatter.order, *_STRUCTURE_KEYS),
collapsed_keys=DEFAULT.frontmatter.collapsed_keys,
),
paths=DEFAULT.paths,
index=replace(DEFAULT.index, facets=FacetPolicy(keys=_STRUCTURE_KEYS)),
ownership=DEFAULT.ownership,
)
# OKF v0.2, as an ADDITIVE profile: `DEFAULT` states commons' ingest-spec §5
# layer and keeps stating it, so nothing here migrates anything. The key order
# is DEFAULT's followed by the §5 families v0.2 adds, which is also the order

154
tests/test_faceted_index.py Normal file
View file

@ -0,0 +1,154 @@
"""A faceted index: the entry carries the metadata its document carries.
The measured defect this closes, as data. ms-ai-architect ran a pre-registered
bake-off on 2026-08-26 over 55 documents and 40 gold questions; every arm hit
40/40, so retrieval quality could not separate them. The only axis that did
separate them was trap exposure — 18/20 for the OKF-index arm against 8/20 for
a frontmatter head-scan — and the reason was measured on both sides
independently: the flat DEFAULT index carries title/date/status/supersedes
0 times while the documents in the same bundle carry them 55/55/55/5.
The metadata IS in the bundle. The index throws it away. A facet policy is what
lets an index keep it, at a cost the index arm can still afford.
DEFAULT is untouched, and that is not caution: DEFAULT states commons'
ingest-spec §6 layer, so changing its bytes from here would be this repo
editing another repo's contract. Facets arrive as a new profile, additively.
"""
from __future__ import annotations
import re
import pytest
from llm_ingestion_okf.profiles import DEFAULT, STRICT_V1, STRUCTURED_V1, FacetPolicy, IndexPolicy
def test_default_carries_no_facets_and_renders_exactly_what_it_did() -> None:
# The byte-level proof that this feature is additive.
assert DEFAULT.index.facets is None
assert DEFAULT.index.render_link("A Title", "inbox-a.md") == "- [A Title](inbox-a.md)"
assert DEFAULT.index.entry_pattern is DEFAULT.index.link_pattern
assert STRICT_V1.index.facets is None
def test_a_faceted_policy_renders_present_facets_in_policy_order() -> None:
policy = STRUCTURED_V1.index
line = policy.render_link(
"N500 Vegbygging",
"inbox-n500-vegbygging.md",
facets={"status": "gjeldende", "number": "N500"},
)
assert line == (
"- [N500 Vegbygging](inbox-n500-vegbygging.md) — number: N500; status: gjeldende"
)
def test_facet_order_follows_the_policy_not_the_mapping() -> None:
# Two callers passing the same facts must emit the same bytes; a dict
# preserves insertion order, so ordering by the mapping would make a golden
# depend on how a caller happened to build its argument.
policy = STRUCTURED_V1.index
forwards = policy.render_link("T", "a.md", facets={"number": "N1", "status": "x"})
backwards = policy.render_link("T", "a.md", facets={"status": "x", "number": "N1"})
assert forwards == backwards
def test_an_entry_with_no_facet_values_renders_as_the_bare_link() -> None:
# A separator with nothing after it is a half-written entry, and it would
# cost every entry in a bundle that declares nothing.
assert STRUCTURED_V1.index.render_link("T", "a.md", facets={}) == "- [T](a.md)"
assert STRUCTURED_V1.index.render_link("T", "a.md") == "- [T](a.md)"
def test_a_facet_the_policy_does_not_name_is_refused() -> None:
# Silently dropping it would put a value in a file no reader of this
# contract looks at — the same posture the root frontmatter takes.
with pytest.raises(ValueError, match="not named"):
STRUCTURED_V1.index.render_link("T", "a.md", facets={"invented": "x"})
def test_offering_facets_to_a_policy_that_has_none_is_refused() -> None:
with pytest.raises(ValueError, match="carries no facets"):
DEFAULT.index.render_link("T", "a.md", facets={"number": "N1"})
@pytest.mark.parametrize("bad", ["a; b", "a — b", "a\nb"])
def test_a_facet_value_that_would_break_the_grammar_is_refused_never_repaired(bad: str) -> None:
# Validation, not repair — the standing posture everywhere in this library.
with pytest.raises(ValueError, match="separator|joiner|single-line"):
STRUCTURED_V1.index.render_link("T", "a.md", facets={"status": bad})
# --- round trip -----------------------------------------------------------
def test_render_and_parse_round_trip() -> None:
policy = STRUCTURED_V1.index
facets = {"number": "N500", "status": "gjeldende", "references": "[N200, N300?]"}
line = policy.render_link("N500 Vegbygging", "inbox-n500.md", facets=facets)
entry = policy.parse_entry(line)
assert entry is not None
assert entry.label == "N500 Vegbygging"
assert entry.target == "inbox-n500.md"
assert entry.facets == facets
def test_a_bare_link_still_parses_under_a_faceted_policy() -> None:
# An index built before facets existed must keep working: its lines are
# managed lines with no facets, not unmanaged prose.
entry = STRUCTURED_V1.index.parse_entry("- [A](inbox-a.md)")
assert entry is not None
assert entry.facets == {}
def test_prose_is_not_an_entry() -> None:
assert STRUCTURED_V1.index.parse_entry("Some curated prose about inbox-a.md.") is None
assert DEFAULT.index.parse_entry("# Heading") is None
def test_entry_pattern_is_anchored_to_the_whole_line() -> None:
# Index maintenance keys on this pattern to decide what it may rewrite;
# a substring match would let it edit curated prose.
assert STRUCTURED_V1.index.entry_pattern.match("prefix - [A](a.md)") is None
# --- construction guards --------------------------------------------------
def test_a_facet_policy_needs_at_least_one_key() -> None:
with pytest.raises(ValueError, match="at least one"):
FacetPolicy(keys=())
def test_a_link_pattern_that_is_not_line_anchored_cannot_carry_facets() -> None:
# The faceted pattern is built from this one, so an unanchored base would
# silently produce an unanchored faceted pattern.
with pytest.raises(ValueError, match="anchored"):
IndexPolicy(
name="index.md",
link_template="- [{label}]({target})",
link_pattern=re.compile(r"^- \[(?P<label>[^\]]*)\]\((?P<target>[^)]+)\)"),
facets=FacetPolicy(keys=("number",)),
)
def test_structured_v1_is_default_in_every_respect_but_the_index() -> None:
# An additive profile, stated as one: the namespaces, the type policy and
# the ownership stamp are DEFAULT's, so a bundle written under this profile
# stays re-runnable under DEFAULT and vice versa.
assert STRUCTURED_V1.paths == DEFAULT.paths
assert STRUCTURED_V1.types == DEFAULT.types
assert STRUCTURED_V1.ownership == DEFAULT.ownership
assert STRUCTURED_V1.index.name == DEFAULT.index.name
def test_structured_v1_names_the_structure_keys_in_its_frontmatter_order() -> None:
# A key the schema does not name is emitted in `emit`'s sorted tail, where
# `derived` would precede `number`: alphabetical order standing in for the
# contract's own.
order = STRUCTURED_V1.frontmatter.order
assert DEFAULT.frontmatter.order == order[: len(DEFAULT.frontmatter.order)]
for key in STRUCTURED_V1.index.facets.keys if STRUCTURED_V1.index.facets else ():
assert key in order