open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(consume): read a concept, with adjudication absence as its own state

Browse source 
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-07 09:08:41 +02:00
commit 1ec3f5289c
2 changed files with 205 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

83
tests/test_okf_consume.py
View file

@ -21,15 +21,20 @@ house pattern rather than new inventions:
from __future__ import annotations
import hashlib
import os
import sys
from pathlib import Path
import pytest
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
import okf_consume # noqa: E402
from llm_ingestion_okf.materialize import parse_frontmatter # noqa: E402
GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
@ -108,3 +113,81 @@ def test_the_ref_covers_the_indexes_too_since_the_walk_reads_them(tmp_path: Path
nested = copy / "krav" / "1-1" / "index.md"
nested.write_text(nested.read_text(encoding="utf-8") + "\nfritekst\n", encoding="utf-8")
assert okf_consume.bundle_ref(copy) != before
# --- Step 2: one concept, read into a record ----------------------------------
PROPOSED_CONCEPT = "krav/1-1/foerste-krav"
ROOT_BUNDLE_ID = "b-golden-segmented-okf-v0-2"
def _read(concept_id: str, root: Path = GOLDEN) -> okf_consume.Concept:
return okf_consume.read_concept(
root / f"{concept_id}.md", bundle_root=root, root_bundle_id=ROOT_BUNDLE_ID
)
def test_a_concept_carrying_adjudication_reads_that_value() -> None:
assert _read(PROPOSED_CONCEPT).adjudication == "proposed"
def test_a_concept_carrying_no_adjudication_key_reads_as_unknown(tmp_path: Path) -> None:
# SS 6.1: `unknown` is written EXPLICITLY. "Not judged" and "we cannot tell
# whether it was judged" are different facts, and only one is about the
# concept.
root = tmp_path / "bundle"
_copy_bundle(GOLDEN, root)
target = root / f"{PROPOSED_CONCEPT}.md"
target.write_text(
target.read_text(encoding="utf-8").replace("adjudication: proposed\n", ""),
encoding="utf-8",
)
concept = _read(PROPOSED_CONCEPT, root)
assert concept.adjudication == "unknown"
assert concept.adjudication_present is False
def test_an_adjudication_value_outside_the_wire_set_is_refused_by_name(tmp_path: Path) -> None:
# Mapping an unrecognised value to `unknown` would report "we cannot tell"
# where the truth is "the bundle said something this consumer does not
# understand" -- a defect laundered into a state.
root = tmp_path / "bundle"
_copy_bundle(GOLDEN, root)
target = root / f"{PROPOSED_CONCEPT}.md"
target.write_text(
target.read_text(encoding="utf-8").replace("adjudication: proposed", "adjudication: seen"),
encoding="utf-8",
)
with pytest.raises(okf_consume.ConsumeError) as raised:
_read(PROPOSED_CONCEPT, root)
assert raised.value.code == "adjudication_unknown_value"
def test_the_digest_is_of_the_concept_file_and_is_not_the_source_sha256() -> None:
concept = _read(PROPOSED_CONCEPT)
on_disk = hashlib.sha256((GOLDEN / f"{PROPOSED_CONCEPT}.md").read_bytes()).hexdigest()
assert concept.sha256 == on_disk
assert len(concept.sha256) == 64
frontmatter = parse_frontmatter(GOLDEN / f"{PROPOSED_CONCEPT}.md")
assert frontmatter["source_sha256"] != concept.sha256
def test_the_concept_id_keeps_its_slashes_where_import_slug_would_flatten_them() -> None:
# `importer.import_slug` flattens one line below the rule this id follows.
# A flattened id fails a document-prefix match in a way that looks like a
# ranking miss rather than an id-format bug.
assert _read(PROPOSED_CONCEPT).concept_id == "krav/1-1/foerste-krav"
def test_bundle_id_falls_back_to_the_root_index_and_says_that_it_did(tmp_path: Path) -> None:
root = tmp_path / "bundle"
_copy_bundle(GOLDEN, root)
target = root / f"{PROPOSED_CONCEPT}.md"
target.write_text(
target.read_text(encoding="utf-8").replace(f"bundle_id: {ROOT_BUNDLE_ID}\n", ""),
encoding="utf-8",
)
concept = _read(PROPOSED_CONCEPT, root)
assert concept.bundle_id == ROOT_BUNDLE_ID
assert concept.bundle_id_inherited is True
assert _read(PROPOSED_CONCEPT).bundle_id_inherited is False

122
tools/okf_consume.py
View file

@ -33,10 +33,19 @@ from __future__ import annotations
import hashlib
import sys
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Literal
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from llm_ingestion_okf.inbox import ( # noqa: E402
ADJUDICATION_ADJUDICATED,
ADJUDICATION_PROPOSED,
ADJUDICATION_STATES,
)
from llm_ingestion_okf.materialize import parse_frontmatter # noqa: E402
from llm_ingestion_okf.profiles import SEGMENTED_OKF_V0_2, BundleProfile # noqa: E402
#: The profile whose index policy reads a faceted, per-directory index -- the
@ -51,6 +60,10 @@ DEFAULT_PROFILE = SEGMENTED_OKF_V0_2
#: the algorithm inline is what lets a consumer reproduce it without a document.
REF_ALGORITHM = "sha256-tree"
#: The one concept suffix this instrument reads, taken from the profile so a
#: second literal cannot drift from it.
CONCEPT_SUFFIX = DEFAULT_PROFILE.paths.concept_suffix
def _walk_index_tree(
bundle_root: Path, *, profile: BundleProfile
@ -182,3 +195,112 @@ def bundle_ref(bundle_root: Path, *, profile: BundleProfile = DEFAULT_PROFILE) -
lines.append(f"{relative}\t{digest}".encode())
joined = b"\n".join(sorted(lines))
return f"{REF_ALGORITHM}:{hashlib.sha256(joined).hexdigest()}"
class ConsumeError(Exception):
"""A refusal this instrument can name.
Carries a `code` for the same reason `okf_contract_check.Finding` does: one
"invalid" verdict over a dozen defects is a diagnostic no caller can act on.
"""
def __init__(self, message: str, *, code: str) -> None:
super().__init__(message)
self.code = code
#: The three states a CONSUMER must distinguish (SS 6.1), against the two the
#: WIRE carries (`inbox.ADJUDICATION_STATES`). The difference is the whole
#: point: `unknown` is not a value a producer writes, it is what the absence of
#: the key means, and it is written explicitly here so a reader never has to
#: infer it from a missing member.
CONSUMER_ADJUDICATION_STATES = (*ADJUDICATION_STATES, "unknown")
AdjudicationState = Literal["proposed", "adjudicated", "unknown"]
@dataclass(frozen=True)
class Concept:
"""One concept read off disk, with every state written rather than implied."""
path: Path
concept_id: str
bundle_id: str
#: `True` when `bundle_id` came from the root index rather than the concept.
#: Recorded rather than silently defaulted: SS 3.1 makes identity the
#: `(bundle_id, concept_id)` tuple, so where the first half came from is
#: part of what the payload is asserting.
bundle_id_inherited: bool
sha256: str
okf_type: str
title: str
source_file: str
adjudication: AdjudicationState
#: `False` when the key was absent. `adjudication == "unknown"` already says
#: so, but a separate flag keeps the two facts from being one inference.
adjudication_present: bool
frontmatter: Mapping[str, str]
body: str
def read_concept(path: Path, *, bundle_root: Path, root_bundle_id: str) -> Concept:
"""One concept file as a record. Reads; derives nothing about the question.
`sha256` is the digest of the CONCEPT FILE (SS 3.2), never the
`source_sha256` frontmatter key -- that one digests the source document the
concept was extracted from, and conflating them would make the payload's
content identity point at a PDF nobody in the chain reads. Both exist on
every K2 concept, which is what makes the confusion available.
"""
frontmatter = parse_frontmatter(path)
relative = path.relative_to(bundle_root).as_posix()
#: The slash-preserving id: the bundle-relative path minus the suffix. This
#: instrument's own choice, consistent with the RULE at `importer.py:244`
#: and `inbox.py:1101-1102` -- but deliberately NOT `importer.import_slug`,
#: which one line further down flattens the id to a single hyphenated
#: segment. A flattened id fails a document-prefix match in a way that looks
#: like a ranking miss rather than an id-format bug.
concept_id = relative[: -len(CONCEPT_SUFFIX)] if relative.endswith(CONCEPT_SUFFIX) else relative
raw = frontmatter.get("adjudication")
if raw is None:
adjudication: AdjudicationState = "unknown"
elif raw == ADJUDICATION_PROPOSED:
adjudication = "proposed"
elif raw == ADJUDICATION_ADJUDICATED:
adjudication = "adjudicated"
else:
raise ConsumeError(
f"{relative} carries adjudication={raw!r}, outside the wire set "
f"{ADJUDICATION_STATES}; refusing to map it to 'unknown', which "
"would report 'we cannot tell whether it was judged' where the "
"truth is that the bundle said something this consumer does not "
"understand",
code="adjudication_unknown_value",
)
declared = frontmatter.get("bundle_id")
return Concept(
path=path,
concept_id=concept_id,
bundle_id=declared if declared else root_bundle_id,
bundle_id_inherited=not declared,
sha256=hashlib.sha256(path.read_bytes()).hexdigest(),
okf_type=frontmatter.get("type", ""),
title=frontmatter.get("title", ""),
source_file=frontmatter.get("source_file", ""),
adjudication=adjudication,
adjudication_present=raw is not None,
frontmatter=frontmatter,
body=_body(path),
)
def _body(path: Path) -> str:
"""The text after the frontmatter block, or the whole file when there is none."""
text = path.read_text(encoding="utf-8")
lines = text.splitlines()
if not lines or lines[0].strip() != "---":
return text
for offset, line in enumerate(lines[1:], start=2):
if line.strip() == "---":
return "\n".join(lines[offset:])
return text