feat(consume): read a concept, with adjudication absence as its own state

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

tests/test_okf_consume.py

@@ -21,15 +21,20 @@ house pattern rather than new inventions:
 
 from __future__ import annotations
 
+import hashlib
 import os
 import sys
 from pathlib import Path
 
+import pytest
+
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(PROJECT_ROOT / "tools"))
 
 import okf_consume  # noqa: E402
 
+from llm_ingestion_okf.materialize import parse_frontmatter  # noqa: E402
+
 GOLDEN = PROJECT_ROOT / "examples" / "ingest-golden-segmented-okf-v0-2" / "expected-bundle"
 
 
@@ -108,3 +113,81 @@ def test_the_ref_covers_the_indexes_too_since_the_walk_reads_them(tmp_path: Path
     nested = copy / "krav" / "1-1" / "index.md"
     nested.write_text(nested.read_text(encoding="utf-8") + "\nfritekst\n", encoding="utf-8")
     assert okf_consume.bundle_ref(copy) != before
+
+
+# --- Step 2: one concept, read into a record ----------------------------------
+
+PROPOSED_CONCEPT = "krav/1-1/foerste-krav"
+ROOT_BUNDLE_ID = "b-golden-segmented-okf-v0-2"
+
+
+def _read(concept_id: str, root: Path = GOLDEN) -> okf_consume.Concept:
+    return okf_consume.read_concept(
+        root / f"{concept_id}.md", bundle_root=root, root_bundle_id=ROOT_BUNDLE_ID
+    )
+
+
+def test_a_concept_carrying_adjudication_reads_that_value() -> None:
+    assert _read(PROPOSED_CONCEPT).adjudication == "proposed"
+
+
+def test_a_concept_carrying_no_adjudication_key_reads_as_unknown(tmp_path: Path) -> None:
+    # SS 6.1: `unknown` is written EXPLICITLY. "Not judged" and "we cannot tell
+    # whether it was judged" are different facts, and only one is about the
+    # concept.
+    root = tmp_path / "bundle"
+    _copy_bundle(GOLDEN, root)
+    target = root / f"{PROPOSED_CONCEPT}.md"
+    target.write_text(
+        target.read_text(encoding="utf-8").replace("adjudication: proposed\n", ""),
+        encoding="utf-8",
+    )
+    concept = _read(PROPOSED_CONCEPT, root)
+    assert concept.adjudication == "unknown"
+    assert concept.adjudication_present is False
+
+
+def test_an_adjudication_value_outside_the_wire_set_is_refused_by_name(tmp_path: Path) -> None:
+    # Mapping an unrecognised value to `unknown` would report "we cannot tell"
+    # where the truth is "the bundle said something this consumer does not
+    # understand" -- a defect laundered into a state.
+    root = tmp_path / "bundle"
+    _copy_bundle(GOLDEN, root)
+    target = root / f"{PROPOSED_CONCEPT}.md"
+    target.write_text(
+        target.read_text(encoding="utf-8").replace("adjudication: proposed", "adjudication: seen"),
+        encoding="utf-8",
+    )
+    with pytest.raises(okf_consume.ConsumeError) as raised:
+        _read(PROPOSED_CONCEPT, root)
+    assert raised.value.code == "adjudication_unknown_value"
+
+
+def test_the_digest_is_of_the_concept_file_and_is_not_the_source_sha256() -> None:
+    concept = _read(PROPOSED_CONCEPT)
+    on_disk = hashlib.sha256((GOLDEN / f"{PROPOSED_CONCEPT}.md").read_bytes()).hexdigest()
+    assert concept.sha256 == on_disk
+    assert len(concept.sha256) == 64
+    frontmatter = parse_frontmatter(GOLDEN / f"{PROPOSED_CONCEPT}.md")
+    assert frontmatter["source_sha256"] != concept.sha256
+
+
+def test_the_concept_id_keeps_its_slashes_where_import_slug_would_flatten_them() -> None:
+    # `importer.import_slug` flattens one line below the rule this id follows.
+    # A flattened id fails a document-prefix match in a way that looks like a
+    # ranking miss rather than an id-format bug.
+    assert _read(PROPOSED_CONCEPT).concept_id == "krav/1-1/foerste-krav"
+
+
+def test_bundle_id_falls_back_to_the_root_index_and_says_that_it_did(tmp_path: Path) -> None:
+    root = tmp_path / "bundle"
+    _copy_bundle(GOLDEN, root)
+    target = root / f"{PROPOSED_CONCEPT}.md"
+    target.write_text(
+        target.read_text(encoding="utf-8").replace(f"bundle_id: {ROOT_BUNDLE_ID}\n", ""),
+        encoding="utf-8",
+    )
+    concept = _read(PROPOSED_CONCEPT, root)
+    assert concept.bundle_id == ROOT_BUNDLE_ID
+    assert concept.bundle_id_inherited is True
+    assert _read(PROPOSED_CONCEPT).bundle_id_inherited is False

tools/okf_consume.py

@@ -33,10 +33,19 @@ from __future__ import annotations
 
 import hashlib
 import sys
+from collections.abc import Mapping
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Literal
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
 
+from llm_ingestion_okf.inbox import (  # noqa: E402
+    ADJUDICATION_ADJUDICATED,
+    ADJUDICATION_PROPOSED,
+    ADJUDICATION_STATES,
+)
+from llm_ingestion_okf.materialize import parse_frontmatter  # noqa: E402
 from llm_ingestion_okf.profiles import SEGMENTED_OKF_V0_2, BundleProfile  # noqa: E402
 
 #: The profile whose index policy reads a faceted, per-directory index -- the
@@ -51,6 +60,10 @@ DEFAULT_PROFILE = SEGMENTED_OKF_V0_2
 #: the algorithm inline is what lets a consumer reproduce it without a document.
 REF_ALGORITHM = "sha256-tree"
 
+#: The one concept suffix this instrument reads, taken from the profile so a
+#: second literal cannot drift from it.
+CONCEPT_SUFFIX = DEFAULT_PROFILE.paths.concept_suffix
+
 
 def _walk_index_tree(
     bundle_root: Path, *, profile: BundleProfile
@@ -182,3 +195,112 @@ def bundle_ref(bundle_root: Path, *, profile: BundleProfile = DEFAULT_PROFILE) -
         lines.append(f"{relative}\t{digest}".encode())
     joined = b"\n".join(sorted(lines))
     return f"{REF_ALGORITHM}:{hashlib.sha256(joined).hexdigest()}"
+
+
+class ConsumeError(Exception):
+    """A refusal this instrument can name.
+
+    Carries a `code` for the same reason `okf_contract_check.Finding` does: one
+    "invalid" verdict over a dozen defects is a diagnostic no caller can act on.
+    """
+
+    def __init__(self, message: str, *, code: str) -> None:
+        super().__init__(message)
+        self.code = code
+
+
+#: The three states a CONSUMER must distinguish (SS 6.1), against the two the
+#: WIRE carries (`inbox.ADJUDICATION_STATES`). The difference is the whole
+#: point: `unknown` is not a value a producer writes, it is what the absence of
+#: the key means, and it is written explicitly here so a reader never has to
+#: infer it from a missing member.
+CONSUMER_ADJUDICATION_STATES = (*ADJUDICATION_STATES, "unknown")
+
+AdjudicationState = Literal["proposed", "adjudicated", "unknown"]
+
+
+@dataclass(frozen=True)
+class Concept:
+    """One concept read off disk, with every state written rather than implied."""
+
+    path: Path
+    concept_id: str
+    bundle_id: str
+    #: `True` when `bundle_id` came from the root index rather than the concept.
+    #: Recorded rather than silently defaulted: SS 3.1 makes identity the
+    #: `(bundle_id, concept_id)` tuple, so where the first half came from is
+    #: part of what the payload is asserting.
+    bundle_id_inherited: bool
+    sha256: str
+    okf_type: str
+    title: str
+    source_file: str
+    adjudication: AdjudicationState
+    #: `False` when the key was absent. `adjudication == "unknown"` already says
+    #: so, but a separate flag keeps the two facts from being one inference.
+    adjudication_present: bool
+    frontmatter: Mapping[str, str]
+    body: str
+
+
+def read_concept(path: Path, *, bundle_root: Path, root_bundle_id: str) -> Concept:
+    """One concept file as a record. Reads; derives nothing about the question.
+
+    `sha256` is the digest of the CONCEPT FILE (SS 3.2), never the
+    `source_sha256` frontmatter key -- that one digests the source document the
+    concept was extracted from, and conflating them would make the payload's
+    content identity point at a PDF nobody in the chain reads. Both exist on
+    every K2 concept, which is what makes the confusion available.
+    """
+    frontmatter = parse_frontmatter(path)
+    relative = path.relative_to(bundle_root).as_posix()
+    #: The slash-preserving id: the bundle-relative path minus the suffix. This
+    #: instrument's own choice, consistent with the RULE at `importer.py:244`
+    #: and `inbox.py:1101-1102` -- but deliberately NOT `importer.import_slug`,
+    #: which one line further down flattens the id to a single hyphenated
+    #: segment. A flattened id fails a document-prefix match in a way that looks
+    #: like a ranking miss rather than an id-format bug.
+    concept_id = relative[: -len(CONCEPT_SUFFIX)] if relative.endswith(CONCEPT_SUFFIX) else relative
+    raw = frontmatter.get("adjudication")
+    if raw is None:
+        adjudication: AdjudicationState = "unknown"
+    elif raw == ADJUDICATION_PROPOSED:
+        adjudication = "proposed"
+    elif raw == ADJUDICATION_ADJUDICATED:
+        adjudication = "adjudicated"
+    else:
+        raise ConsumeError(
+            f"{relative} carries adjudication={raw!r}, outside the wire set "
+            f"{ADJUDICATION_STATES}; refusing to map it to 'unknown', which "
+            "would report 'we cannot tell whether it was judged' where the "
+            "truth is that the bundle said something this consumer does not "
+            "understand",
+            code="adjudication_unknown_value",
+        )
+    declared = frontmatter.get("bundle_id")
+    return Concept(
+        path=path,
+        concept_id=concept_id,
+        bundle_id=declared if declared else root_bundle_id,
+        bundle_id_inherited=not declared,
+        sha256=hashlib.sha256(path.read_bytes()).hexdigest(),
+        okf_type=frontmatter.get("type", ""),
+        title=frontmatter.get("title", ""),
+        source_file=frontmatter.get("source_file", ""),
+        adjudication=adjudication,
+        adjudication_present=raw is not None,
+        frontmatter=frontmatter,
+        body=_body(path),
+    )
+
+
+def _body(path: Path) -> str:
+    """The text after the frontmatter block, or the whole file when there is none."""
+    text = path.read_text(encoding="utf-8")
+    lines = text.splitlines()
+    if not lines or lines[0].strip() != "---":
+        return text
+    for offset, line in enumerate(lines[1:], start=2):
+        if line.strip() == "---":
+            return "\n".join(lines[offset:])
+    return text