feat(okf): read_provenance names WHY a signal is unreadable, on the SkippedLink shape

Co-Authored-By: Claude <claude-opus-5>

src/portfolio_optimiser/okf.py

@@ -29,6 +29,7 @@ never written.
 
 from __future__ import annotations
 
+import itertools
 import json
 import posixpath
 import re
@@ -370,6 +371,96 @@ class SkippedLink:
     reason: SkipReason
 
 
+#: WHY a provenance value could not be read. The tokens name the SHAPE the value is written in
+#: and NOTHING else — the same discipline ``SkipReason`` carries. A block sequence and a block
+#: mapping are both CONFORMANT OKF (SPEC §5.2 writes ``verified`` in exactly those forms); they are
+#: simply outside the accepted single-line subset, and this decoder is not entitled to an opinion
+#: about whether the author erred. ``unsupported-flow`` is the one token that does denote a
+#: malformation, and it says so by naming the flow FORM rather than the author.
+ProvenanceReason = Literal["block-sequence", "block-mapping", "unsupported-flow"]
+
+
+@dataclass(frozen=True)
+class UnreadableProvenance:
+    """A provenance key that IS present and could NOT be read, and why — never silence.
+
+    Mirrors ``SkippedLink``: structured rather than a rendered string, for the reason
+    ``BudgetExceeded`` carries ``kind``/``limit``/``observed`` as fields (kø-(y)). "Which shape is
+    this written in" and "how many entries were there" are two separate operative questions, and a
+    caller forced to re-parse a token to tell them apart has been handed a diagnostic it cannot act
+    on. That is why the COUNT is its own field and is never folded into the token.
+
+    ``value`` is the offending text VERBATIM as written in the file, indentation included — the
+    operator fixing the document edits that text, and a normalised form would send them looking for
+    a string their file does not contain."""
+
+    #: Path of the document the key was read from.
+    file: str
+    #: The frontmatter key that could not be read.
+    key: str
+    #: The offending text exactly as it appears in that file.
+    value: str
+    reason: ProvenanceReason
+    #: Entries seen. ``0`` when nothing countable was there — an empty key, or a flow value the
+    #: decoder refused and therefore never enumerated.
+    items_seen: int
+
+
+def read_provenance(
+    path: str | Path, key: str
+) -> tuple[dict[str, str], ...] | UnreadableProvenance | None:
+    """Read one provenance key into entries, or say why it could not be read.
+
+    Three outcomes, and the three-way split is the whole point:
+
+    * ``None`` — the document does not carry the key. **Absence is ``None``, never a default**:
+      the F2 principle one layer down, where an unread signal must not become an asserted absent
+      one.
+    * a tuple of entries — the value was in the accepted flow subset and decoded.
+    * ``UnreadableProvenance`` — the key is THERE and could not be read, with the shape and the
+      count that say what was seen.
+
+    Driven by ``_split_frontmatter``: this is a second READER of the one parse, never a second
+    parser. Only top-level (unindented) frontmatter lines are considered as key sites, so an
+    indented ``by:`` inside a block entry can never be mistaken for a document-level key.
+
+    Gated by ``tests/test_provenance_decoder_loadbearing.py``."""
+    lines = _split_frontmatter(Path(path).read_text(encoding="utf-8"))[0]
+    for i, line in enumerate(lines):
+        if line[:1] in (" ", "\t"):
+            continue
+        name, sep, raw = line.partition(":")
+        if not sep or name.strip() != key:
+            continue
+        value = raw.strip()
+        if value:
+            try:
+                return decode_flow_value(value, key=key)
+            except FlowDecodeError:
+                return UnreadableProvenance(
+                    file=str(path), key=key, value=value, reason="unsupported-flow", items_seen=0
+                )
+        continuation = list(itertools.takewhile(lambda ln: ln[:1] in (" ", "\t"), lines[i + 1 :]))
+        if not continuation:
+            return UnreadableProvenance(
+                file=str(path), key=key, value="", reason="block-mapping", items_seen=0
+            )
+        # An ITEM is a continuation line whose STRIPPED form opens with ``- ``. A ``- `` occurring
+        # inside a value is text, not an item, and counting it would inflate the number the caller
+        # acts on.
+        items = sum(1 for ln in continuation if ln.strip().startswith("- "))
+        return UnreadableProvenance(
+            file=str(path),
+            key=key,
+            value="\n".join(continuation),
+            # SPEC §5.2's one-element MUST: a bare mapping IS one entry, so a mapping that is
+            # present counts 1 rather than 0 — 0 is reserved for "nothing was there at all".
+            reason="block-sequence" if items else "block-mapping",
+            items_seen=items or 1,
+        )
+    return None
+
+
 @dataclass(frozen=True)
 class Bundle:
     """A navigated OKF bundle: ``index.md`` plus every cross-linked file that resolves — and, in

tests/test_provenance_decoder_loadbearing.py

@@ -18,6 +18,8 @@ means unchanged against a measurement rather than against a recollection.
 from __future__ import annotations
 
 import ast
+import itertools
+import tempfile
 from pathlib import Path
 
 import pytest
@@ -67,6 +69,17 @@ _EXPECTED_BODY: dict[str, str] = {
 }
 
 
+_FIXTURE_COUNTER = itertools.count()
+
+
+def _fixture(text: str) -> Path:
+    """A throwaway document on disk. The accessor reads FILES, so an in-memory string would test a
+    different function than the one shipped."""
+    path = Path(tempfile.mkdtemp(prefix="okf-provenance-")) / f"c{next(_FIXTURE_COUNTER)}.md"
+    path.write_text(text, encoding="utf-8")
+    return path
+
+
 def _write(tmp_path: Path, shape: str) -> Path:
     path = tmp_path / f"{shape}.md"
     path.write_text(_SHAPES[shape], encoding="utf-8")
@@ -161,13 +174,14 @@ def test_load_file_reads_each_file_once(tmp_path: Path) -> None:
 def test_the_parsed_dict_loses_what_the_accessor_recovers(tmp_path: Path) -> None:
     """AMENDMENT C — the leak, shown by putting both readers on the SAME file in ONE arm.
 
-    ``parse_frontmatter`` hands back ``""`` for a block-form ``verified``; the provenance accessor
+    ``parse_frontmatter`` hands back ``""`` for a block-form ``verified``; the accessor hands back
-    hands back the shape and the entry count. Asserting only the accessor's answer would leave the
+    the SHAPE and the COUNT. Asserting only the accessor's answer would leave the *reason the
-    *reason the accessor exists* undocumented, and the reason is the whole of condition 2.
+    accessor exists* undocumented, and the reason is the whole of condition 2.
 
-    ``read_provenance`` arrives in Step 4. This arm is AUTHORED here and ENABLES ITSELF the moment
+    Authored in Step 2 against an accessor that did not exist yet, and it ENABLED ITSELF when the
-    the symbol exists — a self-enabling skip rather than a TODO, because a note in prose is a note
+    symbol arrived in Step 4 — a self-enabling skip rather than a TODO. What the arm asserts was
-    somebody has to remember to act on.
+    corrected at that point: an unreadable value yields ``UnreadableProvenance``, not entries, and
+    a test that had guessed otherwise would have been a test written against an imagined API.
     """
     read_provenance = getattr(okf, "read_provenance", None)
     if read_provenance is None:
@@ -176,9 +190,9 @@ def test_the_parsed_dict_loses_what_the_accessor_recovers(tmp_path: Path) -> Non
     path = _write(tmp_path, "block")
     assert parse_frontmatter(path)["verified"] == ""
 
-    provenance = read_provenance(path, key="verified")
+    provenance = read_provenance(path, "verified")
-    assert provenance.entries, "the accessor recovered nothing the parser had already lost"
+    assert isinstance(provenance, okf.UnreadableProvenance)
-    assert len(provenance.entries) == 2
+    assert (provenance.reason, provenance.items_seen) == ("block-sequence", 2)
 
 
 # --- Step 3: the flow-form decoder ------------------------------------------------------------
@@ -315,3 +329,105 @@ def test_a_verified_entry_with_no_actor_is_refused() -> None:
     assert okf.decode_flow_value("{ at: 2026-01-01T00:00:00Z }", key="sources") == (
         {"at": "2026-01-01T00:00:00Z"},
     )
+
+
+# --- Step 4: the provenance accessor and its reason vocabulary --------------------------------
+
+_FIXTURE_BUNDLE = (
+    Path(__file__).resolve().parents[1] / "tests" / "golden" / "block-form-provenance" / "bundle"
+)
+
+
+def test_a_single_entry_block_sequence_reports_shape_and_count() -> None:
+    """The committed fixture's one-entry block ``verified``: the PAIR is asserted, not the token.
+
+    ``reason`` alone cannot tell a one-entry list from a five-entry one, and the count alone cannot
+    say which shape produced it. Together they are strictly more information than a single
+    ``block-form``/``multi-entry`` token, and neither half mislabels the other.
+    """
+    result = okf.read_provenance(_FIXTURE_BUNDLE / "attested.md", "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert (result.reason, result.items_seen) == ("block-sequence", 1)
+    assert result.key == "verified"
+
+
+def test_a_two_entry_block_sequence_counts_both() -> None:
+    """The same shape, a different count — which is what makes ``items_seen`` a real discriminator."""
+    result = okf.read_provenance(_FIXTURE_BUNDLE / "multi-verified.md", "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert (result.reason, result.items_seen) == ("block-sequence", 2)
+
+
+def test_a_block_mapping_is_named_a_mapping_not_a_sequence() -> None:
+    """Both shapes are CONFORMANT OKF and merely outside the accepted subset.
+
+    The tokens name the shape the value is written in and nothing else — the decoder is not
+    entitled to an opinion about whether the author erred, and a token that implied one would file
+    this repo's own SPEC-canonical fixture as a defect.
+    """
+    path = _fixture(
+        "---\ntype: concept\nverified:\n  by: human:a\n  at: 2026-01-01T00:00:00Z\n---\nbody\n"
+    )
+    result = okf.read_provenance(path, "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert (result.reason, result.items_seen) == ("block-mapping", 1)
+
+
+def test_a_key_with_an_empty_value_and_no_continuation_counts_nothing() -> None:
+    """``verified:`` with nothing under it saw zero entries, and says so."""
+    path = _fixture("---\ntype: concept\nverified:\ntitle: x\n---\nbody\n")
+    result = okf.read_provenance(path, "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert (result.reason, result.items_seen) == ("block-mapping", 0)
+
+
+def test_a_flow_value_the_decoder_refuses_is_named_unsupported_flow() -> None:
+    """``unsupported-flow`` is the ONE token that denotes a malformation, and it names the FORM."""
+    path = _fixture("---\ntype: concept\nsources: [ a.pdf, b.pdf ]\n---\nbody\n")
+    result = okf.read_provenance(path, "sources")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert result.reason == "unsupported-flow"
+    assert result.value == "[ a.pdf, b.pdf ]"
+
+
+def test_a_readable_flow_value_comes_back_decoded() -> None:
+    """CONTROL — an accessor that only ever answered ``UnreadableProvenance`` would pass every
+    negative arm above and be worthless."""
+    path = _fixture(
+        "---\ntype: concept\nsources: [{ id: golden-v0-2-sales, resource: fixture }]\n---\nbody\n"
+    )
+    assert okf.read_provenance(path, "sources") == (
+        {"id": "golden-v0-2-sales", "resource": "fixture"},
+    )
+
+
+def test_an_absent_key_is_None_never_a_default() -> None:
+    """Absence is ``None`` — the F2 principle one layer down.
+
+    An unread signal must not become an asserted absent one: ``None`` says "this document does not
+    carry the key", which is a different fact from "the key is there and unreadable" and from
+    "the key is there and empty".
+    """
+    path = _fixture("---\ntype: concept\ntitle: x\n---\nbody\n")
+    assert okf.read_provenance(path, "sources") is None
+
+
+def test_a_literal_dash_inside_a_value_does_not_inflate_the_count() -> None:
+    """An item is a continuation line whose STRIPPED form starts with ``- ``, not a substring."""
+    path = _fixture("---\ntype: concept\nverified:\n  by: human:a - reviewed - twice\n---\nbody\n")
+    result = okf.read_provenance(path, "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert (result.reason, result.items_seen) == ("block-mapping", 1)
+
+
+def test_the_offending_text_is_carried_verbatim() -> None:
+    """``value`` is what the operator will edit, indentation included — never a normalised form
+    that would send them looking for a string their file does not contain (the ``SkippedLink``
+    rule, applied one layer down)."""
+    path = _fixture(
+        "---\ntype: concept\nverified:\n  - { by: human:a, at: 2026-01-01T00:00:00Z }\n---\nbody\n"
+    )
+    result = okf.read_provenance(path, "verified")
+    assert isinstance(result, okf.UnreadableProvenance)
+    assert result.value == "  - { by: human:a, at: 2026-01-01T00:00:00Z }"
+    assert result.file == str(path)