portfolio-optimiser-claude/tests/test_okf.py

"""OKF navigation + read-context rendering (method-spec §3 Step 1, §11).

Load-bearing seams proved here:

- **Verdict-layer exclusion:** the realization signal must NEVER appear in the
  rendered read-context — prior verdicts reach the prompt ONLY via the gated
  experience fold. The test is RED if ``type: verdict`` files leak into rendering.
- **Context-seam purity (import guard):** the navigation/context module imports
  no agent toolkit — and stays pure stdlib (D7-portable by design).
"""

from __future__ import annotations

import ast
import sys
from pathlib import Path

import pytest

from portfolio_optimiser_claude.okf import (
    ConceptFile,
    bundle_context,
    navigate_bundle,
    parse_concept_file,
)

BUNDLE = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
NAV_GOLDENS = Path(__file__).resolve().parents[1] / "shared" / "examples"
SRC_PKG = Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser_claude"


def _write(path: Path, text: str) -> None:
    path.write_text(text, encoding="utf-8")


def _make_bundle(tmp_path: Path, index_body: str, files: dict[str, str]) -> Path:
    _write(tmp_path / "index.md", f"---\ntype: index\ntitle: Test\n---\n{index_body}\n")
    for name, text in files.items():
        _write(tmp_path / name, text)
    return tmp_path


class TestNavigation:
    """§3 Step 1: navigate from index.md — deterministic, tolerant, boundary-checked."""

    def test_missing_index_is_an_error(self, tmp_path: Path) -> None:
        # A bundle has no entry point without index.md.
        with pytest.raises(FileNotFoundError):
            navigate_bundle(tmp_path)

    def test_order_is_index_first_then_first_seen_deduplicated(self, tmp_path: Path) -> None:
        bundle = _make_bundle(
            tmp_path,
            "See [b](b.md) then [a](a.md) then [b again](b.md).",
            {
                "a.md": "---\ntype: project\ntitle: A\n---\nBody A.",
                "b.md": "---\ntype: reference\ntitle: B\n---\nBody B.",
            },
        )
        names = [c.path.name for c in navigate_bundle(bundle)]
        assert names == ["index.md", "b.md", "a.md"]

    def test_frontmatterless_index_navigates(self, tmp_path: Path) -> None:
        # method-spec §3 Step 1: the index contributes "the index body (the summary)"
        # and is NOT a "non-index concept file" rendered as a `## {type}: {title}`
        # section — so a generated index carrying only the summary + links (ingest
        # spec §6 shape, frozen in the ingest golden) is valid. The index MAY omit
        # frontmatter; okf must navigate it via its body, never raise.
        _write(tmp_path / "index.md", "Summary line.\n- [A](a.md)\n")
        _write(tmp_path / "a.md", "---\ntype: project\ntitle: A\n---\nBody A.")
        concepts = navigate_bundle(tmp_path)
        assert [c.path.name for c in concepts] == ["index.md", "a.md"]
        index = concepts[0]
        assert index.type == "index"  # default type, no KeyError for downstream .type reads
        assert index.body == "Summary line.\n- [A](a.md)"  # body = the whole file
        # The summary flows into the read-context as the leading section.
        assert bundle_context(tmp_path).startswith("Summary line.")

    def test_index_with_frontmatter_lacking_type_navigates(self, tmp_path: Path) -> None:
        # The index is the bundle ENTRY POINT, not a "non-index concept file" — so
        # `type` is not required OF IT, whether or not it carries a frontmatter
        # block. Keying the tolerance on the ABSENCE of frontmatter was the defect:
        # an index that carries frontmatter for some OTHER reason (an OKF v0.2
        # generated index stamps `okf_version` there) reached `parse_concept_file`
        # and killed the whole navigation — not just the index read. RED if the
        # tolerance is keyed on absence-of-frontmatter again.
        _write(tmp_path / "index.md", "---\nokf_version: 0.2\n---\nSummary line.\n- [A](a.md)\n")
        _write(tmp_path / "a.md", "---\ntype: project\ntitle: A\n---\nBody A.")
        concepts = navigate_bundle(tmp_path)
        assert [c.path.name for c in concepts] == ["index.md", "a.md"]
        index = concepts[0]
        assert index.type == "index"  # defaulted, not read from the file
        assert index.frontmatter["okf_version"] == "0.2"  # unknown fields preserved
        assert index.body == "Summary line.\n- [A](a.md)"  # frontmatter stripped from body
        assert bundle_context(tmp_path).startswith("Summary line.")

    def test_frontmatterless_tolerance_is_scoped_to_the_index(self, tmp_path: Path) -> None:
        # LOAD-BEARING (honesty): the relaxation is for the index ENTRY POINT only.
        # A NON-index concept file without frontmatter is still malformed and MUST
        # raise — else the relaxation has silently weakened the `type`-required rule
        # for concept files (method-spec §3 Step 1). RED if the tolerance leaks.
        _write(tmp_path / "index.md", "Summary.\n- [A](a.md)\n")
        _write(tmp_path / "a.md", "no frontmatter here\n")
        with pytest.raises(ValueError, match="frontmatter"):
            navigate_bundle(tmp_path)

    def test_out_of_bundle_and_escaping_targets_are_skipped(self, tmp_path: Path) -> None:
        # ../-escapes never resolve outside the bundle (fail-closed), and are skipped
        # rather than raised. `sub/inner.md` is skipped here because it does not
        # EXIST, not because it carries a separator — the separator heuristic is
        # retired (see TestNavigationBoundary); depth is legal, escape is not.
        outside = tmp_path / "outside.md"
        _write(outside, "---\ntype: project\ntitle: Outside\n---\nSecret.")
        bundle_dir = tmp_path / "bundle"
        bundle_dir.mkdir()
        bundle = _make_bundle(
            bundle_dir,
            "Links: [up](../outside.md), [sub](sub/inner.md), [ok](a.md).",
            {"a.md": "---\ntype: project\ntitle: A\n---\nBody A."},
        )
        names = [c.path.name for c in navigate_bundle(bundle)]
        assert names == ["index.md", "a.md"]

    def test_broken_link_is_tolerated_never_raised(self, tmp_path: Path) -> None:
        bundle = _make_bundle(
            tmp_path,
            "Links: [gone](missing.md), [ok](a.md).",
            {"a.md": "---\ntype: project\ntitle: A\n---\nBody A."},
        )
        names = [c.path.name for c in navigate_bundle(bundle)]
        assert names == ["index.md", "a.md"]

    def test_null_byte_target_is_tolerated_never_raised(self, tmp_path: Path) -> None:
        # LOAD-BEARING (§11 "Navigation boundary": a malformed target MUST be
        # skipped, not raised). A NUL byte is an INVALID PATH COMPONENT, so
        # ``resolve``/``is_file`` raises ``ValueError: embedded null byte`` inside
        # ``_resolve_target``. That ValueError MUST be caught and the link skipped —
        # a NUL target is a broken link, not a fatal error. It is the malformed
        # sub-class the escape golden's README delegates here, precisely because a
        # literal NUL byte does not belong in a committed text fixture.
        # RED (ValueError propagates) if the resolve guard is detached.
        bundle = _make_bundle(
            tmp_path,
            "Links: [bad](ba\x00d.md), [ok](a.md).",
            {"a.md": "---\ntype: project\ntitle: A\n---\nBody A."},
        )
        names = [c.path.name for c in navigate_bundle(bundle)]
        assert names == ["index.md", "a.md"]

    def test_shared_bundle_navigates_all_linked_concepts(self) -> None:
        # Integration on the shared example bundle: every index-linked file is
        # reached exactly once; the out-of-bundle ../../README.md link is skipped.
        names = [c.path.name for c in navigate_bundle(BUNDLE)]
        assert names[0] == "index.md"
        assert len(names) == len(set(names))
        assert set(names) == {
            "index.md",
            "bygg-kontor-nord.md",
            "tiltak-led-retrofit.md",
            "metode-ipmvp-a.md",
            "kilder-realiseringsgap.md",
            "verdict-led-fro.md",
        }


class TestNavigationBoundary:
    """LOAD-BEARING (§11 seam "Navigation boundary"): escape, not depth, is forbidden.

    Gated by the COMMONS-OWNED nav-goldens, whose shape is ``bundle/`` in →
    ``expected-read-context.md`` out — the only shape that can express §3 Step 1's
    property "two conformant implementations produce an identical read-context from
    the same bundle". The positive case pins hierarchy, both link forms, depth-first
    first-seen order, resolved-path dedup, cycle termination, recursive verdict
    exclusion and flat rendering; the negative case pins escape, and exists so the
    gate can go RED at all (a gate that can only pass proves nothing).

    RED if the retired "a path separator means out-of-bundle" heuristic returns (the
    hierarchy case loses every nested concept), if dedup re-keys on the raw target,
    if a leading ``/`` is read as filesystem-absolute, or if nested index bodies
    start rendering as content.
    """

    def _read_context(self, case: str) -> tuple[str, str]:
        root = NAV_GOLDENS / case
        expected = (root / "expected-read-context.md").read_text(encoding="utf-8")
        # The fixture README pins the shape and permits trailing-whitespace
        # normalization; only the trailing newline is normalized here.
        return bundle_context(root / "bundle").rstrip("\n"), expected.rstrip("\n")

    def test_hierarchy_golden_renders_the_expected_read_context(self) -> None:
        rendered, expected = self._read_context("nav-golden-hierarchy")
        assert rendered == expected

    def test_escape_golden_renders_the_expected_read_context(self) -> None:
        rendered, expected = self._read_context("nav-golden-escape")
        assert rendered == expected

    def test_escape_golden_never_reads_the_out_of_bundle_decoy(self) -> None:
        # Stated separately from the golden compare because the decoy REALLY exists
        # one level above ``bundle/``: a ``..`` escape would succeed, so its absence
        # is a boundary proof rather than a missing-file accident.
        rendered, _ = self._read_context("nav-golden-escape")
        # Positive control: the decoy file REALLY carries both strings, in EXACTLY the
        # form the negatives search for. The comment above asserted this in prose; a
        # fixture that lost the file (or reworded it) would satisfy both `not in`
        # checks without the boundary doing anything.
        decoy = (NAV_GOLDENS / "nav-golden-escape" / "SHOULD-NOT-BE-READ.md").read_text(
            encoding="utf-8"
        )
        assert "MUST never be reached" in decoy
        assert "Decoy" in decoy
        assert "MUST never be reached" not in rendered
        assert "Decoy" not in rendered

    def test_leading_slash_resolves_to_the_bundle_root_and_is_followed(
        self, tmp_path: Path
    ) -> None:
        # The root-relative form must be FOLLOWED, not merely survived. The escape
        # golden's `/etc/passwd` trap is skipped either way (as a bundle-root miss
        # under the rule, as a boundary violation under a filesystem reading), so it
        # cannot distinguish the two; a root-relative target that EXISTS can.
        bundle = _make_bundle(tmp_path, "Root-relative: [T](/deep/target.md).", {})
        (bundle / "deep").mkdir()
        _write(
            bundle / "deep" / "target.md",
            "---\ntype: project\ntitle: T\n---\nRoot-relative body.",
        )
        assert [c.path.name for c in navigate_bundle(bundle)] == ["index.md", "target.md"]

    def test_dedup_is_keyed_on_the_resolved_path_not_the_raw_target(self, tmp_path: Path) -> None:
        # `./a.md` and `a.md` are two raw strings and one resolved path — RED if the
        # dedup key regresses to the raw target.
        bundle = _make_bundle(
            tmp_path,
            "See [a](a.md) then [a again](./a.md).",
            {"a.md": "---\ntype: project\ntitle: A\n---\nBody A."},
        )
        assert [c.path.name for c in navigate_bundle(bundle)] == ["index.md", "a.md"]

    def test_a_nested_directory_without_its_own_index_is_not_an_error(self, tmp_path: Path) -> None:
        # The missing-index rule binds the bundle ROOT alone: an intermediate
        # directory is navigated only through the links its files carry, so a nested
        # directory lacking index.md is unreachable, never fatal.
        bundle = _make_bundle(tmp_path, "Down: [d](sub/doc.md).", {})
        (bundle / "sub").mkdir()
        _write(bundle / "sub" / "doc.md", "---\ntype: project\ntitle: D\n---\nNested body.")
        assert [c.path.name for c in navigate_bundle(bundle)] == ["index.md", "doc.md"]


class TestFrontmatter:
    """§3 Step 1: leading ``---`` block, line-oriented ``key: value``; ``type`` required."""

    def test_missing_type_is_an_error(self, tmp_path: Path) -> None:
        # LOAD-BEARING (honesty), and the scope guard for the index tolerance: the
        # vehicle MUST be a NON-index concept file. `type` is required of concept
        # files, never of the entry point — an index.md vehicle would only re-assert
        # the entry-point behaviour, which is now (correctly) the opposite. RED if
        # the index's `type` default ever leaks onto concept files.
        _write(tmp_path / "index.md", "Summary. See [a](a.md).")
        _write(tmp_path / "a.md", "---\ntitle: No type\n---\nBody.")
        with pytest.raises(ValueError, match="type"):
            navigate_bundle(tmp_path)

    def test_unknown_fields_preserved_and_quotes_stripped(self, tmp_path: Path) -> None:
        _write(
            tmp_path / "index.md",
            '---\ntype: index\ntitle: "Quoted title"\ncustom_field: kept\n---\nBody.',
        )
        index = navigate_bundle(tmp_path)[0]
        assert isinstance(index, ConceptFile)
        assert index.type == "index"
        assert index.title == "Quoted title"
        assert index.frontmatter["custom_field"] == "kept"


class TestRendering:
    """§3 Step 1: index body first, then ``## {type}: {title}`` sections."""

    def test_sections_are_typed_and_titled_after_index_body(self, tmp_path: Path) -> None:
        bundle = _make_bundle(
            tmp_path,
            "The summary.\n\nSee [a](a.md).",
            {"a.md": "---\ntype: project\ntitle: Building A\n---\nProject body."},
        )
        context = bundle_context(bundle)
        assert context.startswith("The summary.")
        assert "## project: Building A" in context
        assert "Project body." in context

    def test_empty_sections_are_dropped(self, tmp_path: Path) -> None:
        bundle = _make_bundle(
            tmp_path,
            "Summary. See [empty](empty.md) and [full](full.md).",
            {
                "empty.md": "---\ntype: reference\ntitle: Empty\n---\n",
                "full.md": "---\ntype: reference\ntitle: Full\n---\nBody.",
            },
        )
        context = bundle_context(bundle)
        # Positive control: a reference WITH a body renders in EXACTLY the heading form
        # the negative searches for — so the absence below is the empty-drop, not a
        # renamed heading or an unnavigated file.
        assert "## reference: Full" in context
        assert "## reference: Empty" not in context


class TestVerdictLayerExclusion:
    """LOAD-BEARING (§3 Step 1, §11): verdicts never leak via the read-context."""

    def test_navigable_verdict_is_excluded_from_rendering(self, tmp_path: Path) -> None:
        # Control + seam in one: the verdict file IS navigable (so exclusion is
        # doing real work), yet its unique marker never reaches the read-context.
        # RED if rendering stops gating on ``type: verdict``.
        marker = "UNIQUE-REALIZATION-MARKER-0.82"
        bundle = _make_bundle(
            tmp_path,
            "Summary. See [v](v.md) and [a](a.md).",
            {
                "v.md": f"---\ntype: verdict\ntitle: Seed\n---\nSignal: {marker}.",
                "a.md": "---\ntype: project\ntitle: A\n---\nBody A.",
            },
        )
        navigated = [c.path.name for c in navigate_bundle(bundle)]
        assert "v.md" in navigated
        context = bundle_context(bundle)
        assert marker not in context
        assert "## verdict" not in context
        assert "## project: A" in context

    def test_nested_block_cannot_forge_the_type_that_gates_exclusion(self, tmp_path: Path) -> None:
        # The exclusion in `bundle_context` gates on `.type`, and `.type` is whatever
        # the frontmatter parse produced. A line-oriented parse that lets an INDENTED
        # `type:` reach the same dict makes the gate forgeable: the file still declares
        # `type: verdict` at column 0, but a later nested key overwrites it and the
        # verdict body renders. Spec-legal input — method-spec §2 calls this YAML
        # frontmatter, and ingest-spec §7 (`:153`, `:216`) says unknown keys MAY follow
        # and ride through navigation.
        # RED if nested keys participate in the frontmatter mapping.
        marker = "NESTED-FORGERY-MARKER-0.91"
        bundle = _make_bundle(
            tmp_path,
            "Summary. See [v](v.md) and [a](a.md).",
            {
                "v.md": (
                    "---\ntype: verdict\ntitle: Seed\n"
                    "provenance:\n  type: project\n---\n"
                    f"Signal: {marker}."
                ),
                "a.md": "---\ntype: project\ntitle: A\n---\nBody A.",
            },
        )
        # Positive control #1 — the file IS navigated, so a marker-absence below is the
        # exclusion doing work, not an unreachable file. Stands alone, behind no other
        # assert (session 17: a control can itself hide behind a preceding assertion).
        assert "v.md" in [c.path.name for c in navigate_bundle(bundle)]
        context = bundle_context(bundle)
        # Positive control #2 — a rendered concept proves the context is non-empty and
        # that `## {type}: {title}` is EXACTLY the form the negatives search for.
        assert "## project: A" in context
        assert marker not in context
        assert "## project: Seed" not in context

    def test_nested_block_does_not_collide_with_a_top_level_key(self, tmp_path: Path) -> None:
        # Two sibling blocks sharing an inner key (OKF §10 Attested Computation:
        # `executor.resource` + `attester.resource`) must not collapse into one
        # top-level `resource`, silently losing the first and promoting the second.
        # RED if indented keys land in the frontmatter mapping.
        _write(
            tmp_path / "c.md",
            "---\ntype: project\ntitle: C\nresource: top-level-value\n"
            "executor:\n  resource: references/skills/run-on-bq.md\n"
            "attester:\n  resource: references/attesters/revenue.py\n---\nBody C.",
        )
        frontmatter = parse_concept_file(tmp_path / "c.md").frontmatter
        # Positive control — a COLUMN-0 key of the very name under test survives the
        # parse. Without it, "the nested value is absent" would also hold for a parser
        # that dropped `resource` entirely.
        assert frontmatter["resource"] == "top-level-value"
        assert "references/attesters/revenue.py" not in frontmatter.values()
        assert "references/skills/run-on-bq.md" not in frontmatter.values()

    def test_shared_bundle_context_carries_no_realization_signal(self) -> None:
        # The seed verdict's learning signal (realization rate 0.82, expected
        # actual 24 600 NOK) must be absent from the rendered context — it may
        # reach the prompt ONLY via the gated experience fold.
        context = bundle_context(BUNDLE)
        # Positive control: every leak string IS live in the bundle's verdict file, in
        # EXACTLY the form the loop below searches for. Without it "absent from the
        # context" would also hold for a bundle that never carried the signal at all.
        leaks = ("0.82", "0,82", "24600", "24 600", "realization_rate")
        verdict_text = (BUNDLE / "verdict-led-fro.md").read_text(encoding="utf-8")
        for leak in leaks:
            assert leak in verdict_text
        assert "## verdict" not in context
        for leak in leaks:
            assert leak not in context
        # ...while the non-verdict concept layers ARE rendered:
        assert "## project:" in context
        assert "## hypothesis:" in context
        assert "## methodology:" in context
        assert "## reference:" in context


def _imported_module_names(module_path: Path) -> set[str]:
    tree = ast.parse(module_path.read_text(encoding="utf-8"))
    names: set[str] = set()
    for node in ast.walk(tree):
        if isinstance(node, ast.Import):
            names.update(alias.name.split(".")[0] for alias in node.names)
        elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
            names.add(node.module.split(".")[0])
    return names


class TestContextSeamPurity:
    """LOAD-BEARING (§11): the context seam imports no agent toolkit."""

    @pytest.mark.parametrize("module", ["okf.py", "experience.py"])
    def test_context_seam_never_imports_an_agent_toolkit(self, module: str) -> None:
        names = _imported_module_names(SRC_PKG / module)
        # Positive control: the scanner resolved real imports from this module.
        # An empty `names` satisfies the intersection below exactly as well as
        # purity does — and `test_okf_is_pure_stdlib`'s subset check is likewise
        # trivially true of the empty set, so neither guards the other.
        assert "dataclasses" in names
        assert not names & {"claude_agent_sdk", "anthropic"}

    def test_okf_is_pure_stdlib(self) -> None:
        # D7-portable by design: navigation/rendering depends on nothing beyond
        # the standard library (relative imports would show as level > 0).
        names = _imported_module_names(SRC_PKG / "okf.py")
        assert names <= set(sys.stdlib_module_names)