feat(fase2): wire Step-1 ExpeL retrieval into the hypothesis prompt

Closes maalbilde §5 gap #1 (the one missing "feedback-into-prompt" dataflow)
for the OKF-bundle path. Before, ExpeL was computed AFTER generation into a
discarded SessionContext, so a prior verdict could not influence any hypothesis
(context_providers=0).

- New okf.py: framework-neutral OKF bundle navigation (index + frontmatter +
  cross-links), pure stdlib, no agent_framework/mcp (D7-portable), enforced by
  test_okf_is_maf_free.
- verdicts.py: seed_store_from_bundle + bundle_candidate_features build the
  ExpeL substrate + the pre-hypothesis query key from a bundle.
- run_project(bundle_dir=...): folds the candidate's prior verdicts into the
  generation context BEFORE generate_via_llm; the road path is unchanged.

Load-bearing (maalbilde §7): test_step1_expel_loadbearing proves a prior verdict
reaches the hypothesis prompt and goes RED when the fold is detached (shown via
TDD red->green). The marker is the minted verdict id (content hash) because
docs_dir==bundle_dir lets keyword chunk-stuffing leak the realization rate;
clean layer separation is Fase 2b.

Suite 121->133 passed; mypy + ruff check clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MHR8iKxJRxDiDfNw8HZmWE
This commit is contained in:
Kjell Tore Guttormsen 2026-06-29 10:56:48 +02:00
commit d6d83d42b5
9 changed files with 529 additions and 7 deletions

View file

@ -163,6 +163,43 @@ def make_portfolio_client_factory() -> Callable[..., Callable[[str], BaseChatCli
return _make
class _RecordingChatClient(SyntheticUsageChatClient):
"""Records the incoming prompt blob per call into a SHARED sink, then returns a fixed valid
reply. Lets a test assert exactly what text reached the prompt the probe the Step-1 ExpeL
wiring is made load-bearing against (does a prior verdict reach the hypothesis prompt?)."""
def __init__(self, sink: list[str], reply: str, *, tokens_per_reply: int = 8) -> None:
super().__init__(default_reply=reply, tokens_per_reply=tokens_per_reply)
self._sink = sink
def _inner_get_response(
self, *, messages: Sequence[Message], stream: bool, options: Any, **kwargs: Any
) -> Any:
self._sink.append(" ".join(getattr(m, "text", "") or "" for m in messages))
return super()._inner_get_response(
messages=messages, stream=stream, options=options, **kwargs
)
@pytest.fixture()
def make_recording_client_factory() -> Callable[
[str], tuple[Callable[[str], BaseChatClient], list[str]]
]:
"""Return a maker that builds a per-role client factory recording every prompt blob into a
shared list. Returns ``(factory, recorded_prompts)`` so the test inspects what reached the
prompt across the whole run (debate rounds + generation)."""
def _make(reply: str) -> tuple[Callable[[str], BaseChatClient], list[str]]:
sink: list[str] = []
def factory(role: str) -> BaseChatClient:
return _RecordingChatClient(sink, reply)
return factory, sink
return _make
@pytest.fixture()
def fresh_store() -> VerdictStore:
return VerdictStore(verdicts=[])

96
tests/test_okf.py Normal file
View file

@ -0,0 +1,96 @@
"""OKF bundle navigation (okf.py) — the framework-neutral, D7-portable context seam.
These tests pin the minimal navigation the Step-1 ExpeL wiring depends on: read ``index.md``,
follow intra-bundle cross-links, parse each file's frontmatter, classify by ``type``, and locate
the candidate IR projection. Robustness (OKF SPEC §4): broken links are tolerated, never raised.
No ``agent_framework``/``mcp`` import is allowed in ``okf`` guarded by ``test_okf_is_maf_free``.
"""
from __future__ import annotations
from pathlib import Path
from portfolio_optimiser import okf
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
def test_navigate_classifies_every_typed_file() -> None:
"""The energi bundle resolves to its six typed OKF files (index + project + hypothesis +
methodology + reference + verdict), each carrying its declared ``type``."""
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
types = {f.name: f.type for f in bundle.files}
assert types == {
"index.md": "index",
"bygg-kontor-nord.md": "project",
"tiltak-led-retrofit.md": "hypothesis",
"metode-ipmvp-a.md": "methodology",
"kilder-realiseringsgap.md": "reference",
"verdict-led-fro.md": "verdict",
}
def test_navigate_exposes_verdicts_and_hypothesis() -> None:
"""The navigation surfaces exactly one ``type: verdict`` file (the ExpeL seed) and the
candidate hypothesis the two the Step-1 wiring keys on."""
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
assert [f.name for f in bundle.verdicts] == ["verdict-led-fro.md"]
assert bundle.verdicts[0].frontmatter["realization_rate"] == "0.82"
assert bundle.hypothesis is not None
assert bundle.hypothesis.name == "tiltak-led-retrofit.md"
def test_navigate_index_summary_is_progressive_disclosure() -> None:
"""``index_summary`` is the index body (the progressive-disclosure entry point), not stuffing
the whole bundle."""
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
assert "progressiv disclosure" in bundle.index_summary.lower()
def test_navigate_tolerates_broken_links(tmp_path) -> None:
"""OKF SPEC §4: a consumer MUST tolerate broken links. An index linking a missing file
navigates without raising, simply omitting the absent target."""
(tmp_path / "index.md").write_text(
"---\ntype: index\n---\n\nSee [gone](missing.md) and [here](real.md).\n",
encoding="utf-8",
)
(tmp_path / "real.md").write_text("---\ntype: project\n---\n\nbody\n", encoding="utf-8")
bundle = okf.navigate_bundle(str(tmp_path))
names = {f.name for f in bundle.files}
assert names == {"index.md", "real.md"} # missing.md silently skipped, no raise
def test_load_ir_projection_returns_candidate_ir() -> None:
"""The bundle's IR projection (``validator-input.json``) is the candidate measure's cost-IR —
the pre-hypothesis ExpeL query key source."""
ir = okf.load_ir_projection(str(BUNDLE_DIR))
assert ir["project_id"] == "BYGG-KONTOR-NORD"
assert [a["code"] for a in ir["affected_items"]] == ["ENERGI-TOTAL-EL"]
assert ir["claimed_saving_nok"] == 30000
def test_parse_frontmatter_reads_scalar_fields() -> None:
"""The minimal frontmatter reader returns the leading ``---`` block as key:value strings."""
fm = okf.parse_frontmatter(BUNDLE_DIR / "verdict-led-fro.md")
assert fm["type"] == "verdict"
assert fm["decision"] == "approved_with_adjustment"
def test_okf_is_maf_free() -> None:
"""D7 portability: ``okf.py`` IMPORTS no ``agent_framework`` / ``mcp`` (the docstring may name
them to document the constraint, exactly as ``retrieval.py`` does) checked via the AST, not a
raw substring, so the prose claim doesn't trip the guard."""
import ast
src = (
Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "okf.py"
).read_text(encoding="utf-8")
imported: list[str] = []
for node in ast.walk(ast.parse(src)):
if isinstance(node, ast.Import):
imported += [a.name for a in node.names]
elif isinstance(node, ast.ImportFrom):
imported.append(node.module or "")
forbidden = [m for m in imported if m.split(".")[0] in {"agent_framework", "mcp"}]
assert forbidden == [], f"okf.py must not import MAF/mcp, found: {forbidden}"

View file

@ -0,0 +1,93 @@
"""Step-1 load-bearing seam (målbilde §5 / §7): a prior expert verdict MUST reach the next
hypothesis prompt.
The diagnosis (fot-i-bakken, verified in code): ExpeL retrieval was computed AFTER generation and
written into a discarded ``SessionContext`` so the seed verdict could not influence any
hypothesis (``context_providers`` = 0). Fase 2a wires it: before ``generate_via_llm``, the
candidate's prior verdicts are folded into the generation context.
These two tests are a load-bearing PAIR, not a smoke test (per the Fase-2 green-but-dead trap):
- the positive test asserts the seed verdict's realization signal ("0.82") IS in the generation
prompt it goes RED the moment the fold is detached;
- the empty-store test asserts the signal is ABSENT without a seed proving the signal's presence
is CAUSED by the seam, not incidental to the fixture.
"""
from __future__ import annotations
from pathlib import Path
from portfolio_optimiser.run import run_project
from portfolio_optimiser.validator import ValidatedProposal
from portfolio_optimiser.verdicts import VerdictStore, seed_store_from_bundle
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
# A VALID BYGG-KONTOR-NORD proposal: affected total = 300000 x 1.0; no assumptions -> degenerate
# Monte Carlo P90 = 0.30 x 300000 = 90000 >= claimed 30000 -> validates on the first attempt.
_ENERGY_REPLY = (
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
)
_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
# The load-bearing marker is the seed verdict's MINTED id (a 16-char content hash that
# ``ExpeLContextProvider.format_fewshot`` emits as ``- [<id>] ...``). It cannot appear in the
# bundle's raw text, so its presence in the prompt isolates the STRUCTURAL ExpeL path. (The
# realization rate "0.82" is NOT a usable marker here: ``docs_dir == bundle_dir``, so keyword
# chunk-stuffing reads the verdict file and would leak "0.82" regardless of the fold. Layer
# separation — verdicts out of the keyword docs folder — is Fase 2b.)
_SEED_ID = seed_store_from_bundle(str(BUNDLE_DIR)).verdicts[0].id
def _generation_prompts(sink: list[str]) -> list[str]:
"""The generation-call prompts (``generate._build_messages`` embeds 'SavingsProposal'),
isolated from the debate-round prompts also captured in the sink."""
return [p for p in sink if "SavingsProposal" in p]
async def test_prior_verdict_reaches_hypothesis_prompt(make_recording_client_factory) -> None:
"""LOAD-BEARING: with the energi bundle's seed verdict in the store, the realization signal
reaches the hypothesis-generation prompt. Detach the fold in ``run.py`` and this goes red."""
store = seed_store_from_bundle(str(BUNDLE_DIR))
assert store.verdicts, "precondition: the bundle seeds exactly one verdict"
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
result = await run_project(
"BYGG-KONTOR-NORD",
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=store,
client_factory=factory,
)
assert isinstance(result.outcome, ValidatedProposal)
gen_prompts = _generation_prompts(recorded)
assert gen_prompts, "the generation call must have happened"
assert any(_SEED_ID in p for p in gen_prompts), (
"the seed verdict did not reach the hypothesis prompt — the ExpeL->prompt fold is detached"
)
async def test_empty_store_leaves_no_signal_in_prompt(make_recording_client_factory) -> None:
"""CAUSALITY CONTROL: the same run with an EMPTY store carries no realization signal into the
prompt proving the signal above is caused by the seam, making the positive test load-bearing
(not merely green against the fixture's mere presence)."""
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
await run_project(
"BYGG-KONTOR-NORD",
"local",
docs_dir=str(BUNDLE_DIR),
bundle_dir=str(BUNDLE_DIR),
verdict_input=_VERDICT_INPUT,
store=VerdictStore(verdicts=[]),
client_factory=factory,
)
assert all(_SEED_ID not in p for p in recorded), (
"no seed verdict in the store, yet its id appeared in a prompt — "
"the assertion is not actually load-bearing"
)

View file

@ -7,6 +7,8 @@ genuine two-arg ``extend_instructions(source_id, instructions)`` GA signature
Critical Fase 1 risk. Pattern: tests/spikes/test_d_verdictstore.py + real SessionContext.
"""
from pathlib import Path
import pytest
from agent_framework import SessionContext
@ -15,9 +17,13 @@ from portfolio_optimiser.verdicts import (
ProposalFeatures,
Verdict,
VerdictStore,
bundle_candidate_features,
capture_verdict,
seed_store_from_bundle,
)
_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
_QUERY = ProposalFeatures(
affected_codes=frozenset({"05.2", "03.1"}),
measure_type="scope_reduction",
@ -106,3 +112,36 @@ async def test_before_run_populates_real_sessioncontext_two_arg() -> None:
ctx = SessionContext(input_messages=[], instructions=[])
await provider.before_run(agent=None, session=None, context=ctx, state={})
assert any(true_id in instr for instr in ctx.instructions)
# --- OKF-bundle seeding (Fase 2a): the pre-hypothesis ExpeL query key + the seed store ---
def test_bundle_candidate_features_keys_on_the_ir_projection() -> None:
"""The pre-hypothesis ExpeL query is the candidate measure's cost-IR features (from the
bundle's ``validator-input.json``) — available BEFORE any proposal is generated."""
features = bundle_candidate_features(str(_BUNDLE_DIR))
assert features.affected_codes == frozenset({"ENERGI-TOTAL-EL"})
assert "LED-retrofit" in features.measure_type
assert features.claimed_saving_nok == 30000
def test_seed_store_from_bundle_carries_the_realization_signal() -> None:
"""Each ``type: verdict`` file becomes a structurally-keyed ``Verdict`` whose rationale carries
the learning signal the validator cannot compute (the realization rate 0.82)."""
store = seed_store_from_bundle(str(_BUNDLE_DIR))
assert len(store.verdicts) == 1
seed = store.verdicts[0]
assert seed.proposal_features.affected_codes == frozenset({"ENERGI-TOTAL-EL"})
assert "approved" in seed.decision
assert "0.82" in seed.rationale
def test_seed_store_retrieval_matches_the_candidate() -> None:
"""A3: seed and query derive from the SAME IR -> similarity 1.0 -> the lone seed is retrieved
for the candidate (the structural match the Step-1 wiring relies on)."""
store = seed_store_from_bundle(str(_BUNDLE_DIR))
query = bundle_candidate_features(str(_BUNDLE_DIR))
hits = store.retrieve(query, k=3)
assert len(hits) == 1
assert "0.82" in hits[0].rationale