feat(fase2): wire Step-1 ExpeL retrieval into the hypothesis prompt
Closes maalbilde §5 gap #1 (the one missing "feedback-into-prompt" dataflow) for the OKF-bundle path. Before, ExpeL was computed AFTER generation into a discarded SessionContext, so a prior verdict could not influence any hypothesis (context_providers=0). - New okf.py: framework-neutral OKF bundle navigation (index + frontmatter + cross-links), pure stdlib, no agent_framework/mcp (D7-portable), enforced by test_okf_is_maf_free. - verdicts.py: seed_store_from_bundle + bundle_candidate_features build the ExpeL substrate + the pre-hypothesis query key from a bundle. - run_project(bundle_dir=...): folds the candidate's prior verdicts into the generation context BEFORE generate_via_llm; the road path is unchanged. Load-bearing (maalbilde §7): test_step1_expel_loadbearing proves a prior verdict reaches the hypothesis prompt and goes RED when the fold is detached (shown via TDD red->green). The marker is the minted verdict id (content hash) because docs_dir==bundle_dir lets keyword chunk-stuffing leak the realization rate; clean layer separation is Fase 2b. Suite 121->133 passed; mypy + ruff check clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MHR8iKxJRxDiDfNw8HZmWE
This commit is contained in:
parent
2b1a1832ef
commit
d6d83d42b5
9 changed files with 529 additions and 7 deletions
|
|
@ -163,6 +163,43 @@ def make_portfolio_client_factory() -> Callable[..., Callable[[str], BaseChatCli
|
|||
return _make
|
||||
|
||||
|
||||
class _RecordingChatClient(SyntheticUsageChatClient):
|
||||
"""Records the incoming prompt blob per call into a SHARED sink, then returns a fixed valid
|
||||
reply. Lets a test assert exactly what text reached the prompt — the probe the Step-1 ExpeL
|
||||
wiring is made load-bearing against (does a prior verdict reach the hypothesis prompt?)."""
|
||||
|
||||
def __init__(self, sink: list[str], reply: str, *, tokens_per_reply: int = 8) -> None:
|
||||
super().__init__(default_reply=reply, tokens_per_reply=tokens_per_reply)
|
||||
self._sink = sink
|
||||
|
||||
def _inner_get_response(
|
||||
self, *, messages: Sequence[Message], stream: bool, options: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
self._sink.append(" ".join(getattr(m, "text", "") or "" for m in messages))
|
||||
return super()._inner_get_response(
|
||||
messages=messages, stream=stream, options=options, **kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def make_recording_client_factory() -> Callable[
|
||||
[str], tuple[Callable[[str], BaseChatClient], list[str]]
|
||||
]:
|
||||
"""Return a maker that builds a per-role client factory recording every prompt blob into a
|
||||
shared list. Returns ``(factory, recorded_prompts)`` so the test inspects what reached the
|
||||
prompt across the whole run (debate rounds + generation)."""
|
||||
|
||||
def _make(reply: str) -> tuple[Callable[[str], BaseChatClient], list[str]]:
|
||||
sink: list[str] = []
|
||||
|
||||
def factory(role: str) -> BaseChatClient:
|
||||
return _RecordingChatClient(sink, reply)
|
||||
|
||||
return factory, sink
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fresh_store() -> VerdictStore:
|
||||
return VerdictStore(verdicts=[])
|
||||
|
|
|
|||
96
tests/test_okf.py
Normal file
96
tests/test_okf.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
"""OKF bundle navigation (okf.py) — the framework-neutral, D7-portable context seam.
|
||||
|
||||
These tests pin the minimal navigation the Step-1 ExpeL wiring depends on: read ``index.md``,
|
||||
follow intra-bundle cross-links, parse each file's frontmatter, classify by ``type``, and locate
|
||||
the candidate IR projection. Robustness (OKF SPEC §4): broken links are tolerated, never raised.
|
||||
|
||||
No ``agent_framework``/``mcp`` import is allowed in ``okf`` — guarded by ``test_okf_is_maf_free``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from portfolio_optimiser import okf
|
||||
|
||||
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
|
||||
def test_navigate_classifies_every_typed_file() -> None:
|
||||
"""The energi bundle resolves to its six typed OKF files (index + project + hypothesis +
|
||||
methodology + reference + verdict), each carrying its declared ``type``."""
|
||||
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
|
||||
types = {f.name: f.type for f in bundle.files}
|
||||
assert types == {
|
||||
"index.md": "index",
|
||||
"bygg-kontor-nord.md": "project",
|
||||
"tiltak-led-retrofit.md": "hypothesis",
|
||||
"metode-ipmvp-a.md": "methodology",
|
||||
"kilder-realiseringsgap.md": "reference",
|
||||
"verdict-led-fro.md": "verdict",
|
||||
}
|
||||
|
||||
|
||||
def test_navigate_exposes_verdicts_and_hypothesis() -> None:
|
||||
"""The navigation surfaces exactly one ``type: verdict`` file (the ExpeL seed) and the
|
||||
candidate hypothesis — the two the Step-1 wiring keys on."""
|
||||
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
|
||||
assert [f.name for f in bundle.verdicts] == ["verdict-led-fro.md"]
|
||||
assert bundle.verdicts[0].frontmatter["realization_rate"] == "0.82"
|
||||
assert bundle.hypothesis is not None
|
||||
assert bundle.hypothesis.name == "tiltak-led-retrofit.md"
|
||||
|
||||
|
||||
def test_navigate_index_summary_is_progressive_disclosure() -> None:
|
||||
"""``index_summary`` is the index body (the progressive-disclosure entry point), not stuffing
|
||||
the whole bundle."""
|
||||
bundle = okf.navigate_bundle(str(BUNDLE_DIR))
|
||||
assert "progressiv disclosure" in bundle.index_summary.lower()
|
||||
|
||||
|
||||
def test_navigate_tolerates_broken_links(tmp_path) -> None:
|
||||
"""OKF SPEC §4: a consumer MUST tolerate broken links. An index linking a missing file
|
||||
navigates without raising, simply omitting the absent target."""
|
||||
(tmp_path / "index.md").write_text(
|
||||
"---\ntype: index\n---\n\nSee [gone](missing.md) and [here](real.md).\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "real.md").write_text("---\ntype: project\n---\n\nbody\n", encoding="utf-8")
|
||||
bundle = okf.navigate_bundle(str(tmp_path))
|
||||
names = {f.name for f in bundle.files}
|
||||
assert names == {"index.md", "real.md"} # missing.md silently skipped, no raise
|
||||
|
||||
|
||||
def test_load_ir_projection_returns_candidate_ir() -> None:
|
||||
"""The bundle's IR projection (``validator-input.json``) is the candidate measure's cost-IR —
|
||||
the pre-hypothesis ExpeL query key source."""
|
||||
ir = okf.load_ir_projection(str(BUNDLE_DIR))
|
||||
assert ir["project_id"] == "BYGG-KONTOR-NORD"
|
||||
assert [a["code"] for a in ir["affected_items"]] == ["ENERGI-TOTAL-EL"]
|
||||
assert ir["claimed_saving_nok"] == 30000
|
||||
|
||||
|
||||
def test_parse_frontmatter_reads_scalar_fields() -> None:
|
||||
"""The minimal frontmatter reader returns the leading ``---`` block as key:value strings."""
|
||||
fm = okf.parse_frontmatter(BUNDLE_DIR / "verdict-led-fro.md")
|
||||
assert fm["type"] == "verdict"
|
||||
assert fm["decision"] == "approved_with_adjustment"
|
||||
|
||||
|
||||
def test_okf_is_maf_free() -> None:
|
||||
"""D7 portability: ``okf.py`` IMPORTS no ``agent_framework`` / ``mcp`` (the docstring may name
|
||||
them to document the constraint, exactly as ``retrieval.py`` does) — checked via the AST, not a
|
||||
raw substring, so the prose claim doesn't trip the guard."""
|
||||
import ast
|
||||
|
||||
src = (
|
||||
Path(__file__).resolve().parents[1] / "src" / "portfolio_optimiser" / "okf.py"
|
||||
).read_text(encoding="utf-8")
|
||||
imported: list[str] = []
|
||||
for node in ast.walk(ast.parse(src)):
|
||||
if isinstance(node, ast.Import):
|
||||
imported += [a.name for a in node.names]
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
imported.append(node.module or "")
|
||||
forbidden = [m for m in imported if m.split(".")[0] in {"agent_framework", "mcp"}]
|
||||
assert forbidden == [], f"okf.py must not import MAF/mcp, found: {forbidden}"
|
||||
93
tests/test_step1_expel_loadbearing.py
Normal file
93
tests/test_step1_expel_loadbearing.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
"""Step-1 load-bearing seam (målbilde §5 / §7): a prior expert verdict MUST reach the next
|
||||
hypothesis prompt.
|
||||
|
||||
The diagnosis (fot-i-bakken, verified in code): ExpeL retrieval was computed AFTER generation and
|
||||
written into a discarded ``SessionContext`` — so the seed verdict could not influence any
|
||||
hypothesis (``context_providers`` = 0). Fase 2a wires it: before ``generate_via_llm``, the
|
||||
candidate's prior verdicts are folded into the generation context.
|
||||
|
||||
These two tests are a load-bearing PAIR, not a smoke test (per the Fase-2 green-but-dead trap):
|
||||
- the positive test asserts the seed verdict's realization signal ("0.82") IS in the generation
|
||||
prompt — it goes RED the moment the fold is detached;
|
||||
- the empty-store test asserts the signal is ABSENT without a seed — proving the signal's presence
|
||||
is CAUSED by the seam, not incidental to the fixture.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from portfolio_optimiser.run import run_project
|
||||
from portfolio_optimiser.validator import ValidatedProposal
|
||||
from portfolio_optimiser.verdicts import VerdictStore, seed_store_from_bundle
|
||||
|
||||
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
# A VALID BYGG-KONTOR-NORD proposal: affected total = 300000 x 1.0; no assumptions -> degenerate
|
||||
# Monte Carlo P90 = 0.30 x 300000 = 90000 >= claimed 30000 -> validates on the first attempt.
|
||||
_ENERGY_REPLY = (
|
||||
'{"measure":"LED-retrofit av kontorbelysning","affected_items":'
|
||||
'[{"code":"ENERGI-TOTAL-EL","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
|
||||
)
|
||||
_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
|
||||
|
||||
# The load-bearing marker is the seed verdict's MINTED id (a 16-char content hash that
|
||||
# ``ExpeLContextProvider.format_fewshot`` emits as ``- [<id>] ...``). It cannot appear in the
|
||||
# bundle's raw text, so its presence in the prompt isolates the STRUCTURAL ExpeL path. (The
|
||||
# realization rate "0.82" is NOT a usable marker here: ``docs_dir == bundle_dir``, so keyword
|
||||
# chunk-stuffing reads the verdict file and would leak "0.82" regardless of the fold. Layer
|
||||
# separation — verdicts out of the keyword docs folder — is Fase 2b.)
|
||||
_SEED_ID = seed_store_from_bundle(str(BUNDLE_DIR)).verdicts[0].id
|
||||
|
||||
|
||||
def _generation_prompts(sink: list[str]) -> list[str]:
|
||||
"""The generation-call prompts (``generate._build_messages`` embeds 'SavingsProposal'),
|
||||
isolated from the debate-round prompts also captured in the sink."""
|
||||
return [p for p in sink if "SavingsProposal" in p]
|
||||
|
||||
|
||||
async def test_prior_verdict_reaches_hypothesis_prompt(make_recording_client_factory) -> None:
|
||||
"""LOAD-BEARING: with the energi bundle's seed verdict in the store, the realization signal
|
||||
reaches the hypothesis-generation prompt. Detach the fold in ``run.py`` and this goes red."""
|
||||
store = seed_store_from_bundle(str(BUNDLE_DIR))
|
||||
assert store.verdicts, "precondition: the bundle seeds exactly one verdict"
|
||||
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
|
||||
|
||||
result = await run_project(
|
||||
"BYGG-KONTOR-NORD",
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
store=store,
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, ValidatedProposal)
|
||||
gen_prompts = _generation_prompts(recorded)
|
||||
assert gen_prompts, "the generation call must have happened"
|
||||
assert any(_SEED_ID in p for p in gen_prompts), (
|
||||
"the seed verdict did not reach the hypothesis prompt — the ExpeL->prompt fold is detached"
|
||||
)
|
||||
|
||||
|
||||
async def test_empty_store_leaves_no_signal_in_prompt(make_recording_client_factory) -> None:
|
||||
"""CAUSALITY CONTROL: the same run with an EMPTY store carries no realization signal into the
|
||||
prompt — proving the signal above is caused by the seam, making the positive test load-bearing
|
||||
(not merely green against the fixture's mere presence)."""
|
||||
factory, recorded = make_recording_client_factory(_ENERGY_REPLY)
|
||||
|
||||
await run_project(
|
||||
"BYGG-KONTOR-NORD",
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
store=VerdictStore(verdicts=[]),
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert all(_SEED_ID not in p for p in recorded), (
|
||||
"no seed verdict in the store, yet its id appeared in a prompt — "
|
||||
"the assertion is not actually load-bearing"
|
||||
)
|
||||
|
|
@ -7,6 +7,8 @@ genuine two-arg ``extend_instructions(source_id, instructions)`` GA signature
|
|||
Critical Fase 1 risk. Pattern: tests/spikes/test_d_verdictstore.py + real SessionContext.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from agent_framework import SessionContext
|
||||
|
||||
|
|
@ -15,9 +17,13 @@ from portfolio_optimiser.verdicts import (
|
|||
ProposalFeatures,
|
||||
Verdict,
|
||||
VerdictStore,
|
||||
bundle_candidate_features,
|
||||
capture_verdict,
|
||||
seed_store_from_bundle,
|
||||
)
|
||||
|
||||
_BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
_QUERY = ProposalFeatures(
|
||||
affected_codes=frozenset({"05.2", "03.1"}),
|
||||
measure_type="scope_reduction",
|
||||
|
|
@ -106,3 +112,36 @@ async def test_before_run_populates_real_sessioncontext_two_arg() -> None:
|
|||
ctx = SessionContext(input_messages=[], instructions=[])
|
||||
await provider.before_run(agent=None, session=None, context=ctx, state={})
|
||||
assert any(true_id in instr for instr in ctx.instructions)
|
||||
|
||||
|
||||
# --- OKF-bundle seeding (Fase 2a): the pre-hypothesis ExpeL query key + the seed store ---
|
||||
|
||||
|
||||
def test_bundle_candidate_features_keys_on_the_ir_projection() -> None:
|
||||
"""The pre-hypothesis ExpeL query is the candidate measure's cost-IR features (from the
|
||||
bundle's ``validator-input.json``) — available BEFORE any proposal is generated."""
|
||||
features = bundle_candidate_features(str(_BUNDLE_DIR))
|
||||
assert features.affected_codes == frozenset({"ENERGI-TOTAL-EL"})
|
||||
assert "LED-retrofit" in features.measure_type
|
||||
assert features.claimed_saving_nok == 30000
|
||||
|
||||
|
||||
def test_seed_store_from_bundle_carries_the_realization_signal() -> None:
|
||||
"""Each ``type: verdict`` file becomes a structurally-keyed ``Verdict`` whose rationale carries
|
||||
the learning signal the validator cannot compute (the realization rate 0.82)."""
|
||||
store = seed_store_from_bundle(str(_BUNDLE_DIR))
|
||||
assert len(store.verdicts) == 1
|
||||
seed = store.verdicts[0]
|
||||
assert seed.proposal_features.affected_codes == frozenset({"ENERGI-TOTAL-EL"})
|
||||
assert "approved" in seed.decision
|
||||
assert "0.82" in seed.rationale
|
||||
|
||||
|
||||
def test_seed_store_retrieval_matches_the_candidate() -> None:
|
||||
"""A3: seed and query derive from the SAME IR -> similarity 1.0 -> the lone seed is retrieved
|
||||
for the candidate (the structural match the Step-1 wiring relies on)."""
|
||||
store = seed_store_from_bundle(str(_BUNDLE_DIR))
|
||||
query = bundle_candidate_features(str(_BUNDLE_DIR))
|
||||
hits = store.retrieve(query, k=3)
|
||||
assert len(hits) == 1
|
||||
assert "0.82" in hits[0].rationale
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue