From 22b039235a9b79b7c23ea5815331770243f99998 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 13:29:02 +0200 Subject: [PATCH] feat(fase2): VerdictStore + ExpeL + capture_verdict (two-arg, real SessionContext test) --- src/portfolio_optimiser/verdicts.py | 198 ++++++++++++++++++++++++++++ tests/test_verdicts.py | 108 +++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 src/portfolio_optimiser/verdicts.py create mode 100644 tests/test_verdicts.py diff --git a/src/portfolio_optimiser/verdicts.py b/src/portfolio_optimiser/verdicts.py new file mode 100644 index 0000000..c80d9d5 --- /dev/null +++ b/src/portfolio_optimiser/verdicts.py @@ -0,0 +1,198 @@ +"""VerdictStore + ExpeL retrieval + Layer-2 out-of-band verdict capture (B2). + +The learning substrate: a structurally-ranked store of historical expert verdicts, surfaced +as ExpeL few-shots for the next run. **Similarity is structural, not textual** — a weighted +score over the affected cost-code set (Jaccard) + measure-type match + a magnitude bucket; +raw ``description`` text is deliberately excluded, so a true match with different wording +beats surface-text decoys. + +Two corrections vs the Fase 1 spike: + +1. **Two-arg ``extend_instructions``** (the Critical Fase 1 bug): ``before_run`` calls + ``context.extend_instructions(self.source_id, [...])`` — the genuine GA signature + (``_sessions.py:253``), exercised against a REAL ``SessionContext`` in the tests. +2. **Layer-2 capture** (``capture_verdict``): mints a stable content-hash ``Verdict.id`` so a + structurally identical proposal maps to the same id (the learning-loop key). The store is + **in-memory only** for the MVP — durable persistence is deferred to Fase 3. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass + +from agent_framework import ContextProvider, SessionContext + +# Weights: the affected cost-code overlap dominates, then measure type, then magnitude. +_W_CODES, _W_MEASURE, _W_MAGNITUDE = 0.60, 0.25, 0.15 +_MAGNITUDE_BUCKETS = [(0.0, 1e5), (1e5, 5e5), (5e5, 1e6), (1e6, float("inf"))] + + +@dataclass(frozen=True) +class ProposalFeatures: + """The *structured* features retrieval ranks on. ``description`` is surface text and is + deliberately NOT part of the similarity score (nor the minted id).""" + + affected_codes: frozenset[str] + measure_type: str + claimed_saving_nok: float + description: str = "" + + +@dataclass(frozen=True) +class Verdict: + """One historical expert verdict in the store.""" + + id: str + proposal_features: ProposalFeatures + decision: str # "approved" | "rejected" + rationale: str + + +def _magnitude_bucket(value: float) -> int: + for i, (low, high) in enumerate(_MAGNITUDE_BUCKETS): + if low <= value < high: + return i + return len(_MAGNITUDE_BUCKETS) - 1 + + +def _jaccard(a: frozenset[str], b: frozenset[str]) -> float: + union = a | b + return len(a & b) / len(union) if union else 1.0 + + +def similarity(query: ProposalFeatures, candidate: ProposalFeatures) -> float: + """Weighted structural similarity in [0, 1] — text is ignored by design.""" + codes = _jaccard(query.affected_codes, candidate.affected_codes) + measure = 1.0 if query.measure_type == candidate.measure_type else 0.0 + magnitude = ( + 1.0 + if _magnitude_bucket(query.claimed_saving_nok) + == _magnitude_bucket(candidate.claimed_saving_nok) + else 0.0 + ) + return _W_CODES * codes + _W_MEASURE * measure + _W_MAGNITUDE * magnitude + + +def _mint_id(features: ProposalFeatures) -> str: + """Stable content-hash id over the STRUCTURAL fields (not description), so a + structurally identical proposal maps to the same id.""" + canonical = json.dumps( + { + "affected_codes": sorted(features.affected_codes), + "measure_type": features.measure_type, + "claimed_saving_nok": features.claimed_saving_nok, + }, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] + + +def capture_verdict(features: ProposalFeatures, decision: str, rationale: str) -> Verdict: + """Layer-2 out-of-band verdict constructor: mint a stable content-hash id (the + learning-loop key) and build the ``Verdict`` to persist in the store.""" + return Verdict( + id=_mint_id(features), + proposal_features=features, + decision=decision, + rationale=rationale, + ) + + +@dataclass +class VerdictStore: + """Minimal in-memory store of historical verdicts (MVP — no file persistence).""" + + verdicts: list[Verdict] + + def retrieve(self, query: ProposalFeatures, k: int) -> list[Verdict]: + """Return the top-``k`` verdicts by structural similarity. Deterministic: ties break + by verdict id, so ordering is stable across runs.""" + if k <= 0: + raise ValueError(f"k must be positive, got {k}") + ranked = sorted( + self.verdicts, + key=lambda v: (-similarity(query, v.proposal_features), v.id), + ) + return ranked[:k] + + def add(self, verdict: Verdict) -> None: + """Persist a captured verdict in-memory (idempotent on content-hash id).""" + if all(v.id != verdict.id for v in self.verdicts): + self.verdicts.append(verdict) + + +class ExpeLContextProvider(ContextProvider): + """Wraps ``VerdictStore.retrieve`` for ExpeL few-shot injection via the GA + ``ContextProvider`` hook.""" + + def __init__(self, store: VerdictStore, query: ProposalFeatures, *, k: int = 3) -> None: + super().__init__(source_id="expel-verdictstore") + self._store = store + self._query = query + self._k = k + + def format_fewshot(self) -> str: + hits = self._store.retrieve(self._query, self._k) + body = "\n".join(f"- [{v.id}] {v.decision}: {v.rationale}" for v in hits) + return f"Relevant prior verdicts (ExpeL few-shot):\n{body}" + + async def before_run( + self, + *, + agent: object, + session: object, + context: SessionContext, + state: dict, + ) -> None: + # Two-arg GA signature (_sessions.py:253) — the Fase 1 single-arg bug is fixed. + context.extend_instructions(self.source_id, [self.format_fewshot()]) + + +def seed_store() -> VerdictStore: + """Seed 12 synthetic verdicts spanning the reference domain's cost codes, measure types, + magnitudes, and decisions (B2 store of 10-20).""" + rows = [ + ("V01", {"05.2", "03.1"}, "scope_reduction", 180_000, "approved", + "asphalt + base course trimmed within feasible range"), + ("V02", {"05.2"}, "rate_renegotiation", 60_000, "approved", + "renegotiated asphalt unit rate"), + ("V03", {"07.4"}, "material_substitution", 120_000, "rejected", + "granite kerb substitution unsafe"), + ("V04", {"09.1"}, "scope_reduction", 240_000, "approved", + "fewer LED masts on low-traffic stretch"), + ("V05", {"02.3", "03.1"}, "scope_reduction", 350_000, "rejected", + "soil replacement is load-bearing, cannot cut"), + ("V06", {"21.2"}, "rate_renegotiation", 90_000, "approved", + "blasting rate renegotiated"), + ("V07", {"22.4"}, "material_substitution", 700_000, "rejected", + "fiber shotcrete spec is mandated"), + ("V08", {"88.2"}, "scope_reduction", 150_000, "approved", + "concrete repair area re-measured smaller"), + ("V09", {"87.3"}, "material_substitution", 130_000, "approved", + "alternative membrane qualified"), + ("V10", {"05.2", "03.1"}, "rate_renegotiation", 200_000, "approved", + "combined paving rate discount"), + ("V11", {"01.1"}, "scope_reduction", 95_000, "rejected", + "rigging is fixed cost, no scope to cut"), + ("V12", {"31.3"}, "scope_reduction", 110_000, "approved", + "drainage length reduced after survey"), + ] + return VerdictStore( + verdicts=[ + Verdict( + id=vid, + proposal_features=ProposalFeatures( + affected_codes=frozenset(codes), + measure_type=mtype, + claimed_saving_nok=saving, + description=desc, + ), + decision=decision, + rationale=desc, + ) + for vid, codes, mtype, saving, decision, desc in rows + ] + ) diff --git a/tests/test_verdicts.py b/tests/test_verdicts.py new file mode 100644 index 0000000..e063081 --- /dev/null +++ b/tests/test_verdicts.py @@ -0,0 +1,108 @@ +"""Step 3 tests — non-tautological top-K retrieval + REAL-SessionContext two-arg injection. + +The true match shares the *structured* similarity fields with the query but uses different +description text; the decoys share surface text but differ structurally. The injection test +uses a REAL ``agent_framework.SessionContext`` (not a single-arg fake), exercising the +genuine two-arg ``extend_instructions(source_id, instructions)`` GA signature — retiring the +Critical Fase 1 risk. Pattern: tests/spikes/test_d_verdictstore.py + real SessionContext. +""" + +import pytest +from agent_framework import SessionContext + +from portfolio_optimiser.verdicts import ( + ExpeLContextProvider, + ProposalFeatures, + Verdict, + VerdictStore, + capture_verdict, +) + +_QUERY = ProposalFeatures( + affected_codes=frozenset({"05.2", "03.1"}), + measure_type="scope_reduction", + claimed_saving_nok=220_000, # bucket [100k, 500k) + description="asphalt base course reduction near school", +) + + +def _store_with_true_match_and_decoys() -> tuple[VerdictStore, str]: + true_match = Verdict( + id="TRUE", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"05.2", "03.1"}), # same codes + measure_type="scope_reduction", # same measure type + claimed_saving_nok=200_000, # same magnitude bucket + description="zzz totally unrelated wording alpha beta", # DIFFERENT text + ), + decision="approved", + rationale="prior scope reduction on the same codes was approved", + ) + decoy_low = Verdict( + id="DECOY-LOW", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"09.1"}), + measure_type="rate_renegotiation", + claimed_saving_nok=50_000, + description="asphalt base course reduction near school", # same words as query + ), + decision="rejected", + rationale="surface-text decoy", + ) + decoy_high = Verdict( + id="DECOY-HIGH", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"21.2"}), + measure_type="material_substitution", + claimed_saving_nok=700_000, + description="asphalt base course reduction extra words", + ), + decision="rejected", + rationale="surface-text decoy", + ) + return VerdictStore(verdicts=[decoy_low, true_match, decoy_high]), "TRUE" + + +def test_retrieve_finds_structural_match_over_text_decoys() -> None: + store, true_id = _store_with_true_match_and_decoys() + hits = store.retrieve(_QUERY, k=3) + assert hits[0].id == true_id # structural match ranks #1 despite different wording + + +def test_retrieve_is_deterministic() -> None: + store, _ = _store_with_true_match_and_decoys() + assert [h.id for h in store.retrieve(_QUERY, k=3)] == [ + h.id for h in store.retrieve(_QUERY, k=3) + ] + + +def test_retrieve_rejects_non_positive_k() -> None: + store, _ = _store_with_true_match_and_decoys() + with pytest.raises(ValueError): + store.retrieve(_QUERY, k=0) + + +def test_capture_verdict_mints_stable_id() -> None: + a = capture_verdict(_QUERY, "approved", "ok") + b = capture_verdict( + ProposalFeatures( + affected_codes=frozenset({"03.1", "05.2"}), # same set, different order + measure_type="scope_reduction", + claimed_saving_nok=220_000, + description="DIFFERENT surface wording entirely", # text excluded from the id + ), + "approved", + "ok", + ) + assert a.id == b.id # structurally identical -> stable id + assert len(a.id) == 16 + + +async def test_before_run_populates_real_sessioncontext_two_arg() -> None: + store, true_id = _store_with_true_match_and_decoys() + provider = ExpeLContextProvider(store, _QUERY, k=2) + # A REAL SessionContext (not a single-arg fake) — exercises the genuine GA two-arg + # extend_instructions(source_id, instructions) signature. + ctx = SessionContext(input_messages=[], instructions=[]) + await provider.before_run(agent=None, session=None, context=ctx, state={}) + assert any(true_id in instr for instr in ctx.instructions)