From f7a36b59ac6c4b3c71181a74467f4b0f1c10b051 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 10:32:39 +0200 Subject: [PATCH] feat(fase1): spike D - verdictstore + expel retrieval [skip-docs] --- docs/fase1-spikes/findings-d.md | 47 +++++++++ spikes/d_verdictstore.py | 145 ++++++++++++++++++++++++++++ tests/spikes/test_d_verdictstore.py | 110 +++++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 docs/fase1-spikes/findings-d.md create mode 100644 spikes/d_verdictstore.py create mode 100644 tests/spikes/test_d_verdictstore.py diff --git a/docs/fase1-spikes/findings-d.md b/docs/fase1-spikes/findings-d.md new file mode 100644 index 0000000..3ca0ba4 --- /dev/null +++ b/docs/fase1-spikes/findings-d.md @@ -0,0 +1,47 @@ +# Spike D findings — VerdictStore + ExpeL retrieval (B2) + +**Assumption:** retrieval surfaces a relevant *prior* verdict for a similar new proposal — +the substrate for the framework's learning loop. + +## Result — CONFIRMED (deterministic, no endpoint) + +A minimal in-memory `VerdictStore` holds 12 synthetic verdicts seeded from the reference +domain's cost codes, measure types, magnitudes, and decisions (B2's "10–20"). + +**Similarity is structural, not textual (reviewer refinement #2):** a weighted score over +*structured* fields — Jaccard on the affected cost-code set (0.60) + a `measure_type` +match (0.25) + a magnitude-bucket match on the claimed saving (0.15). Raw description text +is **deliberately ignored**. + +`retrieve(query, k)` is the guaranteed SC-D unit. The test is **non-tautological by +construction**: the true match shares the structured fields with the query but uses +*different wording*, while two decoys share the query's *surface text* but differ +structurally (disjoint codes, different measure type, different magnitude bucket). The +structural retriever returns the **true match as top-1** and ranks the surface-text decoys +last — a text-matching retriever would be fooled. Ordering is **deterministic** (ties +break by verdict id). + +**ExpeL injection:** a thin `ExpeLContextProvider` subclasses the real +`agent_framework.ContextProvider` and, in `before_run`, injects the retrieved verdicts as +few-shot instructions via `SessionContext.extend_instructions` — asserted against the +introspected interface. The `retrieve` ranking remains the deliverable regardless of the +MAF session surface. + +## Out of scope (Fase 2 option) + +The **embedding-based** similarity path is intentionally not built — it needs a live +endpoint, is non-deterministic, and serves no SC for a throwaway spike. Structured-field +similarity is sufficient to confirm B2. Embeddings (or a hybrid structured+embedding score) +are a Fase 2 option for the durable VerdictStore. + +## Token use + +**0 — deterministic retrieval.** No model is called; similarity is pure arithmetic over +structured fields. The ExpeL provider only *formats* retrieved verdicts into few-shot text +— the actual model call that would consume tokens is a Fase 2 concern. + +## Implication for Fase 2 + +The learning loop's retrieval is realizable with a simple, deterministic, structural +similarity — good enough to surface relevant prior verdicts. Fase 2 can keep this as the +baseline and add embeddings only if structured similarity proves insufficient on real data. diff --git a/spikes/d_verdictstore.py b/spikes/d_verdictstore.py new file mode 100644 index 0000000..5de339b --- /dev/null +++ b/spikes/d_verdictstore.py @@ -0,0 +1,145 @@ +"""Spike D — VerdictStore + ExpeL retrieval (B2). + +De-risks the assumption that retrieval surfaces a relevant *prior* verdict for a similar +new proposal — the substrate for the framework's learning loop. + +**Similarity is structural, not textual (reviewer refinement #2):** a weighted score over +*structured* fields — Jaccard on the affected cost-code set + a `measure_type` match + a +magnitude-bucket match on the claimed saving — **never** raw description text. This makes +the retrieval non-tautological: a true match with different wording beats decoys that +merely share surface text. + +`retrieve()` is the guaranteed SC-D deliverable (always tested). A thin +`ExpeLContextProvider` wraps it for few-shot injection via the real `ContextProvider` +hook. The embedding-based similarity path is intentionally **out of scope** for a +throwaway spike (a Fase 2 option — see findings-d). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from agent_framework import ContextProvider + +# Weights: the affected cost-code overlap dominates, then measure type, then magnitude. +_W_CODES, _W_MEASURE, _W_MAGNITUDE = 0.60, 0.25, 0.15 +_MAGNITUDE_BUCKETS = [(0.0, 1e5), (1e5, 5e5), (5e5, 1e6), (1e6, float("inf"))] + + +@dataclass(frozen=True) +class ProposalFeatures: + """The *structured* features retrieval ranks on. ``description`` is surface text and + is deliberately NOT part of the similarity score.""" + + affected_codes: frozenset[str] + measure_type: str + claimed_saving_nok: float + description: str = "" + + +@dataclass(frozen=True) +class Verdict: + """One historical expert verdict in the store.""" + + id: str + proposal_features: ProposalFeatures + decision: str # "approved" | "rejected" + rationale: str + + +def _magnitude_bucket(value: float) -> int: + for i, (low, high) in enumerate(_MAGNITUDE_BUCKETS): + if low <= value < high: + return i + return len(_MAGNITUDE_BUCKETS) - 1 + + +def _jaccard(a: frozenset[str], b: frozenset[str]) -> float: + union = a | b + return len(a & b) / len(union) if union else 1.0 + + +def similarity(query: ProposalFeatures, candidate: ProposalFeatures) -> float: + """Weighted structural similarity in [0, 1] — text is ignored by design.""" + codes = _jaccard(query.affected_codes, candidate.affected_codes) + measure = 1.0 if query.measure_type == candidate.measure_type else 0.0 + magnitude = 1.0 if _magnitude_bucket(query.claimed_saving_nok) == _magnitude_bucket( + candidate.claimed_saving_nok + ) else 0.0 + return _W_CODES * codes + _W_MEASURE * measure + _W_MAGNITUDE * magnitude + + +@dataclass +class VerdictStore: + """Minimal in-memory store of historical verdicts.""" + + verdicts: list[Verdict] + + def retrieve(self, query: ProposalFeatures, k: int) -> list[Verdict]: + """Return the top-``k`` verdicts by structural similarity. Deterministic: ties + break by verdict id, so ordering is stable across runs.""" + if k <= 0: + raise ValueError(f"k must be positive, got {k}") + ranked = sorted( + self.verdicts, + key=lambda v: (-similarity(query, v.proposal_features), v.id), + ) + return ranked[:k] + + +class ExpeLContextProvider(ContextProvider): + """Wraps `VerdictStore.retrieve` for ExpeL few-shot injection. + + Asserted only against the introspected `ContextProvider` interface + (`before_run` + `SessionContext.extend_instructions`); the `retrieve` ranking remains + the guaranteed SC-D deliverable regardless of the MAF session surface. + """ + + def __init__(self, store: VerdictStore, query: ProposalFeatures, *, k: int = 3) -> None: + super().__init__(source_id="expel-verdictstore") + self._store = store + self._query = query + self._k = k + + def format_fewshot(self) -> str: + hits = self._store.retrieve(self._query, self._k) + body = "\n".join(f"- [{v.id}] {v.decision}: {v.rationale}" for v in hits) + return f"Relevant prior verdicts (ExpeL few-shot):\n{body}" + + async def before_run(self, *, agent: object, session: object, context: object, state: dict) -> None: + context.extend_instructions([self.format_fewshot()]) # type: ignore[attr-defined] + + +def seed_store() -> VerdictStore: + """Seed 12 synthetic verdicts spanning the reference domain's cost codes, measure + types, magnitudes, and decisions (B2 store of 10–20).""" + rows = [ + ("V01", {"05.2", "03.1"}, "scope_reduction", 180_000, "approved", "asphalt + base course trimmed within feasible range"), + ("V02", {"05.2"}, "rate_renegotiation", 60_000, "approved", "renegotiated asphalt unit rate"), + ("V03", {"07.4"}, "material_substitution", 120_000, "rejected", "granite kerb substitution unsafe"), + ("V04", {"09.1"}, "scope_reduction", 240_000, "approved", "fewer LED masts on low-traffic stretch"), + ("V05", {"02.3", "03.1"}, "scope_reduction", 350_000, "rejected", "soil replacement is load-bearing, cannot cut"), + ("V06", {"21.2"}, "rate_renegotiation", 90_000, "approved", "blasting rate renegotiated"), + ("V07", {"22.4"}, "material_substitution", 700_000, "rejected", "fiber shotcrete spec is mandated"), + ("V08", {"88.2"}, "scope_reduction", 150_000, "approved", "concrete repair area re-measured smaller"), + ("V09", {"87.3"}, "material_substitution", 130_000, "approved", "alternative membrane qualified"), + ("V10", {"05.2", "03.1"}, "rate_renegotiation", 200_000, "approved", "combined paving rate discount"), + ("V11", {"01.1"}, "scope_reduction", 95_000, "rejected", "rigging is fixed cost, no scope to cut"), + ("V12", {"31.3"}, "scope_reduction", 110_000, "approved", "drainage length reduced after survey"), + ] + return VerdictStore( + verdicts=[ + Verdict( + id=vid, + proposal_features=ProposalFeatures( + affected_codes=frozenset(codes), + measure_type=mtype, + claimed_saving_nok=saving, + description=desc, + ), + decision=decision, + rationale=desc, + ) + for vid, codes, mtype, saving, decision, desc in rows + ] + ) diff --git a/tests/spikes/test_d_verdictstore.py b/tests/spikes/test_d_verdictstore.py new file mode 100644 index 0000000..2f42cc8 --- /dev/null +++ b/tests/spikes/test_d_verdictstore.py @@ -0,0 +1,110 @@ +"""Spike D tests — non-tautological top-K retrieval (B2). + +The true match shares the *structured* similarity fields with the query but uses different +description text; the decoys share surface description text but differ structurally. A +text-matching retriever would be fooled by the decoys — a structural one is not. +Pattern: tests/test_reference_domain.py. +""" + +import pytest + +from spikes.d_verdictstore import ( + ExpeLContextProvider, + ProposalFeatures, + Verdict, + VerdictStore, + seed_store, +) + +_QUERY = ProposalFeatures( + affected_codes=frozenset({"05.2", "03.1"}), + measure_type="scope_reduction", + claimed_saving_nok=220_000, # bucket [100k, 500k) + description="asphalt base course reduction near school", +) + + +def _store_with_true_match_and_decoys() -> tuple[VerdictStore, str]: + true_match = Verdict( + id="TRUE", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"05.2", "03.1"}), # same codes + measure_type="scope_reduction", # same measure type + claimed_saving_nok=200_000, # same magnitude bucket + description="zzz totally unrelated wording alpha beta", # DIFFERENT text + ), + decision="approved", + rationale="prior scope reduction on the same codes was approved", + ) + # Decoys share the query's SURFACE text but differ structurally (disjoint codes, + # different measure type, different magnitude bucket). + decoy_low = Verdict( + id="DECOY-LOW", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"09.1"}), + measure_type="rate_renegotiation", + claimed_saving_nok=50_000, # bucket [0, 100k) + description="asphalt base course reduction near school", # same words as query + ), + decision="rejected", + rationale="surface-text decoy", + ) + decoy_high = Verdict( + id="DECOY-HIGH", + proposal_features=ProposalFeatures( + affected_codes=frozenset({"21.2"}), + measure_type="material_substitution", + claimed_saving_nok=700_000, # bucket [500k, 1M) + description="asphalt base course reduction extra words", + ), + decision="rejected", + rationale="surface-text decoy", + ) + # Order deliberately not putting the true match first. + return VerdictStore(verdicts=[decoy_low, true_match, decoy_high]), "TRUE" + + +def test_retrieve_finds_structural_match_over_text_decoys() -> None: + store, true_id = _store_with_true_match_and_decoys() + hits = store.retrieve(_QUERY, k=3) + assert hits[0].id == true_id # structural match ranks #1 despite different wording + assert true_id in {h.id for h in hits[:1]} # within top-K (top-1 here) + + +def test_retrieve_is_deterministic() -> None: + store, _ = _store_with_true_match_and_decoys() + assert [h.id for h in store.retrieve(_QUERY, k=3)] == [h.id for h in store.retrieve(_QUERY, k=3)] + + +def test_retrieve_rejects_non_positive_k() -> None: + store, _ = _store_with_true_match_and_decoys() + with pytest.raises(ValueError): + store.retrieve(_QUERY, k=0) + + +def test_seed_store_has_10_to_20_verdicts() -> None: + store = seed_store() + assert 10 <= len(store.verdicts) <= 20 + + +class _RecordingContext: + """Duck-typed stand-in for SessionContext — records injected instructions without + depending on the (private) SessionContext internals.""" + + def __init__(self) -> None: + self.instructions: list[str] = [] + + def extend_instructions(self, items: list[str]) -> None: + self.instructions.extend(items) + + +async def test_expel_provider_injects_retrieved_fewshot() -> None: + store, true_id = _store_with_true_match_and_decoys() + provider = ExpeLContextProvider(store, _QUERY, k=2) + # The deterministic deliverable: the few-shot text carries the retrieved true match. + assert true_id in provider.format_fewshot() + # The injection hook (real ContextProvider.before_run signature) extends instructions. + ctx = _RecordingContext() + await provider.before_run(agent=None, session=None, context=ctx, state={}) + assert len(ctx.instructions) == 1 + assert true_id in ctx.instructions[0]