From 0abc7083df7adf973ccf7080c42611627c0ea419 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Wed, 24 Jun 2026 13:48:54 +0200 Subject: [PATCH] feat(fase2): LLM->IR generation with validator-as-retry --- src/portfolio_optimiser/generate.py | 125 ++++++++++++++++++++++++++++ tests/test_generate.py | 63 ++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 src/portfolio_optimiser/generate.py create mode 100644 tests/test_generate.py diff --git a/src/portfolio_optimiser/generate.py b/src/portfolio_optimiser/generate.py new file mode 100644 index 0000000..903f31a --- /dev/null +++ b/src/portfolio_optimiser/generate.py @@ -0,0 +1,125 @@ +"""LLM->IR generation wired to validator-as-retry (B1 + research 03 Dim 3). + +A NON-STREAMING chat call asks the model for a structured ``SavingsProposal``; the reply is +parsed into the typed IR. Small local models leak text or emit wrong-typed tool calls +(research 03 Dim 3), so a malformed reply is RETRIED — never silently accepted. The +deterministic validator is the reliability mechanism. + +The token bound lives HERE, in the generate loop (``meter`` checked between attempts) — so +``validator.py`` stays the verbatim Step-2 module (it is in this step's ``forbidden_paths``). + +Two entry points, because the LLM call is async while ``validator.self_repair`` is sync: + +* ``generate_with_validation`` — the SYNC validator-as-retry primitive: it drives + ``validator.self_repair`` over a sync candidate source and adds the meter bound between + attempts. Used for deterministic candidate sources. +* ``generate_via_llm`` — the ASYNC LLM path: an async mirror of the same bounded retry that + awaits the chat call (parse-retry inside the meter budget, then ``validate_proposal``). + Returns ``ValidatedProposal | Rejection``; never a malformed proposal; raises + ``BudgetExceeded`` when the meter cap is crossed. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable + +from agent_framework import BaseChatClient, Message +from pydantic import ValidationError + +from portfolio_optimiser.budget import TokenMeter +from portfolio_optimiser.ir import SavingsProposal +from portfolio_optimiser.reference_domain import Project +from portfolio_optimiser.validator import ( + Rejection, + ValidatedProposal, + self_repair, + validate_proposal, +) + + +class GenerationError(RuntimeError): + """No parseable proposal could be produced within the attempt budget.""" + + +def _build_messages(project: Project, context: str) -> list[Message]: + prompt = ( + "Propose ONE concrete cost-saving measure for this project.\n" + f"Project: {project.id} - {project.name}\n" + f"Context (prior verdicts / cited cost docs):\n{context}\n\n" + "Respond with ONLY a JSON object for a SavingsProposal with keys: project_id, " + "measure, affected_items (list of {code, quantity, unit_cost}), claimed_saving_nok, " + "and optional assumptions." + ) + return [Message(role="user", contents=[prompt])] + + +def _parse_ir(text: str, project: Project) -> SavingsProposal: + """Parse the model's structured reply into the typed IR. Raises on malformed/text-leaked + output (JSON error or Pydantic ``ValidationError``).""" + data = json.loads(text) + if not isinstance(data, dict): + raise ValueError("reply is not a JSON object") + data.setdefault("project_id", project.id) + return SavingsProposal(**data) + + +def _charge_usage(meter: TokenMeter, reply: object) -> None: + usage = getattr(reply, "usage_details", None) + total = usage.get("total_token_count") if usage else None + if total: + meter.charge(int(total)) # raises BudgetExceeded over cap + + +def generate_with_validation( + make_proposal: Callable[[int], SavingsProposal], + meter: TokenMeter, + *, + max_attempts: int = 3, +) -> ValidatedProposal | Rejection: + """Sync validator-as-retry: drive ``validator.self_repair`` over a sync candidate source, + checking the token meter between attempts (the token bound lives HERE, never in + ``validator.py``). Returns ``ValidatedProposal | Rejection``; raises ``BudgetExceeded`` on + a meter cap.""" + + def _attempt(attempt: int) -> SavingsProposal: + meter.tick_round() # between-attempt iteration bound (BudgetExceeded over cap) + return make_proposal(attempt) + + return self_repair(_attempt, max_attempts=max_attempts) + + +async def generate_via_llm( + chat_client: BaseChatClient, + project: Project, + context: str, + meter: TokenMeter, + *, + max_attempts: int = 3, +) -> ValidatedProposal | Rejection: + """Async LLM path: non-streaming chat -> parse -> validate, with bounded retries and the + meter checked in this loop. A malformed/text-leaked reply is retried (never silently + accepted); a validation rejection is retried; returns ``ValidatedProposal | Rejection``; + raises ``BudgetExceeded`` when the meter cap is crossed.""" + messages = _build_messages(project, context) + + async def _fetch_parsed() -> SavingsProposal: + # Parse-robust: a malformed/text-leaked reply is retried; the meter caps total work. + while True: + meter.tick_round() # between-attempt bound (BudgetExceeded over cap) + reply = await chat_client.get_response(messages) # non-streaming + _charge_usage(meter, reply) + try: + return _parse_ir(reply.text, project) + except (ValidationError, ValueError, TypeError): + continue + + last: Rejection | None = None + for _ in range(max_attempts): + candidate = await _fetch_parsed() + result = validate_proposal(candidate) + if isinstance(result, ValidatedProposal): + return result + last = result + assert last is not None # max_attempts >= 1, so at least one validation ran + return last # validation never passed within the attempt budget -> typed Rejection diff --git a/tests/test_generate.py b/tests/test_generate.py new file mode 100644 index 0000000..39789e7 --- /dev/null +++ b/tests/test_generate.py @@ -0,0 +1,63 @@ +"""Step 10 tests — LLM->IR generation + validator-as-retry (FakeChatClient, no LLM). + +A well-formed reply validates; a malformed/text-leaked reply is retried (not silently +accepted); exhausting attempts OR crossing the meter cap is a typed failure (never a +malformed proposal). Pattern: tests/spikes/test_harness.py + tests/spikes/test_c_validator.py. +""" + +import pytest +from spikes._harness import FakeChatClient + +from portfolio_optimiser.budget import Budget, BudgetExceeded, TokenMeter +from portfolio_optimiser.generate import generate_via_llm, generate_with_validation +from portfolio_optimiser.ir import SavingsProposal +from portfolio_optimiser.reference_domain import load_reference_projects +from portfolio_optimiser.validator import Rejection, ValidatedProposal, proposal_for + +# A feasible proposal for FV42-GSV-E1: 200k <= ~445k (30% of the affected total). +_VALID = ( + '{"project_id":"FV42-GSV-E1","measure":"Reduce scope",' + '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},' + '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}' +) + + +@pytest.fixture(scope="module") +def project(): + return load_reference_projects()[0] # FV42-GSV-E1 + + +def _meter() -> TokenMeter: + return TokenMeter(Budget(max_tokens=10**9, max_rounds=20)) + + +async def test_wellformed_reply_yields_validated_proposal(project) -> None: + client = FakeChatClient(scripted=[_VALID], default_reply=_VALID) + result = await generate_via_llm(client, project, "", _meter(), max_attempts=3) + assert isinstance(result, ValidatedProposal) + assert isinstance(result.proposal, SavingsProposal) + + +async def test_malformed_reply_is_retried_not_silently_accepted(project) -> None: + client = FakeChatClient(scripted=["not json at all {{{", _VALID], default_reply=_VALID) + result = await generate_via_llm(client, project, "", _meter(), max_attempts=3) + assert isinstance(result, ValidatedProposal) # the malformed reply was NOT accepted + assert client.call_count >= 2 # it retried past the malformed reply + + +def test_generate_with_validation_is_bounded_and_typed(project) -> None: + calls = {"n": 0} + + def always_out_of_range(_attempt: int) -> SavingsProposal: + calls["n"] += 1 + return proposal_for(project, ["05.2", "03.1"], claimed_saving_nok=800_000) + + # Exhausting attempts -> typed Rejection (never a malformed/validated proposal); self_repair bounded. + result = generate_with_validation(always_out_of_range, _meter(), max_attempts=3) + assert isinstance(result, Rejection) + assert calls["n"] == 3 + + # Crossing the meter cap -> typed failure (BudgetExceeded). + tiny = TokenMeter(Budget(max_tokens=10**9, max_rounds=1)) + with pytest.raises(BudgetExceeded): + generate_with_validation(always_out_of_range, tiny, max_attempts=3)