feat(fase2): LLM->IR generation with validator-as-retry

src/portfolio_optimiser/generate.py

@@ -0,0 +1,125 @@
+"""LLM->IR generation wired to validator-as-retry (B1 + research 03 Dim 3).
+
+A NON-STREAMING chat call asks the model for a structured ``SavingsProposal``; the reply is
+parsed into the typed IR. Small local models leak text or emit wrong-typed tool calls
+(research 03 Dim 3), so a malformed reply is RETRIED — never silently accepted. The
+deterministic validator is the reliability mechanism.
+
+The token bound lives HERE, in the generate loop (``meter`` checked between attempts) — so
+``validator.py`` stays the verbatim Step-2 module (it is in this step's ``forbidden_paths``).
+
+Two entry points, because the LLM call is async while ``validator.self_repair`` is sync:
+
+* ``generate_with_validation`` — the SYNC validator-as-retry primitive: it drives
+  ``validator.self_repair`` over a sync candidate source and adds the meter bound between
+  attempts. Used for deterministic candidate sources.
+* ``generate_via_llm`` — the ASYNC LLM path: an async mirror of the same bounded retry that
+  awaits the chat call (parse-retry inside the meter budget, then ``validate_proposal``).
+  Returns ``ValidatedProposal | Rejection``; never a malformed proposal; raises
+  ``BudgetExceeded`` when the meter cap is crossed.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+
+from agent_framework import BaseChatClient, Message
+from pydantic import ValidationError
+
+from portfolio_optimiser.budget import TokenMeter
+from portfolio_optimiser.ir import SavingsProposal
+from portfolio_optimiser.reference_domain import Project
+from portfolio_optimiser.validator import (
+    Rejection,
+    ValidatedProposal,
+    self_repair,
+    validate_proposal,
+)
+
+
+class GenerationError(RuntimeError):
+    """No parseable proposal could be produced within the attempt budget."""
+
+
+def _build_messages(project: Project, context: str) -> list[Message]:
+    prompt = (
+        "Propose ONE concrete cost-saving measure for this project.\n"
+        f"Project: {project.id} - {project.name}\n"
+        f"Context (prior verdicts / cited cost docs):\n{context}\n\n"
+        "Respond with ONLY a JSON object for a SavingsProposal with keys: project_id, "
+        "measure, affected_items (list of {code, quantity, unit_cost}), claimed_saving_nok, "
+        "and optional assumptions."
+    )
+    return [Message(role="user", contents=[prompt])]
+
+
+def _parse_ir(text: str, project: Project) -> SavingsProposal:
+    """Parse the model's structured reply into the typed IR. Raises on malformed/text-leaked
+    output (JSON error or Pydantic ``ValidationError``)."""
+    data = json.loads(text)
+    if not isinstance(data, dict):
+        raise ValueError("reply is not a JSON object")
+    data.setdefault("project_id", project.id)
+    return SavingsProposal(**data)
+
+
+def _charge_usage(meter: TokenMeter, reply: object) -> None:
+    usage = getattr(reply, "usage_details", None)
+    total = usage.get("total_token_count") if usage else None
+    if total:
+        meter.charge(int(total))  # raises BudgetExceeded over cap
+
+
+def generate_with_validation(
+    make_proposal: Callable[[int], SavingsProposal],
+    meter: TokenMeter,
+    *,
+    max_attempts: int = 3,
+) -> ValidatedProposal | Rejection:
+    """Sync validator-as-retry: drive ``validator.self_repair`` over a sync candidate source,
+    checking the token meter between attempts (the token bound lives HERE, never in
+    ``validator.py``). Returns ``ValidatedProposal | Rejection``; raises ``BudgetExceeded`` on
+    a meter cap."""
+
+    def _attempt(attempt: int) -> SavingsProposal:
+        meter.tick_round()  # between-attempt iteration bound (BudgetExceeded over cap)
+        return make_proposal(attempt)
+
+    return self_repair(_attempt, max_attempts=max_attempts)
+
+
+async def generate_via_llm(
+    chat_client: BaseChatClient,
+    project: Project,
+    context: str,
+    meter: TokenMeter,
+    *,
+    max_attempts: int = 3,
+) -> ValidatedProposal | Rejection:
+    """Async LLM path: non-streaming chat -> parse -> validate, with bounded retries and the
+    meter checked in this loop. A malformed/text-leaked reply is retried (never silently
+    accepted); a validation rejection is retried; returns ``ValidatedProposal | Rejection``;
+    raises ``BudgetExceeded`` when the meter cap is crossed."""
+    messages = _build_messages(project, context)
+
+    async def _fetch_parsed() -> SavingsProposal:
+        # Parse-robust: a malformed/text-leaked reply is retried; the meter caps total work.
+        while True:
+            meter.tick_round()  # between-attempt bound (BudgetExceeded over cap)
+            reply = await chat_client.get_response(messages)  # non-streaming
+            _charge_usage(meter, reply)
+            try:
+                return _parse_ir(reply.text, project)
+            except (ValidationError, ValueError, TypeError):
+                continue
+
+    last: Rejection | None = None
+    for _ in range(max_attempts):
+        candidate = await _fetch_parsed()
+        result = validate_proposal(candidate)
+        if isinstance(result, ValidatedProposal):
+            return result
+        last = result
+    assert last is not None  # max_attempts >= 1, so at least one validation ran
+    return last  # validation never passed within the attempt budget -> typed Rejection

tests/test_generate.py

@@ -0,0 +1,63 @@
+"""Step 10 tests — LLM->IR generation + validator-as-retry (FakeChatClient, no LLM).
+
+A well-formed reply validates; a malformed/text-leaked reply is retried (not silently
+accepted); exhausting attempts OR crossing the meter cap is a typed failure (never a
+malformed proposal). Pattern: tests/spikes/test_harness.py + tests/spikes/test_c_validator.py.
+"""
+
+import pytest
+from spikes._harness import FakeChatClient
+
+from portfolio_optimiser.budget import Budget, BudgetExceeded, TokenMeter
+from portfolio_optimiser.generate import generate_via_llm, generate_with_validation
+from portfolio_optimiser.ir import SavingsProposal
+from portfolio_optimiser.reference_domain import load_reference_projects
+from portfolio_optimiser.validator import Rejection, ValidatedProposal, proposal_for
+
+# A feasible proposal for FV42-GSV-E1: 200k <= ~445k (30% of the affected total).
+_VALID = (
+    '{"project_id":"FV42-GSV-E1","measure":"Reduce scope",'
+    '"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
+    '{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
+)
+
+
+@pytest.fixture(scope="module")
+def project():
+    return load_reference_projects()[0]  # FV42-GSV-E1
+
+
+def _meter() -> TokenMeter:
+    return TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
+
+
+async def test_wellformed_reply_yields_validated_proposal(project) -> None:
+    client = FakeChatClient(scripted=[_VALID], default_reply=_VALID)
+    result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
+    assert isinstance(result, ValidatedProposal)
+    assert isinstance(result.proposal, SavingsProposal)
+
+
+async def test_malformed_reply_is_retried_not_silently_accepted(project) -> None:
+    client = FakeChatClient(scripted=["not json at all {{{", _VALID], default_reply=_VALID)
+    result = await generate_via_llm(client, project, "", _meter(), max_attempts=3)
+    assert isinstance(result, ValidatedProposal)  # the malformed reply was NOT accepted
+    assert client.call_count >= 2  # it retried past the malformed reply
+
+
+def test_generate_with_validation_is_bounded_and_typed(project) -> None:
+    calls = {"n": 0}
+
+    def always_out_of_range(_attempt: int) -> SavingsProposal:
+        calls["n"] += 1
+        return proposal_for(project, ["05.2", "03.1"], claimed_saving_nok=800_000)
+
+    # Exhausting attempts -> typed Rejection (never a malformed/validated proposal); self_repair bounded.
+    result = generate_with_validation(always_out_of_range, _meter(), max_attempts=3)
+    assert isinstance(result, Rejection)
+    assert calls["n"] == 3
+
+    # Crossing the meter cap -> typed failure (BudgetExceeded).
+    tiny = TokenMeter(Budget(max_tokens=10**9, max_rounds=1))
+    with pytest.raises(BudgetExceeded):
+        generate_with_validation(always_out_of_range, tiny, max_attempts=3)