"""LLM->IR generation wired to validator-as-retry (B1 + research 03 Dim 3). A NON-STREAMING chat call asks the model for a structured ``SavingsProposal``; the reply is parsed into the typed IR. Small local models leak text or emit wrong-typed tool calls (research 03 Dim 3), so a malformed reply is RETRIED — never silently accepted. The deterministic validator is the reliability mechanism. The token bound lives HERE, in the generate loop (``meter`` checked between attempts) — so ``validator.py`` stays the verbatim Step-2 module (it is in this step's ``forbidden_paths``). Two entry points, because the LLM call is async while ``validator.self_repair`` is sync: * ``generate_with_validation`` — the SYNC validator-as-retry primitive: it drives ``validator.self_repair`` over a sync candidate source and adds the meter bound between attempts. Used for deterministic candidate sources. * ``generate_via_llm`` — the ASYNC LLM path: an async mirror of the same bounded retry that awaits the chat call (parse-retry inside the meter budget, then ``validate_proposal``). Returns a ``GenerationResult`` (the outcome PLUS the falsifications that informed it); never a malformed proposal; raises ``BudgetExceeded`` when the meter cap is crossed. """ from __future__ import annotations import json from collections.abc import Callable, Mapping from dataclasses import dataclass, field, replace from typing import Any from agent_framework import BaseChatClient, Message from pydantic import BaseModel, ValidationError from portfolio_optimiser.budget import TokenMeter from portfolio_optimiser.ir import CostBaseline, SavingsProposal from portfolio_optimiser.mandate import Approach from portfolio_optimiser.proposal_review import ( ProposalReview, ProposalReviewer, ProposalReviewRequest, ) from portfolio_optimiser.reference_domain import Project from portfolio_optimiser.validator import ( Grounding, Rejection, ValidatedProposal, self_repair, validate_proposal, ) class GenerationError(RuntimeError): """No parseable proposal could be produced within the attempt budget.""" class StructuredOutputUnsupported(TypeError): """A schema node cannot be expressed in the provider's strict structured-output subset. Fail-closed, and deliberately so (mirrors ``write_concept_file`` / ``promote_verdict``: validation, never repair). The alternative — silently dropping what cannot be expressed — would stop commissioning a field without saying so, and the field it would have dropped first is ``assumptions``, whose absence makes the Monte Carlo falsifier inert while it still reports percentiles. A schema this module cannot express is a decision for a human, not a default. """ #: Type-specific JSON Schema keywords the provider's structured-output subset does NOT support, #: transcribed from Azure's published table (Structured outputs -> "Unsupported type-specific #: keywords", https://learn.microsoft.com/azure/foundry/openai/how-to/structured-outputs), which #: states it is the same subset OpenAI accepts. #: #: ``exclusiveMinimum``/``exclusiveMaximum`` are NOT literally in that table — it names #: ``minimum maximum multipleOf`` — but they are the same family, and pydantic emits them for #: ``Field(gt=...)``/``Field(lt=...)``, which is exactly how this repo's IR spells its bounds. Being #: stricter than the table costs nothing here: every constraint stripped is re-applied by pydantic in #: ``_parse_ir`` and by ``validate_proposal``. The schema's job is SHAPE; the validator's job is #: VALUES. ``default`` is stripped for a different reason — strict mode requires every property to be #: required, so a default can never apply. UNSUPPORTED_SCHEMA_KEYWORDS = frozenset( { # String "minLength", "maxLength", "pattern", "format", # Number "minimum", "maximum", "multipleOf", "exclusiveMinimum", "exclusiveMaximum", # Objects "patternProperties", "unevaluatedProperties", "propertyNames", "minProperties", "maxProperties", # Arrays "unevaluatedItems", "contains", "minContains", "maxContains", "minItems", "maxItems", "uniqueItems", # Meaningless once every property is required "default", } ) #: The strict-legal stand-in for ``SavingsProposal.assumptions``. #: #: The IR spells the uncertainty bands as ``dict[str, tuple[float, float]]`` — a free-form map whose #: values are tuples. Neither half is expressible: strict mode requires ``additionalProperties: #: false`` in every object (so a map with arbitrary keys cannot be described), and tuples arrive as #: ``prefixItems``, which is outside the supported type list. Dropping the field instead would be #: silent damage: ``validator._monte_carlo`` falls back to the item's stated ``unit_cost`` for every #: code with no band, so with no bands at all the samples are identical and P10 == P50 == P90 — the #: stochastic falsifier goes inert while still reporting percentiles. #: #: So the WIRE carries an array of named entries and ``_parse_ir`` folds it back into the IR's map. #: The IR itself is untouched; the entry names spell out what the tuple positions mean, which the #: model would otherwise have to guess. _ASSUMPTIONS_WIRE_NODE: dict[str, Any] = { "type": "array", "description": ( "Uncertainty band per affected cost line: the low and high unit cost the true price is " "expected to fall between. The band MUST enclose that item's own unit_cost. Omit an entry " "for a line whose unit cost is certain; an empty list means no uncertainty is claimed." ), "items": { "type": "object", "properties": { "code": {"type": "string"}, "low_unit_cost": {"type": "number"}, "high_unit_cost": {"type": "number"}, }, }, } #: Dotted paths (from the root model's own properties) whose node is replaced before sanitising. _PROPOSAL_SCHEMA_OVERRIDES: Mapping[str, dict[str, Any]] = {"assumptions": _ASSUMPTIONS_WIRE_NODE} def _sanitise_schema_node(node: Any, *, path: str, overrides: Mapping[str, dict[str, Any]]) -> Any: """Rewrite one JSON Schema node into the strict subset, or raise ``StructuredOutputUnsupported``. An override is applied FIRST, so a declared replacement is what gets checked and emitted — that is how the one inexpressible node in this repo's IR (``assumptions``) is expressed rather than excused. The replacement is then sanitised by the same code as everything else, so an override cannot smuggle in an illegal node. """ if not isinstance(node, Mapping): return node if path in overrides: node = overrides[path] if "prefixItems" in node: raise StructuredOutputUnsupported( f"{path or ''}: tuple types (prefixItems) are outside the strict subset" ) for combinator in ("oneOf", "allOf"): if combinator in node: raise StructuredOutputUnsupported( f"{path or ''}: {combinator} is outside the strict subset (anyOf is the " "only supported combinator)" ) if isinstance(node.get("additionalProperties"), Mapping): raise StructuredOutputUnsupported( f"{path or ''}: a free-form map cannot be expressed — strict mode requires " "additionalProperties: false in every object. Declare an override that spells the " "entries out as an array." ) out: dict[str, Any] = {} for key, value in node.items(): if key in UNSUPPORTED_SCHEMA_KEYWORDS: continue if key == "properties" and isinstance(value, Mapping): out[key] = { name: _sanitise_schema_node( sub, path=f"{path}.{name}" if path else name, overrides=overrides ) for name, sub in value.items() } elif key == "$defs" and isinstance(value, Mapping): out[key] = { name: _sanitise_schema_node(sub, path=f"$defs.{name}", overrides=overrides) for name, sub in value.items() } elif key == "items": out[key] = _sanitise_schema_node(value, path=f"{path}[]", overrides=overrides) elif key == "anyOf" and isinstance(value, list): out[key] = [_sanitise_schema_node(sub, path=path, overrides=overrides) for sub in value] else: out[key] = value if "properties" in out: # Strict mode's two structural demands, applied to EVERY object rather than the root only: # no undeclared keys, and every declared key required. out["additionalProperties"] = False out["required"] = sorted(out["properties"]) return out def strict_json_schema( model: type[BaseModel], *, overrides: Mapping[str, dict[str, Any]] | None = None ) -> dict[str, Any]: """Derive a strict-structured-output schema from ``model``'s own pydantic schema. DERIVED rather than hand-written on purpose: a hand-written copy of a shape that already exists in ``ir.py`` is the second copy that drifts (kø-(p)), and it drifts silently — the model would keep being commissioned for the old shape. ``$defs``/``$ref`` are kept (the published subset supports definitions), so nested models need no inlining. """ schema = _sanitise_schema_node(model.model_json_schema(), path="", overrides=overrides or {}) assert isinstance(schema, dict) # a model's root schema is always an object return schema def proposal_response_format() -> dict[str, Any]: """The ``response_format`` mapping commissioning a ``SavingsProposal`` from the proposer. A MAPPING, not the ``type[BaseModel]`` the option also accepts, and the reason is measured: given a class, the client converts it with ``type_to_response_format_param``, which emits ``minimum`` / ``exclusiveMinimum`` / ``minItems`` / ``prefixItems`` and an ``assumptions`` node whose ``additionalProperties`` is a schema — four things the published subset rules out. Our own mapping is the only way to control what reaches the wire. ONE mapping serves both wired profiles (measured against agent-framework-openai 1.8.2 / agent-framework-foundry 1.8.2): the Chat Completions client passes it through verbatim, and the Responses client — which ``FoundryChatClient`` delegates to — converts this exact envelope into ``text.format``. """ return { "type": "json_schema", "json_schema": { "name": SavingsProposal.__name__, "strict": True, "schema": strict_json_schema(SavingsProposal, overrides=_PROPOSAL_SCHEMA_OVERRIDES), }, } @dataclass(frozen=True) class ParseFailure: """One model reply that did NOT parse into the typed IR, kept VERBATIM (Fase 1b, funn 1). ``text`` is the reply exactly as the model produced it — never truncated, stripped or summarised. It is the thing the run PAID for and the only evidence of *why* the reply did not parse; a paraphrase would make the next paid run a guess again, which is the defect this type exists to close. ``error`` names the parse error itself (``json.JSONDecodeError`` vs a pydantic ``ValidationError`` are very different diagnoses: leaked prose vs a wrong-shaped object). Collected into a CALLER-OWNED sink rather than returned — see ``generate_via_llm``. """ text: str error: str @dataclass(frozen=True) class GenerationResult: """What one ``generate_via_llm`` call produced: the outcome, and the falsification history that informed it (Step 5, målbilde §5/§7). A TYPED RETURN VALUE rather than an out-parameter or a callback, deliberately: the informed refinement loop already computed this history internally and then dropped it, so Step 5 was the one step of the eight with no observable output. A returned value cannot be silently lost by a caller that forgets to pass a collector, and it forces every call site to acknowledge the seam. ``refinements`` holds ONLY the rejections that were actually fed back into a later attempt's prompt — the honest reading of "informed refinement". When the attempt budget runs out, the final rejection IS ``outcome``: it informed nothing and is not repeated here. So the total number of validator falsifications this call produced is ``len(refinements)`` plus one when ``outcome`` is itself a ``Rejection``. It is empty on the common single-attempt path, which is honest rather than merely convenient: nothing was falsified, so there is nothing to show. """ outcome: ValidatedProposal | Rejection refinements: tuple[Rejection, ...] = field(default=()) def _build_messages( project: Project, context: str, prior_rejection: Rejection | None = None, *, approach: Approach | None = None, prior_feedback: str | None = None, parse_error: str | None = None, ) -> list[Message]: """Build the hypothesis prompt. When ``prior_rejection`` is set (Step 5, målbilde §5/§7), append a revision block carrying ONLY the falsification *reason* verbatim — never the prior proposal JSON (minimal honest payload: the model must address the falsification, not parrot the rejected candidate back). ``None`` -> the byte-identical base prompt, so attempt 1 is unchanged. The reason carries only the rejected claim/feasible figures, which deliberately do not collide with other load-bearing prompt markers. When ``approach`` is set (Trekk A3, krav 1) the opening instruction switches from *find one* to *quantify THIS one*: the domain expert has already decided what shall be evaluated, and the model's job is the numbers, not the direction. The expert's ``label`` and ``description`` are carried VERBATIM — the description is the reason the approach is worth trying, which is exactly the part the model cannot infer from the cost data. ``None`` -> the byte-identical base prompt, so an un-commissioned run is untouched (mirrors ``prior_rejection``). When ``prior_feedback`` is set (MAJOR-2) a THIRD block carries a domain expert's own words VERBATIM — and only the words, never the previous proposal JSON, for exactly the reason ``prior_rejection`` carries only the reason. This block describes a candidate the validator ACCEPTED and a human nonetheless asked to change, which is a different instruction from "your numbers were refuted"; conflating the two would tell the model the machine objected when a person did. ``None`` -> the byte-identical base prompt, like the other two. When ``parse_error`` is set (P20/C1) a FOURTH block carries the reason the PREVIOUS reply could not be parsed — the same "only the reason, never the JSON" rule the other two follow. It is a different instruction from both: a rejection means the numbers were refuted and a feedback means a person objected, while this one means nothing was ever read. MEASURED (P19 F4): ``_fetch_parsed`` retried with the byte-identical prompt, and one round-3 run (``kontrakt-sorasen-04``) spent ELEVEN of its twelve rounds on replies that all failed the same way — ``claimed_saving_nok: 0`` — because nothing ever told the model what was wrong. ``None`` -> the byte-identical base prompt, like the other three. All four are composable, and the ORDER is fixed: base -> approach head -> rejection -> feedback -> parse error. A prompt can legitimately carry a rejection AND a feedback at once — that is the attempt after a revise whose bought attempt the validator then rejected: the human's instruction STANDS until the human next answers, while the machine's reason is per-attempt (only the most recent, as today). """ if approach is None: head = "Propose ONE concrete cost-saving measure for this project.\n" else: head = ( "A domain expert has commissioned ONE specific approach for this project. " "Quantify THAT approach as a concrete cost-saving measure — do not substitute a " "different measure. If it does not apply to this project, say so through the " "numbers rather than proposing something else.\n" f"Approach: {approach.label}\n" ) if approach.description: head += f"Why the expert wants it evaluated: {approach.description}\n" # P19 A3: the ONE requirement of the knowledge base that binds this direction, when one was # declared. The line exists only when the field does, so every prompt written before today # — the demo's included, which is what keeps the golden transcript byte-identical — is # unchanged by construction. ``ref`` leads because that is what the model is asked to # restate; the path follows so the claim can be checked against what the run opened. if approach.requirement is not None: head += ( f"Binding requirement: {approach.requirement.ref} " f"({approach.requirement.path})\n" "Name that requirement verbatim in 'measure'.\n" ) prompt = ( f"{head}" f"Project: {project.id} - {project.name}\n" f"Context (prior verdicts / cited cost docs):\n{context}\n\n" "Respond with ONLY a JSON object for a SavingsProposal with keys: project_id, " "measure, affected_items (list of {code, quantity, unit_cost}), claimed_saving_nok, " "and optional assumptions.\n" "Each entry in affected_items must restate a cost line as the project's price schedule " "already carries it: quantity and unit_cost are the unchanged baseline figures, not the " "reduced quantity or unit cost your measure would produce. The effect of the measure " "belongs in claimed_saving_nok." ) if prior_rejection is not None: prompt += ( "\n\nYour previous proposal was REJECTED by the deterministic validator.\n" f"Reason: {prior_rejection.reason}\n" "Produce a REVISED SavingsProposal that resolves this." ) if prior_feedback is not None: prompt += ( "\n\nA domain expert reviewed your previous proposal, which the deterministic " "validator had accepted, and asked for a revision.\n" f"Expert feedback: {prior_feedback}\n" "Produce a REVISED SavingsProposal that follows this feedback." ) if parse_error is not None: prompt += ( "\n\nYour previous reply could not be PARSED as a SavingsProposal, so it was " "discarded before any validator saw it.\n" f"Reason: {parse_error}\n" "Reply with a SavingsProposal whose claimed_saving_nok is greater than 0 and whose " "affected_items each carry code, quantity and unit_cost." ) return [Message(role="user", contents=[prompt])] def _normalise_assumptions(data: dict[str, Any]) -> None: """Fold the WIRE's array-of-entries assumption bands back into the IR's ``code -> (low, high)`` map, in place. ADDITIVE, never a replacement: a reply that already uses the IR's map form (every scripted reply in the suite, and any model that answers without honouring the schema) is left untouched. A malformed entry is raised as ``ValueError`` rather than ``KeyError`` on purpose — ``ValueError`` is what ``_fetch_parsed`` catches, so a bad band is captured as the parse failure it is instead of escaping the loop and killing the run. """ entries = data.get("assumptions") if not isinstance(entries, list): return bands: dict[str, tuple[Any, Any]] = {} for entry in entries: if ( not isinstance(entry, Mapping) or not { "code", "low_unit_cost", "high_unit_cost", } <= entry.keys() ): raise ValueError( f"each assumption entry needs code, low_unit_cost and high_unit_cost; got {entry!r}" ) bands[entry["code"]] = (entry["low_unit_cost"], entry["high_unit_cost"]) data["assumptions"] = bands def _parse_ir(text: str, project: Project) -> SavingsProposal: """Parse the model's structured reply into the typed IR. Raises on malformed/text-leaked output (JSON error or Pydantic ``ValidationError``).""" data = json.loads(text) if not isinstance(data, dict): raise ValueError("reply is not a JSON object") data.setdefault("project_id", project.id) _normalise_assumptions(data) return SavingsProposal(**data) def _charge_usage(meter: TokenMeter, reply: object) -> None: usage = getattr(reply, "usage_details", None) total = usage.get("total_token_count") if usage else None if total: meter.charge(int(total)) # raises BudgetExceeded over cap def generate_with_validation( make_proposal: Callable[[int], SavingsProposal], meter: TokenMeter, *, max_attempts: int = 3, ) -> ValidatedProposal | Rejection: """Sync validator-as-retry: drive ``validator.self_repair`` over a sync candidate source, checking the token meter between attempts (the token bound lives HERE, never in ``validator.py``). Returns ``ValidatedProposal | Rejection``; raises ``BudgetExceeded`` on a meter cap.""" def _attempt(attempt: int) -> SavingsProposal: meter.tick_round() # between-attempt iteration bound (BudgetExceeded over cap) return make_proposal(attempt) return self_repair(_attempt, max_attempts=max_attempts) def _grounding_text( project: Project, baseline: CostBaseline | None, delivered: Grounding ) -> Grounding: """P7: compose the ONE text a candidate's identifiers must be grounded in — the run's non-model-authored input, and nothing else. Three sources, each of which the run can point at without asking the model: **P18/B1: the result carries the DOCUMENT BOUNDARIES, not only the text.** ``code in text`` cannot tell "this project has such a line" from "this word is in every letterhead" — P16 measured a base's own name, ``R761``, carrying a fabricated 250 000 NOK line to ``validated``. The two later sources join as ONE-LINE documents rather than being appended to a blob: each IS one cost line, and the share rule then reads them exactly as it reads a concept file. * ``delivered`` — what the CALLER can prove this run was GIVEN. ``run_project`` fills it from the delivered rendered context (the pre-pass cut, the bundle pointer, or the road path's retrieved chunks) PLUS the navigated base's ``context_files`` — never ``files``, because that is the property which drops the ``type: verdict`` layer at every level, and grounding a proposal in a prior verdict would route the ExpeL fold's own material around its gate; * the project's OWN cost lines. The road path's estimate IS the project, so a code it carries is real whether or not any text restated it. MEASURED: ``_project_from_bundle`` builds ``cost_items=()``, so this source contributes nothing on the bundle path and the gate stays exactly as sharp where fabrication was measured (K2, økt 108); * the baseline's codes when a run is anchored. Stage 0 has already ruled every code that reaches stage 0b a REAL line of this project; the weaker stage must not overrule the stronger falsifier because a prose summary happened not to repeat the code. **The rendered PROMPT is deliberately NOT a source, and that is a measurement, not taste.** Two of the prompt's parts are the model's own words fed back to it: on the S2c bundle path ``gen_context`` is the DEBATE OUTPUT (measured — a scripted proposer that names a code in a debate turn then grounds its own proposal in that turn), and from attempt 2 onward the prompt carries the previous ``Rejection.reason`` VERBATIM (Step 5) — which, for this stage, quotes the very identifier it just refused. Grounding in the prompt would therefore let the gate's own refusal ground the next attempt: a falsifier that disarms itself on its second round. """ return Grounding( documents=( *delivered.documents, *(item.code for item in project.cost_items), *(() if baseline is None else baseline.items), ), # P20/B: the base's own vocabulary of clause numbers travels WITH the text it was read # off. Carried through rather than recomposed: ``run_project`` walks the base once and # composes both halves there, and a second derivation here would be free to disagree with # the documents it is supposed to describe (kø-(p)). The two later sources are cost CODES, # which declare nothing, so they contribute none. declared_references=delivered.declared_references, ) @dataclass(frozen=True) class GroundingOffer: """P8: what the DELIVERED input of one run can lawfully ground an ``affected_item`` code in. A REPORT, never a gate. It blocks nothing — a run whose offer is null still runs — because a blocking requirement is exactly ``--require-cost-baseline``, which F4 settled as opt-in. The PAIR is the whole diagnosis, and neither number says it alone. MEASURED on K2: the delivered text offers 50 citable identifiers and ZERO cost lines, while the proposer prompt asks each ``affected_item`` to "restate a cost line as the project's price schedule already carries it". An operator reading "identifiers: 50, cost lines: 0" learns that no attempt could have succeeded; reading either number by itself, they learn nothing of the sort. ``chars`` is the size of the exact text P7's gate will measure against — carried so the report and the gate can be seen to be talking about the same input, which is the only defence against a second rendering free to disagree with the one that was sent. """ #: Length of ``_grounding_text``'s output — the text the gate itself will search. chars: int #: Distinct tokens of any ``_IDENTIFIER_FORMS`` shape the text carries. identifiers: int #: Cost lines this run can anchor one of them AS. 0 when the run is un-anchored, which is the #: state every free K2 arm measured. Read off the SAME ``baseline`` binding ``_grounding_text`` #: takes as its third source, so the count and the anchoring can never disagree. cost_lines: int def grounding_offer( project: Project, baseline: CostBaseline | None, delivered: Grounding ) -> GroundingOffer: """Measure what ``delivered`` can ground, on the EXACT text the gate will see. Composed THROUGH ``_grounding_text`` — the one composer ``generate_via_llm`` passes to ``validate_proposal`` — never re-assembled here. A second composition would be free to drift from the one that was actually sent, and a report about a text nobody was given is worse than no report: it reads as evidence. Deterministic and free: no model call, no network, and no second walk of the bundle. """ grounding = _grounding_text(project, baseline, delivered) return GroundingOffer( chars=len(grounding.text), # ONE reader, shared with P19/B3's gate (kø-(p)): a report and a gate that counted # identifiers differently would disagree about one run's own input. identifiers=len(grounding.identifiers), cost_lines=0 if baseline is None else len(baseline.items), ) async def generate_via_llm( chat_client: BaseChatClient, project: Project, context: str, meter: TokenMeter, *, max_attempts: int = 3, baseline: CostBaseline | None = None, approach: Approach | None = None, parse_failures: list[ParseFailure] | None = None, reviewer: ProposalReviewer | None = None, reviews: list[ProposalReview] | None = None, review_key: tuple[str | None, str | None] = (None, None), checker_verdict: str = "absent", grounding: Grounding | None = None, ) -> GenerationResult: """Async LLM path: non-streaming chat -> parse -> validate, with TWO bounded retry kinds, the meter checked in this loop: * malformed/text-leaked reply -> BLIND parse-retry (inner loop): re-fetch until the reply parses; never silently accepted. * validator rejection -> INFORMED refinement (outer ``max_attempts`` loop; Step 5, målbilde §5/§7): the previous attempt's ``Rejection.reason`` is fed into the next attempt's prompt (``_build_messages(prior_rejection=...)``) so the proposer can correct rather than re-answer blindly. Bounded by ``max_attempts`` + the meter (no new loop; §6). The only per-attempt falsifier here is the deterministic validator (the numbers). The checker is a run-level, one-shot signal (run.py, before generation); seeding generation with the checker critique is separately scoped and NOT done here. ``approach`` (Trekk A3, krav 1) binds every attempt of this call to ONE expert-commissioned approach. It changes only the prompt: ``validate_proposal`` is called exactly as before, so a commissioned approach gets **no discount at the deterministic gate** — the expert directs what is evaluated, never what is approved. A commissioned proposal that fails is refined through the same informed-refinement path, still bound to that approach, and returns a typed ``Rejection`` when the attempt budget runs out. ``baseline`` (S4.0) is handed straight to ``validate_proposal``, so a fabricated cost line is falsified per ATTEMPT like any other rejection — and its reason feeds the next attempt's prompt through the SAME informed-refinement path (Step 5), which is why no new loop appears here. ``parse_failures`` (Fase 1b, funn 1) is a CALLER-OWNED sink: every reply that fails to parse is appended to it VERBATIM, at the moment it fails. It is an out-parameter and not part of the return value ON PURPOSE, and the reason is measured rather than stylistic. ``meter.tick_round`` raises ``BudgetExceeded`` inside the inner fetch loop, so on the path this capture exists for — a model whose replies never parse, which burns the round ledger — this function raises and returns NOTHING. That is exactly the live Fase-1b failure. A field on ``GenerationResult`` (the Step-5 ``refinements`` shape) would be blind to it, as would any artefact written by the caller *after* a successful return. The sink mirrors ``meter`` instead: a caller-owned accumulator this loop mutates, whose contents the caller still holds however the loop ended. Step 5's "a returned value cannot be silently lost by a caller that forgets to pass a collector" governs a value that REACHES the caller; here it does not, so the rule is cited and departed from deliberately. That a caller can forget is answered by a test on the wiring, not by a shape that cannot work. ``reviewer`` (MAJOR-2) is the THIRD falsifier — a human, and the only one that gates nothing. It is called synchronously the moment ``validate_proposal`` ACCEPTS a candidate, never on a rejected one (a rejection is already fed back informed above; asking a person to comment on numbers the machine just refuted spends the person on the machine's job). It answers ``approve`` — take it as it stands — or ``revise(feedback)``, which buys exactly ONE more attempt out of the budget this loop already has: no new loop, no second cap, and the words go into the next prompt through ``prior_feedback`` (verbatim, never the proposal JSON). **D6: the outcome is the validator's LAST ruling, and the reviewer never selects among attempts.** An honoured revise whose follow-up the validator then rejects, with nothing left to buy, ends in that ``Rejection`` — exactly as an exhausted loop always has. Falling back to the earlier validated proposal would hand the run the very candidate the expert asked to change. The candidate they were LOOKING at is not lost: it travels in the review record. ``reviews`` is a CALLER-OWNED sink for the same measured reason ``parse_failures`` is one, one level sharper: ``meter.tick_round`` raises inside ``_fetch_parsed`` on the attempt a revise bought, so on exactly the run whose record matters most this function returns NOTHING. A field on ``GenerationResult`` would be blind to it. ``review_key`` is the caller's ``(approach_id, approach_label)`` — the loop records what the caller keyed, so the artefact can say WHICH candidate a human answered about; a ``None`` label falls back to the project id so a direct library caller never sees an empty header. ``checker_verdict`` (D3) is the run-level reasoning gate's answer, passed down READ-ONLY: it informs the expert, and never enters the record. **Honesty limit inherited by Step 5:** a revise consumes one of the same ``max_attempts`` iterations a validator rejection would, so a run can spend its attempts on expert revisions and never reach a second validator falsification — ``refinements`` then under-reports by construction on that run. Stated, not repaired. Returns a ``GenerationResult``: the ``ValidatedProposal | Rejection`` outcome plus every rejection that was fed back into a later attempt's prompt. Surfacing that history changes nothing about the loop's BOUND — ``max_attempts`` and ``meter.tick_round`` are exactly as before ("refine until good enough" without a cap stays forbidden, §6); it only stops the loop from discarding what it already knew. Never a malformed proposal; raises ``BudgetExceeded`` when the meter cap is crossed.""" async def _fetch_parsed(build: Callable[[str | None], list[Message]]) -> SavingsProposal: # Parse-robust: a malformed/text-leaked reply is retried; the meter caps total work. # # P20/C1: the retry is no longer BLIND. It takes a BUILDER rather than a finished message # list, because the whole defect was that the same bytes were re-sent: measured, one # round-3 run burned 11 of its 12 rounds on replies that all failed identically. The # builder is the caller's own ``_build_messages`` binding, so this loop cannot compose a # prompt the outer loop would not have composed (kø-(p)); the reason is per-RETRY, like # ``prior_rejection`` is per-attempt, and starts empty so attempt 1 is byte-identical. parse_error: str | None = None while True: meter.tick_round() # between-attempt bound (BudgetExceeded over cap) messages = build(parse_error) # Fase 1b, funn 1b: hand the model a GRAMMAR, not a prose request. The prompt's # "Respond with ONLY a JSON object" line stays — a provider that ignores # ``response_format`` (or a local model that does not implement it) must still be told # what is wanted, and the parse-retry below remains the backstop either way. reply = await chat_client.get_response( # non-streaming messages, options={"response_format": proposal_response_format()} ) _charge_usage(meter, reply) try: return _parse_ir(reply.text, project) except (ValidationError, ValueError, TypeError) as exc: # Capture BEFORE the retry: this reply was paid for, and once ``continue`` runs the # only record of what the model actually said is gone (Fase 1b, funn 1). Verbatim — # the operator is diagnosing a format failure, so any shortening removes evidence. reason = f"{type(exc).__name__}: {exc}" if parse_failures is not None: parse_failures.append(ParseFailure(text=reply.text, error=reason)) parse_error = reason continue last: Rejection | None = None # The falsifications that were FED BACK, in attempt order. ``last`` still drives the PROMPT and # is still overwritten each round -- only the most-recent falsification reaches the model, so # prompt growth is unchanged. This list is a record for the CALLER, appended to only once a # rejection is about to inform a further attempt; it is never read back into a prompt. fed_back: list[Rejection] = [] # The most recent ruling of EITHER kind. Before MAJOR-2 the exit rested on ``last``, which only # a validator REJECTION sets. The plan predicted that "validated -> revise" on every attempt # would therefore reach the end of the loop with ``last is None`` and die on # ``assert last is not None``; MEASUREMENT FALSIFIED THAT (MAJOR-2 mutation M29 stayed GREEN # against the whole suite). D1(a) RETURNS inside the loop when ``remaining == 0``, and on the # final attempt ``max_attempts - i - 1`` is always 0 -- so the tail is reachable only after a # validator REJECTION, which sets ``last`` too. This carrier is therefore UNWITNESSED (the # ``budget_stop`` precedent): it stands because it makes D6 explicit and removes an assert that # rested on a non-local invariant, not because a test holds it. D6 reads straight off it: # whatever the validator ruled LAST is what the run carries, never a proposal the reviewer # picked. last_ruling: ValidatedProposal | Rejection | None = None # The expert's STANDING instruction. Unlike ``last`` it is not cleared per attempt: it holds # until the reviewer next answers, because a human's request survives one machine round trip. feedback: str | None = None # Index into ``reviews`` of a revise whose bought attempt has not fetched yet. ``honoured`` # means the follow-up actually FETCHED, not that it was bought -- so it is promoted only once # ``_fetch_parsed`` returns, and a revise the ledger cut before then stays false. pending_revise: int | None = None approach_id, approach_label = review_key for i in range(max_attempts): # Informed refinement: feed the PREVIOUS attempt's validator rejection into this # attempt's prompt. ``last`` is None on attempt 1 -> the unchanged base prompt; it is # overwritten each round -> only the most-recent falsification ("forrige"), never an # accumulated history (bounded prompt growth). if last is not None: fed_back.append(last) candidate = await _fetch_parsed( lambda parse_error: _build_messages( project, context, prior_rejection=last, approach=approach, prior_feedback=feedback, parse_error=parse_error, ) ) if pending_revise is not None and reviews is not None: reviews[pending_revise] = replace(reviews[pending_revise], honoured=True) pending_revise = None # P7: ground the candidate in the EXACT text this attempt sent. ``messages`` is the # proposer's entire input for this call -- on the S2c bundle path the debate navigates and # generation sees only the debate output, so an identifier the candidate carries but the # messages do not came from the model's weights, not from this run. Rendered from the ONE # ``_build_messages`` result rather than recomposed here: a second rendering of the prompt # would be free to disagree with the one that was actually sent. result = validate_proposal( candidate, baseline=baseline, grounding=_grounding_text( project, baseline, # ``None`` -> the retrieval ``context`` this caller handed in, which for a # caller that declared nothing else IS the input it declared. ``run_project`` # always passes it EXPLICITLY, because on the debate path ``context`` has been # replaced by the model's OWN summary of what it read. Grounding.of(context) if grounding is None else grounding, ), ) last_ruling = result if isinstance(result, Rejection): last = result continue if reviewer is None: return GenerationResult(outcome=result, refinements=tuple(fed_back)) # What a ``revise`` can ACTUALLY buy: the smaller of this call's own attempt headroom and # the SHARED round ledger's. The ledger (``max_rounds*4`` at run level) spans every approach # of a mandate and is frequently what binds; a number computed from ``max_attempts`` alone # would be a claim the terminal makes about itself that the meter then refutes. remaining = min(max_attempts - i - 1, meter.budget.max_rounds - meter.rounds) decision = reviewer( ProposalReviewRequest( project_id=project.id, approach_id=approach_id, approach_label=approach_label or project.id, attempt=i, attempts_remaining=remaining, proposal=result, checker_verdict=checker_verdict, ) ) if decision.feedback is None: if reviews is not None: reviews.append( ProposalReview( approach_id=approach_id, attempt=i, decision="approve", feedback="", honoured=True, proposal=result, ) ) return GenerationResult(outcome=result, refinements=tuple(fed_back)) if reviews is not None: reviews.append( ProposalReview( approach_id=approach_id, attempt=i, decision="revise", feedback=decision.feedback, honoured=False, proposal=result, ) ) if remaining <= 0: # An un-honoured revise buys nothing, so the last ruling -- the validated one -- stands # (criterion 12). One rule with D6, both cases. return GenerationResult(outcome=result, refinements=tuple(fed_back)) pending_revise = None if reviews is None else len(reviews) - 1 feedback = decision.feedback # The machine's reason does not apply to a proposal the machine ACCEPTED. last = None if last_ruling is None: # pragma: no cover - every call site passes max_attempts >= 1 raise ValueError(f"max_attempts must be positive, got {max_attempts}") # The validator's LAST ruling, whichever kind it was. When it is a ``Rejection`` it was never # fed back, so it is deliberately absent from ``refinements``. return GenerationResult(outcome=last_ruling, refinements=tuple(fed_back))