Kjell Tore Guttormsen
ce5b1151c8
Post-hoc /trekreview of S4.2 surfaced two confirmed findings; both closed via TDD. S42-001 (MAJOR): the new --live-dry-run CLI tests read PORTFOLIO_MODEL_MAP / PORTFOLIO_FOUNDRY_PROJECT_ENDPOINT via resolve_model/AzureFoundryBackend but did not isolate them, so both arms inverted their rc in a Foundry-configured env. Add an autouse fixture that delenvs both, mirroring test_backends/test_preflight. S42-002 (MINOR): the --live-dry-run except ValueError attached the azure-preflight remediation to every offline-path ValueError (unknown project_id, empty docs_dir, bundle mismatch). Scope the hint to args.profile == "azure"; structured refusal + rc 1 preserved for all. New test proves a LOCAL unknown-project refusal carries no azure hint. Gate: pytest 358 passed / 4 skipped, ruff check + format clean, mypy 24 files.
695 lines
34 KiB
Python
695 lines
34 KiB
Python
"""Vertical-slice orchestrator + single-command entry (two-layer HITL wiring).
|
|
|
|
``run_project`` composes the whole method for ONE synthetic project on real (or injected)
|
|
chat clients:
|
|
|
|
1. ``load_contracts`` — fail-fast: every config (incl. the verdict-feedback shape) is
|
|
validated BEFORE any chat client is built.
|
|
2. load the project + retrieve cited chunks via the Step-7 data source -> first-class
|
|
``provenance.Citation`` list.
|
|
3. a FRESH maker-checker ``GroupChat`` debate (``fresh_workflow``), round-capped
|
|
(``with_max_rounds``); **Layer-1 HITL** = the optional in-run ``with_request_info`` gate.
|
|
4. ``generate_via_llm`` -> blocking ``validate_proposal`` -> ``ValidatedProposal | Rejection``
|
|
(the token bound is the ``meter`` checked in the generate loop).
|
|
5. attach a first-class ``ProvenanceStamp``.
|
|
6. **Layer-2 (out-of-band)**: ``capture_verdict`` mints a stable id from the proposal's
|
|
features; the decision/rationale come from ``verdict_input`` (function arg / CLI / fixture).
|
|
The B11 expert notification is a STUB (``notify``) in Fase 2.
|
|
7. persist to the ``VerdictStore`` -> the next run's ``ExpeLContextProvider`` retrieval (the
|
|
learning loop; this run also exercises the two-arg ``extend_instructions`` injection).
|
|
|
|
The two HITL layers are deliberately distinct: **Layer-1** is the optional synchronous in-run
|
|
review gate (no checkpoint — research 01: durable resume is fragile); **Layer-2** is the
|
|
durable learned verdict captured out-of-band in the VerdictStore (D7-portable).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable, Sequence
|
|
from dataclasses import dataclass, replace
|
|
from decimal import ROUND_HALF_UP, Decimal
|
|
from typing import Any, Literal, cast
|
|
|
|
from agent_framework import BaseChatClient, SessionContext
|
|
|
|
from portfolio_optimiser.backends import Profile, get_backend, resolve_model
|
|
from portfolio_optimiser.budget import Budget, BudgetMiddleware, TokenMeter
|
|
from portfolio_optimiser.contracts import GoalConfig, GoalContract, load_contracts
|
|
from portfolio_optimiser.ledger import SavingsLedger
|
|
from portfolio_optimiser.datasource import (
|
|
bundle_citations,
|
|
chunk_dict_to_citation,
|
|
make_retrieval_tool,
|
|
retrieve_chunks,
|
|
)
|
|
from portfolio_optimiser.dimension import Dimension, admits
|
|
from portfolio_optimiser.generate import generate_via_llm
|
|
from portfolio_optimiser.ir import SavingsProposal
|
|
from portfolio_optimiser.provenance import ProvenanceStamp
|
|
from portfolio_optimiser.reference_domain import Project, load_reference_projects
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
|
from portfolio_optimiser import okf, outbox
|
|
from portfolio_optimiser.verdicts import (
|
|
ExpeLContextProvider,
|
|
ProposalFeatures,
|
|
Verdict,
|
|
VerdictStore,
|
|
bundle_candidate_features,
|
|
capture_verdict,
|
|
load_verdicts_from_dir,
|
|
)
|
|
from portfolio_optimiser.workflow import _MAKER_CHECKER_ROLES, fresh_workflow
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RunResult:
|
|
"""The outcome of one project run: the validated/rejected proposal, its first-class
|
|
provenance, the captured (Layer-2) verdict, the ExpeL hits surfaced for it, the store, the
|
|
debate's converged output that the candidate was generated from (F1 traceability), and the
|
|
checker's gate decision (Step 3/4: ``"approve" | "reject" | "absent"``). ``checker_verdict``
|
|
records the checker's decision distinctly from ``provenance.validator_decision`` so the two
|
|
falsifiers (reasoning vs numbers) are never conflated."""
|
|
|
|
outcome: ValidatedProposal | Rejection
|
|
provenance: ProvenanceStamp
|
|
verdict: Verdict
|
|
retrieved: list[Verdict]
|
|
store: VerdictStore
|
|
debate_output: str
|
|
checker_verdict: str = "absent"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DryRunReport:
|
|
"""S4.2 offline ``--live-dry-run`` outcome (comparison protocol §4 pkt 3): everything a real run
|
|
would use — profile, the resolved model-id per BUILT role, and the round/token parameters —
|
|
captured WITHOUT a model call. A DISTINCT type from ``RunResult``, whose post-generation fields
|
|
(outcome/provenance/verdict) do not exist yet on a run that stopped before the first model call."""
|
|
|
|
profile: str
|
|
resolved_models: dict[str, str]
|
|
max_rounds: int
|
|
max_tokens: int
|
|
top_k: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GoalReached:
|
|
"""A savings-goal signal VALUE (Step 8, SC6) — NOT an exception. Structured like
|
|
``BudgetExceeded`` (``budget.py:22-34``) but semantically SUCCESS (the goal was reached), not
|
|
resource exhaustion (H1). Used as a ``stop_reason`` value + a loop ``break``, never ``raise``d.
|
|
``scope`` is ``"portfolio"`` (the whole pass) or ``"project"`` (one pid); ``limit_ore`` is the
|
|
threshold that was met, ``observed_ore`` the accumulated realized sum that met it (``>=``)."""
|
|
|
|
scope: Literal["project", "portfolio"]
|
|
project_id: str | None
|
|
limit_ore: int
|
|
observed_ore: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PortfolioResult:
|
|
"""The outcome of a sequential fan-out over N projects (SC2).
|
|
|
|
``runs`` is one ``RunResult`` per project in input order; ``store`` is the ONE shared
|
|
``VerdictStore`` threaded across every run (the cross-project ExpeL learning loop).
|
|
The remaining fields are a thin aggregate over ``runs``: ``validated_count`` /
|
|
``rejected_count`` partition the outcomes; ``sum_claimed_saving_nok`` totals the claimed
|
|
saving of the validated proposals only; ``sum_token_usage`` totals every run's
|
|
provenance token usage. ``stopped_early`` / ``stop_reason`` record a Step-8 goal-stop: they
|
|
default so the frozen aggregate and every existing constructor call are unaffected."""
|
|
|
|
runs: tuple[RunResult, ...]
|
|
store: VerdictStore
|
|
validated_count: int
|
|
rejected_count: int
|
|
sum_claimed_saving_nok: float
|
|
sum_token_usage: int
|
|
stopped_early: bool = False
|
|
stop_reason: GoalReached | None = None
|
|
|
|
|
|
def _authored_texts(result: Any, name: str) -> list[str]:
|
|
"""The texts of ``get_outputs()`` entries authored by participant ``name`` (proposer/checker),
|
|
in surfaced order. MAF surfaces ``author_name`` on each output's ``messages`` — NOT on the
|
|
``AgentResponse`` itself (verified against 1.9.0) — so we match through ``messages``. This
|
|
separates the proposer's converged output (fed to generation, F1) from the checker's gate
|
|
verdict (Step 3/4); the orchestrator's termination notice is authored by neither, so it is
|
|
excluded automatically."""
|
|
texts: list[str] = []
|
|
for out in result.get_outputs():
|
|
if not any(getattr(m, "author_name", None) == name for m in getattr(out, "messages", [])):
|
|
continue
|
|
text = out if isinstance(out, str) else getattr(out, "text", None)
|
|
if text:
|
|
texts.append(text)
|
|
return texts
|
|
|
|
|
|
def _debate_text(result: Any) -> str:
|
|
"""The PROPOSER's converged output (fed into generation, F1). With ``output_from=agents`` both
|
|
participants surface, so we select proposer-authored outputs specifically — taking the last of
|
|
ALL surfaced outputs would feed the checker's verdict to generation at even round counts.
|
|
Returns ``""`` when the proposer produced no surfaced text."""
|
|
proposer_texts = _authored_texts(result, "proposer")
|
|
return proposer_texts[-1] if proposer_texts else ""
|
|
|
|
|
|
def _checker_verdict(result: Any) -> tuple[str, str]:
|
|
"""Parse the checker's gate verdict from its surfaced debate output (Step 3/4, målbilde §2/§6).
|
|
Returns ``(decision, reason)``: ``"reject"`` ONLY on an explicit ``VERDICT: REJECT`` (with the
|
|
trailing reason), ``"approve"`` on an explicit ``VERDICT: APPROVE``, else ``"absent"``. The gate
|
|
is opt-in-reject (fail-open): a missing/unparseable marker never blocks, so the deterministic
|
|
validator stays the sole gate on those runs."""
|
|
checker_texts = _authored_texts(result, "checker")
|
|
text = checker_texts[-1] if checker_texts else ""
|
|
upper = text.upper()
|
|
if "VERDICT: REJECT" in upper:
|
|
reason = text[upper.index("VERDICT: REJECT") + len("VERDICT: REJECT") :]
|
|
return "reject", reason.lstrip(" -:—").strip()
|
|
if "VERDICT: APPROVE" in upper:
|
|
return "approve", ""
|
|
return "absent", ""
|
|
|
|
|
|
def _project_by_id(project_id: str) -> Project:
|
|
for project in load_reference_projects():
|
|
if project.id == project_id:
|
|
return project
|
|
raise ValueError(f"unknown project_id: {project_id!r}")
|
|
|
|
|
|
def _project_from_bundle(
|
|
bundle_dir: str, project_id: str, *, bundle: okf.Bundle | None = None
|
|
) -> Project:
|
|
"""Derive a minimal ``Project`` from an OKF bundle (so a bundle the loop runs need NOT be a
|
|
road reference-domain project). Only ``id`` + ``name`` reach the generation prompt
|
|
(``generate._build_messages``), so ``cost_items`` is empty and ``verdict_input`` is unused here
|
|
(the Layer-2 decision flows via the ``verdict_input`` argument). Fail-fast: the bundle's IR
|
|
``project_id`` must match the requested id. ``bundle`` reuses an already-navigated bundle to
|
|
avoid a second navigation."""
|
|
ir = okf.load_ir_projection(bundle_dir)
|
|
if ir["project_id"] != project_id:
|
|
raise ValueError(f"bundle project_id {ir['project_id']!r} != requested {project_id!r}")
|
|
nav = bundle if bundle is not None else okf.navigate_bundle(bundle_dir)
|
|
project_file = next((f for f in nav.files if f.type == "project"), None)
|
|
name = (
|
|
project_file.frontmatter.get("title", project_id).strip('"')
|
|
if project_file is not None
|
|
else project_id
|
|
)
|
|
return Project(
|
|
id=project_id,
|
|
name=name,
|
|
description="",
|
|
currency="NOK",
|
|
cost_items=(),
|
|
docs_dir=bundle_dir,
|
|
verdict_input={},
|
|
)
|
|
|
|
|
|
def _features_of(proposal: SavingsProposal) -> ProposalFeatures:
|
|
return ProposalFeatures(
|
|
affected_codes=frozenset(item.code for item in proposal.affected_items),
|
|
measure_type=proposal.measure,
|
|
claimed_saving_nok=proposal.claimed_saving_nok,
|
|
description=proposal.measure,
|
|
)
|
|
|
|
|
|
def _default_factory(profile: Profile | str) -> Callable[[str], BaseChatClient]:
|
|
def factory(role: str) -> BaseChatClient:
|
|
return get_backend(profile).create_chat_client(model=resolve_model(profile, role))
|
|
|
|
return factory
|
|
|
|
|
|
async def run_project(
|
|
project_id: str,
|
|
profile: Profile | str = Profile.LOCAL,
|
|
*,
|
|
docs_dir: str,
|
|
verdict_input: dict[str, str],
|
|
bundle_dir: str | None = None,
|
|
dimension: Dimension | None = None,
|
|
store: VerdictStore | None = None,
|
|
verdict_dir: str | None = None,
|
|
outbox_dir: str | None = None,
|
|
run_id: str | None = None,
|
|
client_factory: Callable[[str], BaseChatClient] | None = None,
|
|
max_rounds: int = 3,
|
|
max_tokens: int = 100_000,
|
|
top_k: int = 3,
|
|
enable_layer1_hitl: bool = False,
|
|
notify: Callable[[Verdict], None] | None = None,
|
|
meter: TokenMeter | None = None,
|
|
live_dry_run: bool = False,
|
|
) -> RunResult | DryRunReport:
|
|
"""Run the vertical slice for ONE project. ``client_factory`` is the test-injection seam
|
|
(defaults to the real backend). ``verdict_input`` carries the expert decision/rationale
|
|
(Layer-2). ``bundle_dir`` (Fase 2a) makes the run OKF-bundle-driven: the project is derived
|
|
from the bundle and, before generation, the candidate's prior verdicts in ``store`` are folded
|
|
into the hypothesis prompt (Step-1 ExpeL wiring, målbilde §5/§7). ``verdict_dir`` (Fase 5,
|
|
Steg 7, målbilde §3/§7) is the async file inbox: a folder of expert/persona-authored verdict
|
|
files (plain JSON, R2 raw layer) MERGED into the store BEFORE the Step-1 fold, so a verdict
|
|
dropped after an earlier run is consumed by this separate, later run — the long feedback loop,
|
|
fully resumable across runs separated in time. The system READS this folder; it does not write
|
|
to it (the expert/persona writes, målbilde §3). ``outbox_dir`` (Fase 2a, Steg 7 output layer,
|
|
målbilde §3, R2) is the RAW OUTBOX: when set, the run's proposal + outcome artefacts are written
|
|
there via ``outbox.write_outbox`` (``run_id`` is then REQUIRED — no wall-clock/uuid default, for
|
|
byte-determinism). The outbox dir should be DISTINCT from any ``verdict_dir`` inbox: writing the
|
|
outbox into a folder later read as an inbox would re-ingest raw agent output and bypass the
|
|
Step-8 promotion gate (self-contamination) — documented here, not enforced. Raises
|
|
``pydantic.ValidationError`` on a bad contract and ``BudgetExceeded`` when the token/round cap is
|
|
crossed, and ``ValueError`` when ``outbox_dir`` is set without a ``run_id``. ``live_dry_run``
|
|
(S4.2, comparison protocol §4 pkt 2/3) is the offline drill: it walks the whole path up to the
|
|
EAGER client build, writes the run-config artefact (when ``outbox_dir`` is set), and returns a
|
|
``DryRunReport`` BEFORE the first model call (``debate.run``) — zero chat calls."""
|
|
# 0. Fail-fast: an outbox write is byte-deterministic and keyed on run_id — no wall-clock default.
|
|
if outbox_dir is not None and run_id is None:
|
|
raise ValueError(
|
|
"run_id is required when outbox_dir is set (no wall-clock/uuid default — the outbox "
|
|
"artefacts are byte-deterministic and keyed on run_id)"
|
|
)
|
|
|
|
# 1. Fail-fast: validate ALL contracts (incl. the verdict-feedback shape) before any client.
|
|
load_contracts(
|
|
{"docs_dir": docs_dir, "top_k": top_k},
|
|
{"max_rounds": max_rounds, "max_tokens": max_tokens},
|
|
verdict_input,
|
|
)
|
|
|
|
# 1b. Long loop (Steg 7): ingest the async verdict inbox INTO the store before the Step-1 fold.
|
|
# Merge (not replace) into the passed store so run_portfolio's cross-project threading stays
|
|
# intact; store.add is idempotent on the content-hash id. A verdict that landed after an earlier
|
|
# run thus reaches THIS run's hypothesis via the existing fold below — no change to the fold.
|
|
if verdict_dir is not None:
|
|
store = store if store is not None else VerdictStore(verdicts=[])
|
|
for dropped in load_verdicts_from_dir(verdict_dir):
|
|
store.add(dropped)
|
|
|
|
# 2-3. Project + agent read-context + first-class citations. A bundle run derives ALL THREE from
|
|
# the navigated OKF bundle via progressive disclosure (verdict layer EXCLUDED — målbilde §2/§4),
|
|
# NOT keyword chunk-stuffing; the road path keeps the chunk-retrieval data source. ``debate_tools``
|
|
# is the query-time retrieval surface — empty on the bundle path (navigation already placed the
|
|
# curated context in the prompt, and a docs_dir==bundle_dir tool would re-leak the verdict layer).
|
|
if bundle_dir is not None:
|
|
bundle = okf.navigate_bundle(bundle_dir)
|
|
project = _project_from_bundle(bundle_dir, project_id, bundle=bundle)
|
|
# §4.1a context-scope: agents read ONLY dimension-scoped bundle knowledge (Step-3 filter);
|
|
# dimension=None keeps the full context, byte-identical to before.
|
|
context = okf.bundle_context(bundle, dimension=dimension.id if dimension else None)
|
|
citations = bundle_citations(bundle)
|
|
debate_tools: list[Any] = []
|
|
else:
|
|
project = _project_by_id(project_id)
|
|
chunks = retrieve_chunks("cost saving measure", docs_dir, top_k)
|
|
citations = [chunk_dict_to_citation(c) for c in chunks]
|
|
context = "\n".join(c["snippet"] for c in chunks)
|
|
debate_tools = [make_retrieval_tool(docs_dir, top_k=top_k)]
|
|
if not citations:
|
|
raise ValueError(f"no citable content in docs_dir: {docs_dir!r}")
|
|
|
|
# 4. Budget + maker-checker debate (round-capped; Layer-1 HITL optional). The shared meter is
|
|
# driven on the debate's chat calls by the BudgetMiddleware (the brief's named short-circuit
|
|
# mechanism); ``debate_tools`` exposes the citation-bearing data source on the road path.
|
|
meter = (
|
|
meter
|
|
if meter is not None
|
|
else TokenMeter(Budget(max_tokens=max_tokens, max_rounds=max(max_rounds * 4, 4)))
|
|
)
|
|
factory = client_factory if client_factory is not None else _default_factory(profile)
|
|
budget_mw = BudgetMiddleware(meter)
|
|
debate = fresh_workflow(
|
|
factory,
|
|
max_rounds=max_rounds,
|
|
enable_layer1_hitl=enable_layer1_hitl,
|
|
tools=debate_tools,
|
|
middleware=[budget_mw],
|
|
)
|
|
# S4.2 cut (comparison protocol §4 pkt 2/3): everything above is offline — contracts, budget, and
|
|
# the EAGER client build (fresh_workflow constructs the proposer+checker clients, workflow.py:64).
|
|
# Capture the run-config (resolved model per BUILT role, profile, params, token cap) and, for a
|
|
# ``--live-dry-run``, STOP HERE — before the first (paid) model call at ``debate.run`` below.
|
|
if outbox_dir is not None or live_dry_run:
|
|
# ``resolved_models`` reflects the configured MAP (the default factory's model-ids for the M2
|
|
# run). Under an injected ``client_factory`` the built clients may differ (e.g. "synthetic");
|
|
# ``provenance.model`` (below) stays the authority on the client actually built.
|
|
resolved_models = {role: resolve_model(profile, role) for role in _MAKER_CHECKER_ROLES}
|
|
if outbox_dir is not None:
|
|
assert run_id is not None # narrowed by the step-0 guard (no wall-clock default)
|
|
outbox.write_run_config(
|
|
outbox_dir,
|
|
run_id,
|
|
profile=Profile(profile).value,
|
|
resolved_models=resolved_models,
|
|
max_rounds=max_rounds,
|
|
max_tokens=max_tokens,
|
|
top_k=top_k,
|
|
)
|
|
if live_dry_run:
|
|
return DryRunReport(
|
|
profile=Profile(profile).value,
|
|
resolved_models=resolved_models,
|
|
max_rounds=max_rounds,
|
|
max_tokens=max_tokens,
|
|
top_k=top_k,
|
|
)
|
|
result = await debate.run(f"Find a cost-saving measure for {project.id}.\nContext:\n{context}")
|
|
# F1: the candidate must derive from the DEBATE. Feed the proposer's converged output into
|
|
# generation (retrieval context is the last-resort fallback only). The checker's verdict
|
|
# (Step 3/4) is parsed from the SAME debate result and gates the outcome below.
|
|
debate_output = _debate_text(result)
|
|
checker_decision, checker_reason = _checker_verdict(result)
|
|
gen_context = debate_output or context
|
|
|
|
# Step-1 ExpeL wiring (Fase 2a, målbilde §5/§7): fold the candidate's prior verdicts INTO the
|
|
# hypothesis context BEFORE generation, keyed on the OKF bundle's candidate features (available
|
|
# pre-hypothesis). THIS is the one missing dataflow — previously ExpeL was computed
|
|
# post-generation into a discarded SessionContext (step 7 below), so a prior verdict could not
|
|
# reach the next hypothesis. Bundle-driven path with a populated store only; the road path is
|
|
# untouched (its post-hoc, proposal-keyed retrieval below is unchanged).
|
|
if bundle_dir is not None and store is not None and store.verdicts:
|
|
expel_query = bundle_candidate_features(bundle_dir)
|
|
fewshot = ExpeLContextProvider(store, expel_query, k=top_k).format_fewshot()
|
|
gen_context = f"{fewshot}\n\n{gen_context}"
|
|
|
|
# 5. Structured candidate -> blocking validation on the NUMBERS; token bound = the meter.
|
|
proposer_client = factory("proposer")
|
|
validator_outcome = await generate_via_llm(proposer_client, project, gen_context, meter)
|
|
proposal = validator_outcome.proposal
|
|
|
|
# 6. First-class provenance stamp (authoritative; independent of MAF Annotation).
|
|
# F1: an injected client_factory stamps the injected client's REAL model ("unknown" is the
|
|
# neutral fallback for a client that doesn't surface one -- never a fabricated name); the
|
|
# default path keeps the deterministic resolve_model. validator_decision reflects the VALIDATOR
|
|
# (the numbers) ONLY -- stamped from validator_outcome BEFORE the checker override below, so a
|
|
# checker-gated proposal whose numbers passed is never mislabelled as validator-rejected.
|
|
model = (
|
|
(getattr(proposer_client, "model", None) or "unknown")
|
|
if client_factory is not None
|
|
else resolve_model(profile, "proposer")
|
|
)
|
|
stamp = ProvenanceStamp(
|
|
citations=citations,
|
|
model=model,
|
|
role="proposer",
|
|
validator_decision=(
|
|
"validated" if isinstance(validator_outcome, ValidatedProposal) else "rejected"
|
|
),
|
|
token_usage=meter.tokens,
|
|
)
|
|
|
|
# 6b. Step 3/4 checker gate (målbilde §2/§6): the validator falsifies the numbers, the checker
|
|
# falsifies the reasoning. An explicit checker REJECT blocks an otherwise-validated proposal; a
|
|
# validator rejection (the numbers) already stands. Fail-open: APPROVE/absent never blocks.
|
|
outcome: ValidatedProposal | Rejection
|
|
if isinstance(validator_outcome, ValidatedProposal) and checker_decision == "reject":
|
|
outcome = Rejection(proposal=proposal, reason=f"checker rejected: {checker_reason}")
|
|
else:
|
|
outcome = validator_outcome
|
|
|
|
# 6c. Step 2 dimension scope gate (§4.1b): a candidate whose measure_type/codes fall OUTSIDE the
|
|
# run's dimension is rejected. A scope/type gate placed AFTER the checker override (preserves
|
|
# test_checker_gate_loadbearing) — NOT a new numeric gate: validate_proposal stays the only
|
|
# blocking numeric gate and provenance.validator_decision (the numbers) is untouched. Mirrors the
|
|
# override form: only an otherwise-standing ValidatedProposal can be flipped to a Rejection.
|
|
if dimension is not None and isinstance(outcome, ValidatedProposal):
|
|
feats = _features_of(proposal)
|
|
if not admits(
|
|
measure_type=feats.measure_type, codes=feats.affected_codes, dimension=dimension
|
|
):
|
|
outcome = Rejection(
|
|
proposal=proposal,
|
|
reason=f"outside dimension {dimension.id!r}: measure_type={feats.measure_type!r}",
|
|
)
|
|
|
|
# 7. ExpeL (regression guard + traceability): exercises the two-arg extend_instructions
|
|
# injection on a REAL SessionContext (the Critical Fase-1 GA-signature guard), and surfaces
|
|
# the proposal-keyed retrieval for RunResult.retrieved. On the bundle path the load-bearing
|
|
# ExpeL->prompt dataflow already happened pre-generation (above); this block's SessionContext
|
|
# is NOT what reaches the prompt.
|
|
store = store if store is not None else VerdictStore(verdicts=[])
|
|
features = _features_of(proposal)
|
|
provider = ExpeLContextProvider(store, features, k=top_k)
|
|
sctx = SessionContext(input_messages=[], instructions=[])
|
|
await provider.before_run(agent=None, session=None, context=sctx, state={})
|
|
retrieved = store.retrieve(features, k=top_k) if store.verdicts else []
|
|
|
|
# 8. Layer-2 (out-of-band): capture the durable verdict + persist; B11 notify is a stub.
|
|
verdict = capture_verdict(features, verdict_input["decision"], verdict_input["rationale"])
|
|
store.add(verdict)
|
|
if notify is not None:
|
|
notify(verdict)
|
|
|
|
# S2.1 outbox (RAW output layer, målbilde §3): persist the run's proposal + outcome artefacts
|
|
# when configured. Wired ONLY here — no new consumer (S5.1/S5.2 are Non-Goals this bolk). run_id
|
|
# is guaranteed non-None by the fail-fast guard at the top.
|
|
if outbox_dir is not None:
|
|
assert run_id is not None # narrowed by the step-0 guard; keeps the type checker honest
|
|
outbox.write_outbox(
|
|
outbox_dir,
|
|
run_id,
|
|
outcome=outcome,
|
|
provenance=stamp,
|
|
checker_verdict=checker_decision,
|
|
verdict_id=verdict.id,
|
|
)
|
|
|
|
return RunResult(
|
|
outcome=outcome,
|
|
provenance=stamp,
|
|
verdict=verdict,
|
|
retrieved=retrieved,
|
|
store=store,
|
|
debate_output=debate_output,
|
|
checker_verdict=checker_decision,
|
|
)
|
|
|
|
|
|
def _aggregate(runs: tuple[RunResult, ...], store: VerdictStore) -> PortfolioResult:
|
|
"""Thin aggregate over the per-project runs (SC2): partition validated/rejected, total the
|
|
validated claimed saving, and total every run's provenance token usage."""
|
|
validated = [r for r in runs if isinstance(r.outcome, ValidatedProposal)]
|
|
rejected = [r for r in runs if isinstance(r.outcome, Rejection)]
|
|
return PortfolioResult(
|
|
runs=runs,
|
|
store=store,
|
|
validated_count=len(validated),
|
|
rejected_count=len(rejected),
|
|
sum_claimed_saving_nok=sum(r.outcome.proposal.claimed_saving_nok for r in validated),
|
|
sum_token_usage=sum(r.provenance.token_usage for r in runs),
|
|
)
|
|
|
|
|
|
def _to_ore(nok: float) -> int:
|
|
"""NOK float -> integer øre, deterministically (Decimal, mirrors ``ledger.realize``)."""
|
|
return int((Decimal(str(nok)) * 100).quantize(Decimal("1"), rounding=ROUND_HALF_UP))
|
|
|
|
|
|
def _goal_limit_if_reached(goal: GoalContract, observed_ore: int, baseline_ore: int) -> int | None:
|
|
"""The threshold ``observed_ore`` MET (``>=``), or ``None`` if the goal is not yet reached. An
|
|
absolute-øre target compares directly; a percent target is taken against ``baseline_ore`` (the
|
|
addressable cost in øre). When both are set, reaching EITHER counts as met."""
|
|
if goal.absolute_ore is not None and observed_ore >= goal.absolute_ore:
|
|
return goal.absolute_ore
|
|
if goal.percent is not None:
|
|
if baseline_ore <= 0:
|
|
raise ValueError(
|
|
f"percent goal ({goal.percent}%) is meaningless against a non-positive baseline "
|
|
f"({baseline_ore} øre): int(percent/100 * 0) == 0 would falsely read as 'goal "
|
|
"reached'. Supply an absolute_ore target, or ensure the addressable baseline is > 0."
|
|
)
|
|
threshold = int(goal.percent / 100 * baseline_ore)
|
|
if observed_ore >= threshold:
|
|
return threshold
|
|
return None
|
|
|
|
|
|
async def run_portfolio(
|
|
project_ids: Sequence[str] | None = None,
|
|
profile: Profile | str = Profile.LOCAL,
|
|
*,
|
|
dimension: Dimension | None = None,
|
|
ledger: SavingsLedger | None = None,
|
|
goals: GoalConfig | None = None,
|
|
store: VerdictStore | None = None,
|
|
client_factory: Callable[[str], BaseChatClient] | None = None,
|
|
max_rounds: int = 3,
|
|
max_tokens: int = 100_000,
|
|
top_k: int = 3,
|
|
meter_factory: Callable[[], TokenMeter] | None = None,
|
|
) -> PortfolioResult:
|
|
"""Fan out over a portfolio of independent projects SEQUENTIALLY, composing ``run_project``
|
|
as-is (every project's execution state — meter, debate, retrieval context — is built fresh
|
|
per call, so sequential reuse is inherently isolated). ONE ``VerdictStore`` is threaded
|
|
across every run, AND each project's ``bundle_dir``/``verdict_dir`` are threaded into
|
|
``run_project`` (Fase 2a S2.0): a bundle-backed project's ``bundle_dir``-gated Step-1 ExpeL fold
|
|
then folds the store's prior verdicts into its hypothesis prompt, so a verdict captured on
|
|
project k materially reaches project k+1's hypothesis — the cross-project learning loop is
|
|
delivered, not merely asserted. ``verdict_dir`` also lets each project merge its async file inbox
|
|
(Steg 7) on the portfolio path. ``project_ids`` defaults to every loaded project; an unknown id
|
|
raises ``ValueError``. ``meter_factory`` (test seam) supplies a per-project meter — inject a
|
|
shared meter to make the isolation guard go red (SC3).
|
|
|
|
Step 8 goal-stop (SC6): ``ledger`` carries EARLIER, out-of-band HITL realizations (the long file
|
|
loop / Steg-7 role split — ``run_project`` never realizes mid-pass, C3), so the check reads an
|
|
ACCUMULATED sum, it does not build one during the pass. BEFORE running each pid, the accumulated
|
|
realized sum is compared to ``goals`` with a ``>=`` boundary (reached, not exceeded — H1): a HARD
|
|
portfolio goal ``break``s the pass (``stopped_early``); a HARD per-project goal SKIPS that pid
|
|
(its further runs are future passes, not more runs here); a SOFT goal flags ``stop_reason`` but
|
|
continues. Because the ledger is static during the pass, a reached goal is observed on the first
|
|
iteration. ``stop_reason`` surfaces the first goal event; a per-project skip is also observable
|
|
as the pid's absence from ``runs``."""
|
|
projects = {p.id: p for p in load_reference_projects()}
|
|
ids = list(project_ids) if project_ids is not None else list(projects)
|
|
store = store if store is not None else VerdictStore(verdicts=[])
|
|
ledger = ledger if ledger is not None else SavingsLedger(entries=[])
|
|
goals = goals if goals is not None else GoalConfig()
|
|
portfolio_baseline_ore = _to_ore(sum(projects[p].total_cost for p in ids if p in projects))
|
|
|
|
runs: list[RunResult] = []
|
|
stopped_early = False
|
|
stop_reason: GoalReached | None = None
|
|
for pid in ids:
|
|
if pid not in projects:
|
|
raise ValueError(f"unknown project_id: {pid!r}")
|
|
project = projects[pid]
|
|
|
|
if goals.portfolio is not None:
|
|
observed = ledger.portfolio_total()
|
|
limit = _goal_limit_if_reached(goals.portfolio, observed, portfolio_baseline_ore)
|
|
if limit is not None:
|
|
if goals.portfolio.mode == "hard":
|
|
stopped_early = True
|
|
stop_reason = GoalReached("portfolio", None, limit, observed)
|
|
break
|
|
if stop_reason is None:
|
|
stop_reason = GoalReached("portfolio", None, limit, observed) # soft flag
|
|
|
|
per_project_goal = goals.per_project.get(pid)
|
|
if per_project_goal is not None:
|
|
observed = ledger.per_project_total(pid)
|
|
limit = _goal_limit_if_reached(per_project_goal, observed, _to_ore(project.total_cost))
|
|
if limit is not None:
|
|
if stop_reason is None:
|
|
stop_reason = GoalReached("project", pid, limit, observed)
|
|
if per_project_goal.mode == "hard":
|
|
continue # skip THIS pid; the rest of the pass proceeds
|
|
|
|
# run_portfolio only drives full runs (never dry-run), so the return narrows to RunResult;
|
|
# the cast keeps the widened run_project signature honest without an @overload duplication.
|
|
result = cast(
|
|
RunResult,
|
|
await run_project(
|
|
pid,
|
|
profile,
|
|
docs_dir=project.docs_dir,
|
|
verdict_input=project.verdict_input,
|
|
bundle_dir=project.bundle_dir,
|
|
verdict_dir=project.verdict_dir,
|
|
dimension=dimension,
|
|
store=store,
|
|
client_factory=client_factory,
|
|
max_rounds=max_rounds,
|
|
max_tokens=max_tokens,
|
|
top_k=top_k,
|
|
meter=meter_factory() if meter_factory is not None else None,
|
|
),
|
|
)
|
|
runs.append(result)
|
|
|
|
base = _aggregate(tuple(runs), store)
|
|
if stopped_early or stop_reason is not None:
|
|
return replace(base, stopped_early=stopped_early, stop_reason=stop_reason)
|
|
return base
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
"""Single-command console entry: run the slice for one project against a docs folder."""
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
|
|
parser = argparse.ArgumentParser(description="portfolio-optimiser vertical slice")
|
|
parser.add_argument("project_id")
|
|
parser.add_argument("--profile", default="local")
|
|
parser.add_argument("--docs-dir", required=True)
|
|
parser.add_argument(
|
|
"--bundle-dir", default=None, help="OKF bundle dir (enables the Step-1 fold)"
|
|
)
|
|
parser.add_argument(
|
|
"--verdict-dir",
|
|
default=None,
|
|
help="async verdict inbox: a folder of dropped expert verdicts, ingested before generation "
|
|
"(the long loop — a verdict that landed after an earlier run is consumed by this run)",
|
|
)
|
|
parser.add_argument("--decision", default="approved", choices=["approved", "rejected"])
|
|
parser.add_argument("--rationale", default="reviewed by expert")
|
|
parser.add_argument(
|
|
"--live-dry-run",
|
|
action="store_true",
|
|
help="offline drill: build contracts/clients/budget, STOP before the first model call",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.live_dry_run:
|
|
# S4.2 drill (comparison protocol §4 pkt 2/3): walk the offline path, STOP before the first
|
|
# model call, print the run-config. A misconfigured profile (e.g. AZURE with a
|
|
# REPLACE-WITH-* placeholder) makes resolve_model raise inside the eager factory build —
|
|
# refuse cleanly (mirror preflight.main) instead of tracebacking; S4.1 is the config gate.
|
|
try:
|
|
report = asyncio.run(
|
|
run_project(
|
|
args.project_id,
|
|
args.profile,
|
|
docs_dir=args.docs_dir,
|
|
bundle_dir=args.bundle_dir,
|
|
verdict_dir=args.verdict_dir,
|
|
verdict_input={"decision": args.decision, "rationale": args.rationale},
|
|
live_dry_run=True,
|
|
)
|
|
)
|
|
except ValueError as exc:
|
|
# Structured refusal (rc 1, no traceback) for ANY offline-path ValueError. The
|
|
# azure-preflight remediation is only meaningful for the AZURE config gate (S4.1), so
|
|
# scope it to that profile — a LOCAL-profile ValueError (unknown project_id, empty
|
|
# docs_dir, bundle mismatch) must not carry an irrelevant azure hint.
|
|
msg = f"live-dry-run refused: {exc}"
|
|
if args.profile == "azure":
|
|
msg += (
|
|
"\nkjør 'python -m portfolio_optimiser.preflight --profile azure' først "
|
|
"(S4.1 offline config-gate)"
|
|
)
|
|
print(msg, file=sys.stderr)
|
|
return 1
|
|
assert isinstance(report, DryRunReport) # live_dry_run=True always returns a DryRunReport
|
|
print(
|
|
f"{args.project_id}: LIVE-DRY-RUN OK (profile={report.profile}, "
|
|
f"models={report.resolved_models}, max_rounds={report.max_rounds}, "
|
|
f"max_tokens={report.max_tokens}, top_k={report.top_k}) — "
|
|
"ingen modellkall gjort (stoppet før første debate.run)"
|
|
)
|
|
return 0
|
|
|
|
result = cast(
|
|
RunResult,
|
|
asyncio.run(
|
|
run_project(
|
|
args.project_id,
|
|
args.profile,
|
|
docs_dir=args.docs_dir,
|
|
bundle_dir=args.bundle_dir,
|
|
verdict_dir=args.verdict_dir,
|
|
verdict_input={"decision": args.decision, "rationale": args.rationale},
|
|
)
|
|
),
|
|
)
|
|
kind = type(result.outcome).__name__
|
|
print(f"{args.project_id}: {kind} (verdict id={result.verdict.id}, decision={args.decision})")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover - console entry
|
|
raise SystemExit(main())
|