feat(fase1): dimension in run_project — context scope + candidate constraint (F1)
This commit is contained in:
parent
525ff0dda1
commit
d2029964cc
2 changed files with 203 additions and 1 deletions
|
|
@ -40,6 +40,7 @@ from portfolio_optimiser.datasource import (
|
|||
make_retrieval_tool,
|
||||
retrieve_chunks,
|
||||
)
|
||||
from portfolio_optimiser.dimension import Dimension, admits
|
||||
from portfolio_optimiser.generate import generate_via_llm
|
||||
from portfolio_optimiser.ir import SavingsProposal
|
||||
from portfolio_optimiser.provenance import ProvenanceStamp
|
||||
|
|
@ -198,6 +199,7 @@ async def run_project(
|
|||
docs_dir: str,
|
||||
verdict_input: dict[str, str],
|
||||
bundle_dir: str | None = None,
|
||||
dimension: Dimension | None = None,
|
||||
store: VerdictStore | None = None,
|
||||
verdict_dir: str | None = None,
|
||||
client_factory: Callable[[str], BaseChatClient] | None = None,
|
||||
|
|
@ -243,7 +245,9 @@ async def run_project(
|
|||
if bundle_dir is not None:
|
||||
bundle = okf.navigate_bundle(bundle_dir)
|
||||
project = _project_from_bundle(bundle_dir, project_id, bundle=bundle)
|
||||
context = okf.bundle_context(bundle)
|
||||
# §4.1a context-scope: agents read ONLY dimension-scoped bundle knowledge (Step-3 filter);
|
||||
# dimension=None keeps the full context, byte-identical to before.
|
||||
context = okf.bundle_context(bundle, dimension=dimension.id if dimension else None)
|
||||
citations = bundle_citations(bundle)
|
||||
debate_tools: list[Any] = []
|
||||
else:
|
||||
|
|
@ -326,6 +330,21 @@ async def run_project(
|
|||
else:
|
||||
outcome = validator_outcome
|
||||
|
||||
# 6c. Step 2 dimension scope gate (§4.1b): a candidate whose measure_type/codes fall OUTSIDE the
|
||||
# run's dimension is rejected. A scope/type gate placed AFTER the checker override (preserves
|
||||
# test_checker_gate_loadbearing) — NOT a new numeric gate: validate_proposal stays the only
|
||||
# blocking numeric gate and provenance.validator_decision (the numbers) is untouched. Mirrors the
|
||||
# override form: only an otherwise-standing ValidatedProposal can be flipped to a Rejection.
|
||||
if dimension is not None and isinstance(outcome, ValidatedProposal):
|
||||
feats = _features_of(proposal)
|
||||
if not admits(
|
||||
measure_type=feats.measure_type, codes=feats.affected_codes, dimension=dimension
|
||||
):
|
||||
outcome = Rejection(
|
||||
proposal=proposal,
|
||||
reason=f"outside dimension {dimension.id!r}: measure_type={feats.measure_type!r}",
|
||||
)
|
||||
|
||||
# 7. ExpeL (regression guard + traceability): exercises the two-arg extend_instructions
|
||||
# injection on a REAL SessionContext (the Critical Fase-1 GA-signature guard), and surfaces
|
||||
# the proposal-keyed retrieval for RunResult.retrieved. On the bundle path the load-bearing
|
||||
|
|
@ -374,6 +393,7 @@ async def run_portfolio(
|
|||
project_ids: Sequence[str] | None = None,
|
||||
profile: Profile | str = Profile.LOCAL,
|
||||
*,
|
||||
dimension: Dimension | None = None,
|
||||
store: VerdictStore | None = None,
|
||||
client_factory: Callable[[str], BaseChatClient] | None = None,
|
||||
max_rounds: int = 3,
|
||||
|
|
@ -401,6 +421,7 @@ async def run_portfolio(
|
|||
profile,
|
||||
docs_dir=project.docs_dir,
|
||||
verdict_input=project.verdict_input,
|
||||
dimension=dimension,
|
||||
store=store,
|
||||
client_factory=client_factory,
|
||||
max_rounds=max_rounds,
|
||||
|
|
|
|||
181
tests/test_dimension_loadbearing.py
Normal file
181
tests/test_dimension_loadbearing.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
"""Step 2 load-bearing seam (SC2 + brief §4.1, målbilde §2/§6): a run's ``dimension`` does TWO
|
||||
things, each with a named detach point:
|
||||
|
||||
- **Candidate constraint (§4.1b):** a validator-VALID candidate (P90-valid, empty assumptions)
|
||||
whose ``measure_type`` falls OUTSIDE the run's dimension is rejected WHEN a dimension is set.
|
||||
The proposal validates on the numbers, so the ONLY possible rejecter is the ``admits`` scope
|
||||
gate (closes the green-but-dead trap). RED if ``admits`` is removed ⇒ the foreign candidate
|
||||
slips through. Control: an in-dimension candidate passes.
|
||||
- **Context scope (§4.1a):** a dimension-scoped ``run_project`` feeds ONLY dimension-matched bundle
|
||||
text into the agent prompt — a sentinel from ANOTHER dimension's concept file is ABSENT from the
|
||||
captured prompt. RED if the ``dimension=`` arg to ``bundle_context`` is dropped ⇒ the foreign-
|
||||
dimension context leaks in. Control: ``dimension=None`` ⇒ the sentinel is present.
|
||||
|
||||
Patterns: ``test_checker_gate_loadbearing.py:59/87`` (gate + causality control),
|
||||
``conftest.py:184`` (recording client), ``test_step8_promotion_loadbearing.py:51`` (bundle copy).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from agent_framework import BaseChatClient
|
||||
from conftest import SyntheticUsageChatClient
|
||||
|
||||
from portfolio_optimiser.dimension import Dimension
|
||||
from portfolio_optimiser.run import run_project
|
||||
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
||||
|
||||
BUNDLE_DIR = Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
_ENERGY_DIM = Dimension(
|
||||
id="energi", label="Energi", allowed_measure_types=frozenset({"energy_efficiency"})
|
||||
)
|
||||
_VERDICT_INPUT = {"decision": "approved", "rationale": "expert reviewed (sim)"}
|
||||
|
||||
# A marker that appears ONLY in the asfalt-marked concept file's body, so it can reach the prompt
|
||||
# solely through un-filtered context — its presence/absence is the §4.1a leak probe.
|
||||
_ASFALT_SENTINEL = "ASFALT-LEAK-SENTINEL-x7y8z9"
|
||||
|
||||
|
||||
def _valid_reply(measure: str, code: str) -> str:
|
||||
"""A validator-VALID proposal: affected total 300000, degenerate P90 = 0.30 x 300000 = 90000
|
||||
>= claimed 30000, empty assumptions -> validates on the numbers regardless of ``measure``/``code``
|
||||
(so a rejection can only come from the dimension scope gate)."""
|
||||
return (
|
||||
'{"measure":"' + measure + '","affected_items":'
|
||||
'[{"code":"' + code + '","quantity":300000,"unit_cost":1.0}],"claimed_saving_nok":30000}'
|
||||
)
|
||||
|
||||
|
||||
def _role_factory(proposer_reply: str, checker_reply: str) -> Callable[[str], BaseChatClient]:
|
||||
def factory(role: str) -> BaseChatClient:
|
||||
return SyntheticUsageChatClient(
|
||||
default_reply=checker_reply if role == "checker" else proposer_reply
|
||||
)
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
# --- Candidate constraint (§4.1b) ----------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_foreign_dimension_candidate_rejected_when_dimension_set() -> None:
|
||||
"""LOAD-BEARING: a foreign-dimension candidate that validates on the numbers is rejected by the
|
||||
scope gate. RED if ``admits`` is detached (the foreign candidate slips through as validated)."""
|
||||
factory = _role_factory(
|
||||
_valid_reply("paving_renegotiation", "SENTINEL-FOREIGN"), "VERDICT: APPROVE"
|
||||
)
|
||||
|
||||
result = await run_project(
|
||||
"BYGG-KONTOR-NORD",
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
dimension=_ENERGY_DIM,
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, Rejection), (
|
||||
"a foreign-dimension candidate slipped through — the admits scope gate is not gating"
|
||||
)
|
||||
assert "outside dimension" in result.outcome.reason
|
||||
# Provenance honesty: the VALIDATOR passed (the numbers are feasible); only the scope gate
|
||||
# rejected. validator_decision reflects the numbers ONLY — never the scope gate.
|
||||
assert result.provenance.validator_decision == "validated"
|
||||
|
||||
|
||||
async def test_in_dimension_candidate_passes() -> None:
|
||||
"""CAUSALITY CONTROL: the SAME shape with an in-dimension ``measure_type`` validates normally —
|
||||
proving the rejection above is caused by the dimension scope, not the fixture."""
|
||||
factory = _role_factory(
|
||||
_valid_reply("energy_efficiency", "ENERGI-TOTAL-EL"), "VERDICT: APPROVE"
|
||||
)
|
||||
|
||||
result = await run_project(
|
||||
"BYGG-KONTOR-NORD",
|
||||
"local",
|
||||
docs_dir=str(BUNDLE_DIR),
|
||||
bundle_dir=str(BUNDLE_DIR),
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
dimension=_ENERGY_DIM,
|
||||
client_factory=factory,
|
||||
)
|
||||
|
||||
assert isinstance(result.outcome, ValidatedProposal)
|
||||
|
||||
|
||||
# --- Context scope (§4.1a) -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bundle_with_asfalt_file(tmp_path: Path) -> str:
|
||||
"""A throwaway copy of the shared bundle with one extra asfalt-marked concept file carrying the
|
||||
sentinel in its body, linked from the index so ``navigate_bundle`` reaches it. The shared,
|
||||
framework-neutral fixture is never mutated (mirrors ``_copy_bundle``)."""
|
||||
dst = tmp_path / "bundle"
|
||||
shutil.copytree(BUNDLE_DIR, dst)
|
||||
(dst / "asfalt-note.md").write_text(
|
||||
f"---\ntype: methodology\ndimension: asfalt\n---\n\n{_ASFALT_SENTINEL} — paving method note\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
index = dst / "index.md"
|
||||
index.write_text(
|
||||
index.read_text(encoding="utf-8") + "\n- [asfalt](asfalt-note.md)\n", encoding="utf-8"
|
||||
)
|
||||
return str(dst)
|
||||
|
||||
|
||||
async def _run_and_capture(bundle_dir: str, dimension: Dimension | None) -> str:
|
||||
"""Run the bundle path with a prompt-recording client and return the concatenated prompt text
|
||||
that reached the agents."""
|
||||
sink: list[str] = []
|
||||
|
||||
def factory(role: str) -> BaseChatClient:
|
||||
client = SyntheticUsageChatClient(
|
||||
default_reply=_valid_reply("energy_efficiency", "ENERGI-TOTAL-EL")
|
||||
)
|
||||
_orig = client._inner_get_response
|
||||
|
||||
def _recording(*, messages, stream, options, **kwargs): # type: ignore[no-untyped-def]
|
||||
sink.append(" ".join(getattr(m, "text", "") or "" for m in messages))
|
||||
return _orig(messages=messages, stream=stream, options=options, **kwargs)
|
||||
|
||||
client._inner_get_response = _recording # type: ignore[method-assign]
|
||||
return client
|
||||
|
||||
await run_project(
|
||||
"BYGG-KONTOR-NORD",
|
||||
"local",
|
||||
docs_dir=bundle_dir,
|
||||
bundle_dir=bundle_dir,
|
||||
verdict_input=_VERDICT_INPUT,
|
||||
dimension=dimension,
|
||||
client_factory=factory,
|
||||
)
|
||||
return " ".join(sink)
|
||||
|
||||
|
||||
async def test_dimension_scopes_the_agent_context(tmp_path) -> None:
|
||||
"""LOAD-BEARING (§4.1a): a dimension-scoped run feeds ONLY dimension-matched bundle text into
|
||||
the prompt — the asfalt sentinel is ABSENT. RED if the ``dimension=`` arg to ``bundle_context``
|
||||
is dropped (the foreign-dimension context leaks into the prompt)."""
|
||||
bundle_dir = _bundle_with_asfalt_file(tmp_path)
|
||||
|
||||
scoped_prompt = await _run_and_capture(bundle_dir, _ENERGY_DIM)
|
||||
assert _ASFALT_SENTINEL not in scoped_prompt, (
|
||||
"another dimension's context leaked into the prompt — bundle_context is not dimension-scoped"
|
||||
)
|
||||
|
||||
|
||||
async def test_no_dimension_leaves_context_unscoped(tmp_path) -> None:
|
||||
"""CAUSALITY CONTROL: with ``dimension=None`` the asfalt sentinel IS present — proving its
|
||||
absence above is caused by the dimension scope, not by the file being unreachable."""
|
||||
bundle_dir = _bundle_with_asfalt_file(tmp_path)
|
||||
|
||||
full_prompt = await _run_and_capture(bundle_dir, None)
|
||||
assert _ASFALT_SENTINEL in full_prompt, (
|
||||
"the asfalt file is unreachable even without a filter — the control does not prove causality"
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue