Kjell Tore Guttormsen
37547fe292
The context sets, the packaged knowledge bases and the example bundles are replaced by one fictitious example set about IT operations in an invented organisation: three context sets (serverrom-2027, driftsavtale-2027 and the two-base drift-og-avtale-2027), two synthetic knowledge bases under src/portfolio_optimiser/data/kunnskapsbaser and two example bundles under src/portfolio_optimiser/data/bundles. Numbers, codes and structural values in tests and fixtures are kept; names, ids and wording change. Dated measurement documents that only recorded runs on the replaced material are deleted. Gate figures measured on the new set are not comparable with earlier ones. The exclusion gate from the previous commit is green: 0 tracked files hit outside the shared/ subtree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
100 lines
4.6 KiB
Python
100 lines
4.6 KiB
Python
"""Load-bearing: a COMMISSIONED approach reaches the proposer, and gets no discount from the
|
|
deterministic validator (Trekk A3).
|
|
|
|
The seam this pins is the whole point of krav 1: a domain expert names an approach, and the
|
|
hypothesis prompt must actually carry it — otherwise the run merely *claims* to evaluate what was
|
|
ordered. Two detach points, and each must go RED on its own:
|
|
|
|
* drop the approach block from ``_build_messages`` -> the expert's label/description never reaches
|
|
the model (``test_commissioned_approach_reaches_the_proposer``);
|
|
* let a commissioned approach bypass ``validate_proposal`` -> an expert's wish would outrank the
|
|
deterministic gate (``test_commissioned_approach_gets_no_validator_discount``).
|
|
|
|
The control (``test_no_approach_keeps_the_base_prompt_byte_identical``) proves the addition is
|
|
inert when no mandate is given: a test that can only go green proves nothing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from spikes._harness import FakeChatClient, message_texts
|
|
|
|
from portfolio_optimiser.budget import Budget, TokenMeter
|
|
from portfolio_optimiser.generate import _build_messages, generate_via_llm
|
|
from portfolio_optimiser.mandate import Approach
|
|
from portfolio_optimiser.reference_domain import load_reference_projects
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
|
|
|
_VALID = (
|
|
'{"project_id":"KONTOR-IT-E1","measure":"Reduce scope",'
|
|
'"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
|
|
'{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":200000}'
|
|
)
|
|
#: Same shape and a value the IR itself accepts, but above the method cap (30% of the ~1.48M
|
|
#: affected total = ~445k) -> the DETERMINISTIC VALIDATOR is what must reject it, not the parser.
|
|
_INFEASIBLE = (
|
|
'{"project_id":"KONTOR-IT-E1","measure":"Reduce scope",'
|
|
'"affected_items":[{"code":"05.2","quantity":4300,"unit_cost":215},'
|
|
'{"code":"03.1","quantity":1800,"unit_cost":310}],"claimed_saving_nok":800000}'
|
|
)
|
|
|
|
_APPROACH = Approach(
|
|
id="led-retrofit",
|
|
label="LED retrofit of office lighting",
|
|
description="Operations believes the fixtures are original and run far past their rated life.",
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def project():
|
|
return load_reference_projects()[0] # KONTOR-IT-E1
|
|
|
|
|
|
def _meter() -> TokenMeter:
|
|
return TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
|
|
|
|
|
|
def test_commissioned_approach_reaches_the_proposer(project) -> None:
|
|
"""The expert's own words — label AND description — reach the hypothesis prompt VERBATIM.
|
|
|
|
The description is the part the model cannot infer from cost data (it is the expert's reason
|
|
for wanting this tried), so paraphrasing it away would quietly discard the domain knowledge.
|
|
"""
|
|
text = message_texts(_build_messages(project, "ctx", approach=_APPROACH))[0]
|
|
assert _APPROACH.label in text
|
|
assert _APPROACH.description in text
|
|
|
|
|
|
def test_no_approach_keeps_the_base_prompt_byte_identical(project) -> None:
|
|
"""CONTROL: without a mandate the prompt is byte-identical to the pre-Trekk-A one, so every
|
|
existing run and golden is untouched (mirrors ``prior_rejection=None``)."""
|
|
base = message_texts(_build_messages(project, "ctx"))[0]
|
|
assert base == message_texts(_build_messages(project, "ctx", approach=None))[0]
|
|
assert "Propose ONE concrete cost-saving measure for this project." in base
|
|
assert _APPROACH.label not in base
|
|
|
|
|
|
async def test_commissioned_approach_gets_no_validator_discount(project) -> None:
|
|
"""A commissioned approach whose numbers do not hold is REJECTED — the deterministic gate is
|
|
blocking, and being asked for by a domain expert is not a reason to pass it.
|
|
|
|
This is the one rule krav 1 must not be allowed to erode: the expert directs *what is
|
|
evaluated*, never *what is approved*.
|
|
"""
|
|
client = FakeChatClient(scripted=[_INFEASIBLE], default_reply=_INFEASIBLE)
|
|
result = await generate_via_llm(
|
|
client, project, "", _meter(), max_attempts=1, approach=_APPROACH
|
|
)
|
|
assert isinstance(result.outcome, Rejection)
|
|
|
|
|
|
async def test_commissioned_approach_still_validates_when_the_numbers_hold(project) -> None:
|
|
"""...and the same commissioned path DOES produce a validated proposal when the numbers are
|
|
feasible — so the rejection above is the validator working, not the approach path being
|
|
broken end to end."""
|
|
client = FakeChatClient(scripted=[_VALID], default_reply=_VALID)
|
|
result = await generate_via_llm(
|
|
client, project, "", _meter(), max_attempts=1, approach=_APPROACH
|
|
)
|
|
assert isinstance(result.outcome, ValidatedProposal)
|
|
assert _APPROACH.label in client.received_texts[0][0] # it went through the commissioned prompt
|