fix(proposer): affected_items maa baere BASELINE-linja, ikke den foreslaatte reduksjonen
Funn (a) fra oekt 94s levende K2-maaling (docs/2026-09-06-major2-levende-k2.md
§ 4, rad 3-4). Genererings-prompten beskrev feltet som
affected_items (list of {code, quantity, unit_cost})
og sa ingenting om HVEM sine tall det er. Stage 0 (S4.0) avstemmer hver linje
mot prosjektets egen kostbaseline, saa de eneste tallene som kan passere er
prisskjemaets EGNE - men en modell som blir bedt om aa halvere et volum leser
quantity som feltet tiltaket sitt hoerer i, fyller inn den REDUSERTE verdien
(1000 der baselinen sier 1250) og avvises. Den levende kjoeringen hadde lest det
prisede skjemaet TO ganger gjennom navigasjonsstigen og gjettet likevel: tallene
var tilgjengelige, KONTRAKTEN for feltet var ikke uttalt.
Besparelsen har sitt eget felt, og prompten sier det i samme aandedrag - "dette
er baseline-tallene" uten "og reduksjonen din hoerer der" etterlater modellen med
en verdi den er fortalt aa ikke putte noe sted.
Blokka rir paa BASE-prompten, ikke paa en gren, og det er hva arm (3) finnes for:
defekten ble maalt paa et REVIDERT forsoek, saa en instruksjon som bare naadde
forsoek 1 ville vaert fravaerende fra noeyaktig den prompten den ble maalt i.
prior_rejection / prior_feedback / approach appendes ETTER basen, saa hver
komposisjon baerer den fortsatt.
RoedT foerst: 6 av 7 armer roede (kontrollen groenn - den asserterer at
referanseprosjektet faktisk HAR kostlinjer, saa gaten ikke beskriver noe
imaginaert). Ingen endring i skjema, validator eller stage 0.
Suite 1381 passed / 5 skipped (fra 1368/5; +13, 0 fjernet - strengt supersett).
ruff + mypy rene. Golden demo-transcript.stdout BYTE-UENDRET,
shasum -a 1 av INNHOLDET = ea8c534773acdbe41ae68f2c55724d69aaf8be4f.
Ordre 20260906T212735Z-2448754-from-.claude, funn (a).
Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
parent
446150eecc
commit
c6886ed605
2 changed files with 126 additions and 1 deletions
|
|
@ -331,7 +331,11 @@ def _build_messages(
|
||||||
f"Context (prior verdicts / cited cost docs):\n{context}\n\n"
|
f"Context (prior verdicts / cited cost docs):\n{context}\n\n"
|
||||||
"Respond with ONLY a JSON object for a SavingsProposal with keys: project_id, "
|
"Respond with ONLY a JSON object for a SavingsProposal with keys: project_id, "
|
||||||
"measure, affected_items (list of {code, quantity, unit_cost}), claimed_saving_nok, "
|
"measure, affected_items (list of {code, quantity, unit_cost}), claimed_saving_nok, "
|
||||||
"and optional assumptions."
|
"and optional assumptions.\n"
|
||||||
|
"Each entry in affected_items must restate a cost line as the project's price schedule "
|
||||||
|
"already carries it: quantity and unit_cost are the unchanged baseline figures, not the "
|
||||||
|
"reduced quantity or unit cost your measure would produce. The effect of the measure "
|
||||||
|
"belongs in claimed_saving_nok."
|
||||||
)
|
)
|
||||||
if prior_rejection is not None:
|
if prior_rejection is not None:
|
||||||
prompt += (
|
prompt += (
|
||||||
|
|
|
||||||
121
tests/test_baseline_line_in_prompt_loadbearing.py
Normal file
121
tests/test_baseline_line_in_prompt_loadbearing.py
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
"""The generation prompt says WHICH figures ``affected_items`` must carry: the BASELINE line.
|
||||||
|
|
||||||
|
Measured against a live model on K2 (``docs/2026-09-06-major2-levende-k2.md`` § 4, rows 3-4), and
|
||||||
|
this is finding (a) of that session's three ``src/`` findings. The prompt described the field as
|
||||||
|
|
||||||
|
affected_items (list of {code, quantity, unit_cost})
|
||||||
|
|
||||||
|
and said nothing about whose numbers those are. Stage 0 (S4.0) reconciles every item against the
|
||||||
|
project's own cost baseline, so the only figures that can pass are the price schedule's OWN — but a
|
||||||
|
model asked to halve a volume reads ``quantity`` as the field where its measure goes, fills in the
|
||||||
|
REDUCED figure (1000 where the baseline says 1250) and is refused. The live run had read the priced
|
||||||
|
schedule twice through the navigation ladder and still guessed, which is the point: the numbers
|
||||||
|
were available, the CONTRACT for the field was not stated.
|
||||||
|
|
||||||
|
The saving has its own field. ``claimed_saving_nok`` is where the measure's effect belongs, and the
|
||||||
|
prompt now says so in the same breath, because "these are the baseline numbers" without "and your
|
||||||
|
reduction goes there" leaves the model with a value it has been told not to put anywhere.
|
||||||
|
|
||||||
|
**The block rides on the BASE prompt, not on a branch, and that is what arm (3) exists for.** The
|
||||||
|
failure was measured on a REVISED attempt — the model had already been rejected once and was being
|
||||||
|
re-asked — so an instruction present only on attempt 1 would be absent from precisely the prompt
|
||||||
|
the defect was measured in. ``prior_rejection`` and ``prior_feedback`` are appended AFTER the base,
|
||||||
|
so composing either of them must not displace it.
|
||||||
|
|
||||||
|
Detach point: revert the block in ``generate._build_messages`` -> RED on (1), (2) and (3).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from spikes._harness import message_texts
|
||||||
|
|
||||||
|
from portfolio_optimiser.generate import _build_messages
|
||||||
|
from portfolio_optimiser.ir import SavingsProposal
|
||||||
|
from portfolio_optimiser.mandate import Approach
|
||||||
|
from portfolio_optimiser.reference_domain import load_reference_projects
|
||||||
|
from portfolio_optimiser.validator import Rejection
|
||||||
|
|
||||||
|
#: Pinned VERBATIM, because the wording is the deliverable: the field's contract is prose in a
|
||||||
|
#: prompt, and prose no test reads is prose free to drift (the repo's Fase-3 class -- a claim the
|
||||||
|
#: surface makes about itself).
|
||||||
|
_BASELINE_CLAUSE = "as the project's price schedule already carries it"
|
||||||
|
_UNCHANGED_CLAUSE = "the unchanged baseline figures"
|
||||||
|
_NOT_THE_REDUCTION = "not the reduced quantity or unit cost your measure would produce"
|
||||||
|
_WHERE_THE_SAVING_GOES = "claimed_saving_nok"
|
||||||
|
|
||||||
|
_APPROACH = Approach(id="led", label="LED retrofit", description="fixtures past rated life")
|
||||||
|
_REJECTION = Rejection(
|
||||||
|
proposal=SavingsProposal(
|
||||||
|
project_id="FV42-GSV-E1",
|
||||||
|
measure="Reduce scope",
|
||||||
|
affected_items=[{"code": "05.2", "quantity": 1000, "unit_cost": 215}],
|
||||||
|
claimed_saving_nok=10000,
|
||||||
|
),
|
||||||
|
reason="cost line 05.2 quantity 1000 deviates from baseline 1250",
|
||||||
|
)
|
||||||
|
_FEEDBACK = "Halve the volume rather than the unit price."
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def project():
|
||||||
|
return load_reference_projects()[0] # FV42-GSV-E1
|
||||||
|
|
||||||
|
|
||||||
|
def _base(project) -> str:
|
||||||
|
return message_texts(_build_messages(project, "ctx"))[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_prompt_says_the_items_carry_the_baseline_line(project) -> None:
|
||||||
|
"""(1) ``quantity``/``unit_cost`` are named as the price schedule's OWN, unchanged figures."""
|
||||||
|
text = _base(project)
|
||||||
|
assert _BASELINE_CLAUSE in text, "the prompt does not say whose numbers affected_items carries"
|
||||||
|
assert _UNCHANGED_CLAUSE in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_prompt_separates_the_baseline_from_the_proposed_reduction(project) -> None:
|
||||||
|
"""(2) the reduction is named and sent to its own field, rather than left unaddressed.
|
||||||
|
|
||||||
|
Stating the baseline rule alone leaves the model holding a number it has been told not to put
|
||||||
|
in ``affected_items`` and given nowhere else to put it -- which is the same guess, one step
|
||||||
|
later.
|
||||||
|
"""
|
||||||
|
text = _base(project)
|
||||||
|
assert _NOT_THE_REDUCTION in text
|
||||||
|
assert _WHERE_THE_SAVING_GOES in text
|
||||||
|
assert text.index(_NOT_THE_REDUCTION) < text.index(
|
||||||
|
_WHERE_THE_SAVING_GOES, text.index(_NOT_THE_REDUCTION)
|
||||||
|
), "the reduction must be pointed at its field in the same breath it is refused in this one"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"kwargs",
|
||||||
|
[
|
||||||
|
pytest.param({"prior_rejection": _REJECTION}, id="after-a-validator-rejection"),
|
||||||
|
pytest.param({"prior_feedback": _FEEDBACK}, id="after-expert-feedback"),
|
||||||
|
pytest.param({"approach": _APPROACH}, id="under-a-commissioned-approach"),
|
||||||
|
pytest.param(
|
||||||
|
{"prior_rejection": _REJECTION, "prior_feedback": _FEEDBACK, "approach": _APPROACH},
|
||||||
|
id="all-three-composed",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_every_composable_prompt_still_carries_the_contract(project, kwargs) -> None:
|
||||||
|
"""(3) the block survives every composition -- the measured failure was on attempt 2.
|
||||||
|
|
||||||
|
A block that only reached attempt 1 would be missing from the exact prompt the live K2 run
|
||||||
|
guessed in, and every arm above would still be green.
|
||||||
|
"""
|
||||||
|
text = message_texts(_build_messages(project, "ctx", **kwargs))[0]
|
||||||
|
assert _BASELINE_CLAUSE in text
|
||||||
|
assert _NOT_THE_REDUCTION in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_reference_project_actually_has_a_baseline_to_speak_of(project) -> None:
|
||||||
|
"""CONTROL: the instruction describes something that exists.
|
||||||
|
|
||||||
|
Without this the file could pass against a prompt that names a "price schedule" no run has --
|
||||||
|
a gate whose subject is imaginary is a gate that can only be green.
|
||||||
|
"""
|
||||||
|
assert project.cost_items, "the reference project carries no cost lines"
|
||||||
|
assert all(i.quantity > 0 and i.unit_cost > 0 for i in project.cost_items)
|
||||||
Loading…
Add table
Add a link
Reference in a new issue