Kjell Tore Guttormsen
2e04c00d6e
Pure formatting, no behaviour change. ruff 0.15.18 reported "3 files would be reformatted, 195 files already formatted" on HEAD455d611; after this commit "198 files already formatted". Suite 1570 passed / 5 skipped before and after, golden demo-transcript.stdout content sha1 unchanged (ea8c534773acdbe41ae68f2c55724d69aaf8be4f). Two of the three came from P7 (277bb95), one from P8 (455d611). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
363 lines
18 KiB
Python
363 lines
18 KiB
Python
"""P7 — an identifier the proposal builds on must appear VERBATIM in the input, or the verdict
|
|
falls.
|
|
|
|
The hole this closes was MEASURED (økt 108/109). ``_reconcile_against_baseline`` already carries
|
|
the sentence "the cost code is absent from the baseline — a fabricated line", but it is reached
|
|
only through ``if baseline is not None`` (``validator.py``), and an unanchored run has no baseline.
|
|
So the paid K2 run of økt 108 ended in ``ValidatedProposal`` (verdict ``5fd6272e3725fe68``) on two
|
|
cost codes — ``M-04-01`` / ``M-04-03`` — that appear in NO prompt of that run. The falsifier was
|
|
tied to whether a cost baseline happened to exist. **The input always exists; the baseline does
|
|
not.**
|
|
|
|
Every fixture under ``tests/fixtures/p7-grounding/`` is the VERBATIM generation prompt of a free
|
|
recording — the entire input the proposer saw on the attempt that produced its candidate. They are
|
|
tracked test data on purpose: the recordings live in ``scratchpad/``, which ``git archive HEAD``
|
|
does not carry, so a test reading them would pass here and fail the handover gate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser.ir import (
|
|
AffectedItem,
|
|
CostBaseline,
|
|
CostBaselineLine,
|
|
SavingsProposal,
|
|
)
|
|
from portfolio_optimiser.validator import Rejection, ValidatedProposal, validate_proposal
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures" / "p7-grounding"
|
|
|
|
#: The known positive (premiss (vii), `docs/2026-09-08-n-bundlene-hypoteseform.md` § 11.4): the
|
|
#: model quoted this requirement number VERBATIM in 6 of 6 replies, and it stands in 6 of 6
|
|
#: prompts. Note the EM-DASH (U+2014): the hyphen variant scores 0 of 6 in both.
|
|
KNOWN_POSITIVE = "Krav 3.3.1—13"
|
|
|
|
|
|
def _prompt(name: str) -> str:
|
|
return (FIXTURES / name).read_text(encoding="utf-8")
|
|
|
|
|
|
def _proposal(*codes: str, unit_cost: float = 100.0) -> SavingsProposal:
|
|
"""A minimal proposal carrying exactly ``codes``, cheap enough that every LATER stage passes —
|
|
so a rejection can only have come from the grounding stage."""
|
|
return SavingsProposal(
|
|
project_id="K2",
|
|
measure="reduce marking scope",
|
|
affected_items=[AffectedItem(code=c, quantity=10.0, unit_cost=unit_cost) for c in codes],
|
|
claimed_saving_nok=1.0,
|
|
)
|
|
|
|
|
|
def _project():
|
|
"""The reference project, used only as the prompt's header — the grounding text is the rendered
|
|
prompt, so the project's own cost items play no part here."""
|
|
from portfolio_optimiser.reference_domain import load_reference_projects
|
|
|
|
return load_reference_projects()[0]
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (a)/(b) the two recordings that reached a verdict on fabricated codes
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("fixture", "codes"),
|
|
[
|
|
("p6-k2-generation-prompt.txt", ("M-04-01", "M-04-03")),
|
|
("s7c-k2-generation-prompt.txt", ("PRD-001", "PRD-002")),
|
|
],
|
|
)
|
|
def test_a_fabricated_code_from_a_real_recording_falls(
|
|
fixture: str, codes: tuple[str, ...]
|
|
) -> None:
|
|
"""(a)+(b). Both recordings' codes are absent from the prompt that produced them (denominator
|
|
2 of 2 in each), and the verdict must now fall — with ``baseline=None``, which is exactly the
|
|
configuration under which økt 108 said ``validated``."""
|
|
text = _prompt(fixture)
|
|
for code in codes:
|
|
assert code not in text, f"fixture drifted: {code!r} is IN the prompt"
|
|
ruling = validate_proposal(_proposal(*codes), baseline=None, grounding=text)
|
|
assert isinstance(ruling, Rejection), f"{codes} cleared the gate on an input that names neither"
|
|
for code in codes:
|
|
assert repr(code) in ruling.reason
|
|
|
|
|
|
def test_b_the_reason_names_every_ungrounded_identifier_in_the_proposal_s_own_order() -> None:
|
|
"""Completeness, for ``_reconcile_against_baseline``'s own reason (økt 94, measured live): a
|
|
message naming only the FIRST violation reads as an instruction to fix that one field, and the
|
|
proposer fixes one and rebreaks the other. Same ``"; "`` joiner, same PROPOSAL order."""
|
|
text = _prompt("p6-k2-generation-prompt.txt")
|
|
ruling = validate_proposal(_proposal("M-04-03", "M-04-01"), baseline=None, grounding=text)
|
|
assert isinstance(ruling, Rejection)
|
|
parts = ruling.reason.split("; ")
|
|
assert len(parts) == 2, f"expected one sentence per violation, got {ruling.reason!r}"
|
|
assert "'M-04-03'" in parts[0] and "'M-04-01'" in parts[1], ruling.reason
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (c) THE KNOWN POSITIVE — with its control on the SAME input
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_c_the_known_positive_is_not_flagged() -> None:
|
|
"""``Krav 3.3.1—13`` stands VERBATIM in the N100 generation prompt, so a proposal that cites it
|
|
is grounded and the grounding stage must stay silent. A rule that flags everything code-shaped
|
|
is red here."""
|
|
text = _prompt("p4-n100-generation-prompt.txt")
|
|
assert KNOWN_POSITIVE in text, "fixture drifted: the known positive is not in the prompt"
|
|
ruling = validate_proposal(_proposal(KNOWN_POSITIVE), baseline=None, grounding=text)
|
|
assert isinstance(ruling, ValidatedProposal), getattr(ruling, "reason", "")
|
|
|
|
|
|
def test_c_control_the_rule_can_still_flag_on_that_same_recording() -> None:
|
|
"""THE CONTROL that keeps the arm above from proving nothing. ``CRS-01`` is the one identifier
|
|
the model produced on the N100 recording that no prompt carries (1 of 2 ungrounded). Same
|
|
fixture, same call — only the identifier differs, and this one must fall."""
|
|
text = _prompt("p4-n100-generation-prompt.txt")
|
|
assert "CRS-01" not in text
|
|
ruling = validate_proposal(_proposal("CRS-01"), baseline=None, grounding=text)
|
|
assert isinstance(ruling, Rejection)
|
|
assert "'CRS-01'" in ruling.reason
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (d) a code that IS in the input is never flagged, however fabricated it looks
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_d_a_code_quoted_verbatim_from_the_input_is_not_flagged() -> None:
|
|
"""The discriminator between this rule and "flag anything that looks like a code". The token is
|
|
deliberately shaped like the fabrications above; the ONLY difference is that the input says it."""
|
|
text = "Context:\nPrice schedule line ZZZ-999-01 covers technical marking.\n"
|
|
ruling = validate_proposal(_proposal("ZZZ-999-01"), baseline=None, grounding=text)
|
|
assert isinstance(ruling, ValidatedProposal), getattr(ruling, "reason", "")
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (e) THE HOLE ITSELF: the stage does not depend on a baseline
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_e_the_stage_fires_with_no_baseline_at_all() -> None:
|
|
"""Premiss (iv). The pre-P7 tree returns ``ValidatedProposal`` here — measured on the real P6
|
|
proposal — because stage 0 sits behind ``if baseline is not None``."""
|
|
text = _prompt("p6-k2-generation-prompt.txt")
|
|
assert isinstance(
|
|
validate_proposal(_proposal("M-04-01"), baseline=None, grounding=text), Rejection
|
|
)
|
|
|
|
|
|
def test_e_the_stage_is_not_gated_on_the_absence_of_a_baseline() -> None:
|
|
"""Independence in the other direction, and a statement about the STAGE rather than about the
|
|
product: an anchored call whose baseline carries the code still reaches stage 0b, and stage 0b
|
|
judges only the text it was handed. What counts as "the input" is the CALLER's decision — and
|
|
the caller (``generate._grounding_text``) folds the baseline's own codes in, so no real run
|
|
rejects a line its baseline proves. The pair is deliberate: putting the baseline inside this
|
|
stage would make the stage untestable in isolation, and putting it nowhere would turn an
|
|
anchored run into a wall (see ``test_g_...`` below for the product-level half)."""
|
|
text = _prompt("p6-k2-generation-prompt.txt")
|
|
baseline = CostBaseline(
|
|
project_id="K2", items={"M-04-01": CostBaselineLine(quantity=10.0, unit_cost=100.0)}
|
|
)
|
|
assert isinstance(
|
|
validate_proposal(_proposal("M-04-01"), baseline=baseline, grounding=text), Rejection
|
|
)
|
|
|
|
|
|
def test_e_control_no_grounding_argument_is_the_pre_P7_behaviour() -> None:
|
|
"""``grounding=None`` must be byte-for-byte the old gate — that is what keeps every existing
|
|
caller, every golden and the demo transcript unchanged."""
|
|
ruling = validate_proposal(_proposal("M-04-01", "M-04-03"), baseline=None)
|
|
assert isinstance(ruling, ValidatedProposal)
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (f) THE WIRING — the seam where a model-authored proposal meets its own input
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
async def test_f_generate_via_llm_grounds_the_candidate_in_the_prompt_it_sent() -> None:
|
|
"""LOAD-BEARING WIRING, behavioural. ``generate_via_llm`` is the ONE place a model-authored
|
|
proposal and the exact text the model was given exist at the same moment (premiss (viii)).
|
|
The rendered ``messages`` ARE the proposer's entire input for that call, so an identifier the
|
|
proposal carries but the messages do not came from the model's weights, not from this run.
|
|
|
|
Driven through the canonical ``ScriptedChatClient`` (S2.5) so the seam is exercised, not
|
|
inspected. RED when ``grounding=`` is detached at the call site: the outcome flips to
|
|
``ValidatedProposal`` — which is precisely the økt-108 defect."""
|
|
from portfolio_optimiser.budget import Budget, TokenMeter
|
|
from portfolio_optimiser.generate import generate_via_llm
|
|
from portfolio_optimiser.simulation import ScriptedChatClient
|
|
|
|
project = _project()
|
|
reply = _proposal("GHOST-77-01").model_dump_json()
|
|
meter = TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
|
|
result = await generate_via_llm(
|
|
ScriptedChatClient(reply=reply, tokens_per_reply=8),
|
|
project,
|
|
"The price schedule carries line REAL-11-02 for technical marking.",
|
|
meter,
|
|
max_attempts=1,
|
|
)
|
|
assert isinstance(result.outcome, Rejection), "an identifier no prompt carried was validated"
|
|
assert "'GHOST-77-01'" in result.outcome.reason
|
|
|
|
|
|
async def test_f_control_a_code_the_context_names_survives_the_same_call() -> None:
|
|
"""THE CONTROL. Same client, same call, same meter — only the identifier changes, and this one
|
|
the context states verbatim. Without it the arm above would also pass on a rule that rejects
|
|
every proposal ``generate_via_llm`` ever produces."""
|
|
from portfolio_optimiser.budget import Budget, TokenMeter
|
|
from portfolio_optimiser.generate import generate_via_llm
|
|
from portfolio_optimiser.simulation import ScriptedChatClient
|
|
|
|
project = _project()
|
|
reply = _proposal("REAL-11-02").model_dump_json()
|
|
meter = TokenMeter(Budget(max_tokens=10**9, max_rounds=20))
|
|
result = await generate_via_llm(
|
|
ScriptedChatClient(reply=reply, tokens_per_reply=8),
|
|
project,
|
|
"The price schedule carries line REAL-11-02 for technical marking.",
|
|
meter,
|
|
max_attempts=1,
|
|
)
|
|
assert isinstance(result.outcome, ValidatedProposal), getattr(result.outcome, "reason", "")
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (g) THE SEAM'S OTHER TWO SOURCES — what only the caller can prove the run was given
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_g_the_seam_grounds_a_code_the_baseline_proves_even_when_no_prompt_repeats_it() -> None:
|
|
"""The product-level half of the arm above. Stage 0 has ALREADY ruled this code a real line of
|
|
the project; the weaker stage must not overrule the stronger falsifier just because the debate's
|
|
prose summary did not repeat it. RED on a composer that drops the baseline: every anchored S2c
|
|
run would then reject its own real codes — a wall, not a gate."""
|
|
from portfolio_optimiser.generate import _grounding_text
|
|
|
|
project = _project()
|
|
baseline = CostBaseline(
|
|
project_id="K2", items={"M-04-01": CostBaselineLine(quantity=10.0, unit_cost=100.0)}
|
|
)
|
|
text = _grounding_text(project, baseline, "the debate summarised this in prose, naming no code")
|
|
assert isinstance(
|
|
validate_proposal(_proposal("M-04-01"), baseline=baseline, grounding=text),
|
|
ValidatedProposal,
|
|
)
|
|
|
|
|
|
def test_g_the_seam_grounds_a_code_the_delivered_base_carries() -> None:
|
|
"""``delivered`` is the source only ``run_project`` can supply: on the S2c bundle path the
|
|
debate NAVIGATES, so the base's own text never reaches the generation prompt. RED when the
|
|
threading is detached — the run would then reject every real code of every unanchored base."""
|
|
from portfolio_optimiser.generate import _grounding_text
|
|
|
|
project = _project()
|
|
text = _grounding_text(project, None, "the price schedule line BASE-77-01 is real")
|
|
assert isinstance(
|
|
validate_proposal(_proposal("BASE-77-01"), baseline=None, grounding=text),
|
|
ValidatedProposal,
|
|
)
|
|
assert isinstance(
|
|
validate_proposal(_proposal("BASE-77-02"), baseline=None, grounding=text), Rejection
|
|
), "CONTROL: the delivered base must not ground a code it does not carry"
|
|
|
|
|
|
async def test_g_run_project_grounds_a_candidate_in_the_base_it_delivered(tmp_path) -> None:
|
|
"""BEHAVIOURAL, over the real ``run_project`` bundle arm — the ONE place the delivered base's
|
|
text is composed. Two arms over the same crafted copy of the fixture base:
|
|
|
|
* a code whose ONLY occurrence is inside a ``type: verdict`` document must NOT ground the
|
|
proposal. That is MAJOR-3 N2 / S7a-3's ``context_files``-not-``files`` rule applied to
|
|
grounding: the verdict layer reaches a hypothesis through the GATED ExpeL fold alone, and
|
|
grounding a proposal in a prior verdict routes that material around its own gate. RED when
|
|
the grounding text is built from ``bundle.files``;
|
|
* the CONTROL, an ordinary concept-file code on the same base, must survive — otherwise the arm
|
|
above would also pass on a run that rejects everything, and RED when the threading from
|
|
``run_project`` is detached at all.
|
|
"""
|
|
import shutil
|
|
|
|
from portfolio_optimiser.run import run_project
|
|
from portfolio_optimiser.simulation import ScriptedChatClient
|
|
from portfolio_optimiser.validator import ValidatedProposal as VP
|
|
|
|
base = tmp_path / "base"
|
|
shutil.copytree(
|
|
Path(__file__).resolve().parents[1] / "shared" / "examples" / "bygg-energi-mikro", base
|
|
)
|
|
(base / "dom-tidligere.md").write_text(
|
|
"---\ntype: verdict\ntitle: Prior judgement\ndecision: approved\n---\n\n"
|
|
"The measure on cost line VERDICT-ONLY-9 was approved.\n",
|
|
encoding="utf-8",
|
|
)
|
|
index = base / "index.md"
|
|
index.write_text(
|
|
index.read_text(encoding="utf-8") + "\n- [Prior judgement](dom-tidligere.md)\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
async def _outcome(code: str):
|
|
reply = (
|
|
f'{{"measure":"LED-retrofit","affected_items":'
|
|
f'[{{"code":"{code}","quantity":300000,"unit_cost":1.0}}],'
|
|
f'"claimed_saving_nok":30000}}'
|
|
)
|
|
result = await run_project(
|
|
"BYGG-KONTOR-NORD",
|
|
"local",
|
|
docs_dir=str(base),
|
|
bundle_dir=str(base),
|
|
client_factory=lambda role: ScriptedChatClient(
|
|
"Reasoning holds.\nVERDICT: APPROVE" if role == "checker" else reply, role=role
|
|
),
|
|
)
|
|
return result.outcome
|
|
|
|
control = await _outcome("ENERGI-TOTAL-EL")
|
|
assert isinstance(control, VP), (
|
|
f"CONTROL: a code the delivered base carries was not grounded — {control}"
|
|
)
|
|
leaked = await _outcome("VERDICT-ONLY-9")
|
|
assert isinstance(leaked, Rejection), "a verdict-layer code grounded a proposal"
|
|
assert "'VERDICT-ONLY-9'" in leaked.reason
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------
|
|
# (h) THE GATE MUST NOT DISARM ITSELF ON ITS SECOND ROUND
|
|
# --------------------------------------------------------------------------------------------
|
|
|
|
|
|
async def test_h_the_refusal_does_not_ground_the_next_attempt_that_repeats_the_code() -> None:
|
|
"""MEASURED while building this rule, and the reason the rendered PROMPT is not evidence.
|
|
|
|
Step 5 feeds the previous ``Rejection.reason`` VERBATIM into the next attempt's prompt — and
|
|
this stage's reason QUOTES the identifier it just refused. An implementation that grounded a
|
|
candidate in its own prompt would therefore accept on attempt 2 exactly what it refused on
|
|
attempt 1: a falsifier that disarms itself. Both attempts here repeat the same ungrounded code,
|
|
and both must fall.
|
|
"""
|
|
from portfolio_optimiser.budget import Budget, TokenMeter
|
|
from portfolio_optimiser.generate import generate_via_llm
|
|
from portfolio_optimiser.simulation import ScriptedChatClient
|
|
|
|
reply = _proposal("GHOST-77-01").model_dump_json()
|
|
client = ScriptedChatClient(reply=reply, tokens_per_reply=8)
|
|
result = await generate_via_llm(
|
|
client,
|
|
_project(),
|
|
"The price schedule carries line REAL-11-02 for technical marking.",
|
|
TokenMeter(Budget(max_tokens=10**9, max_rounds=20)),
|
|
max_attempts=2,
|
|
)
|
|
assert client.call_count == 2, "the informed-refinement loop did not reach a second attempt"
|
|
assert isinstance(result.outcome, Rejection), (
|
|
"attempt 2 was accepted after attempt 1 refused the same identifier — the refusal grounded it"
|
|
)
|
|
assert "'GHOST-77-01'" in result.outcome.reason
|