feat(simulation): GO — demoen kjører levert VEGLYS-bundle, forankret på deres tall [skip-docs]
P3 a-c lukket. Pullen hentet commons 002f000+27cdce9: kriterium 8 grønt (nav-goldens byte-uendret, målt både med git diff og shasum) og suiten uendret 785 — ingen abort. Retningen på tallene er SNUDD (P3 b): manus-registerets kostlinjer er skrevet FRA shared/examples/veglys-fv-soer/cost-baseline.json, ikke ved siden av den, og baseline_from_scripted_candidate brukes ikke på denne stien — main() leser levert fil. Overdrivelsen 2100000 er valgt utenfor commons' tall-inventar: målt fraværende fra bundelen og over målt P90 1769915 (deres anslag var ~1770000). 600000/900000 ville klarert gaten og aldri utløst Steg 5. Målingen felte en defekt reserven skjulte: :g slår over i eksponentform ved 7. signifikante siffer, så levert baseline printet 4.38615e+06. Reservens 300000 har seks siffer og nådde aldri overgangen. _num erstatter :g begge steder. Load-bearing MÅLT mot hele suiten, fem mutasjoner alle røde + grønn kontroll: detach main-wiringen · reverter _num til :g · drift registeret ETT siffer (4386151 — innenfor 5 %-toleransen, fanget av ingenting i 792 tester bortsett fra den nye) · sett flip_key til et token som finnes i bundelen · detach forankringen på bundle-stien. 785 -> 793 passed / 4 skipped. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BUjfw4eJdwwnqHSXhfcY6i
This commit is contained in:
parent
ce892be855
commit
02ddc6735f
3 changed files with 306 additions and 15 deletions
|
|
@ -142,9 +142,11 @@ def test_a_candidate_whose_two_replies_disagree_is_refused() -> None:
|
|||
baseline_from_scripted_candidate(skewed)
|
||||
|
||||
|
||||
def test_the_demo_entry_point_runs_the_anchored_reserve() -> None:
|
||||
def test_the_demo_entry_point_runs_the_anchored_delivered_bundle() -> None:
|
||||
"""LOAD-BEARING on the CALL SITE: the thing the operator actually runs on stage must be the
|
||||
anchored variant. Goes RED if ``main`` is pointed back at the plain reserve.
|
||||
anchored variant — and since P3 (GO), the anchor is the DELIVERED bundle's own shipped
|
||||
``cost-baseline.json``, not the script-derived reserve one. Goes RED if ``main`` is pointed back
|
||||
at either the plain reserve or the anchored reserve.
|
||||
|
||||
The declared baseline is printed because an anchoring nobody can see is an anchoring nobody can
|
||||
check: every other line of the demo is byte-identical whether the gate is anchored or not.
|
||||
|
|
@ -153,7 +155,12 @@ def test_the_demo_entry_point_runs_the_anchored_reserve() -> None:
|
|||
``"kostbaseline erklært" in stdout`` — but the un-anchored branch read "ingen kostbaseline
|
||||
erklært", which CONTAINS that substring; and ``"ENERGI-TOTAL-EL" in stdout`` holds either way,
|
||||
because the Step-2 line prints the proposal's own cost lines. Both survived the mutation. The
|
||||
assertions below name the whole declared line and rule the other branch out explicitly."""
|
||||
assertions below name the whole declared line and rule the other branch out explicitly.
|
||||
|
||||
The magnitude is asserted in FULL (``4386150``, not ``4.38615e+06``): ``:g`` switched to exponent
|
||||
notation at the 7th significant digit, which the reserve's six-digit ``300000`` never reached —
|
||||
so the delivered content exposed the formatting defect on its first run, and this assert is what
|
||||
keeps it exposed."""
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-m", "portfolio_optimiser.simulation"],
|
||||
capture_output=True,
|
||||
|
|
@ -161,12 +168,17 @@ def test_the_demo_entry_point_runs_the_anchored_reserve() -> None:
|
|||
check=False,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
assert "kostbaseline erklært (ENERGI-TOTAL-EL 300000 x 1)" in proc.stdout, (
|
||||
"the demo ran against a bundle with no cost baseline — the deterministic gate on stage is "
|
||||
"reasoning only about the numbers the proposal supplied itself"
|
||||
assert "kostbaseline erklært (ENERGI-VEGLYS-EL 4386150 x 1)" in proc.stdout, (
|
||||
"the demo is not anchored on the DELIVERED cost baseline — either it ran a bundle without "
|
||||
"one (the gate then reasons only about the proposal's own numbers), or it fell back to the "
|
||||
"script-derived reserve baseline"
|
||||
)
|
||||
assert "validatorens stage 0 avstemmer" in proc.stdout
|
||||
assert "uten kostbaseline" not in proc.stdout
|
||||
assert "e+06" not in proc.stdout, "a cost magnitude regressed to exponent notation"
|
||||
assert "tallene er syntetiske" not in proc.stdout, (
|
||||
"the reserve's honesty sentence is on stage, so the run is NOT on delivered content"
|
||||
)
|
||||
|
||||
|
||||
def _only_candidate() -> ScriptedCandidate:
|
||||
|
|
|
|||
210
tests/test_go_delivered_bundle_loadbearing.py
Normal file
210
tests/test_go_delivered_bundle_loadbearing.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
"""P3 (GO) — the demo runs the DELIVERED bundle, and its numbers come FROM that bundle.
|
||||
|
||||
P4 pkt. 0 anchored the demo against a repo-local reserve whose ``cost-baseline.json`` was DERIVED in
|
||||
code from the scripted register (``baseline_from_scripted_candidate``): the reserve's numbers are
|
||||
synthetic, so the script was the only ground truth there was. On GO day the direction **reverses** —
|
||||
a domain team delivered ``shared/examples/veglys-fv-soer/`` (commons ``002f000``) with its own
|
||||
``cost-baseline.json``, so the register is written FROM that file and the derivation helper is not
|
||||
used on this path.
|
||||
|
||||
That reversal is what these tests guard. The risk it models is drift: two copies of the same numbers
|
||||
— one in commons' delivered file, one in our register — that stop agreeing without anyone noticing.
|
||||
A drifted register does not crash; it makes the deterministic gate reject the demo's own hypothesis
|
||||
at stage 0 instead of at the P90 stage, which on screen is the SAME ``REJECTED`` line telling a
|
||||
different story. So the agreement is asserted directly (below), and the 10 % test proves the gate
|
||||
would actually catch a disagreement.
|
||||
|
||||
The 10 % test is repeated here against delivered content on purpose. Its twin in
|
||||
``test_anchored_reserve_loadbearing`` proves the MECHANISM; this one measures the CONTENT — the
|
||||
mechanism could be perfect while the delivered file and the register disagree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from portfolio_optimiser import okf
|
||||
from portfolio_optimiser.ir import CostBaseline, CostBaselineLine
|
||||
from portfolio_optimiser.simulation import (
|
||||
_CANDIDATES,
|
||||
_INBOX_MARKER,
|
||||
_VEGLYS_PROJECT_ID,
|
||||
ScriptedCandidate,
|
||||
_delivered_bundle_dir,
|
||||
materialize_anchored_bundle,
|
||||
simulate_learning_loop,
|
||||
)
|
||||
from portfolio_optimiser.persona import load_persona_example
|
||||
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
||||
|
||||
|
||||
def _veglys() -> ScriptedCandidate:
|
||||
"""The registry entry for the delivered project — selected by id, never by index (the registry
|
||||
is a set of DATA entries whose order carries no meaning)."""
|
||||
(candidate,) = [c for c in _CANDIDATES if c.project_id == _VEGLYS_PROJECT_ID]
|
||||
return candidate
|
||||
|
||||
|
||||
def _delivered_baseline() -> CostBaseline:
|
||||
baseline = okf.load_optional_cost_baseline(str(_delivered_bundle_dir()))
|
||||
assert baseline is not None, "the delivered bundle no longer ships a cost-baseline.json"
|
||||
return baseline
|
||||
|
||||
|
||||
def test_the_delivered_bundle_ships_its_own_baseline() -> None:
|
||||
"""The delivered bundle is anchored WITHOUT ``materialize_anchored_bundle`` — which is exactly
|
||||
what let GO day be a call-site change rather than a new seam. RED if commons ever drops the
|
||||
file: the demo would then run un-anchored while every other line stayed byte-identical."""
|
||||
baseline = _delivered_baseline()
|
||||
|
||||
assert baseline.project_id == _VEGLYS_PROJECT_ID
|
||||
assert set(baseline.items) == {"ENERGI-VEGLYS-EL"}
|
||||
|
||||
|
||||
def test_the_register_states_the_delivered_numbers_verbatim() -> None:
|
||||
"""LOAD-BEARING on the DIRECTION of derivation (plan P3 b): the register's cost lines are
|
||||
written FROM the delivered ``cost-baseline.json``, so they must equal it exactly — in BOTH
|
||||
scripted replies.
|
||||
|
||||
RED when the two drift: if commons re-states the baseline (or someone re-types it here), the
|
||||
demo's own hypothesis starts being felled by stage 0's reconciliation instead of by the P90
|
||||
stage it narrates. Same ``REJECTED`` line on screen, different mechanism behind it.
|
||||
|
||||
Both replies are checked, not just the corrected one: the overclaimed hypothesis must differ
|
||||
from it in the CLAIM alone, never in the cost lines."""
|
||||
candidate = _veglys()
|
||||
delivered = _delivered_baseline().items
|
||||
|
||||
for reply in (candidate.overclaimed, candidate.corrected):
|
||||
items = json.loads(reply)["affected_items"]
|
||||
assert {i["code"] for i in items} == set(delivered), (
|
||||
"the scripted reply names cost codes the delivered baseline does not declare"
|
||||
)
|
||||
for item in items:
|
||||
line = delivered[item["code"]]
|
||||
assert (item["quantity"], item["unit_cost"]) == (line.quantity, line.unit_cost), (
|
||||
f"register and delivered cost-baseline.json disagree on {item['code']}: "
|
||||
f"script says {(item['quantity'], item['unit_cost'])}, "
|
||||
f"delivered says {(line.quantity, line.unit_cost)}"
|
||||
)
|
||||
|
||||
overclaimed = json.loads(candidate.overclaimed)
|
||||
corrected = json.loads(candidate.corrected)
|
||||
assert overclaimed["claimed_saving_nok"] != corrected["claimed_saving_nok"]
|
||||
assert overclaimed["affected_items"] == corrected["affected_items"]
|
||||
|
||||
|
||||
def test_the_overclaim_and_the_markers_are_absent_from_the_delivered_bundle() -> None:
|
||||
"""The three tokens the walkthrough traces must not already be IN the content it traces.
|
||||
|
||||
``flip_key`` absent: otherwise attempt 1's prompt carries it and the proposer 'corrects' before
|
||||
anything was falsified — Step 5 would show a correction nothing caused. Both markers absent:
|
||||
otherwise Run A's 'markøren er FRAVÆRENDE' line is false, and the promotion / inbox paths would
|
||||
each appear to work while carrying nothing.
|
||||
|
||||
This is a CONTENT check, and delivered content is the reason it exists: the register's figure was
|
||||
chosen against commons' number inventory, and this is what keeps that choice honest if either
|
||||
side changes."""
|
||||
text = "\n".join(
|
||||
path.read_text(encoding="utf-8")
|
||||
for path in sorted(_delivered_bundle_dir().rglob("*"))
|
||||
if path.is_file()
|
||||
)
|
||||
|
||||
for token in (_veglys().flip_key, load_persona_example().marker, _INBOX_MARKER):
|
||||
assert token not in text, (
|
||||
f"{token!r} already occurs in the delivered bundle — the walkthrough would trace a "
|
||||
"token the content supplied, not one the loop carried"
|
||||
)
|
||||
|
||||
|
||||
async def test_the_delivered_bundle_runs_the_whole_demo(tmp_path) -> None:
|
||||
"""CONTROL, and demo-criterion 2 measured against delivered content: hypothesis #1 is falsified
|
||||
by the P90 stage, the corrected one validates, and both runs end FORESLÅTT.
|
||||
|
||||
A gate that rejects everything proves nothing, so this is what gives the 10 % test below its
|
||||
meaning. It also pins WHICH stage rejects #1 — were stage 0 to start rejecting it, the screen
|
||||
would still show a REJECTED and a VALIDATED line while demonstrating a different mechanism."""
|
||||
result = await simulate_learning_loop(
|
||||
str(_delivered_bundle_dir()), str(tmp_path), project_id=_VEGLYS_PROJECT_ID
|
||||
)
|
||||
|
||||
assert isinstance(result.run_a.outcome, ValidatedProposal)
|
||||
assert isinstance(result.run_b.outcome, ValidatedProposal)
|
||||
assert result.run_a.refinements, "no falsification was fed back — Step 5 is not being shown"
|
||||
assert "exceeds P90 feasible" in result.run_a.refinements[0].reason, (
|
||||
"hypothesis #1 was rejected by some other stage than the P90 one the demo narrates"
|
||||
)
|
||||
# Both learning paths close on delivered content too (Step 8 wiki + Step 7 inbox).
|
||||
assert result.marker_in_run_b_prompt and result.inbox_marker_in_run_b_prompt
|
||||
assert not result.marker_in_run_a_prompt
|
||||
|
||||
|
||||
async def test_a_deviating_delivered_baseline_forkaster_the_run_before_the_solver(tmp_path) -> None:
|
||||
"""LOAD-BEARING (the 10 % test, on DELIVERED content): when the project's declared cost lines
|
||||
deviate 10 % from what the script asserts, the run is FORKASTET at stage 0 — with the
|
||||
reconciliation reason, never the P90 one.
|
||||
|
||||
This is the answer to "the register was copied from the baseline, so of course they agree": the
|
||||
agreement is only worth something if a DISagreement would be caught. The script is byte-identical
|
||||
to the control above; only the declared baseline moves.
|
||||
|
||||
The deviated copy is materialized outside ``shared/`` — the delivered bundle is a pull-only
|
||||
subtree and criterion 8 requires it byte-unchanged."""
|
||||
delivered = _delivered_baseline()
|
||||
deviated = CostBaseline(
|
||||
project_id=delivered.project_id,
|
||||
items={
|
||||
code: CostBaselineLine(quantity=line.quantity * 1.10, unit_cost=line.unit_cost)
|
||||
for code, line in delivered.items.items()
|
||||
},
|
||||
)
|
||||
bundle = materialize_anchored_bundle(
|
||||
tmp_path / "avvikende", source=_delivered_bundle_dir(), baseline=deviated
|
||||
)
|
||||
result = await simulate_learning_loop(str(bundle), str(tmp_path), project_id=_VEGLYS_PROJECT_ID)
|
||||
|
||||
outcome = result.run_a.outcome
|
||||
assert isinstance(outcome, Rejection), (
|
||||
"a proposal 10 % away from the project's declared cost lines was NOT rejected — the "
|
||||
"deterministic gate is not anchored to the delivered baseline"
|
||||
)
|
||||
assert "outside the 5.0% tolerance" in outcome.reason
|
||||
assert "ENERGI-VEGLYS-EL" in outcome.reason
|
||||
assert "P90" not in outcome.reason, (
|
||||
"rejected by the solver stage, not by the reconciliation stage 0 that must run BEFORE it"
|
||||
)
|
||||
|
||||
|
||||
def test_the_delivered_bundle_is_never_mutated_by_a_run() -> None:
|
||||
"""Criterion 8's day-to-day half: the demo copies the bundle, so the commons-owned files under
|
||||
``shared/`` are byte-unchanged after everything above has run."""
|
||||
import subprocess
|
||||
|
||||
proc = subprocess.run(
|
||||
["git", "status", "--porcelain", "--", "shared/examples/veglys-fv-soer"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
assert proc.stdout == "", f"the delivered bundle was modified in place: {proc.stdout!r}"
|
||||
|
||||
|
||||
def test_the_reserve_entry_and_the_delivered_entry_are_both_registered() -> None:
|
||||
"""The abort path stays reachable: pointing ``main`` back at the anchored reserve must remain a
|
||||
three-line revert, which it only is while the reserve's registry entry still exists."""
|
||||
ids = {c.project_id for c in _CANDIDATES}
|
||||
|
||||
assert {"BYGG-KONTOR-NORD", _VEGLYS_PROJECT_ID} <= ids
|
||||
|
||||
|
||||
def test_the_projection_names_the_project_the_register_keys_on() -> None:
|
||||
"""The id must stand VERBATIM in the delivered IR projection: ``run._project_from_bundle``
|
||||
raises on a mismatch, and ``scripted_proposer`` keys on this exact string. Two files, one
|
||||
identifier — so it is asserted, not assumed."""
|
||||
projection = json.loads(
|
||||
(_delivered_bundle_dir() / "validator-input.json").read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
assert projection["project_id"] == _VEGLYS_PROJECT_ID
|
||||
assert _delivered_baseline().project_id == projection["project_id"]
|
||||
Loading…
Add table
Add a link
Reference in a new issue