test(round-builder): the verdict word, the two counts and the amount, each pinned to its own line
RED FIRST, HONESTLY: 0 of the 3 new arms can be red at HEAD, because neither finding is a code
defect. The binder already printed every one of these things correctly; what was missing was an
arm that looked. There is no fix to be red before. The proof that the arms discriminate is the
mutant run, not this commit — and that run is reported here in full.
REPRODUCED AS SURVIVORS FIRST, against the 40 arms as they stood: the checkpoint's N3
(`_status_word`, "rejected" -> "validert") and N2 (the heading's amount truncated to whole
kroner) both passed 40 of 40. Eight more of my own against the same first screen: the verdict
word dropped entirely, the label paired with the NEXT row's word, `not_evaluated` read as
"avvist", `unsupported` read as "validert", the commissioned count off by one, and the heading's
"kroner" changed to "kr" — 8 of 10 survived. Two were caught (the sum off by one øre, the
overview sorted).
AFTER: 12 of 12 felled, every one on an AssertionError about behaviour, 0 on
ImportError/AttributeError/ModuleNotFoundError. Control unmutated: 43 of 43. Mutants lived only
in a scratch clone of `git archive HEAD`; the work tree was never mutated.
THE CAUSE WAS MEASURED, NOT GUESSED. `"60 000,01" in text` also matched the "Berørte
kostnadslinjer" line, which is built from `unit_cost` and which the mutant never touched. Counted
over the fixture's own report, 4 of this file's 11 positive substring-assert sites have their
needle on MORE THAN ONE line, and 3 of those 4 are asserts about one particular line. Those three
now compare the WHOLE line: `- **{label}** — {word}`, `### {label} — {amount} kroner`,
`### {label} — falt på {stage}`. The fourth claims only presence, and presence is what it
measures, so it still says `in`. The docstring's "cannot pass by accident" now says which half it
covers.
The word table is written out HERE rather than imported from `_status_word`: an arm that asks the
binder which word it chose and then agrees with it cannot tell "avvist" from "validert". It is
pinned to the MANDATE's own status list (`ApproachOutcome.status`, four values) the way the stage
vocabulary is pinned to the validator's — `_status_word` falls back on `str(row["status"])`, so a
status added there without a word here reaches the expert as a bare English identifier.
"validert" is a PREFIX of "validert, men uten erklært krav", which is exactly why no substring
assert can separate those two.
Also, two prose claims that promised more than the measurement:
- "Every artefact TYPE a real run leaves in an outbox" -> the types the runs MEASURED left.
`run.py` calls TEN of outbox.py's ten writers; `exploration`, `prepass`, `multibase`,
`plan-review` and `proposal-reviews` are five more types. Re-counted over every outbox in this
repo holding exactly one coverage — 15 directories, out of 25 coverage files in 20 directories
— the union is still 7, so 7 is a measured floor and not a ceiling. The arm is renamed to say
so.
- "Four of the seven are RUN-level" -> FIVE. `_RUN_LEVEL_TYPES` has five members.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c1bdb37b86
commit
000a7fe4ed
1 changed files with 101 additions and 19 deletions
|
|
@ -19,7 +19,7 @@ import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, get_args, get_type_hints
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
@ -27,6 +27,7 @@ from portfolio_optimiser.evals import round_builder as rb
|
||||||
from portfolio_optimiser.evals import v1_gate as gate
|
from portfolio_optimiser.evals import v1_gate as gate
|
||||||
from portfolio_optimiser.ir import AffectedItem, SavingsProposal
|
from portfolio_optimiser.ir import AffectedItem, SavingsProposal
|
||||||
from portfolio_optimiser.ledger import to_ore
|
from portfolio_optimiser.ledger import to_ore
|
||||||
|
from portfolio_optimiser.mandate import ApproachOutcome
|
||||||
from portfolio_optimiser.outbox import (
|
from portfolio_optimiser.outbox import (
|
||||||
write_coverage,
|
write_coverage,
|
||||||
write_debate_tools,
|
write_debate_tools,
|
||||||
|
|
@ -125,17 +126,39 @@ _FELL = tuple(aid for aid, _, status, *_ in _SPEC if status in ("rejected", "uns
|
||||||
_VALIDATED_ORE = sum(to_ore(nok) for _, _, status, nok, _, _ in _SPEC if status == "validated")
|
_VALIDATED_ORE = sum(to_ore(nok) for _, _, status, nok, _, _ in _SPEC if status == "validated")
|
||||||
_STOP_REASON = "tokens"
|
_STOP_REASON = "tokens"
|
||||||
|
|
||||||
#: Every artefact TYPE a real run leaves in an outbox — the denominator, counted here rather
|
#: The verdict word the overview list must print next to each label — written out HERE
|
||||||
#: than remembered. METHOD: over the four archived runs the 19.09 checkpoint read
|
#: rather than imported from ``round_builder._status_word``, because an arm that asks the
|
||||||
#: (``scratchpad/p19..p22-stress/tunnel-hauglia-2027/``, run ids ``…-04/-06/-07/-08``), each
|
#: builder which word it chose and then agrees with it cannot tell «avvist» from «validert».
|
||||||
#: file ``<run_id>-<rest>.json`` is typed as ``proposal``/``outcome`` when ``<rest>`` ends
|
|
||||||
#: there and as ``<rest>`` itself otherwise. Counted 19.09: ``-06``, ``-07`` and ``-08`` hold
|
|
||||||
#: all SEVEN (15 files each); ``-04`` holds six (12 files, no ``parse-failures`` — that file is
|
|
||||||
#: written only when something failed to parse, so its ABSENCE is the signal). The union is
|
|
||||||
#: SEVEN, and two commands write them: ``run.py`` writes six, ``stress.py`` writes
|
|
||||||
#: ``-verdict.json``.
|
|
||||||
#:
|
#:
|
||||||
#: Four of the seven are RUN-level: one file each, no approach id. The builder carries them
|
#: Measured 19.09: reading ``rejected`` as «validert» passed all 40 arms of this file, so a
|
||||||
|
#: REFUSED approach reached the expert's first screen as a validated one. That word is also
|
||||||
|
#: why the assert below is on the WHOLE LINE: «validert» is a PREFIX of the ``unsupported``
|
||||||
|
#: word, so no substring assert can tell those two apart either.
|
||||||
|
_STATUS_WORDS: dict[str, str] = {
|
||||||
|
"validated": "validert",
|
||||||
|
"rejected": "avvist",
|
||||||
|
"unsupported": "validert, men uten erklært krav",
|
||||||
|
"not_evaluated": "ikke vurdert",
|
||||||
|
}
|
||||||
|
|
||||||
|
#: Every artefact type OF THE RUNS MEASURED — the denominator, counted here rather than
|
||||||
|
#: remembered, and deliberately not called «every type a real run leaves». METHOD: each file
|
||||||
|
#: ``<run_id>-<rest>.json`` is typed as ``proposal``/``outcome`` when ``<rest>`` ends there and
|
||||||
|
#: as ``<rest>`` itself otherwise. Counted 19.09 over the four archived runs
|
||||||
|
#: (``scratchpad/p19..p22-stress/tunnel-hauglia-2027/``, run ids ``…-04/-06/-07/-08``): ``-06``,
|
||||||
|
#: ``-07`` and ``-08`` hold all SEVEN (15 files each); ``-04`` holds six (12 files, no
|
||||||
|
#: ``parse-failures`` — that file is written only when something failed to parse, so its ABSENCE
|
||||||
|
#: is the signal). Counted again over EVERY outbox in this repo holding exactly one coverage
|
||||||
|
#: (15 directories, from 25 coverage files in 20 directories): the union is SEVEN there too.
|
||||||
|
#:
|
||||||
|
#: The wider claim would be FALSE. ``run.py`` calls TEN of ``outbox.py``'s ten writers —
|
||||||
|
#: ``exploration``, ``prepass``, ``multibase``, ``plan-review`` and ``proposal-reviews`` are five
|
||||||
|
#: more types, written when the flags that produce them are given, and none of them happens to
|
||||||
|
#: sit in an outbox that also has a coverage today. Seven is what the runs measured left; it is
|
||||||
|
#: not a ceiling. The builder copies on a glob and not on this list, so a type outside it is
|
||||||
|
#: carried anyway — this table is what the FIXTURE must be able to witness, nothing more.
|
||||||
|
#:
|
||||||
|
#: FIVE of the seven are RUN-level: one file each, no approach id. The builder carries them
|
||||||
#: without reading them, and that is the behaviour the fixture has to be able to witness —
|
#: without reading them, and that is the behaviour the fixture has to be able to witness —
|
||||||
#: before 19.09 it wrote three of the seven, so an artefact type could be dropped in silence.
|
#: before 19.09 it wrote three of the seven, so an artefact type could be dropped in silence.
|
||||||
_RUN_LEVEL_TYPES = ("coverage", "debate", "parse-failures", "runconfig", "verdict")
|
_RUN_LEVEL_TYPES = ("coverage", "debate", "parse-failures", "runconfig", "verdict")
|
||||||
|
|
@ -388,11 +411,12 @@ def test_only_this_runs_artefacts_are_copied_and_the_rest_is_reported(tmp_path:
|
||||||
assert set(built.ignored) == {"08.dry.out", "verdict.err"}
|
assert set(built.ignored) == {"08.dry.out", "verdict.err"}
|
||||||
|
|
||||||
|
|
||||||
def test_the_fixture_carries_every_artefact_type_a_real_run_writes(tmp_path: Path) -> None:
|
def test_the_fixture_carries_every_artefact_type_the_measured_runs_wrote(tmp_path: Path) -> None:
|
||||||
"""The DENOMINATOR arm. A fixture that writes three of seven types measures the builder
|
"""The DENOMINATOR arm. A fixture that writes three of seven types measures the builder
|
||||||
against a universe four types smaller than the one it meets, and "every artefact is copied"
|
against a universe four types smaller than the one it meets, and "every artefact is copied"
|
||||||
is then a claim about the fixture. Seven is counted in ``_ARTEFACT_TYPES`` from the four
|
is then a claim about the fixture. Seven is counted in ``_ARTEFACT_TYPES`` from the runs
|
||||||
archived runs, and the types present here are counted from the FILES by the same rule."""
|
MEASURED — not from every run that could exist, see that table — and the types present here
|
||||||
|
are counted from the FILES by the same rule."""
|
||||||
kinds = _types_in(_outbox(tmp_path))
|
kinds = _types_in(_outbox(tmp_path))
|
||||||
assert len(_ARTEFACT_TYPES) == 7
|
assert len(_ARTEFACT_TYPES) == 7
|
||||||
assert set(kinds) == set(_ARTEFACT_TYPES), sorted(set(_ARTEFACT_TYPES) - set(kinds))
|
assert set(kinds) == set(_ARTEFACT_TYPES), sorted(set(_ARTEFACT_TYPES) - set(kinds))
|
||||||
|
|
@ -629,6 +653,18 @@ def _report(tmp_path: Path, **kwargs: Any) -> str:
|
||||||
return (_build(tmp_path, **kwargs).round_dir / "report.md").read_text(encoding="utf-8")
|
return (_build(tmp_path, **kwargs).round_dir / "report.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _overview(tmp_path: Path, **kwargs: Any) -> list[str]:
|
||||||
|
"""The report's FIRST SCREEN — «Hva ble vurdert», cut out of the report as LINES.
|
||||||
|
|
||||||
|
Cut out, and compared line by line, for one measured reason: a needle that is only
|
||||||
|
required to be somewhere ``in text`` is also satisfied by a line further down. Counted
|
||||||
|
19.09 over this file's eleven positive substring-assert sites, against the fixture's own
|
||||||
|
report: FOUR had a needle on more than one line, and THREE of those four were asserts about
|
||||||
|
one particular line. Only the arm that claims presence and nothing else still says ``in``."""
|
||||||
|
section = _report(tmp_path, **kwargs).split("## Hva ble vurdert", 1)[1]
|
||||||
|
return section.split("\n## ", 1)[0].splitlines()
|
||||||
|
|
||||||
|
|
||||||
def test_the_report_names_every_approach_that_was_considered(tmp_path: Path) -> None:
|
def test_the_report_names_every_approach_that_was_considered(tmp_path: Path) -> None:
|
||||||
"""(a) of ``PLAN.md § Målbar endring``. Counted from the table: every label appears, and the
|
"""(a) of ``PLAN.md § Målbar endring``. Counted from the table: every label appears, and the
|
||||||
count of labels present equals the number commissioned."""
|
count of labels present equals the number commissioned."""
|
||||||
|
|
@ -637,24 +673,71 @@ def test_the_report_names_every_approach_that_was_considered(tmp_path: Path) ->
|
||||||
assert len(present) == _COMMISSIONED, sorted({label for _a, label, *_ in _SPEC} - set(present))
|
assert len(present) == _COMMISSIONED, sorted({label for _a, label, *_ in _SPEC} - set(present))
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_overview_gives_every_approach_its_own_verdict_word(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: read ``rejected`` as «validert» in the overview list.
|
||||||
|
|
||||||
|
The word beside the label is the first thing a domain expert reads, and until now no arm
|
||||||
|
counted it — every arm asked whether the LABEL was present. The whole list is compared, in
|
||||||
|
the coverage's own order, against ``_STATUS_WORDS``: a word swapped, a word dropped and a
|
||||||
|
label paired with the NEXT row's word are then three separate failures instead of three ways
|
||||||
|
to pass. All four verdicts occur in ``_SPEC``, and that is asserted rather than assumed."""
|
||||||
|
lines = [line for line in _overview(tmp_path) if line.startswith("- **")]
|
||||||
|
assert lines == [f"- **{label}** — {_STATUS_WORDS[status]}" for _a, label, status, *_ in _SPEC]
|
||||||
|
assert len(lines) == _COMMISSIONED == 6
|
||||||
|
assert len({_STATUS_WORDS[s] for _a, _l, s, *_ in _SPEC}) == 4, "a verdict left the table"
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_verdict_the_coverage_can_carry_has_a_word_of_its_own(tmp_path: Path) -> None:
|
||||||
|
"""The vocabulary is pinned to the MANDATE's own status list, the way the stage vocabulary is
|
||||||
|
pinned to the validator's. ``_status_word`` falls back on ``str(row["status"])``, so a status
|
||||||
|
added there without a word here reaches the expert as a bare English identifier."""
|
||||||
|
statuses = set(get_args(get_type_hints(ApproachOutcome)["status"]))
|
||||||
|
assert statuses == set(_STATUS_WORDS) and len(statuses) == 4
|
||||||
|
for status, word in _STATUS_WORDS.items():
|
||||||
|
assert rb._status_word({"status": status}) == word, status
|
||||||
|
assert rb._status_word({"status": "noe-nytt"}) == "noe-nytt", "the fallback moved"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_overview_says_how_many_were_commissioned_and_how_many_were_reached(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""The two numbers that give the list underneath its meaning: what the commission asked for,
|
||||||
|
and how much of it the run got through. Both counted from ``_SPEC``, and the whole sentence
|
||||||
|
is compared — a mutant that said «5» where the table says 6 passed all 40 arms on 19.09."""
|
||||||
|
prose = [line for line in _overview(tmp_path) if line.startswith("Kommisjonen ba om")]
|
||||||
|
assert prose == [
|
||||||
|
f"Kommisjonen ba om {_COMMISSIONED} tilnærminger, og kjøringen rakk "
|
||||||
|
f"{len(_EVALUATED)} av dem. Kjøringen stoppet før den var ferdig ({_STOP_REASON})."
|
||||||
|
]
|
||||||
|
assert (_COMMISSIONED, len(_EVALUATED)) == (6, 5)
|
||||||
|
|
||||||
|
|
||||||
def test_the_report_states_the_validated_saving_per_approach_and_in_total(tmp_path: Path) -> None:
|
def test_the_report_states_the_validated_saving_per_approach_and_in_total(tmp_path: Path) -> None:
|
||||||
"""(d). MUTANT: sum the wrong field.
|
"""(d). MUTANT: sum the wrong field.
|
||||||
|
|
||||||
The total is the VALIDATED rows' own amounts, quantized per amount (kø-(p)). Summing the
|
The total is the VALIDATED rows' own amounts, quantized per amount (kø-(p)). Summing the
|
||||||
proposals' claims instead would reach 2 670 000,02, and summing the floats before converting
|
proposals' claims instead would reach 2 670 000,02, and summing the floats before converting
|
||||||
would reach 1 370 000,01 — both are spelled out here so the assert cannot pass by accident."""
|
would reach 1 370 000,01 — both are spelled out here so the TOTAL's assert cannot pass by
|
||||||
|
accident.
|
||||||
|
|
||||||
|
The amount PER APPROACH was passing by accident until now, and that is measured: the two
|
||||||
|
60 000,005 rows print «60 000,01» in their heading AND in their «Berørte kostnadslinjer»
|
||||||
|
line, which is built from ``unit_cost``. A mutant that truncated the HEADING to whole kroner
|
||||||
|
therefore left «60 000,01» standing further down and passed all 40 arms. The heading line is
|
||||||
|
compared whole, so the two places are no longer interchangeable."""
|
||||||
built = _build(tmp_path)
|
built = _build(tmp_path)
|
||||||
assert built.validated_ore == _VALIDATED_ORE == 137_000_002
|
assert built.validated_ore == _VALIDATED_ORE == 137_000_002
|
||||||
text = (built.round_dir / "report.md").read_text(encoding="utf-8")
|
text = (built.round_dir / "report.md").read_text(encoding="utf-8")
|
||||||
assert "1 370 000,02" in text
|
assert "1 370 000,02" in text
|
||||||
assert "1 370 000,01" not in text, "the floats were summed before they were quantized"
|
assert "1 370 000,01" not in text, "the floats were summed before they were quantized"
|
||||||
assert "2 670 000,02" not in text, "the claims were summed instead of the validated amounts"
|
assert "2 670 000,02" not in text, "the claims were summed instead of the validated amounts"
|
||||||
|
lines = text.splitlines()
|
||||||
for _aid, label, status, nok, _claimed, _detail in _SPEC:
|
for _aid, label, status, nok, _claimed, _detail in _SPEC:
|
||||||
if status != "validated":
|
if status != "validated":
|
||||||
continue
|
continue
|
||||||
assert label in text, label
|
|
||||||
whole, rest = divmod(to_ore(nok), 100)
|
whole, rest = divmod(to_ore(nok), 100)
|
||||||
assert f"{whole:,}".replace(",", " ") + f",{rest:02d}" in text, label
|
amount = f"{whole:,}".replace(",", " ") + f",{rest:02d}"
|
||||||
|
assert f"### {label} — {amount} kroner" in lines, label
|
||||||
|
|
||||||
|
|
||||||
def test_the_report_says_where_and_why_each_refused_approach_fell(tmp_path: Path) -> None:
|
def test_the_report_says_where_and_why_each_refused_approach_fell(tmp_path: Path) -> None:
|
||||||
|
|
@ -666,10 +749,9 @@ def test_the_report_says_where_and_why_each_refused_approach_fell(tmp_path: Path
|
||||||
for aid, label, status, _nok, _claimed, detail in _SPEC:
|
for aid, label, status, _nok, _claimed, detail in _SPEC:
|
||||||
if status not in ("rejected", "unsupported"):
|
if status not in ("rejected", "unsupported"):
|
||||||
continue
|
continue
|
||||||
assert label in text, aid
|
|
||||||
assert detail in text, aid
|
assert detail in text, aid
|
||||||
stage = "unsupported" if detail == UNSUPPORTED_REASON else "stage4-p90"
|
stage = "unsupported" if detail == UNSUPPORTED_REASON else "stage4-p90"
|
||||||
assert rb.STAGE_PROSE[stage] in text, aid
|
assert f"### {label} — falt på {rb.STAGE_PROSE[stage]}" in text.splitlines(), aid
|
||||||
section = text.split("## Hva falt, og hvorfor", 1)[1].split("\n## ", 1)[0]
|
section = text.split("## Hva falt, og hvorfor", 1)[1].split("\n## ", 1)[0]
|
||||||
assert section.count("### ") == len(_FELL) == 2
|
assert section.count("### ") == len(_FELL) == 2
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue