test(round-builder): 11 arms for the report's CONTENT, and a fixture that holds all seven artefact types
The 19.09 checkpoint planted 20 mutants against this file and SEVEN survived the whole
suite: the report's round number, an artefact type outside proposal/outcome/coverage
dropped in silence, a never-evaluated approach left off its own list, the source quote
removed entirely, the quote's COUNT removed, the affected cost lines removed, and a
refused row's amount counted into the validated total. None of them survived because the
builder was wrong — it does all seven things correctly. They survived because no arm in
709 lines ever looked (grep for citation/cost-line words gave 2 hits, both in the
fixture, none in an assert). An unwitnessed seam is a seam the next edit may delete for
free.
Two fixture defects made three of those blind spots unreachable rather than merely
unmeasured:
* THE DENOMINATOR. A real run leaves SEVEN artefact types; the fixture wrote three.
Counted here, not remembered: over the four archived runs the checkpoint read
(tunnel-hauglia-2027-04/-06/-07/-08), each <run_id>-<rest>.json is typed proposal /
outcome when <rest> ends there and <rest> itself otherwise. -06, -07 and -08 hold all
seven (15 files each); -04 holds six (12 files, no parse-failures — that file is
written only when something failed to parse, so its absence is the signal). Union =
7, written by two commands: run.py writes six, stress.py writes -verdict.json. Six
are now written through the product's own writers; -verdict.json is bytes, because
the builder never reads it and that difference is stated rather than hidden.
* THE CITATIONS. Every proposal carried an identical stamp, so "1 av 1 siterte steder"
could not tell a dropped COUNT from a kept one. Counts are now distinct per approach,
and a shared-list mode reproduces what the real runs actually do.
The refused row now carries an amount, and no archived run does: measured on the same
four outboxes, every rejected row has saving_nok = None. The guard that only a validated
row's figure becomes validated_nok therefore aims at a coverage writer that does not
exist yet — and a fixture that cannot produce the number cannot witness the guard at all.
4 of the 11 new arms are RED on ASSERT, all four on behaviour that does not exist yet:
a dangling symlink as the round directory (exists() follows the link, so the build dies
on the filesystem's own FileExistsError instead of a sentence), the identical citation
list stated once instead of five times, the same cost line named on both sides of the
verdict, and a removed approach shown by the label the expert saw instead of a raw id.
The other 7 pin behaviour that is already correct; they are not red here, and the proof
that they discriminate is the mutant run, not this commit.
The cause of the identical quote was MEASURED before anything was written, because
"the builder reads the wrong field" and "the outbox says the same thing five times" want
opposite fixes: all five proposals in tunnel-hauglia-2027-08 carry byte-identical
270-citation lists (one sha256 across all five), and so do -04, -06 and -07. It is the
run's whole retrieved context stamped once per proposal. The report cannot make that
quote informative; it can stop repeating it.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
207337c8a2
commit
38eaf5a649
1 changed files with 381 additions and 23 deletions
|
|
@ -27,7 +27,13 @@ from portfolio_optimiser.evals import round_builder as rb
|
||||||
from portfolio_optimiser.evals import v1_gate as gate
|
from portfolio_optimiser.evals import v1_gate as gate
|
||||||
from portfolio_optimiser.ir import AffectedItem, SavingsProposal
|
from portfolio_optimiser.ir import AffectedItem, SavingsProposal
|
||||||
from portfolio_optimiser.ledger import to_ore
|
from portfolio_optimiser.ledger import to_ore
|
||||||
from portfolio_optimiser.outbox import write_coverage, write_outbox
|
from portfolio_optimiser.outbox import (
|
||||||
|
write_coverage,
|
||||||
|
write_debate_tools,
|
||||||
|
write_outbox,
|
||||||
|
write_parse_failures,
|
||||||
|
write_run_config,
|
||||||
|
)
|
||||||
from portfolio_optimiser.provenance import Citation, ProvenanceStamp
|
from portfolio_optimiser.provenance import Citation, ProvenanceStamp
|
||||||
from portfolio_optimiser.retrieval import TextSpan
|
from portfolio_optimiser.retrieval import TextSpan
|
||||||
from portfolio_optimiser.validator import (
|
from portfolio_optimiser.validator import (
|
||||||
|
|
@ -43,9 +49,16 @@ _RUN = "demo-kommune-2027-01"
|
||||||
_RAN_AT = "2026-09-18T11:20:00+02:00"
|
_RAN_AT = "2026-09-18T11:20:00+02:00"
|
||||||
_ATTEST_FILE = "attestering.txt"
|
_ATTEST_FILE = "attestering.txt"
|
||||||
|
|
||||||
#: The fixture run, as a table. Columns: id, label, coverage status, validated NOK, the claim the
|
#: The fixture run, as a table. Columns: id, label, coverage status, the ``saving_nok`` the
|
||||||
#: proposal itself carried, the detail the coverage row carries. Everything this file asserts is
|
#: coverage row carries, the claim the proposal itself carried, the detail the coverage row
|
||||||
#: counted from HERE, never from the builder's own answer.
|
#: carries. Everything this file asserts is counted from HERE, never from the builder's answer.
|
||||||
|
#:
|
||||||
|
#: The REJECTED row carries an amount ON PURPOSE, and no archived run does: measured 19.09 over
|
||||||
|
#: the four real outboxes (``tunnel-hauglia-2027-04/-06/-07/-08``), every ``rejected`` row has
|
||||||
|
#: ``saving_nok = None``. The guard that only a ``validated`` row's figure may become
|
||||||
|
#: ``validated_nok`` is therefore aimed at a coverage writer that does not exist yet — and a
|
||||||
|
#: fixture that cannot produce the number cannot witness the guard at all, which is exactly why
|
||||||
|
#: the mutant that counts a refused row's amount survived the whole suite.
|
||||||
#:
|
#:
|
||||||
#: The two 60000.005 amounts are deliberate: quantizing per amount and summing the integers
|
#: The two 60000.005 amounts are deliberate: quantizing per amount and summing the integers
|
||||||
#: (kø-(p), ``ledger.to_ore``) gives 137000002 øre, while summing the floats first gives
|
#: (kø-(p), ``ledger.to_ore``) gives 137000002 øre, while summing the floats first gives
|
||||||
|
|
@ -63,7 +76,7 @@ _SPEC: tuple[tuple[str, str, str, float | None, float, str], ...] = (
|
||||||
"a2-faerre-kummer",
|
"a2-faerre-kummer",
|
||||||
"Færre kummer i kryssene",
|
"Færre kummer i kryssene",
|
||||||
"rejected",
|
"rejected",
|
||||||
None,
|
900_000.0,
|
||||||
900_000.0,
|
900_000.0,
|
||||||
"claimed saving 900000 exceeds P90 feasible 410000",
|
"claimed saving 900000 exceeds P90 feasible 410000",
|
||||||
),
|
),
|
||||||
|
|
@ -106,36 +119,83 @@ _EVALUATED = tuple(aid for aid, _, status, *_ in _SPEC if status != "not_evaluat
|
||||||
_NOT_EVALUATED = tuple(aid for aid, _, status, *_ in _SPEC if status == "not_evaluated")
|
_NOT_EVALUATED = tuple(aid for aid, _, status, *_ in _SPEC if status == "not_evaluated")
|
||||||
_VALIDATED = tuple(aid for aid, _, status, *_ in _SPEC if status == "validated")
|
_VALIDATED = tuple(aid for aid, _, status, *_ in _SPEC if status == "validated")
|
||||||
_FELL = tuple(aid for aid, _, status, *_ in _SPEC if status in ("rejected", "unsupported"))
|
_FELL = tuple(aid for aid, _, status, *_ in _SPEC if status in ("rejected", "unsupported"))
|
||||||
#: Counted from the table, per amount, with the framework's one conversion (kø-(p)).
|
#: Counted from the table, per amount, with the framework's one conversion (kø-(p)). The
|
||||||
|
#: REJECTED row's 900 000 is deliberately OUTSIDE this sum: the validator refused that claim,
|
||||||
|
#: so a builder that let the figure through would report a saving nobody validated.
|
||||||
_VALIDATED_ORE = sum(to_ore(nok) for _, _, status, nok, _, _ in _SPEC if status == "validated")
|
_VALIDATED_ORE = sum(to_ore(nok) for _, _, status, nok, _, _ in _SPEC if status == "validated")
|
||||||
_STOP_REASON = "tokens"
|
_STOP_REASON = "tokens"
|
||||||
|
|
||||||
|
#: Every artefact TYPE a real run leaves in an outbox — the denominator, counted here rather
|
||||||
|
#: than remembered. METHOD: over the four archived runs the 19.09 checkpoint read
|
||||||
|
#: (``scratchpad/p19..p22-stress/tunnel-hauglia-2027/``, run ids ``…-04/-06/-07/-08``), each
|
||||||
|
#: file ``<run_id>-<rest>.json`` is typed as ``proposal``/``outcome`` when ``<rest>`` ends
|
||||||
|
#: there and as ``<rest>`` itself otherwise. Counted 19.09: ``-06``, ``-07`` and ``-08`` hold
|
||||||
|
#: all SEVEN (15 files each); ``-04`` holds six (12 files, no ``parse-failures`` — that file is
|
||||||
|
#: written only when something failed to parse, so its ABSENCE is the signal). The union is
|
||||||
|
#: SEVEN, and two commands write them: ``run.py`` writes six, ``stress.py`` writes
|
||||||
|
#: ``-verdict.json``.
|
||||||
|
#:
|
||||||
|
#: Four of the seven are RUN-level: one file each, no approach id. The builder carries them
|
||||||
|
#: without reading them, and that is the behaviour the fixture has to be able to witness —
|
||||||
|
#: before 19.09 it wrote three of the seven, so an artefact type could be dropped in silence.
|
||||||
|
_RUN_LEVEL_TYPES = ("coverage", "debate", "parse-failures", "runconfig", "verdict")
|
||||||
|
_ARTEFACT_TYPES = ("proposal", "outcome", *_RUN_LEVEL_TYPES)
|
||||||
|
|
||||||
|
#: How many places each proposal cited. DIFFERENT per approach on purpose: the count is what
|
||||||
|
#: separates a grounded proposal from a decorated one, and a fixture where every count is the
|
||||||
|
#: same cannot tell «1 av 3» from «1 av 7» — nor notice a builder that stopped printing it.
|
||||||
|
_CITED: dict[str, int] = {aid: 2 + i for i, (aid, *_rest) in enumerate(_SPEC)}
|
||||||
|
|
||||||
|
|
||||||
def _measure(aid: str) -> str:
|
def _measure(aid: str) -> str:
|
||||||
return f"Tiltak: {dict((a, label) for a, label, *_ in _SPEC)[aid]}"
|
return f"Tiltak: {dict((a, label) for a, label, *_ in _SPEC)[aid]}"
|
||||||
|
|
||||||
|
|
||||||
def _ir(aid: str, claimed: float) -> dict[str, Any]:
|
def _code(aid: str) -> str:
|
||||||
|
return f"DK-{aid[:2].upper()}"
|
||||||
|
|
||||||
|
|
||||||
|
def _ir(aid: str, claimed: float, code: str | None = None) -> dict[str, Any]:
|
||||||
"""The proposal IR a run would have written — built through the real model, so the fixture
|
"""The proposal IR a run would have written — built through the real model, so the fixture
|
||||||
cannot drift from the shape ``write_outbox`` persists and the gate re-mints the key from."""
|
cannot drift from the shape ``write_outbox`` persists and the gate re-mints the key from.
|
||||||
|
|
||||||
|
``code`` is an argument because a real run puts two approaches on the SAME cost line (the
|
||||||
|
19.09 report had ``TUN-LYS-01`` refused under one label and validated under another), and a
|
||||||
|
fixture whose codes are all distinct cannot witness what the report says about that."""
|
||||||
return SavingsProposal(
|
return SavingsProposal(
|
||||||
project_id="demo-kommune",
|
project_id="demo-kommune",
|
||||||
measure=_measure(aid),
|
measure=_measure(aid),
|
||||||
affected_items=[
|
affected_items=[AffectedItem(code=code or _code(aid), quantity=1.0, unit_cost=claimed)],
|
||||||
AffectedItem(code=f"DK-{aid[:2].upper()}", quantity=1.0, unit_cost=claimed)
|
|
||||||
],
|
|
||||||
claimed_saving_nok=claimed,
|
claimed_saving_nok=claimed,
|
||||||
).model_dump()
|
).model_dump()
|
||||||
|
|
||||||
|
|
||||||
def _stamp(decision: str) -> ProvenanceStamp:
|
#: The run-wide citation list every proposal carried in all four archived runs. 270 there, 9
|
||||||
|
#: here — the number is not the point, the IDENTITY across proposals is.
|
||||||
|
_SHARED_CITED = 9
|
||||||
|
|
||||||
|
|
||||||
|
def _snippet(aid: str, k: int) -> str:
|
||||||
|
return f"Kravteksten bak {aid}, sted {k}: restriktiv bruk av kryss anbefales."
|
||||||
|
|
||||||
|
|
||||||
|
def _stamp(decision: str, aid: str, *, shared: bool = False) -> ProvenanceStamp:
|
||||||
|
"""The proposal's own provenance, with a citation list that is ITS OWN.
|
||||||
|
|
||||||
|
Measured 19.09 on all four archived runs: every proposal in a run carried the SAME 270
|
||||||
|
citations, byte for byte — the run's whole retrieved context, stamped once per proposal.
|
||||||
|
That is a property of the outbox, not of the report, and the report now states it once
|
||||||
|
instead of repeating it. A fixture that reproduced it everywhere could only witness the
|
||||||
|
collapsed form, so the default here is per-approach citations and ``cited`` exists for the
|
||||||
|
arm that reproduces the real run's shape."""
|
||||||
return ProvenanceStamp(
|
return ProvenanceStamp(
|
||||||
citations=[
|
citations=[
|
||||||
Citation(
|
Citation(
|
||||||
file="krav/N100/id-1ba48872.md",
|
file=f"krav/N100/id-{'hele-kjoringen' if shared else aid}-{k}.md",
|
||||||
locator=TextSpan(start_index=0, end_index=48),
|
locator=TextSpan(start_index=0, end_index=48),
|
||||||
snippet="Paa motorveger anbefales restriktiv bruk av kryss.",
|
snippet=_snippet("hele-kjoringen" if shared else aid, k),
|
||||||
)
|
)
|
||||||
|
for k in range(_SHARED_CITED if shared else _CITED[aid])
|
||||||
],
|
],
|
||||||
model="synthetic",
|
model="synthetic",
|
||||||
role="proposer",
|
role="proposer",
|
||||||
|
|
@ -154,13 +214,27 @@ def _coverage_rows(spec: Any = _SPEC) -> list[dict[str, Any]]:
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _outbox(root: Path, run_id: str = _RUN, spec: Any = _SPEC) -> Path:
|
def _outbox(
|
||||||
"""One real run's outbox: a proposal/outcome pair per EVALUATED approach plus the coverage."""
|
root: Path,
|
||||||
|
run_id: str = _RUN,
|
||||||
|
spec: Any = _SPEC,
|
||||||
|
*,
|
||||||
|
codes: Any = None,
|
||||||
|
shared_citations: bool = False,
|
||||||
|
) -> Path:
|
||||||
|
"""One real run's outbox: a proposal/outcome pair per EVALUATED approach, the coverage, and
|
||||||
|
the four RUN-level artefacts a real run also leaves behind.
|
||||||
|
|
||||||
|
All seven types (``_ARTEFACT_TYPES``), because an outbox with three of them cannot witness
|
||||||
|
what the builder does with the other four. Six are written by the product's own writers; the
|
||||||
|
seventh, ``-verdict.json``, belongs to a DIFFERENT command (``stress.py`` builds it from a
|
||||||
|
judge's own type) and the builder never reads its content — only that the file is the run's
|
||||||
|
and is carried. It is therefore written here as bytes, and that difference is stated."""
|
||||||
outbox = root / "kjoring"
|
outbox = root / "kjoring"
|
||||||
for aid, _label, status, nok, claimed, detail in spec:
|
for aid, _label, status, nok, claimed, detail in spec:
|
||||||
if status == "not_evaluated":
|
if status == "not_evaluated":
|
||||||
continue
|
continue
|
||||||
ir = _ir(aid, claimed)
|
ir = _ir(aid, claimed, (codes or {}).get(aid))
|
||||||
proposal = SavingsProposal(**ir)
|
proposal = SavingsProposal(**ir)
|
||||||
if status == "validated":
|
if status == "validated":
|
||||||
outcome: Any = ValidatedProposal(
|
outcome: Any = ValidatedProposal(
|
||||||
|
|
@ -180,16 +254,59 @@ def _outbox(root: Path, run_id: str = _RUN, spec: Any = _SPEC) -> Path:
|
||||||
str(outbox),
|
str(outbox),
|
||||||
run_id,
|
run_id,
|
||||||
outcome=outcome,
|
outcome=outcome,
|
||||||
provenance=_stamp("rejected" if status == "rejected" else "validated"),
|
provenance=_stamp(
|
||||||
|
"rejected" if status == "rejected" else "validated",
|
||||||
|
aid,
|
||||||
|
shared=shared_citations,
|
||||||
|
),
|
||||||
checker_verdict="approve",
|
checker_verdict="approve",
|
||||||
verdict_id=verdict_key(features_from_ir(ir)),
|
verdict_id=verdict_key(features_from_ir(ir)),
|
||||||
approach_id=aid,
|
approach_id=aid,
|
||||||
)
|
)
|
||||||
assert nok is None or nok == claimed # the table's own consistency, not the builder's
|
assert nok is None or nok == claimed # the table's own consistency, not the builder's
|
||||||
write_coverage(str(outbox), run_id, rows=_coverage_rows(spec), stop_reason=_STOP_REASON)
|
write_coverage(str(outbox), run_id, rows=_coverage_rows(spec), stop_reason=_STOP_REASON)
|
||||||
|
write_debate_tools(
|
||||||
|
str(outbox),
|
||||||
|
run_id,
|
||||||
|
tool_calls=[{"tool": "read_dir", "argument": "krav/", "round": 1}],
|
||||||
|
requirements=[{"id": "N100-1", "title": "Kryss"}],
|
||||||
|
)
|
||||||
|
write_parse_failures(
|
||||||
|
str(outbox),
|
||||||
|
run_id,
|
||||||
|
failures=[{"role": "proposer", "raw": "dette var ikke JSON"}],
|
||||||
|
)
|
||||||
|
write_run_config(
|
||||||
|
str(outbox),
|
||||||
|
run_id,
|
||||||
|
profile="local",
|
||||||
|
resolved_models={"proposer": "syntetisk"},
|
||||||
|
max_rounds=1,
|
||||||
|
max_tokens=1000,
|
||||||
|
top_k=3,
|
||||||
|
)
|
||||||
|
(outbox / f"{run_id}-verdict.json").write_text(
|
||||||
|
json.dumps({"run_id": run_id, "verdict": "green"}, ensure_ascii=False, sort_keys=True)
|
||||||
|
+ "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
return outbox
|
return outbox
|
||||||
|
|
||||||
|
|
||||||
|
def _types_in(outbox: Path, run_id: str = _RUN) -> dict[str, int]:
|
||||||
|
"""The artefact TYPES an outbox holds, counted by the same rule the denominator was counted
|
||||||
|
with: ``<run_id>-<rest>.json`` is ``proposal``/``outcome`` when ``<rest>`` ends there and
|
||||||
|
``<rest>`` itself otherwise. Counted from the FILES, never from a list the fixture keeps."""
|
||||||
|
kinds: dict[str, int] = {}
|
||||||
|
for path in sorted(outbox.iterdir()):
|
||||||
|
if not (path.is_file() and path.name.startswith(f"{run_id}-") and path.suffix == ".json"):
|
||||||
|
continue
|
||||||
|
rest = path.name[len(run_id) + 1 : -len(".json")]
|
||||||
|
kind = next((k for k in ("proposal", "outcome") if rest.endswith(f"-{k}")), rest)
|
||||||
|
kinds[kind] = kinds.get(kind, 0) + 1
|
||||||
|
return kinds
|
||||||
|
|
||||||
|
|
||||||
def _build(tmp_path: Path, **kwargs: Any) -> rb.Built:
|
def _build(tmp_path: Path, **kwargs: Any) -> rb.Built:
|
||||||
outbox = kwargs.pop("outbox", None) or _outbox(tmp_path)
|
outbox = kwargs.pop("outbox", None) or _outbox(tmp_path)
|
||||||
rounds = kwargs.pop("rounds_dir", None) or tmp_path / "runder"
|
rounds = kwargs.pop("rounds_dir", None) or tmp_path / "runder"
|
||||||
|
|
@ -262,15 +379,80 @@ def test_only_this_runs_artefacts_are_copied_and_the_rest_is_reported(tmp_path:
|
||||||
(source / "08.dry.out").write_text("stdout fra skriptet\n", encoding="utf-8")
|
(source / "08.dry.out").write_text("stdout fra skriptet\n", encoding="utf-8")
|
||||||
(source / "verdict.err").write_text("", encoding="utf-8")
|
(source / "verdict.err").write_text("", encoding="utf-8")
|
||||||
built = _build(tmp_path, outbox=source)
|
built = _build(tmp_path, outbox=source)
|
||||||
expected = {f"{_RUN}-coverage.json"} | {
|
expected = {f"{_RUN}-{kind}.json" for kind in _RUN_LEVEL_TYPES} | {
|
||||||
f"{_RUN}-{aid}-{kind}.json" for aid in _EVALUATED for kind in ("proposal", "outcome")
|
f"{_RUN}-{aid}-{kind}.json" for aid in _EVALUATED for kind in ("proposal", "outcome")
|
||||||
}
|
}
|
||||||
assert len(expected) == 2 * len(_EVALUATED) + 1
|
assert len(expected) == 2 * len(_EVALUATED) + len(_RUN_LEVEL_TYPES)
|
||||||
assert set(built.copied) == expected
|
assert set(built.copied) == expected
|
||||||
assert {p.name for p in (built.round_dir / "outbox").iterdir()} == expected
|
assert {p.name for p in (built.round_dir / "outbox").iterdir()} == expected
|
||||||
assert set(built.ignored) == {"08.dry.out", "verdict.err"}
|
assert set(built.ignored) == {"08.dry.out", "verdict.err"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_fixture_carries_every_artefact_type_a_real_run_writes(tmp_path: Path) -> None:
|
||||||
|
"""The DENOMINATOR arm. A fixture that writes three of seven types measures the builder
|
||||||
|
against a universe four types smaller than the one it meets, and "every artefact is copied"
|
||||||
|
is then a claim about the fixture. Seven is counted in ``_ARTEFACT_TYPES`` from the four
|
||||||
|
archived runs, and the types present here are counted from the FILES by the same rule."""
|
||||||
|
kinds = _types_in(_outbox(tmp_path))
|
||||||
|
assert len(_ARTEFACT_TYPES) == 7
|
||||||
|
assert set(kinds) == set(_ARTEFACT_TYPES), sorted(set(_ARTEFACT_TYPES) - set(kinds))
|
||||||
|
assert kinds["proposal"] == kinds["outcome"] == len(_EVALUATED)
|
||||||
|
assert all(kinds[kind] == 1 for kind in _RUN_LEVEL_TYPES), kinds
|
||||||
|
assert sum(kinds.values()) == 2 * len(_EVALUATED) + len(_RUN_LEVEL_TYPES)
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_artefact_type_of_the_run_is_carried_into_the_round(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: drop an artefact type the builder does not read.
|
||||||
|
|
||||||
|
``-debate.json``, ``-verdict.json``, ``-runconfig.json`` and ``-parse-failures.json`` are the
|
||||||
|
run's own record of what it opened, what a judge said, what it was configured with and what
|
||||||
|
never parsed. The builder reads none of them and must carry all of them: a round is the run,
|
||||||
|
copied. Dropping one silently leaves a round that LOOKS whole. Counted from ``_SPEC`` and
|
||||||
|
``_RUN_LEVEL_TYPES``, on both sides, so this is not the copy agreeing with itself."""
|
||||||
|
source = _outbox(tmp_path)
|
||||||
|
built = _build(tmp_path, outbox=source)
|
||||||
|
expected = {"proposal": len(_EVALUATED), "outcome": len(_EVALUATED)} | {
|
||||||
|
kind: 1 for kind in _RUN_LEVEL_TYPES
|
||||||
|
}
|
||||||
|
assert _types_in(source) == expected
|
||||||
|
assert _types_in(built.round_dir / "outbox") == expected
|
||||||
|
for kind in _RUN_LEVEL_TYPES:
|
||||||
|
assert (built.round_dir / "outbox" / f"{_RUN}-{kind}.json").is_file(), kind
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_refused_rows_amount_never_becomes_a_validated_saving(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: let a coverage row's ``saving_nok`` through regardless of its status.
|
||||||
|
|
||||||
|
Two halves, because there are two places the number could leak: (d) of the outcome file, and
|
||||||
|
the total the report prints. Both are counted from the table — the refused row's 900 000 is
|
||||||
|
added to ``_VALIDATED_ORE`` here to get the figure a leak WOULD produce, and that figure is
|
||||||
|
then required to be absent."""
|
||||||
|
source = _outbox(tmp_path)
|
||||||
|
built = _build(tmp_path, outbox=source)
|
||||||
|
refused_with_amount = {
|
||||||
|
aid for aid, _l, status, nok, *_rest in _SPEC if status != "validated" and nok is not None
|
||||||
|
}
|
||||||
|
assert refused_with_amount == {"a2-faerre-kummer"}, "the table lost the row this arm needs"
|
||||||
|
rows = {
|
||||||
|
row["id"]: row
|
||||||
|
for row in json.loads((built.round_dir / "outcome.json").read_text(encoding="utf-8"))[
|
||||||
|
"approaches"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
for aid in refused_with_amount:
|
||||||
|
assert rows[aid]["validated"] is False, aid
|
||||||
|
assert rows[aid]["validated_nok"] is None, aid
|
||||||
|
leaked = _VALIDATED_ORE + sum(
|
||||||
|
to_ore(nok)
|
||||||
|
for aid, _l, status, nok, *_rest in _SPEC
|
||||||
|
if status != "validated" and nok is not None
|
||||||
|
)
|
||||||
|
assert built.validated_ore == _VALIDATED_ORE == 137_000_002
|
||||||
|
assert leaked == 227_000_002 != _VALIDATED_ORE
|
||||||
|
text = (built.round_dir / "report.md").read_text(encoding="utf-8")
|
||||||
|
assert "2 270 000,02" not in text, "a refused claim was counted as a validated saving"
|
||||||
|
|
||||||
|
|
||||||
def test_the_outcome_is_derived_row_for_row_from_the_runs_own_coverage(tmp_path: Path) -> None:
|
def test_the_outcome_is_derived_row_for_row_from_the_runs_own_coverage(tmp_path: Path) -> None:
|
||||||
"""(a)-(d) per approach, counted from the table. ``feedback_ids`` is empty on every row: no
|
"""(a)-(d) per approach, counted from the table. ``feedback_ids`` is empty on every row: no
|
||||||
run records which feedback item produced which row, and the builder invents no tracking."""
|
run records which feedback item produced which row, and the builder invents no tracking."""
|
||||||
|
|
@ -285,7 +467,9 @@ def test_the_outcome_is_derived_row_for_row_from_the_runs_own_coverage(tmp_path:
|
||||||
for aid, _label, status, nok, _claimed, detail in _SPEC:
|
for aid, _label, status, nok, _claimed, detail in _SPEC:
|
||||||
row = rows[aid]
|
row = rows[aid]
|
||||||
assert row["validated"] is (status == "validated"), aid
|
assert row["validated"] is (status == "validated"), aid
|
||||||
assert row["validated_nok"] == nok, aid
|
# Only a VALIDATED row's figure may reach (d). The refused row in the table carries an
|
||||||
|
# amount precisely so that this is a measurement rather than a tautology.
|
||||||
|
assert row["validated_nok"] == (nok if status == "validated" else None), aid
|
||||||
assert row["feedback_ids"] == [], aid
|
assert row["feedback_ids"] == [], aid
|
||||||
expected_stage = {"validated": "", "not_evaluated": "not_evaluated"}.get(
|
expected_stage = {"validated": "", "not_evaluated": "not_evaluated"}.get(
|
||||||
status, "unsupported" if detail == UNSUPPORTED_REASON else "stage4-p90"
|
status, "unsupported" if detail == UNSUPPORTED_REASON else "stage4-p90"
|
||||||
|
|
@ -361,6 +545,50 @@ def test_an_existing_round_is_never_overwritten(tmp_path: Path) -> None:
|
||||||
assert (built.round_dir / "report.md").read_text(encoding="utf-8") == "# min egen retting\n"
|
assert (built.round_dir / "report.md").read_text(encoding="utf-8") == "# min egen retting\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_round_directory_that_is_a_dangling_symlink_is_refused_with_a_reason(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""``exists()`` FOLLOWS a link, so a dangling one answers False and slips past the
|
||||||
|
"never overwritten" guard — measured 19.09: the build then died on the filesystem's own
|
||||||
|
``FileExistsError`` traceback instead of a sentence the operator can act on. A refusal has to
|
||||||
|
be a refusal in the form the operator sees, with the same exit code as every other one."""
|
||||||
|
source = _outbox(tmp_path)
|
||||||
|
rounds = tmp_path / "runder"
|
||||||
|
rounds.mkdir()
|
||||||
|
target = tmp_path / "finnes-ikke"
|
||||||
|
(rounds / "0").symlink_to(target)
|
||||||
|
try:
|
||||||
|
rb.build_round(source, rounds, 0, ran_at=_RAN_AT)
|
||||||
|
except rb.RoundBuildError as refused:
|
||||||
|
assert "lenke" in str(refused).lower(), str(refused)
|
||||||
|
except OSError as raw:
|
||||||
|
raise AssertionError(
|
||||||
|
f"the filesystem's own error reached the operator instead of a reason: {raw!r}"
|
||||||
|
) from raw
|
||||||
|
else:
|
||||||
|
raise AssertionError("a dangling symlink as the round directory was not refused")
|
||||||
|
assert (rounds / "0").is_symlink(), "the link itself must be left for the operator"
|
||||||
|
assert not target.exists(), "a refusal must not create the link's target"
|
||||||
|
assert sorted(p.name for p in rounds.iterdir()) == ["0"]
|
||||||
|
assert (
|
||||||
|
rb.main(
|
||||||
|
[
|
||||||
|
"--outbox",
|
||||||
|
str(source),
|
||||||
|
"--round",
|
||||||
|
"0",
|
||||||
|
"--rounds-dir",
|
||||||
|
str(rounds),
|
||||||
|
"--ran-at",
|
||||||
|
_RAN_AT,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
== 1
|
||||||
|
)
|
||||||
|
# rc-0 control: the same call against a name that is not a link builds.
|
||||||
|
assert (rb.build_round(source, tmp_path / "andre", 0, ran_at=_RAN_AT).round_dir).is_dir()
|
||||||
|
|
||||||
|
|
||||||
def test_ran_at_must_carry_a_timezone(tmp_path: Path) -> None:
|
def test_ran_at_must_carry_a_timezone(tmp_path: Path) -> None:
|
||||||
"""The gate reads ``ran_at`` as the run's time and refuses a stamp without a zone. Catching it
|
"""The gate reads ``ran_at`` as the run's time and refuses a stamp without a zone. Catching it
|
||||||
here costs one message instead of a round that builds and then reads unparseable."""
|
here costs one message instead of a round that builds and then reads unparseable."""
|
||||||
|
|
@ -511,6 +739,114 @@ def test_the_report_follows_the_runs_own_order_not_a_sorted_one(tmp_path: Path)
|
||||||
assert forward == sorted(forward, reverse=True), "the report sorted instead of following it"
|
assert forward == sorted(forward, reverse=True), "the report sorted instead of following it"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_report_heading_names_the_round_it_is(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: head the report with round ``n + 1``.
|
||||||
|
|
||||||
|
The round number is the expert's only way to place the report in the sequence they are
|
||||||
|
correcting, and an off-by-one there is invisible from inside the file. The heading is compared
|
||||||
|
to a string built from the number the TEST passed, never to the builder's own answer."""
|
||||||
|
rounds = _round_zero(tmp_path)
|
||||||
|
zero = (rounds / "0" / "report.md").read_text(encoding="utf-8")
|
||||||
|
assert zero.splitlines()[0] == f"# Rapport fra runde 0 — kjøring {_RUN}"
|
||||||
|
built = rb.build_round(
|
||||||
|
_outbox(tmp_path / "k1"),
|
||||||
|
rounds,
|
||||||
|
1,
|
||||||
|
ran_at=_RAN_AT,
|
||||||
|
feedback=_feedback_file(tmp_path / "f1.json", "f1"),
|
||||||
|
)
|
||||||
|
first = (built.round_dir / "report.md").read_text(encoding="utf-8").splitlines()[0]
|
||||||
|
assert first == f"# Rapport fra runde 1 — kjøring {_RUN}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_report_lists_every_approach_the_run_never_reached(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: keep the COUNT of approaches the run never reached and drop the list.
|
||||||
|
|
||||||
|
"1 tilnærminger ble aldri vurdert" followed by nothing is a report contradicting itself in
|
||||||
|
the one section about work that did not happen — and the expert cannot ask for an approach
|
||||||
|
they were never shown. Counted from ``_NOT_EVALUATED``."""
|
||||||
|
section = _report(tmp_path).split("## Hva kjøringen aldri rakk", 1)[1].split("\n## ", 1)[0]
|
||||||
|
assert f"{len(_NOT_EVALUATED)} tilnærminger ble aldri vurdert" in section
|
||||||
|
assert section.count("- **") == len(_NOT_EVALUATED) == 1
|
||||||
|
for aid, label, status, *_rest in _SPEC:
|
||||||
|
if status == "not_evaluated":
|
||||||
|
assert label in section, aid
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_report_shows_each_proposals_source_and_how_many_places_it_cited(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""MUTANT: drop the citation entirely, and MUTANT: drop its COUNT.
|
||||||
|
|
||||||
|
A proposal without its source cannot be checked against the knowledge base at all, and a
|
||||||
|
single quote without the count cannot tell a proposal grounded in one place from one that
|
||||||
|
swept 270. The counts in ``_CITED`` are DISTINCT per approach on purpose: a builder printing a
|
||||||
|
constant would satisfy a fixture where every count was the same."""
|
||||||
|
text = _report(tmp_path)
|
||||||
|
assert len({_CITED[aid] for aid in _EVALUATED}) == len(_EVALUATED), "the counts must differ"
|
||||||
|
for aid in _EVALUATED:
|
||||||
|
assert f"Kilde (1 av {_CITED[aid]} siterte steder)" in text, aid
|
||||||
|
assert _snippet(aid, 0) in text, aid
|
||||||
|
assert f"krav/N100/id-{aid}-0.md" in text, aid
|
||||||
|
assert text.count("Kilde (1 av ") == len(_EVALUATED)
|
||||||
|
assert "Felles kilde" not in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_report_shows_the_cost_lines_each_proposal_touches(tmp_path: Path) -> None:
|
||||||
|
"""MUTANT: drop the affected cost lines.
|
||||||
|
|
||||||
|
The cost line with its quantity and unit price is what makes the saving checkable against the
|
||||||
|
project's own budget; without it the report states an amount and no way to arrive at it. Each
|
||||||
|
expected string is built from ``_SPEC`` with the framework's one conversion."""
|
||||||
|
text = _report(tmp_path)
|
||||||
|
for aid, _label, status, _nok, claimed, _detail in _SPEC:
|
||||||
|
if status == "not_evaluated":
|
||||||
|
continue
|
||||||
|
whole, rest = divmod(to_ore(claimed), 100)
|
||||||
|
amount = f"{whole:,}".replace(",", " ") + f",{rest:02d}"
|
||||||
|
assert f"{_code(aid)} (1 × {amount} kroner)" in text, aid
|
||||||
|
assert text.count("Berørte kostnadslinjer: ") == len(_EVALUATED)
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_citation_list_shared_by_every_proposal_is_stated_once(tmp_path: Path) -> None:
|
||||||
|
"""Measured 19.09 on all four archived runs: every proposal carried the SAME citation list,
|
||||||
|
byte for byte (270 places, same order) — the run's whole retrieved context, stamped once per
|
||||||
|
proposal. The cause is in the OUTBOX, not in the builder reading a wrong field, so the report
|
||||||
|
cannot make the quote informative. What it can do is stop repeating it: say it once, say that
|
||||||
|
it is the run's list and not the measure's, and drop the per-proposal copies."""
|
||||||
|
source = _outbox(tmp_path / "felles", shared_citations=True)
|
||||||
|
text = (_build(tmp_path / "felles", outbox=source).round_dir / "report.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert f"Alle {len(_EVALUATED)} forslagene" in text
|
||||||
|
assert text.count(f"Felles kilde (1 av {_SHARED_CITED} siterte steder)") == 1
|
||||||
|
assert text.count(_snippet("hele-kjoringen", 0)) == 1
|
||||||
|
assert "Kilde (1 av " not in text, "the identical quote was repeated per proposal anyway"
|
||||||
|
# Control: the per-approach fixture must NOT take this branch.
|
||||||
|
assert "Felles kilde" not in _report(tmp_path / "egne")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_same_cost_line_on_both_sides_of_the_verdict_is_named_where_it_happens(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""The 19.09 report refused ``TUN-LYS-01`` under one label and validated the SAME cost line
|
||||||
|
under another, and said nothing about it — so the reader met two figures for one budget line
|
||||||
|
with no way to see they collided. Said where it happens, on both sides, or not at all."""
|
||||||
|
label_of = {aid: label for aid, label, *_rest in _SPEC}
|
||||||
|
held, fell = _VALIDATED[0], _FELL[0]
|
||||||
|
source = _outbox(tmp_path / "kollisjon", codes={held: "DK-FELLES", fell: "DK-FELLES"})
|
||||||
|
text = (_build(tmp_path / "kollisjon", outbox=source).round_dir / "report.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert text.count("kostnadslinjen DK-FELLES står også i") == 2
|
||||||
|
held_part = text.split(f"### {label_of[held]}", 1)[1].split("\n### ", 1)[0]
|
||||||
|
assert label_of[fell] in held_part, "the validated side does not name the refused one"
|
||||||
|
fell_part = text.split(f"### {label_of[fell]}", 1)[1].split("\n### ", 1)[0]
|
||||||
|
assert label_of[held] in fell_part, "the refused side does not name the validated one"
|
||||||
|
# Control: distinct cost lines must produce no such sentence at all.
|
||||||
|
assert "står også i" not in _report(tmp_path / "rent")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------
|
||||||
# Round n >= 1 — measured against the round before it
|
# Round n >= 1 — measured against the round before it
|
||||||
# ---------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------
|
||||||
|
|
@ -584,6 +920,28 @@ def test_round_one_reports_what_changed_and_admits_when_nothing_explains_it(
|
||||||
assert rb.STAGE_PROSE["stage4-p90"].split(":")[0] in section
|
assert rb.STAGE_PROSE["stage4-p90"].split(":")[0] in section
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_removed_approach_is_named_by_the_label_the_expert_saw(tmp_path: Path) -> None:
|
||||||
|
"""A removed row is the ONE row whose human name is not in this run's coverage — it is only in
|
||||||
|
the round it disappeared from. Writing the bare id there hands the expert an identifier they
|
||||||
|
have no way to look up, in the section that exists for them to judge what moved. Derived from
|
||||||
|
the previous round's own outbox; ``outcome.json`` keeps its four columns."""
|
||||||
|
rounds = _round_zero(tmp_path)
|
||||||
|
gone = _VALIDATED[-1]
|
||||||
|
shorter = tuple(row for row in _SPEC if row[0] != gone)
|
||||||
|
built = rb.build_round(
|
||||||
|
_outbox(tmp_path / "k1", spec=shorter),
|
||||||
|
rounds,
|
||||||
|
1,
|
||||||
|
ran_at=_RAN_AT,
|
||||||
|
feedback=_feedback_file(tmp_path / "f1.json", "f1"),
|
||||||
|
)
|
||||||
|
text = (built.round_dir / "report.md").read_text(encoding="utf-8")
|
||||||
|
section = text.split("## Endret siden forrige runde", 1)[1].split("\n## ", 1)[0]
|
||||||
|
label = {aid: lab for aid, lab, *_rest in _SPEC}[gone]
|
||||||
|
assert f"**{label}** ({gone})" in section
|
||||||
|
assert f"- **{gone}** —" not in section, "the raw id reached the expert without its label"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------
|
||||||
# The one file the builder may never write
|
# The one file the builder may never write
|
||||||
# ---------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------
|
||||||
|
|
@ -633,7 +991,7 @@ def test_the_builder_writes_exactly_the_names_it_is_allowed_to(tmp_path: Path) -
|
||||||
made = {str(p.relative_to(built.round_dir)) for p in built.round_dir.rglob("*")}
|
made = {str(p.relative_to(built.round_dir)) for p in built.round_dir.rglob("*")}
|
||||||
expected = (
|
expected = (
|
||||||
{"outcome.json", "report.md", "feedback.json", "outbox"}
|
{"outcome.json", "report.md", "feedback.json", "outbox"}
|
||||||
| {f"outbox/{_RUN}-coverage.json"}
|
| {f"outbox/{_RUN}-{kind}.json" for kind in _RUN_LEVEL_TYPES}
|
||||||
| {
|
| {
|
||||||
f"outbox/{_RUN}-{aid}-{kind}.json"
|
f"outbox/{_RUN}-{aid}-{kind}.json"
|
||||||
for aid in _EVALUATED
|
for aid in _EVALUATED
|
||||||
|
|
@ -641,7 +999,7 @@ def test_the_builder_writes_exactly_the_names_it_is_allowed_to(tmp_path: Path) -
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
assert made == expected
|
assert made == expected
|
||||||
assert len(expected) == 4 + 1 + 2 * len(_EVALUATED)
|
assert len(expected) == 4 + len(_RUN_LEVEL_TYPES) + 2 * len(_EVALUATED)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue