test(v1-gate): the PM's three consistent forgeries, as red tests — a family that agrees with itself is still not a witness
M-6 moved the price of a forgery from `touch` to reproducing the product's own artefact
set. The PM checkpoint of 18.09 then paid that price three ways, and row 2 read 3 of 3
GREEN each time:
F1 the whole artefact family written by hand and made internally consistent — coverage,
one proposal/outcome pair per approach, `verdict_id` minted with the product's own
rule. About sixty lines of script. `RUN_ATTESTATION` stood in the output; the row was
green anyway.
F2 four REAL runs' artefacts under a handwritten feedback file.
F4 `<n>/outbox` made a symlink to a real run's directory somewhere else, so a DERIVED
path was redirected by the filesystem.
F4 is a hole. F1 and F2 are not: no arrangement of files can be told apart from a round
that happened, because what is missing is not a check but a WITNESS. These tests say so.
Red here, 16 of them:
- rows 1 and 2 must stop at `gate.FORM_OK` — 0 against the criterion, exit still 1 — on a
tree where every computed check passes but nobody has attested the rounds (F1, F2);
- F2 is built through `outbox.write_outbox` and `write_coverage`, the exact bytes a real
run leaves behind, so it doubles as the control that `verify_run` reads the PRODUCT's
output and not merely the shape this test file writes;
- an outbox that is a symlink out of the round, and an artefact symlinked into one, are
refused by name (F4);
- the step from FORM OK to GREEN is `<n>/attestering.txt`, per round, round 0 included
because row 2 measures round 1 against it: present and matching -> GREEN (the rc-0
control, asserted first in every arm), missing -> FORM OK, present but naming another
round, another run, no date, an unparsable date or a date before the run -> RED;
- nothing in `src/` may write that file: a product that can produce a witness to its own
run has produced exactly the thing rows 1-2 cannot.
One more arm, green on arrival, and that is the finding: the guard at `v1_gate.py:415`
(`keyed != {(run_id, aid)}`), pinned to a constant-false branch, left the suite at 68
passed, identical to baseline. The guard bit, nothing read it — so
`_mut_the_family_is_labelled_for_another_run` gives the pair the right FILENAME and
another run's labels inside.
The fixture now writes the operator's attestation on every round it builds, and the
file's name is pinned in the test file as well (`_ATTEST_FILE`), the way M-5 pins the
rest of the contract: renaming it in the gate alone must fail a test.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
a362504108
commit
ab00016ed5
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
1 changed files with 233 additions and 4 deletions
|
|
@ -26,7 +26,10 @@ import pytest
|
|||
from portfolio_optimiser import frozen_bundles
|
||||
from portfolio_optimiser.evals import v1_gate as gate
|
||||
from portfolio_optimiser.ir import AffectedItem, SavingsProposal
|
||||
from portfolio_optimiser.validator import UNSUPPORTED_REASON
|
||||
from portfolio_optimiser.outbox import write_coverage, write_outbox
|
||||
from portfolio_optimiser.provenance import Citation, ProvenanceStamp
|
||||
from portfolio_optimiser.retrieval import TextSpan
|
||||
from portfolio_optimiser.validator import UNSUPPORTED_REASON, Rejection, ValidatedProposal
|
||||
from portfolio_optimiser.verdicts import features_from_ir, verdict_key
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
|
|
@ -36,6 +39,11 @@ _ALL_NODEIDS = [n for spec in _CONFIG["feedback_types"].values() for n in spec["
|
|||
_CONFIG["row6_evidence"]
|
||||
)
|
||||
_T0 = 1_780_000_000 # a fixed epoch: every fixture run and feedback is ordered against it
|
||||
#: The day the operator attests a fixture round on: the fixture runs are all 2026-05-28.
|
||||
_ATTEST_DATE = "2026-05-29"
|
||||
#: The operator's attestation file, pinned HERE as well: it is the one name in the contract a
|
||||
#: person types by hand, so renaming it in the gate alone must show up as a failing test.
|
||||
_ATTEST_FILE = "attestering.txt"
|
||||
|
||||
#: The reason text a rejection at each stage carries, so a fixture run's coverage produces the
|
||||
#: stage ``validator.rejection_stage`` would read off a real run.
|
||||
|
|
@ -181,15 +189,31 @@ def _outcome(
|
|||
)
|
||||
|
||||
|
||||
def _attest(round_dir: Path, *, run_id: str | None = None, on: str = _ATTEST_DATE) -> None:
|
||||
"""The operator's attestation for one round — the one file in the contract no machine may be
|
||||
able to produce. Written here only because a fixture has to stand in for the operator; the
|
||||
gate never writes one, and neither does anything else in the product."""
|
||||
_write(
|
||||
round_dir / _ATTEST_FILE,
|
||||
f"runde: {round_dir.name}\n"
|
||||
f"kjøring: {run_id or 'r' + round_dir.name}\n"
|
||||
f"dato: {on}\n"
|
||||
"Jeg bekrefter at denne runden ble holdt slik artefaktene beskriver.\n",
|
||||
)
|
||||
|
||||
|
||||
def _green_rounds(root: Path) -> Path:
|
||||
"""Three traced rounds, each run after its feedback, and a round 3 report the expert kept
|
||||
whole while adding a line of their own."""
|
||||
"""Three traced rounds, each run after its feedback, a round 3 report the expert kept whole
|
||||
while adding a line of their own — and the operator's attestation on every round, including
|
||||
the baseline round 0 that round 1 is measured against."""
|
||||
_outcome(root / "0", [_row("a1", False, None, stage="stage0-baseline")])
|
||||
_write(root / "0" / "report.md", "# Rapport 0\n\nlinje\n")
|
||||
_attest(root / "0")
|
||||
for n in (1, 2, 3):
|
||||
_feedback(root / str(n), (f"f{n}", 1, f"Tallet for linje {n} er feil, bruk kontrakten."))
|
||||
_outcome(root / str(n), [_row("a1", True, 1000.0 * n, f"f{n}")])
|
||||
_write(root / str(n) / "report.md", f"# Rapport {n}\n\nlinje\n")
|
||||
_attest(root / str(n))
|
||||
_write(root / "3" / "report.kept.md", "# Rapport 3\n\nlinje\nmin egen merknad\n")
|
||||
return root
|
||||
|
||||
|
|
@ -580,6 +604,15 @@ def _mut_an_unevaluated_approach_has_artefacts(root: Path) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _mut_the_family_is_labelled_for_another_run(root: Path) -> None:
|
||||
"""The right FILENAME, another run's labels INSIDE. The guard on ``keyed`` has stood since
|
||||
M-6, but nothing read it: pinned to a constant-false branch it left the suite at 68 passed,
|
||||
identical to baseline (PM checkpoint 18.09). A guard no test reads is a guard the next edit
|
||||
deletes."""
|
||||
for kind in ("proposal", "outcome"):
|
||||
_edit(root / "1" / "outbox" / f"r1-a1-{kind}.json", run_id="r9", approach_id="a9")
|
||||
|
||||
|
||||
def _mut_the_run_time_has_no_zone(root: Path) -> None:
|
||||
_edit(root / "1" / "outcome.json", ran_at="2026-09-18T10:00:00")
|
||||
|
||||
|
|
@ -597,6 +630,7 @@ def _mut_the_run_time_has_no_zone(root: Path) -> None:
|
|||
(_mut_the_reason_is_not_the_rows, "utfallets grunn"),
|
||||
(_mut_an_artefact_belongs_to_another_run, "coverage ikke nevner"),
|
||||
(_mut_an_unevaluated_approach_has_artefacts, "ble ikke evaluert, men kjøringen skrev"),
|
||||
(_mut_the_family_is_labelled_for_another_run, "artefaktene er merket"),
|
||||
(_mut_the_run_time_has_no_zone, "ran_at er ikke et ISO"),
|
||||
],
|
||||
)
|
||||
|
|
@ -623,10 +657,204 @@ def test_m6_an_mtime_is_not_evidence_and_no_longer_decides_anything(tmp_path: Pa
|
|||
os.utime(path, (_T0 - n * 1000, _T0 - n * 1000))
|
||||
row = gate.score_changes(root, 3, _AI)
|
||||
assert (row.k, row.status) == (3, gate.GREEN), row.exceptions
|
||||
assert row.attests == (gate.RUN_ATTESTATION,)
|
||||
assert row.attests == (gate.RUN_ATTESTATION, gate.ATTEST_RULE)
|
||||
assert "bekrefter operatøren" in gate.RUN_ATTESTATION
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# M-7 — the 18.09 PM check: three CONSISTENT forgeries still read 3 of 3 GREEN
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
#
|
||||
# M-6 moved the price of a forgery from ``touch`` to reproducing the product's own artefact set.
|
||||
# The PM then paid that price three ways and row 2 read 3 of 3 GREEN each time: F1 a whole family
|
||||
# written by hand and made internally consistent, verdict key minted with the product's own rule
|
||||
# (about sixty lines of script) · F2 four REAL runs' artefacts under a handwritten feedback file ·
|
||||
# F4 ``<n>/outbox`` symlinked out of the round to a real run somewhere else.
|
||||
#
|
||||
# F4 is a hole and is closed below. F1 and F2 are not: no arrangement of files can be told from a
|
||||
# round that happened, because what is missing is not a check but a WITNESS. So the computation
|
||||
# stops at FORM OK — 0 against the criterion, exit still 1 — and the step to GREEN is a statement
|
||||
# the operator makes, per round, in a file the gate never writes.
|
||||
|
||||
|
||||
def _unattested(root: Path) -> Path:
|
||||
for path in sorted(root.rglob(_ATTEST_FILE)):
|
||||
path.unlink()
|
||||
return root
|
||||
|
||||
|
||||
def test_m7_a_consistent_family_is_form_ok_and_counts_zero(tmp_path: Path) -> None:
|
||||
"""F1. Everything ``verify_run`` recomputes agrees — because the forger recomputed it too.
|
||||
Rows 1 and 2 must read FORM OK, count 0 of 3, and leave the gate's exit at 1."""
|
||||
root = _unattested(_green_rounds(tmp_path))
|
||||
rounds, changes = gate.score_rounds(root, 3, _AI), gate.score_changes(root, 3, _AI)
|
||||
assert (rounds.k, rounds.status) == (0, gate.FORM_OK), rounds.exceptions
|
||||
assert (changes.k, changes.status) == (0, gate.FORM_OK), changes.exceptions
|
||||
assert gate.exit_code([rounds, changes]) == 1
|
||||
assert any(_ATTEST_FILE in x for x in rounds.exceptions), rounds.exceptions
|
||||
|
||||
|
||||
def _stamp(decision: str) -> ProvenanceStamp:
|
||||
return ProvenanceStamp(
|
||||
citations=[
|
||||
Citation(file="f.md", locator=TextSpan(start_index=0, end_index=5), snippet="h")
|
||||
],
|
||||
model="synthetic",
|
||||
role="proposer",
|
||||
validator_decision=decision, # type: ignore[arg-type]
|
||||
token_usage=8,
|
||||
cost_baseline_anchored=True,
|
||||
bundle_id_source=None,
|
||||
code_forms={},
|
||||
)
|
||||
|
||||
|
||||
def _real_run_family(round_dir: Path) -> None:
|
||||
"""F2. The round's outbox rewritten BY THE PRODUCT — ``outbox.write_outbox`` and
|
||||
``write_coverage``, the exact bytes a real run leaves behind — for the coverage the round
|
||||
already declares. This is also the control that ``verify_run`` reads the product's own
|
||||
output and not merely the shape this test file happens to write: if the two ever part, the
|
||||
gate is checking a fixture."""
|
||||
run_id = f"r{round_dir.name}"
|
||||
outbox = round_dir / "outbox"
|
||||
coverage = json.loads((outbox / f"{run_id}-coverage.json").read_text(encoding="utf-8"))["rows"]
|
||||
shutil.rmtree(outbox)
|
||||
for cov in coverage:
|
||||
if cov["status"] == "not_evaluated":
|
||||
continue
|
||||
ir = _ir(cov["id"], cov.get("saving_nok"))
|
||||
proposal = SavingsProposal(**ir)
|
||||
validated = cov["status"] == "validated"
|
||||
outcome: Any = (
|
||||
ValidatedProposal(proposal=proposal, p10=1.0, p50=2.0, p90=3.0, nominal_feasible=2.0)
|
||||
if validated
|
||||
else Rejection(proposal=proposal, reason=cov["detail"])
|
||||
)
|
||||
write_outbox(
|
||||
str(outbox),
|
||||
run_id,
|
||||
outcome=outcome,
|
||||
provenance=_stamp("validated" if validated else "rejected"),
|
||||
checker_verdict="approve",
|
||||
verdict_id=verdict_key(features_from_ir(ir)),
|
||||
approach_id=cov["id"],
|
||||
)
|
||||
write_coverage(str(outbox), run_id, rows=coverage, stop_reason="")
|
||||
|
||||
|
||||
def test_m7_the_products_own_artefacts_do_not_prove_a_round_either(tmp_path: Path) -> None:
|
||||
"""F2. Real artefacts, handwritten feedback. The rc-0 control comes first: with the operator's
|
||||
attestation in place the SAME product-written tree is GREEN, so this arm fails on the missing
|
||||
witness and not on a fixture the gate cannot read."""
|
||||
root = _green_rounds(tmp_path)
|
||||
for n in (0, 1, 2, 3):
|
||||
_real_run_family(root / str(n))
|
||||
assert gate.score_changes(root, 3, _AI).status == gate.GREEN, gate.score_changes(
|
||||
root, 3, _AI
|
||||
).exceptions
|
||||
row = gate.score_changes(_unattested(root), 3, _AI)
|
||||
assert (row.k, row.status) == (0, gate.FORM_OK), row.exceptions
|
||||
|
||||
|
||||
def test_m7_an_outbox_that_leaves_the_round_is_refused(tmp_path: Path) -> None:
|
||||
"""F4. ``<n>/outbox`` made a symlink to a real run's directory elsewhere, and the round read
|
||||
that run as its own. A derived path is only derived if the filesystem cannot redirect it."""
|
||||
root = _green_rounds(tmp_path / "rounds")
|
||||
elsewhere = tmp_path / "en-ekte-kjoering"
|
||||
shutil.copytree(root / "1" / "outbox", elsewhere)
|
||||
shutil.rmtree(root / "1" / "outbox")
|
||||
(root / "1" / "outbox").symlink_to(elsewhere, target_is_directory=True)
|
||||
row = gate.score_changes(root, 3, _AI)
|
||||
assert (row.k, row.status) == (1, gate.RED), row.exceptions
|
||||
assert any("lenke ut av runden" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
||||
def test_m7_an_artefact_symlinked_into_the_round_is_refused(tmp_path: Path) -> None:
|
||||
"""The same move one level down: the outbox is the round's own directory, but a file in it
|
||||
points at another run's artefact."""
|
||||
root = _green_rounds(tmp_path / "rounds")
|
||||
target = tmp_path / "r1-a1-outcome.json"
|
||||
artefact = root / "1" / "outbox" / "r1-a1-outcome.json"
|
||||
shutil.copy(artefact, target)
|
||||
artefact.unlink()
|
||||
artefact.symlink_to(target)
|
||||
row = gate.score_changes(root, 3, _AI)
|
||||
assert row.status == gate.RED
|
||||
assert any("lenke" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
||||
def test_m7_the_attestation_is_the_step_from_form_ok_to_green(tmp_path: Path) -> None:
|
||||
"""The control in both directions: the same tree is GREEN with the operator's attestation and
|
||||
FORM OK without it. A gate that refuses everything is as useless as one that refuses nothing
|
||||
(P21 C1), and a row that can only read FORM OK would be exactly that."""
|
||||
root = _green_rounds(tmp_path)
|
||||
assert gate.score_rounds(root, 3, _AI).status == gate.GREEN
|
||||
assert gate.score_changes(root, 3, _AI).status == gate.GREEN
|
||||
(root / "2" / _ATTEST_FILE).unlink()
|
||||
rounds, changes = gate.score_rounds(root, 3, _AI), gate.score_changes(root, 3, _AI)
|
||||
assert (rounds.k, rounds.status) == (2, gate.FORM_OK), rounds.exceptions
|
||||
assert (changes.k, changes.status) == (2, gate.FORM_OK), changes.exceptions
|
||||
assert any("mangler" in x for x in rounds.exceptions), rounds.exceptions
|
||||
|
||||
|
||||
def test_m7_row2_needs_the_base_run_attested_too(tmp_path: Path) -> None:
|
||||
"""Round 1 is measured against round 0's run, so an unattested baseline is an unattested
|
||||
comparison and row 2 counts nothing. Row 1, which reads no run at all, is untouched."""
|
||||
root = _green_rounds(tmp_path)
|
||||
(root / "0" / _ATTEST_FILE).unlink()
|
||||
assert gate.score_rounds(root, 3, _AI).status == gate.GREEN
|
||||
row = gate.score_changes(root, 3, _AI)
|
||||
assert (row.k, row.status) == (0, gate.FORM_OK), row.exceptions
|
||||
assert any("grunnkjøringen" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "marker"),
|
||||
[
|
||||
("runde: 1\nkjøring: r9\ndato: 2026-05-29\n", "navngir kjøring"),
|
||||
("runde: 3\nkjøring: r1\ndato: 2026-05-29\n", "attesterer runde"),
|
||||
("runde: 1\nkjøring: r1\n", "mangler dato"),
|
||||
("kjøring: r1\ndato: 2026-05-29\n", "mangler runde"),
|
||||
("runde: 1\nkjøring: r1\ndato: i går\n", "er ikke en ISO-dato"),
|
||||
("runde: 1\nkjøring: r1\ndato: 2020-01-01\n", "før kjøringen"),
|
||||
("Jeg bekrefter at runde 1 ble holdt.\n", "mangler runde, kjøring, dato"),
|
||||
],
|
||||
)
|
||||
def test_m7_an_attestation_that_contradicts_the_round_is_red(
|
||||
tmp_path: Path, text: str, marker: str
|
||||
) -> None:
|
||||
"""A MISSING attestation is FORM OK — nobody has confirmed anything yet. A PRESENT one that
|
||||
does not match the round is something else: a statement about a round this is not, and the
|
||||
row goes red rather than waiting."""
|
||||
root = _green_rounds(tmp_path)
|
||||
assert gate.score_rounds(root, 3, _AI).status == gate.GREEN
|
||||
_write(root / "1" / _ATTEST_FILE, text)
|
||||
row = gate.score_rounds(root, 3, _AI)
|
||||
assert (row.k, row.status) == (2, gate.RED), row.exceptions
|
||||
assert any(marker in x for x in row.exceptions), (marker, row.exceptions)
|
||||
|
||||
|
||||
def test_m7_the_attestation_file_is_pinned() -> None:
|
||||
assert gate.ATTEST_FILE == _ATTEST_FILE
|
||||
|
||||
|
||||
def test_m7_the_gate_never_writes_an_attestation(tmp_path: Path) -> None:
|
||||
"""The file is the operator's word. A product that can produce one has produced a witness to
|
||||
its own run, which is the whole thing rows 1-2 cannot do: so no source file in the package
|
||||
writes ``ATTEST_FILE``, and scoring a tree leaves none behind."""
|
||||
root = _unattested(_green_rounds(tmp_path))
|
||||
gate.score_rounds(root, 3, _AI)
|
||||
gate.score_changes(root, 3, _AI)
|
||||
assert list(root.rglob(_ATTEST_FILE)) == []
|
||||
writes = [
|
||||
f"{path.name}:{i}"
|
||||
for path in sorted((_REPO / "src" / "portfolio_optimiser").rglob("*.py"))
|
||||
for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1)
|
||||
if _ATTEST_FILE in line and "write_text" in line
|
||||
]
|
||||
assert writes == []
|
||||
|
||||
|
||||
def _outcome_obj(rows: list[dict[str, Any]], removed: dict[str, set[str]] | None = None) -> Any:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
|
@ -1018,6 +1246,7 @@ def test_wrong_usage_is_exit_two(tmp_path: Path) -> None:
|
|||
assert tracked.returncode == 2 and "gitignored" in tracked.stderr
|
||||
help_text = _cli("--help").stdout
|
||||
assert "report.kept.md" in help_text and "given_at" in help_text and "outbox" in help_text
|
||||
assert _ATTEST_FILE in help_text and "FORM OK" in help_text
|
||||
|
||||
|
||||
def test_the_command_is_red_today_with_every_row_in_its_output(tmp_path: Path) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue