test(v1-gate): row 2 goes green on four handwritten files — the attack, as a red test
The 18.09 re-measurement of 9825b26 made row 2 read 3 of 3 GREEN without a run existing
anywhere: four handwritten outcome.json, four handwritten <run_id>-coverage.json in an outbox
named by those same files, and os.utime for the ordering. read_outcome does cross-check the
outcome against the coverage — but `outbox` is a free path from the round file and the coverage
file is written by the same hand, so "round 0 must be a named real run" is implemented as "a file
with that name exists", which touch satisfies.
The attack is rebuilt in the test rather than driven through the _outcome fixture, so the
fixtures can be raised to a whole run family without the attack drifting with them. Row 1 is
asserted GREEN on the same tree: the feedback IS well formed there, which is what makes this an
attack on row 2 and not a broken fixture.
RED as committed (3, 'GRØNN') != (0, 'RØD') — the fix is the next commit.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
6e7c9c6d5d
commit
b769537830
1 changed files with 68 additions and 0 deletions
|
|
@ -383,6 +383,74 @@ def test_m1_the_feedback_must_come_between_the_two_runs(tmp_path: Path) -> None:
|
||||||
assert any("ikke gitt mellom kjøring" in x for x in row.exceptions)
|
assert any("ikke gitt mellom kjøring" in x for x in row.exceptions)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------
|
||||||
|
# M-6 — the 18.09 re-measurement: row 2 went green on four handwritten files
|
||||||
|
# ---------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _forged_rounds(root: Path) -> Path:
|
||||||
|
"""The 18.09 re-measurement's attack, rebuilt HERE rather than through ``_outcome`` so it
|
||||||
|
cannot drift with the fixtures: four handwritten ``outcome.json``, four handwritten
|
||||||
|
``<run_id>-coverage.json`` in an outbox the forger named in those same files, and ``os.utime``
|
||||||
|
for the ordering. No run of this product ever touched this tree."""
|
||||||
|
chosen = root / "min-utboks"
|
||||||
|
for n in (0, 1, 2, 3):
|
||||||
|
run_id = f"r{n}"
|
||||||
|
validated = n > 0
|
||||||
|
nok = 1000.0 * n if validated else None
|
||||||
|
_write(
|
||||||
|
chosen / f"{run_id}-coverage.json",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"stop_reason": "",
|
||||||
|
"rows": [
|
||||||
|
{
|
||||||
|
"id": "a1",
|
||||||
|
"status": "validated" if validated else "rejected",
|
||||||
|
"detail": "" if validated else _DETAIL["stage0-baseline"],
|
||||||
|
"saving_nok": nok,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
stamp = _T0 + n * 20
|
||||||
|
os.utime(chosen / f"{run_id}-coverage.json", (stamp, stamp))
|
||||||
|
_write(
|
||||||
|
root / str(n) / "outcome.json",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"outbox": str(chosen),
|
||||||
|
"approaches": [
|
||||||
|
_row(
|
||||||
|
"a1",
|
||||||
|
validated,
|
||||||
|
nok,
|
||||||
|
*([f"f{n}"] if validated else []),
|
||||||
|
stage="" if validated else "stage0-baseline",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
"removed": [],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_write(root / str(n) / "report.md", f"# Rapport {n}\n\nlinje\n")
|
||||||
|
if n:
|
||||||
|
_feedback(
|
||||||
|
root / str(n), (f"f{n}", 1, f"Tallet for linje {n} er feil, bruk kontrakten.")
|
||||||
|
)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def test_m6_a_handwritten_outbox_is_not_a_run(tmp_path: Path) -> None:
|
||||||
|
"""Row 2 must not be satisfiable by files a forger wrote. Row 1 stays GREEN on the same tree —
|
||||||
|
the feedback there IS well formed, and that is what makes this an attack on row 2 rather than
|
||||||
|
a broken fixture. The control that the row can still go green is ``_green_rounds``, which
|
||||||
|
carries a whole run family (``test_row2_a_traced_change_counts``)."""
|
||||||
|
root = _forged_rounds(tmp_path)
|
||||||
|
assert gate.score_rounds(root, 3, _AI).status == gate.GREEN
|
||||||
|
row = gate.score_changes(root, 3, _AI)
|
||||||
|
assert (row.k, row.status) == (0, gate.RED), row.exceptions
|
||||||
|
|
||||||
|
|
||||||
def _outcome_obj(rows: list[dict[str, Any]], removed: dict[str, set[str]] | None = None) -> Any:
|
def _outcome_obj(rows: list[dict[str, Any]], removed: dict[str, set[str]] | None = None) -> Any:
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue