"""B-gatens egne tester: hver rad KAN bli grønn og KAN bli rød — og kan ikke FAKES grønn. Rammen er operatørbeslutningen 19.09.2026: i utvikling og test LEDER Claude Code, og portfolio-optimiser er verktøykassen. po kaller ALDRI Claude. Gaten måler avstanden dit, rad for rad, uten et eneste modellkall. Tre ting gjør testene load-bearing i stedet for grønne-men-døde: * **Nevneren telles uavhengig HER.** Rad 1 og rad 2 får ikke sine tall fra gatens egne funksjoner: testen parser kilden selv (``ast``) eller importerer modulen og teller, og sammenligner. En gate som hardkoder nevneren felles av arm ``M-1``, som kjører den mot en KONSTRUERT kilde der svaret er et annet enn repoets. * **Hver vakt har en kjent-positiv.** Et mønster som ikke treffer sin egen kjent-positive er ikke en måling, og rad 3 nekter å telle det (``M-3``). Et fravær («0 treff») uttales med nevner. * **Rad 6 kan ikke bli grønn av en fil gaten selv skriver** (``M-5``): armen kjører hele ``evaluate`` og krever at attesteringsfila fortsatt ikke finnes etterpå. """ from __future__ import annotations import ast import base64 import json import re import subprocess import sys from pathlib import Path from typing import Any from portfolio_optimiser import workflow from portfolio_optimiser.evals import b_gate as gate from portfolio_optimiser.evals import v1_gate _REPO = Path(__file__).resolve().parents[1] _SRC = _REPO / "src" / "portfolio_optimiser" _CONFIG = gate.load_config() #: Repoets egne tall ved skriving (19.09.2026), talt av armene under mot KILDEN. De står her for #: at en stille endring i kjørestien skal vise seg som en rød test og ikke som et nytt tall. _STEPS_TODAY = 13 _EXTERNAL_TODAY = 3 _ROLES_TODAY = ("proposer", "checker") _PATTERNS_TODAY = 6 def _write(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") def _probes(outcome: str = "missing") -> Any: return lambda nodeids: {n: outcome for n in nodeids} def _row(rows: list[v1_gate.Row], key: str) -> v1_gate.Row: return next(r for r in rows if r.key == key) # --------------------------------------------------------------------------------------------- # Rad 1 — verktøykassen er komplett # --------------------------------------------------------------------------------------------- def _independently_counted_steps() -> tuple[int, int]: """(steg som oppløses i kilden, steg med et kallsted) — talt HER, av testens egen parser. Kilden er ``src/portfolio_optimiser/``; ingen av gatens funksjoner røres.""" resolved = called = 0 for step in _CONFIG["steps"]: tree = ast.parse((_SRC / step["module"]).read_text(encoding="utf-8")) defs = { n.name for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) } if step["symbol"] in defs: resolved += 1 driver = ast.parse((_SRC / step["driver"]["module"]).read_text(encoding="utf-8")) scopes = [ n for n in ast.walk(driver) if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == step["driver"]["scope"] ] names = { c.func.id if isinstance(c.func, ast.Name) else c.func.attr for s in scopes for c in ast.walk(s) if isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute)) } if step["symbol"] in names: called += 1 return resolved, called def test_the_step_denominator_equals_an_independent_count_of_the_source() -> None: resolved, called = _independently_counted_steps() assert resolved == called == _STEPS_TODAY, "kjørestien har endret seg — oppdater nevneren" row = gate.score_toolbox(_CONFIG["steps"], _SRC) assert row.n == _STEPS_TODAY def test_row1_today_is_red_with_three_of_thirteen_steps_callable_from_outside() -> None: row = gate.score_toolbox(_CONFIG["steps"], _SRC) assert (row.k, row.n, row.status) == (_EXTERNAL_TODAY, _STEPS_TODAY, gate.RED) named = {x.split(":")[0] for x in row.exceptions} assert "rundebinding" not in named and "gate" not in named assert "validering" in named and "utboks" in named def _fake_src(root: Path, *, chat_client: bool, symbol: str = "gjoer_noe") -> Path: """En konstruert kilde med ETT steg: en inngang ``main`` som når steget. ``chat_client=True`` legger en chatklient på veien dit — steget er da fortsatt kallbart, men ikke uten en modell, og skal ikke telle.""" src = root / "fakesrc" body = " return create_chat_client()\n" if chat_client else " return 1\n" _write(src / "steg.py", f"def {symbol}():\n return 42\n") _write( src / "driver.py", "from fakesrc.steg import " + symbol + "\n\n" "def _hjelper():\n" + body + "\n\n" "def kjor():\n" f" _hjelper()\n return {symbol}()\n\n\n" "def main(argv=None):\n return kjor()\n", ) return src def _one_step(symbol: str = "gjoer_noe") -> list[dict[str, Any]]: return [ { "id": "steg", "label": "ett steg", "module": "steg.py", "symbol": symbol, "driver": {"module": "driver.py", "scope": "kjor"}, "entry": {"kind": "cli", "module": "driver.py", "scope": "main"}, } ] def test_a_step_reachable_from_a_cli_entry_without_a_chat_client_counts(tmp_path: Path) -> None: src = _fake_src(tmp_path, chat_client=False) row = gate.score_toolbox(_one_step(), src) assert (row.k, row.n, row.status) == (1, 1, gate.GREEN) def test_m2_a_step_that_needs_a_chat_client_is_not_callable_from_outside(tmp_path: Path) -> None: """M-2: fjernes chatklient-sjekken, hopper k fra 0 til 1 og denne armen feller mutanten.""" src = _fake_src(tmp_path, chat_client=True) row = gate.score_toolbox(_one_step(), src) assert (row.k, row.n, row.status) == (0, 1, gate.RED) assert any("chatklient" in x for x in row.exceptions) def test_m1_the_denominator_follows_the_source_and_is_not_the_config_length( tmp_path: Path, ) -> None: """M-1: en hardkodet nevner overlever repoet, men ikke en kilde med ett steg mindre.""" src = _fake_src(tmp_path, chat_client=False) steps = _one_step() + [ { "id": "borte", "label": "et steg som ikke finnes i kilden", "module": "steg.py", "symbol": "finnes_ikke", "driver": {"module": "driver.py", "scope": "kjor"}, "entry": {"kind": "cli", "module": "driver.py", "scope": "main"}, } ] row = gate.score_toolbox(steps, src) assert row.n == 1, "et steg uten symbol i kilden er ikke et steg" assert any("borte" in x for x in row.exceptions) def test_a_step_the_driver_no_longer_calls_leaves_the_denominator(tmp_path: Path) -> None: src = _fake_src(tmp_path, chat_client=False) _write(src / "steg.py", "def gjoer_noe():\n return 42\n\n\ndef ubrukt():\n return 0\n") steps = _one_step() + [ { "id": "ubrukt", "label": "definert, men ikke i kjørestien", "module": "steg.py", "symbol": "ubrukt", "driver": {"module": "driver.py", "scope": "kjor"}, "entry": {"kind": "cli", "module": "driver.py", "scope": "main"}, } ] row = gate.score_toolbox(steps, src) assert row.n == 1 def test_an_entry_without_a_main_is_no_entry(tmp_path: Path) -> None: src = _fake_src(tmp_path, chat_client=False) text = (src / "driver.py").read_text(encoding="utf-8").replace("def main(", "def _main(") _write(src / "driver.py", text) row = gate.score_toolbox(_one_step(), src) assert (row.k, row.n) == (0, 1) assert any("main" in x for x in row.exceptions) # --------------------------------------------------------------------------------------------- # Rad 2 — det modellen leverte, kan leveres utenfra # --------------------------------------------------------------------------------------------- def test_the_role_denominator_is_read_from_workflow_and_matches_the_imported_module() -> None: assert workflow._MAKER_CHECKER_ROLES == _ROLES_TODAY # uavhengig telling: modulen selv assert gate.run_roles(_SRC, _CONFIG["roles"]["source"]) == _ROLES_TODAY def test_the_role_denominator_follows_a_changed_source(tmp_path: Path) -> None: src = tmp_path / "fakesrc" _write(src / "workflow.py", '_MAKER_CHECKER_ROLES = ("a", "b", "c")\n') assert gate.run_roles(src, _CONFIG["roles"]["source"]) == ("a", "b", "c") def test_a_role_counts_only_when_every_named_probe_passes() -> None: evidence = _CONFIG["roles"]["evidence"] outcomes = {n: "passed" for ns in evidence.values() for n in ns} row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC) assert (row.k, row.n, row.status) == (2, 2, gate.GREEN) outcomes[evidence["checker"][0]] = "failed" row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC) assert (row.k, row.n, row.status) == (1, 2, gate.RED) def test_row2_today_is_red_because_the_named_probes_do_not_exist_yet() -> None: row = gate.score_roles(_CONFIG["roles"], {}, _SRC) assert (row.k, row.n, row.status) == (0, 2, gate.RED) assert all("missing" in x for x in row.exceptions) def test_m4_row2_demands_byte_identical_artefacts_not_merely_present_ones() -> None: """M-4: byttes kravet til «finnes», står kontrakten igjen uten den ene setningen som skiller en ekte likhet fra to filer med samme navn.""" assert "byte-identisk" in gate.BYTE_IDENTICAL_RULE row = gate.score_roles(_CONFIG["roles"], {}, _SRC) assert any("byte-identisk" in a for a in row.attests) # --------------------------------------------------------------------------------------------- # Rad 3 — po har ingen vei til Claude # --------------------------------------------------------------------------------------------- def _patterns() -> list[dict[str, str]]: return list(_CONFIG["no_claude_path"]["patterns"]) def test_every_pattern_hits_its_own_known_positive_and_misses_its_known_negative() -> None: """Fravær er et måleresultat: en vakt som ikke kan treffe, måler ikke null — den måler ingenting. Nevner: alle mønstrene i kontrakten.""" assert len(_patterns()) == _PATTERNS_TODAY for spec in _patterns(): rx = re.compile(gate.decode(spec["pattern_b64"])) assert rx.search(gate.decode(spec["known_positive_b64"])), spec["id"] assert not rx.search(gate.decode(spec["known_negative_b64"])), spec["id"] def test_m3_the_denominator_is_the_number_of_patterns_that_can_measure(tmp_path: Path) -> None: """M-3: fjernes et mønster, faller nevneren — og et mønster som ikke treffer sin egen kjent-positive telles ikke som en vakt.""" cfg = json.loads(json.dumps(_CONFIG["no_claude_path"])) cfg["roots"] = ["src"] _write(tmp_path / "src" / "rent.py", "print('ingenting her')\n") row = gate.score_no_claude_path(cfg, tmp_path) assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN) cfg["patterns"][0]["pattern_b64"] = base64.b64encode(b"finnes-aldri-xyzzy").decode() row = gate.score_no_claude_path(cfg, tmp_path) assert (row.k, row.n, row.status) == (_PATTERNS_TODAY - 1, _PATTERNS_TODAY, gate.RED) assert any("kjent-positiv" in x for x in row.exceptions) cfg["patterns"] = cfg["patterns"][1:] row = gate.score_no_claude_path(cfg, tmp_path) assert row.n == _PATTERNS_TODAY - 1 def test_a_planted_path_to_claude_turns_the_row_red_for_that_pattern(tmp_path: Path) -> None: cfg = json.loads(json.dumps(_CONFIG["no_claude_path"])) cfg["roots"] = ["src"] for spec in cfg["patterns"]: _write( tmp_path / "src" / f"{spec['id']}.py", gate.decode(spec["known_positive_b64"]) + "\n" ) row = gate.score_no_claude_path(cfg, tmp_path) assert (row.k, row.status) == (0, gate.RED) assert len(row.exceptions) == _PATTERNS_TODAY def test_markdown_counts_only_inside_fenced_blocks(tmp_path: Path) -> None: """«docs med kjørbare kommandoer»: en pakkenavn-omtale i brødtekst er ikke en vei til Claude; den samme linja i en kodeblokk er en kommando noen kan kjøre.""" cfg = json.loads(json.dumps(_CONFIG["no_claude_path"])) cfg["roots"] = ["docs"] spec = next(s for s in cfg["patterns"] if s["id"] == "maf-anthropic") line = gate.decode(spec["known_positive_b64"]) _write(tmp_path / "docs" / "prosa.md", f"- vurdert og forkastet: `{line}` (aldri tatt inn)\n") assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN _write(tmp_path / "docs" / "kjorbar.md", f"```bash\n{line}\n```\n") row = gate.score_no_claude_path(cfg, tmp_path) assert row.status == gate.RED assert any("kjorbar.md" in x for x in row.exceptions) def test_row3_on_the_real_published_surface_is_green_today() -> None: row = gate.score_no_claude_path(_CONFIG["no_claude_path"], _REPO) assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN), ( row.exceptions ) def test_the_surface_names_its_denominator_in_files_not_in_prose() -> None: files = gate.surface_files(_REPO, _CONFIG["no_claude_path"]["roots"]) assert len(files) > 100, "en tom flate ville gitt 0 treff uten å måle noe" assert any(f.name == "run.py" for f in files) # --------------------------------------------------------------------------------------------- # Rad 4 — ingen modellkall i verktøykasse-modus # --------------------------------------------------------------------------------------------- def test_row4_is_red_today_and_names_all_three_checks() -> None: row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {}) assert (row.k, row.n, row.status) == (0, 3, gate.RED) assert len(row.exceptions) == 3 def test_row4_counts_only_checks_whose_probes_pass() -> None: checks = _CONFIG["no_model_calls"]["checks"] outcomes = {n: "passed" for ns in checks.values() for n in ns} assert gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes).status == gate.GREEN outcomes[checks["fullfører"][0]] = "skipped" row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes) assert (row.k, row.status) == (2, gate.RED) def test_row4_states_what_the_budget_guard_is_in_b_instead_of_pretending() -> None: row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {}) assert any("budsjett" in a.casefold() for a in row.attests) assert "valgt" in gate.BUDGET_IN_B.casefold() # --------------------------------------------------------------------------------------------- # Rad 5 — Foundry-veien urørt # --------------------------------------------------------------------------------------------- def test_the_foundry_path_is_intact_in_the_source_today() -> None: ok, missing = gate.foundry_intact(_CONFIG["foundry"], _SRC) assert (ok, missing) == (True, ()) def test_the_foundry_check_fails_when_the_azure_profile_is_gone(tmp_path: Path) -> None: src = tmp_path / "fakesrc" _write(src / "backends.py", "class Profile:\n LOCAL = 'local'\n") _write(src / "run.py", "def run_project(client_factory=None):\n return 1\n") ok, missing = gate.foundry_intact(_CONFIG["foundry"], src) assert ok is False assert any("AZURE" in m for m in missing) def test_row5_is_red_until_the_schema_comparison_probe_passes() -> None: row = gate.score_foundry(_CONFIG["foundry"], {}, _SRC) assert (row.k, row.n, row.status) == (1, 2, gate.RED) nodeid = _CONFIG["foundry"]["evidence"]["samme-skjema"][0] row = gate.score_foundry(_CONFIG["foundry"], {nodeid: "passed"}, _SRC) assert (row.k, row.n, row.status) == (2, 2, gate.GREEN) # --------------------------------------------------------------------------------------------- # Rad 6 — kjøreboka finnes # --------------------------------------------------------------------------------------------- def test_row6_is_not_measured_while_the_runbook_is_missing(tmp_path: Path) -> None: row = gate.score_runbook(_CONFIG["runbook"], tmp_path) assert (row.k, row.n, row.status) == (0, 2, gate.NOT_MEASURED) assert row.failing is True def test_row6_is_still_not_measured_without_the_operators_attestation(tmp_path: Path) -> None: _write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n\nsteg 1 …\n") row = gate.score_runbook(_CONFIG["runbook"], tmp_path) assert (row.k, row.n, row.status) == (1, 2, gate.NOT_MEASURED) def test_row6_goes_green_only_with_every_required_key(tmp_path: Path) -> None: cfg = _CONFIG["runbook"] _write(tmp_path / cfg["path"], "# kjørebok\n") attest = tmp_path / cfg["attestation"] _write(attest, "kjørebok: docs/kjoerebok-verktoeykassen.md\nkjørt: operatøren\n") assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED _write(attest, "kjørebok: x\nkjørt: operatøren\ndato: 2026-09-20\n") row = gate.score_runbook(cfg, tmp_path) assert (row.k, row.n, row.status) == (2, 2, gate.GREEN) def test_a_duplicated_key_in_the_attestation_is_red(tmp_path: Path) -> None: cfg = _CONFIG["runbook"] _write(tmp_path / cfg["path"], "# kjørebok\n") _write(tmp_path / cfg["attestation"], "kjørebok: x\nkjørt: a\nkjørt: b\ndato: 2026-09-20\n") assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED def test_m5_the_gate_never_writes_the_attestation_itself(tmp_path: Path) -> None: """M-5: kan gaten skrive fila den leser, er rad 6 en sløyfe som bekrefter seg selv.""" _write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n") attest = tmp_path / _CONFIG["runbook"]["attestation"] rows = gate.evaluate(config=_CONFIG, repo_root=tmp_path, src=_SRC, probe_runner=_probes()) assert not attest.exists(), "gaten skrev operatørens attestering — da beviser rad 6 ingenting" assert _row(rows, "kjørebok").status == gate.NOT_MEASURED assert any("aldri" in a for a in _row(rows, "kjørebok").attests) # --------------------------------------------------------------------------------------------- # Hele gaten # --------------------------------------------------------------------------------------------- def test_the_gate_is_red_today_and_every_row_is_red_for_a_measured_reason() -> None: rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes()) assert gate.exit_code(rows) == 1 assert [r.key for r in rows] == [ "verktøykasse", "roller", "ingen-claude-vei", "ingen-modellkall", "foundry", "kjørebok", ] for row in rows: assert row.status in {gate.GREEN, gate.RED, gate.NOT_MEASURED} if row.status != gate.GREEN: assert row.k is not None and row.n is not None, f"{row.key} mangler k av N" assert row.exceptions, f"{row.key} er ikke grønn uten å si hvorfor" def test_the_probe_runner_separates_a_test_that_exists_from_one_that_does_not() -> None: """Kjent-positiv for selve probe-mekanismen: «missing» over hele lista ville vært et måleresultat uten nevner. Nevneren er to nodeid-er — én som finnes i suiten og én som ikke gjør det — og mekanismen må skille dem. Proben peker med vilje UT av denne fila: en nodeid herfra ville fått barne-pytest til å kjøre denne testen igjen, i det uendelige.""" lever = "tests/test_ledger.py::test_totals_per_project_and_portfolio" dod = "tests/test_ledger.py::test_finnes_ikke_xyzzy" outcomes = v1_gate.run_probes([lever, dod], _REPO) assert outcomes[lever] == "passed" assert outcomes[dod] == "missing" def test_render_names_every_row_the_budget_choice_and_the_runbook_rule() -> None: rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes()) text = gate.render(rows) for row in rows: assert row.title in text assert gate.BUDGET_IN_B in text assert gate.RUNBOOK_RULE in text def test_the_command_runs_offline_and_exits_one() -> None: proc = subprocess.run( [sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--json"], cwd=_REPO, capture_output=True, text=True, ) assert proc.returncode == 1, proc.stderr payload = json.loads(proc.stdout) assert payload["exit"] == 1 assert len(payload["rows"]) == 6 def test_a_bad_attestation_path_is_usage_error_not_a_silent_green(tmp_path: Path) -> None: proc = subprocess.run( [sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--attest", str(tmp_path / "x")], cwd=_REPO, capture_output=True, text=True, ) assert proc.returncode == 2