Kjell Tore Guttormsen
0a784065d0
Operator decision 19.09.2026: in development and test Claude Code LEADS and portfolio-optimiser is the toolbox. po never calls Claude; production stays on Foundry. This commit writes the measurement RED — the contract first, the capability later. Six rows, each with a denominator taken from the SOURCE and counted independently here: 1. toolbox complete — every deterministic step of the run path, anchored to (module, symbol) and to the scope that calls it; k = steps reachable from a CLI entry WITHOUT a chat client. 2. what the model delivered can be delivered from outside — denominator read off workflow._MAKER_CHECKER_ROLES, k = roles with passing named probes. 3. po has no path to Claude — six patterns, each carrying its own known-positive AND known-negative fixture. Pattern text is base64 in the config so the gate's own contract cannot register as a hit against the surface it scans. 4. no model calls in toolbox mode — three named probes. 5. the Foundry path untouched — profiles, factory and injection seam read from source, plus a schema-comparison probe. 6. the runbook exists — NOT MEASURED until the operator attests, and the gate never writes that attestation itself. Red now for the reason that matters: the tests fail to import a module that does not exist yet. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
478 lines
20 KiB
Python
478 lines
20 KiB
Python
"""B-gatens egne tester: hver rad KAN bli grønn og KAN bli rød — og kan ikke FAKES grønn.
|
|
|
|
Rammen er operatørbeslutningen 19.09.2026: i utvikling og test LEDER Claude Code, og
|
|
portfolio-optimiser er verktøykassen. po kaller ALDRI Claude. Gaten måler avstanden dit, rad for
|
|
rad, uten et eneste modellkall.
|
|
|
|
Tre ting gjør testene load-bearing i stedet for grønne-men-døde:
|
|
|
|
* **Nevneren telles uavhengig HER.** Rad 1 og rad 2 får ikke sine tall fra gatens egne funksjoner:
|
|
testen parser kilden selv (``ast``) eller importerer modulen og teller, og sammenligner. En gate
|
|
som hardkoder nevneren felles av arm ``M-1``, som kjører den mot en KONSTRUERT kilde der svaret
|
|
er et annet enn repoets.
|
|
* **Hver vakt har en kjent-positiv.** Et mønster som ikke treffer sin egen kjent-positive er ikke
|
|
en måling, og rad 3 nekter å telle det (``M-3``). Et fravær («0 treff») uttales med nevner.
|
|
* **Rad 6 kan ikke bli grønn av en fil gaten selv skriver** (``M-5``): armen kjører hele
|
|
``evaluate`` og krever at attesteringsfila fortsatt ikke finnes etterpå.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import base64
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from portfolio_optimiser import workflow
|
|
from portfolio_optimiser.evals import b_gate as gate
|
|
from portfolio_optimiser.evals import v1_gate
|
|
|
|
_REPO = Path(__file__).resolve().parents[1]
|
|
_SRC = _REPO / "src" / "portfolio_optimiser"
|
|
_CONFIG = gate.load_config()
|
|
|
|
#: Repoets egne tall ved skriving (19.09.2026), talt av armene under mot KILDEN. De står her for
|
|
#: at en stille endring i kjørestien skal vise seg som en rød test og ikke som et nytt tall.
|
|
_STEPS_TODAY = 13
|
|
_EXTERNAL_TODAY = 3
|
|
_ROLES_TODAY = ("proposer", "checker")
|
|
_PATTERNS_TODAY = 6
|
|
|
|
|
|
def _write(path: Path, text: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def _probes(outcome: str = "missing") -> Any:
|
|
return lambda nodeids: {n: outcome for n in nodeids}
|
|
|
|
|
|
def _row(rows: list[v1_gate.Row], key: str) -> v1_gate.Row:
|
|
return next(r for r in rows if r.key == key)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 1 — verktøykassen er komplett
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def _independently_counted_steps() -> tuple[int, int]:
|
|
"""(steg som oppløses i kilden, steg med et kallsted) — talt HER, av testens egen parser.
|
|
|
|
Kilden er ``src/portfolio_optimiser/``; ingen av gatens funksjoner røres."""
|
|
resolved = called = 0
|
|
for step in _CONFIG["steps"]:
|
|
tree = ast.parse((_SRC / step["module"]).read_text(encoding="utf-8"))
|
|
defs = {
|
|
n.name
|
|
for n in tree.body
|
|
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
|
|
}
|
|
if step["symbol"] in defs:
|
|
resolved += 1
|
|
driver = ast.parse((_SRC / step["driver"]["module"]).read_text(encoding="utf-8"))
|
|
scopes = [
|
|
n
|
|
for n in ast.walk(driver)
|
|
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
and n.name == step["driver"]["scope"]
|
|
]
|
|
names = {
|
|
c.func.id if isinstance(c.func, ast.Name) else c.func.attr
|
|
for s in scopes
|
|
for c in ast.walk(s)
|
|
if isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute))
|
|
}
|
|
if step["symbol"] in names:
|
|
called += 1
|
|
return resolved, called
|
|
|
|
|
|
def test_the_step_denominator_equals_an_independent_count_of_the_source() -> None:
|
|
resolved, called = _independently_counted_steps()
|
|
assert resolved == called == _STEPS_TODAY, "kjørestien har endret seg — oppdater nevneren"
|
|
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
|
|
assert row.n == _STEPS_TODAY
|
|
|
|
|
|
def test_row1_today_is_red_with_three_of_thirteen_steps_callable_from_outside() -> None:
|
|
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
|
|
assert (row.k, row.n, row.status) == (_EXTERNAL_TODAY, _STEPS_TODAY, gate.RED)
|
|
named = {x.split(":")[0] for x in row.exceptions}
|
|
assert "rundebinding" not in named and "gate" not in named
|
|
assert "validering" in named and "utboks" in named
|
|
|
|
|
|
def _fake_src(root: Path, *, chat_client: bool, symbol: str = "gjoer_noe") -> Path:
|
|
"""En konstruert kilde med ETT steg: en inngang ``main`` som når steget.
|
|
|
|
``chat_client=True`` legger en chatklient på veien dit — steget er da fortsatt kallbart, men
|
|
ikke uten en modell, og skal ikke telle."""
|
|
src = root / "fakesrc"
|
|
body = " return create_chat_client()\n" if chat_client else " return 1\n"
|
|
_write(src / "steg.py", f"def {symbol}():\n return 42\n")
|
|
_write(
|
|
src / "driver.py",
|
|
"from fakesrc.steg import " + symbol + "\n\n"
|
|
"def _hjelper():\n" + body + "\n\n"
|
|
"def kjor():\n"
|
|
f" _hjelper()\n return {symbol}()\n\n\n"
|
|
"def main(argv=None):\n return kjor()\n",
|
|
)
|
|
return src
|
|
|
|
|
|
def _one_step(symbol: str = "gjoer_noe") -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": "steg",
|
|
"label": "ett steg",
|
|
"module": "steg.py",
|
|
"symbol": symbol,
|
|
"driver": {"module": "driver.py", "scope": "kjor"},
|
|
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
|
}
|
|
]
|
|
|
|
|
|
def test_a_step_reachable_from_a_cli_entry_without_a_chat_client_counts(tmp_path: Path) -> None:
|
|
src = _fake_src(tmp_path, chat_client=False)
|
|
row = gate.score_toolbox(_one_step(), src)
|
|
assert (row.k, row.n, row.status) == (1, 1, gate.GREEN)
|
|
|
|
|
|
def test_m2_a_step_that_needs_a_chat_client_is_not_callable_from_outside(tmp_path: Path) -> None:
|
|
"""M-2: fjernes chatklient-sjekken, hopper k fra 0 til 1 og denne armen feller mutanten."""
|
|
src = _fake_src(tmp_path, chat_client=True)
|
|
row = gate.score_toolbox(_one_step(), src)
|
|
assert (row.k, row.n, row.status) == (0, 1, gate.RED)
|
|
assert any("chatklient" in x for x in row.exceptions)
|
|
|
|
|
|
def test_m1_the_denominator_follows_the_source_and_is_not_the_config_length(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""M-1: en hardkodet nevner overlever repoet, men ikke en kilde med ett steg mindre."""
|
|
src = _fake_src(tmp_path, chat_client=False)
|
|
steps = _one_step() + [
|
|
{
|
|
"id": "borte",
|
|
"label": "et steg som ikke finnes i kilden",
|
|
"module": "steg.py",
|
|
"symbol": "finnes_ikke",
|
|
"driver": {"module": "driver.py", "scope": "kjor"},
|
|
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
|
}
|
|
]
|
|
row = gate.score_toolbox(steps, src)
|
|
assert row.n == 1, "et steg uten symbol i kilden er ikke et steg"
|
|
assert any("borte" in x for x in row.exceptions)
|
|
|
|
|
|
def test_a_step_the_driver_no_longer_calls_leaves_the_denominator(tmp_path: Path) -> None:
|
|
src = _fake_src(tmp_path, chat_client=False)
|
|
_write(src / "steg.py", "def gjoer_noe():\n return 42\n\n\ndef ubrukt():\n return 0\n")
|
|
steps = _one_step() + [
|
|
{
|
|
"id": "ubrukt",
|
|
"label": "definert, men ikke i kjørestien",
|
|
"module": "steg.py",
|
|
"symbol": "ubrukt",
|
|
"driver": {"module": "driver.py", "scope": "kjor"},
|
|
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
|
}
|
|
]
|
|
row = gate.score_toolbox(steps, src)
|
|
assert row.n == 1
|
|
|
|
|
|
def test_an_entry_without_a_main_is_no_entry(tmp_path: Path) -> None:
|
|
src = _fake_src(tmp_path, chat_client=False)
|
|
text = (src / "driver.py").read_text(encoding="utf-8").replace("def main(", "def _main(")
|
|
_write(src / "driver.py", text)
|
|
row = gate.score_toolbox(_one_step(), src)
|
|
assert (row.k, row.n) == (0, 1)
|
|
assert any("main" in x for x in row.exceptions)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 2 — det modellen leverte, kan leveres utenfra
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_role_denominator_is_read_from_workflow_and_matches_the_imported_module() -> None:
|
|
assert workflow._MAKER_CHECKER_ROLES == _ROLES_TODAY # uavhengig telling: modulen selv
|
|
assert gate.run_roles(_SRC, _CONFIG["roles"]["source"]) == _ROLES_TODAY
|
|
|
|
|
|
def test_the_role_denominator_follows_a_changed_source(tmp_path: Path) -> None:
|
|
src = tmp_path / "fakesrc"
|
|
_write(src / "workflow.py", '_MAKER_CHECKER_ROLES = ("a", "b", "c")\n')
|
|
assert gate.run_roles(src, _CONFIG["roles"]["source"]) == ("a", "b", "c")
|
|
|
|
|
|
def test_a_role_counts_only_when_every_named_probe_passes() -> None:
|
|
evidence = _CONFIG["roles"]["evidence"]
|
|
outcomes = {n: "passed" for ns in evidence.values() for n in ns}
|
|
row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC)
|
|
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
|
|
outcomes[evidence["checker"][0]] = "failed"
|
|
row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC)
|
|
assert (row.k, row.n, row.status) == (1, 2, gate.RED)
|
|
|
|
|
|
def test_row2_today_is_red_because_the_named_probes_do_not_exist_yet() -> None:
|
|
row = gate.score_roles(_CONFIG["roles"], {}, _SRC)
|
|
assert (row.k, row.n, row.status) == (0, 2, gate.RED)
|
|
assert all("missing" in x for x in row.exceptions)
|
|
|
|
|
|
def test_m4_row2_demands_byte_identical_artefacts_not_merely_present_ones() -> None:
|
|
"""M-4: byttes kravet til «finnes», står kontrakten igjen uten den ene setningen som skiller
|
|
en ekte likhet fra to filer med samme navn."""
|
|
assert "byte-identisk" in gate.BYTE_IDENTICAL_RULE
|
|
row = gate.score_roles(_CONFIG["roles"], {}, _SRC)
|
|
assert any("byte-identisk" in a for a in row.attests)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 3 — po har ingen vei til Claude
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def _patterns() -> list[dict[str, str]]:
|
|
return list(_CONFIG["no_claude_path"]["patterns"])
|
|
|
|
|
|
def test_every_pattern_hits_its_own_known_positive_and_misses_its_known_negative() -> None:
|
|
"""Fravær er et måleresultat: en vakt som ikke kan treffe, måler ikke null — den måler
|
|
ingenting. Nevner: alle mønstrene i kontrakten."""
|
|
assert len(_patterns()) == _PATTERNS_TODAY
|
|
for spec in _patterns():
|
|
rx = re.compile(gate.decode(spec["pattern_b64"]))
|
|
assert rx.search(gate.decode(spec["known_positive_b64"])), spec["id"]
|
|
assert not rx.search(gate.decode(spec["known_negative_b64"])), spec["id"]
|
|
|
|
|
|
def test_m3_the_denominator_is_the_number_of_patterns_that_can_measure(tmp_path: Path) -> None:
|
|
"""M-3: fjernes et mønster, faller nevneren — og et mønster som ikke treffer sin egen
|
|
kjent-positive telles ikke som en vakt."""
|
|
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
|
cfg["roots"] = ["src"]
|
|
_write(tmp_path / "src" / "rent.py", "print('ingenting her')\n")
|
|
row = gate.score_no_claude_path(cfg, tmp_path)
|
|
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN)
|
|
|
|
cfg["patterns"][0]["pattern_b64"] = base64.b64encode(b"finnes-aldri-xyzzy").decode()
|
|
row = gate.score_no_claude_path(cfg, tmp_path)
|
|
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY - 1, _PATTERNS_TODAY, gate.RED)
|
|
assert any("kjent-positiv" in x for x in row.exceptions)
|
|
|
|
cfg["patterns"] = cfg["patterns"][1:]
|
|
row = gate.score_no_claude_path(cfg, tmp_path)
|
|
assert row.n == _PATTERNS_TODAY - 1
|
|
|
|
|
|
def test_a_planted_path_to_claude_turns_the_row_red_for_that_pattern(tmp_path: Path) -> None:
|
|
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
|
cfg["roots"] = ["src"]
|
|
for spec in cfg["patterns"]:
|
|
_write(tmp_path / "src" / f"{spec['id']}.py", gate.decode(spec["known_positive_b64"]) + "\n")
|
|
row = gate.score_no_claude_path(cfg, tmp_path)
|
|
assert (row.k, row.status) == (0, gate.RED)
|
|
assert len(row.exceptions) == _PATTERNS_TODAY
|
|
|
|
|
|
def test_markdown_counts_only_inside_fenced_blocks(tmp_path: Path) -> None:
|
|
"""«docs med kjørbare kommandoer»: en pakkenavn-omtale i brødtekst er ikke en vei til Claude;
|
|
den samme linja i en kodeblokk er en kommando noen kan kjøre."""
|
|
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
|
cfg["roots"] = ["docs"]
|
|
spec = next(s for s in cfg["patterns"] if s["id"] == "maf-anthropic")
|
|
line = gate.decode(spec["known_positive_b64"])
|
|
_write(tmp_path / "docs" / "prosa.md", f"- vurdert og forkastet: `{line}` (aldri tatt inn)\n")
|
|
assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN
|
|
_write(tmp_path / "docs" / "kjorbar.md", f"```bash\n{line}\n```\n")
|
|
row = gate.score_no_claude_path(cfg, tmp_path)
|
|
assert row.status == gate.RED
|
|
assert any("kjorbar.md" in x for x in row.exceptions)
|
|
|
|
|
|
def test_row3_on_the_real_published_surface_is_green_today() -> None:
|
|
row = gate.score_no_claude_path(_CONFIG["no_claude_path"], _REPO)
|
|
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN), (
|
|
row.exceptions
|
|
)
|
|
|
|
|
|
def test_the_surface_names_its_denominator_in_files_not_in_prose() -> None:
|
|
files = gate.surface_files(_REPO, _CONFIG["no_claude_path"]["roots"])
|
|
assert len(files) > 100, "en tom flate ville gitt 0 treff uten å måle noe"
|
|
assert any(f.name == "run.py" for f in files)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 4 — ingen modellkall i verktøykasse-modus
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_row4_is_red_today_and_names_all_three_checks() -> None:
|
|
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
|
|
assert (row.k, row.n, row.status) == (0, 3, gate.RED)
|
|
assert len(row.exceptions) == 3
|
|
|
|
|
|
def test_row4_counts_only_checks_whose_probes_pass() -> None:
|
|
checks = _CONFIG["no_model_calls"]["checks"]
|
|
outcomes = {n: "passed" for ns in checks.values() for n in ns}
|
|
assert gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes).status == gate.GREEN
|
|
outcomes[checks["fullfører"][0]] = "skipped"
|
|
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes)
|
|
assert (row.k, row.status) == (2, gate.RED)
|
|
|
|
|
|
def test_row4_states_what_the_budget_guard_is_in_b_instead_of_pretending() -> None:
|
|
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
|
|
assert any("budsjett" in a.casefold() for a in row.attests)
|
|
assert "valgt" in gate.BUDGET_IN_B.casefold()
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 5 — Foundry-veien urørt
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_foundry_path_is_intact_in_the_source_today() -> None:
|
|
ok, missing = gate.foundry_intact(_CONFIG["foundry"], _SRC)
|
|
assert (ok, missing) == (True, ())
|
|
|
|
|
|
def test_the_foundry_check_fails_when_the_azure_profile_is_gone(tmp_path: Path) -> None:
|
|
src = tmp_path / "fakesrc"
|
|
_write(src / "backends.py", "class Profile:\n LOCAL = 'local'\n")
|
|
_write(src / "run.py", "def run_project(client_factory=None):\n return 1\n")
|
|
ok, missing = gate.foundry_intact(_CONFIG["foundry"], src)
|
|
assert ok is False
|
|
assert any("AZURE" in m for m in missing)
|
|
|
|
|
|
def test_row5_is_red_until_the_schema_comparison_probe_passes() -> None:
|
|
row = gate.score_foundry(_CONFIG["foundry"], {}, _SRC)
|
|
assert (row.k, row.n, row.status) == (1, 2, gate.RED)
|
|
nodeid = _CONFIG["foundry"]["evidence"]["samme-skjema"][0]
|
|
row = gate.score_foundry(_CONFIG["foundry"], {nodeid: "passed"}, _SRC)
|
|
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Rad 6 — kjøreboka finnes
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_row6_is_not_measured_while_the_runbook_is_missing(tmp_path: Path) -> None:
|
|
row = gate.score_runbook(_CONFIG["runbook"], tmp_path)
|
|
assert (row.k, row.n, row.status) == (0, 2, gate.NOT_MEASURED)
|
|
assert row.failing is True
|
|
|
|
|
|
def test_row6_is_still_not_measured_without_the_operators_attestation(tmp_path: Path) -> None:
|
|
_write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n\nsteg 1 …\n")
|
|
row = gate.score_runbook(_CONFIG["runbook"], tmp_path)
|
|
assert (row.k, row.n, row.status) == (1, 2, gate.NOT_MEASURED)
|
|
|
|
|
|
def test_row6_goes_green_only_with_every_required_key(tmp_path: Path) -> None:
|
|
cfg = _CONFIG["runbook"]
|
|
_write(tmp_path / cfg["path"], "# kjørebok\n")
|
|
attest = tmp_path / cfg["attestation"]
|
|
_write(attest, "kjørebok: docs/kjoerebok-verktoeykassen.md\nkjørt: operatøren\n")
|
|
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
|
|
_write(attest, "kjørebok: x\nkjørt: operatøren\ndato: 2026-09-20\n")
|
|
row = gate.score_runbook(cfg, tmp_path)
|
|
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
|
|
|
|
|
|
def test_a_duplicated_key_in_the_attestation_is_red(tmp_path: Path) -> None:
|
|
cfg = _CONFIG["runbook"]
|
|
_write(tmp_path / cfg["path"], "# kjørebok\n")
|
|
_write(tmp_path / cfg["attestation"], "kjørebok: x\nkjørt: a\nkjørt: b\ndato: 2026-09-20\n")
|
|
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
|
|
|
|
|
|
def test_m5_the_gate_never_writes_the_attestation_itself(tmp_path: Path) -> None:
|
|
"""M-5: kan gaten skrive fila den leser, er rad 6 en sløyfe som bekrefter seg selv."""
|
|
_write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n")
|
|
attest = tmp_path / _CONFIG["runbook"]["attestation"]
|
|
rows = gate.evaluate(
|
|
config=_CONFIG, repo_root=tmp_path, src=_SRC, probe_runner=_probes()
|
|
)
|
|
assert not attest.exists(), "gaten skrev operatørens attestering — da beviser rad 6 ingenting"
|
|
assert _row(rows, "kjørebok").status == gate.NOT_MEASURED
|
|
assert any("aldri" in a for a in _row(rows, "kjørebok").attests)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# Hele gaten
|
|
# ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_gate_is_red_today_and_every_row_is_red_for_a_measured_reason() -> None:
|
|
rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes())
|
|
assert gate.exit_code(rows) == 1
|
|
assert [r.key for r in rows] == [
|
|
"verktøykasse",
|
|
"roller",
|
|
"ingen-claude-vei",
|
|
"ingen-modellkall",
|
|
"foundry",
|
|
"kjørebok",
|
|
]
|
|
for row in rows:
|
|
assert row.status in {gate.GREEN, gate.RED, gate.NOT_MEASURED}
|
|
if row.status != gate.GREEN:
|
|
assert row.k is not None and row.n is not None, f"{row.key} mangler k av N"
|
|
assert row.exceptions, f"{row.key} er ikke grønn uten å si hvorfor"
|
|
|
|
|
|
def test_the_probe_runner_can_return_passed_for_a_test_that_exists() -> None:
|
|
"""Kjent-positiv for selve probe-mekanismen: «missing» over alt ville vært et måleresultat
|
|
uten nevner. Nevner: én nodeid som finnes i denne suiten."""
|
|
nodeid = "tests/test_b_gate.py::test_the_probe_runner_can_return_passed_for_a_test_that_exists"
|
|
assert v1_gate.run_probes([nodeid], _REPO)[nodeid] == "passed"
|
|
|
|
|
|
def test_render_names_every_row_the_budget_choice_and_the_runbook_rule() -> None:
|
|
rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes())
|
|
text = gate.render(rows)
|
|
for row in rows:
|
|
assert row.title in text
|
|
assert gate.BUDGET_IN_B in text
|
|
assert gate.RUNBOOK_RULE in text
|
|
|
|
|
|
def test_the_command_runs_offline_and_exits_one() -> None:
|
|
proc = subprocess.run(
|
|
[sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--json"],
|
|
cwd=_REPO,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert proc.returncode == 1, proc.stderr
|
|
payload = json.loads(proc.stdout)
|
|
assert payload["exit"] == 1
|
|
assert len(payload["rows"]) == 6
|
|
|
|
|
|
def test_a_bad_attestation_path_is_usage_error_not_a_silent_green(tmp_path: Path) -> None:
|
|
proc = subprocess.run(
|
|
[sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--attest", str(tmp_path / "x")],
|
|
cwd=_REPO,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert proc.returncode == 2
|