open/portfolio-optimiser
1
0
Fork 0
Code Issues Releases 3 Activity

test(b-gate): 34 red tests for the gate that measures po as a toolbox Claude Code drives

Browse source 
Operator decision 19.09.2026: in development and test Claude Code LEADS and portfolio-optimiser
is the toolbox. po never calls Claude; production stays on Foundry. This commit writes the
measurement RED — the contract first, the capability later.

Six rows, each with a denominator taken from the SOURCE and counted independently here:

1. toolbox complete — every deterministic step of the run path, anchored to (module, symbol) and
   to the scope that calls it; k = steps reachable from a CLI entry WITHOUT a chat client.
2. what the model delivered can be delivered from outside — denominator read off
   workflow._MAKER_CHECKER_ROLES, k = roles with passing named probes.
3. po has no path to Claude — six patterns, each carrying its own known-positive AND
   known-negative fixture. Pattern text is base64 in the config so the gate's own contract cannot
   register as a hit against the surface it scans.
4. no model calls in toolbox mode — three named probes.
5. the Foundry path untouched — profiles, factory and injection seam read from source, plus a
   schema-comparison probe.
6. the runbook exists — NOT MEASURED until the operator attests, and the gate never writes that
   attestation itself.

Red now for the reason that matters: the tests fail to import a module that does not exist yet.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-19 08:01:29 +02:00
commit 0a784065d0
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 795 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

317
src/portfolio_optimiser/evals/b_gate.json Normal file
View file

@ -0,0 +1,317 @@
{
"outcome": "En Claude Code-økt driver en hel analyse med po som verktøykasse, uten at po gjør ett eneste modellkall.",
"steps": [
{
"id": "pakke",
"label": "last kunnskapspakken og naviger den",
"module": "okf.py",
"symbol": "navigate_bundle",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "kostnadsgrunnlag",
"label": "slå opp kostnadsgrunnlaget",
"module": "okf.py",
"symbol": "derive_cost_baseline",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "kontekst",
"label": "hent kontekst-utdrag til forslaget",
"module": "datasource.py",
"symbol": "retrieve_chunks",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "prepass",
"label": "slipp inn prepass-nyttelasten",
"module": "prepass.py",
"symbol": "admit_payload",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "validering",
"label": "valider forslaget, metodetak inkludert",
"module": "validator.py",
"symbol": "validate_proposal",
"driver": {
"module": "run.py",
"scope": "evaluate_mandate_candidates"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "verdict-id",
"label": "mint verdict_id fra IR-ens trekk",
"module": "verdicts.py",
"symbol": "verdict_key",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "dom",
"label": "fang fagpersonens dom som en Verdict",
"module": "verdicts.py",
"symbol": "capture_verdict",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "kjørekonfig",
"label": "skriv kjøringens konfigurasjon",
"module": "outbox.py",
"symbol": "write_run_config",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "coverage",
"label": "skriv coverage-raden per tilnærming",
"module": "outbox.py",
"symbol": "write_coverage",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "utboks",
"label": "skriv utboksen",
"module": "outbox.py",
"symbol": "write_outbox",
"driver": {
"module": "run.py",
"scope": "run_project"
},
"entry": {
"kind": "cli",
"module": "run.py",
"scope": "main"
}
},
{
"id": "rundebinding",
"label": "bind en utboks til en rundekatalog",
"module": "evals/round_builder.py",
"symbol": "build_round",
"driver": {
"module": "evals/round_builder.py",
"scope": "main"
},
"entry": {
"kind": "cli",
"module": "evals/round_builder.py",
"scope": "main"
}
},
{
"id": "rapport",
"label": "skriv rundens rapport på norsk",
"module": "evals/round_builder.py",
"symbol": "build_report",
"driver": {
"module": "evals/round_builder.py",
"scope": "build_round"
},
"entry": {
"kind": "cli",
"module": "evals/round_builder.py",
"scope": "main"
}
},
{
"id": "gate",
"label": "døm rundene med v1-gaten",
"module": "evals/v1_gate.py",
"symbol": "evaluate",
"driver": {
"module": "evals/v1_gate.py",
"scope": "main"
},
"entry": {
"kind": "cli",
"module": "evals/v1_gate.py",
"scope": "main"
}
}
],
"roles": {
"source": {
"module": "workflow.py",
"symbol": "_MAKER_CHECKER_ROLES"
},
"evidence": {
"proposer": [
"tests/test_b_toolbox_probes.py::test_proposer_from_outside_mints_byte_identical_artefacts",
"tests/test_b_toolbox_probes.py::test_proposer_from_outside_is_rejected_with_the_same_reason"
],
"checker": [
"tests/test_b_toolbox_probes.py::test_checker_from_outside_mints_byte_identical_artefacts",
"tests/test_b_toolbox_probes.py::test_checker_from_outside_is_rejected_with_the_same_reason"
]
}
},
"no_claude_path": {
"roots": [
"src",
"scripts",
"shared",
"docs",
"tests",
"pyproject.toml",
"README.md",
"CHANGELOG.md",
"DEPLOY.md",
"CONTRIBUTING.md",
"SECURITY.md"
],
"patterns": [
{
"id": "prosess",
"label": "prosess-start av claude fra kode",
"pattern_b64": "KHN1YnByb2Nlc3NcLihydW58UG9wZW58Y2FsbHxjaGVja19vdXRwdXQpfG9zXC5zeXN0ZW18b3NcLmV4ZWNcdyspXHMqXChccypcWz9ccypbIiddY2xhdWRlWyInXQ==",
"known_positive_b64": "c3VicHJvY2Vzcy5ydW4oWyJjbGF1ZGUiLCAiLXAiLCAiYW5hbHlzZXIgcHJvc2pla3RldCJdKQ==",
"known_negative_b64": "c3VicHJvY2Vzcy5ydW4oWyJweXRob24iLCAiLW0iLCAicG9ydGZvbGlvX29wdGltaXNlci5ydW4iLCAiUDEiXSk="
},
{
"id": "kommando",
"label": "claude som kommando i en kjørbar linje",
"pattern_b64": "KF58W1xzOyZ8YCQoXSljbGF1ZGVccysoLXBcYnwtLXByaW50XGJ8LS1wZXJtaXNzaW9uLW1vZGVcYnwtLWRhbmdlcm91c2x5LXNraXAtcGVybWlzc2lvbnNcYnwtLW1vZGVsXHMp",
"known_positive_b64": "Y2xhdWRlIC1wICJrasO4ciBhbmFseXNlbiIgLS1tb2RlbCBvcHVz",
"known_negative_b64": "cHl0aG9uIC1tIHBvcnRmb2xpb19vcHRpbWlzZXIucnVuIFAxIC0tcHJvZmlsZSBsb2NhbCAtLW1heC1yb3VuZHMgMg=="
},
{
"id": "sdk-import",
"label": "import av Claude Agent SDK",
"pattern_b64": "XlxzKihmcm9tfGltcG9ydClccysoY2xhdWRlX2FnZW50X3Nka3xjbGF1ZGVfY29kZV9zZGspXGI=",
"known_positive_b64": "aW1wb3J0IGNsYXVkZV9hZ2VudF9zZGs=",
"known_negative_b64": "aW1wb3J0IGFnZW50X2ZyYW1ld29yaw=="
},
{
"id": "maf-anthropic",
"label": "MAF sin Anthropic-provider",
"pattern_b64": "KGFnZW50X2ZyYW1ld29ya1wuYW50aHJvcGljfGFnZW50W18tXWZyYW1ld29ya1tfLV1jbGF1ZGV8QW50aHJvcGljQ2hhdENsaWVudCk=",
"known_positive_b64": "ZnJvbSBhZ2VudF9mcmFtZXdvcmsuYW50aHJvcGljIGltcG9ydCBBbnRocm9waWNDaGF0Q2xpZW50",
"known_negative_b64": "ZnJvbSBhZ2VudF9mcmFtZXdvcmsuYXp1cmUgaW1wb3J0IEF6dXJlQUlBZ2VudENsaWVudA=="
},
{
"id": "oauth",
"label": "lesing av Claude-abonnementets legitimasjon",
"pattern_b64": "KENMQVVERV9DT0RFX09BVVRIX1RPS0VOfFwuY2xhdWRlL1wuY3JlZGVudGlhbHNcLmpzb258Y2xhdWRlXC5haS9vYXV0aCk=",
"known_positive_b64": "dG9rZW4gPSBvcy5lbnZpcm9uWyJDTEFVREVfQ09ERV9PQVVUSF9UT0tFTiJd",
"known_negative_b64": "dG9rZW4gPSBvcy5lbnZpcm9uWyJBWlVSRV9PUEVOQUlfQVBJX0tFWSJd"
},
{
"id": "proxy",
"label": "proxy mot Anthropics API",
"pattern_b64": "KEFOVEhST1BJQ19CQVNFX1VSTHxBTlRIUk9QSUNfQVBJX0tFWXxhcGlcLmFudGhyb3BpY1wuY29tKQ==",
"known_positive_b64": "b3MuZW52aXJvblsiQU5USFJPUElDX0JBU0VfVVJMIl0gPSBlbmRwb2ludA==",
"known_negative_b64": "b3MuZW52aXJvblsiQVpVUkVfT1BFTkFJX0VORFBPSU5UIl0gPSBlbmRwb2ludA=="
}
]
},
"no_model_calls": {
"checks": {
"fullfører": [
"tests/test_b_toolbox_probes.py::test_the_whole_run_completes_with_the_chat_client_raising"
],
"rundekatalog": [
"tests/test_b_toolbox_probes.py::test_the_toolbox_outbox_binds_to_a_scratch_round_directory"
],
"gate-form-ok": [
"tests/test_b_toolbox_probes.py::test_v1_gate_reads_that_round_as_form_ok_not_proven"
]
}
},
"foundry": {
"profiles": [
"AZURE",
"LOCAL"
],
"factory": {
"module": "run.py",
"symbol": "_default_factory"
},
"seam": {
"module": "run.py",
"scope": "run_project",
"parameter": "client_factory"
},
"evidence": {
"samme-skjema": [
"tests/test_b_toolbox_probes.py::test_both_modes_write_the_same_artefact_family"
]
}
},
"runbook": {
"path": "docs/kjoerebok-verktoeykassen.md",
"attestation": "v1-rounds/kjoerebok-attestering.txt",
"keys": [
"kjørebok",
"kjørt",
"dato"
]
}
}

478
tests/test_b_gate.py Normal file
View file

@ -0,0 +1,478 @@
"""B-gatens egne tester: hver rad KAN bli grønn og KAN bli rød — og kan ikke FAKES grønn.
Rammen er operatørbeslutningen 19.09.2026: i utvikling og test LEDER Claude Code, og
portfolio-optimiser er verktøykassen. po kaller ALDRI Claude. Gaten måler avstanden dit, rad for
rad, uten et eneste modellkall.
Tre ting gjør testene load-bearing i stedet for grønne-men-døde:
* **Nevneren telles uavhengig HER.** Rad 1 og rad 2 får ikke sine tall fra gatens egne funksjoner:
testen parser kilden selv (``ast``) eller importerer modulen og teller, og sammenligner. En gate
som hardkoder nevneren felles av arm ``M-1``, som kjører den mot en KONSTRUERT kilde der svaret
er et annet enn repoets.
* **Hver vakt har en kjent-positiv.** Et mønster som ikke treffer sin egen kjent-positive er ikke
en måling, og rad 3 nekter å telle det (``M-3``). Et fravær («0 treff») uttales med nevner.
* **Rad 6 kan ikke bli grønn av en fil gaten selv skriver** (``M-5``): armen kjører hele
``evaluate`` og krever at attesteringsfila fortsatt ikke finnes etterpå.
"""
from __future__ import annotations
import ast
import base64
import json
import re
import subprocess
import sys
from pathlib import Path
from typing import Any
from portfolio_optimiser import workflow
from portfolio_optimiser.evals import b_gate as gate
from portfolio_optimiser.evals import v1_gate
_REPO = Path(__file__).resolve().parents[1]
_SRC = _REPO / "src" / "portfolio_optimiser"
_CONFIG = gate.load_config()
#: Repoets egne tall ved skriving (19.09.2026), talt av armene under mot KILDEN. De står her for
#: at en stille endring i kjørestien skal vise seg som en rød test og ikke som et nytt tall.
_STEPS_TODAY = 13
_EXTERNAL_TODAY = 3
_ROLES_TODAY = ("proposer", "checker")
_PATTERNS_TODAY = 6
def _write(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def _probes(outcome: str = "missing") -> Any:
return lambda nodeids: {n: outcome for n in nodeids}
def _row(rows: list[v1_gate.Row], key: str) -> v1_gate.Row:
return next(r for r in rows if r.key == key)
# ---------------------------------------------------------------------------------------------
# Rad 1 — verktøykassen er komplett
# ---------------------------------------------------------------------------------------------
def _independently_counted_steps() -> tuple[int, int]:
"""(steg som oppløses i kilden, steg med et kallsted) — talt HER, av testens egen parser.
Kilden er ``src/portfolio_optimiser/``; ingen av gatens funksjoner røres."""
resolved = called = 0
for step in _CONFIG["steps"]:
tree = ast.parse((_SRC / step["module"]).read_text(encoding="utf-8"))
defs = {
n.name
for n in tree.body
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
}
if step["symbol"] in defs:
resolved += 1
driver = ast.parse((_SRC / step["driver"]["module"]).read_text(encoding="utf-8"))
scopes = [
n
for n in ast.walk(driver)
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
and n.name == step["driver"]["scope"]
]
names = {
c.func.id if isinstance(c.func, ast.Name) else c.func.attr
for s in scopes
for c in ast.walk(s)
if isinstance(c, ast.Call) and isinstance(c.func, (ast.Name, ast.Attribute))
}
if step["symbol"] in names:
called += 1
return resolved, called
def test_the_step_denominator_equals_an_independent_count_of_the_source() -> None:
resolved, called = _independently_counted_steps()
assert resolved == called == _STEPS_TODAY, "kjørestien har endret seg — oppdater nevneren"
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
assert row.n == _STEPS_TODAY
def test_row1_today_is_red_with_three_of_thirteen_steps_callable_from_outside() -> None:
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
assert (row.k, row.n, row.status) == (_EXTERNAL_TODAY, _STEPS_TODAY, gate.RED)
named = {x.split(":")[0] for x in row.exceptions}
assert "rundebinding" not in named and "gate" not in named
assert "validering" in named and "utboks" in named
def _fake_src(root: Path, *, chat_client: bool, symbol: str = "gjoer_noe") -> Path:
"""En konstruert kilde med ETT steg: en inngang ``main`` som når steget.
``chat_client=True`` legger en chatklient på veien dit — steget er da fortsatt kallbart, men
ikke uten en modell, og skal ikke telle."""
src = root / "fakesrc"
body = " return create_chat_client()\n" if chat_client else " return 1\n"
_write(src / "steg.py", f"def {symbol}():\n return 42\n")
_write(
src / "driver.py",
"from fakesrc.steg import " + symbol + "\n\n"
"def _hjelper():\n" + body + "\n\n"
"def kjor():\n"
f" _hjelper()\n return {symbol}()\n\n\n"
"def main(argv=None):\n return kjor()\n",
)
return src
def _one_step(symbol: str = "gjoer_noe") -> list[dict[str, Any]]:
return [
{
"id": "steg",
"label": "ett steg",
"module": "steg.py",
"symbol": symbol,
"driver": {"module": "driver.py", "scope": "kjor"},
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
}
]
def test_a_step_reachable_from_a_cli_entry_without_a_chat_client_counts(tmp_path: Path) -> None:
src = _fake_src(tmp_path, chat_client=False)
row = gate.score_toolbox(_one_step(), src)
assert (row.k, row.n, row.status) == (1, 1, gate.GREEN)
def test_m2_a_step_that_needs_a_chat_client_is_not_callable_from_outside(tmp_path: Path) -> None:
"""M-2: fjernes chatklient-sjekken, hopper k fra 0 til 1 og denne armen feller mutanten."""
src = _fake_src(tmp_path, chat_client=True)
row = gate.score_toolbox(_one_step(), src)
assert (row.k, row.n, row.status) == (0, 1, gate.RED)
assert any("chatklient" in x for x in row.exceptions)
def test_m1_the_denominator_follows_the_source_and_is_not_the_config_length(
tmp_path: Path,
) -> None:
"""M-1: en hardkodet nevner overlever repoet, men ikke en kilde med ett steg mindre."""
src = _fake_src(tmp_path, chat_client=False)
steps = _one_step() + [
{
"id": "borte",
"label": "et steg som ikke finnes i kilden",
"module": "steg.py",
"symbol": "finnes_ikke",
"driver": {"module": "driver.py", "scope": "kjor"},
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
}
]
row = gate.score_toolbox(steps, src)
assert row.n == 1, "et steg uten symbol i kilden er ikke et steg"
assert any("borte" in x for x in row.exceptions)
def test_a_step_the_driver_no_longer_calls_leaves_the_denominator(tmp_path: Path) -> None:
src = _fake_src(tmp_path, chat_client=False)
_write(src / "steg.py", "def gjoer_noe():\n return 42\n\n\ndef ubrukt():\n return 0\n")
steps = _one_step() + [
{
"id": "ubrukt",
"label": "definert, men ikke i kjørestien",
"module": "steg.py",
"symbol": "ubrukt",
"driver": {"module": "driver.py", "scope": "kjor"},
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
}
]
row = gate.score_toolbox(steps, src)
assert row.n == 1
def test_an_entry_without_a_main_is_no_entry(tmp_path: Path) -> None:
src = _fake_src(tmp_path, chat_client=False)
text = (src / "driver.py").read_text(encoding="utf-8").replace("def main(", "def _main(")
_write(src / "driver.py", text)
row = gate.score_toolbox(_one_step(), src)
assert (row.k, row.n) == (0, 1)
assert any("main" in x for x in row.exceptions)
# ---------------------------------------------------------------------------------------------
# Rad 2 — det modellen leverte, kan leveres utenfra
# ---------------------------------------------------------------------------------------------
def test_the_role_denominator_is_read_from_workflow_and_matches_the_imported_module() -> None:
assert workflow._MAKER_CHECKER_ROLES == _ROLES_TODAY # uavhengig telling: modulen selv
assert gate.run_roles(_SRC, _CONFIG["roles"]["source"]) == _ROLES_TODAY
def test_the_role_denominator_follows_a_changed_source(tmp_path: Path) -> None:
src = tmp_path / "fakesrc"
_write(src / "workflow.py", '_MAKER_CHECKER_ROLES = ("a", "b", "c")\n')
assert gate.run_roles(src, _CONFIG["roles"]["source"]) == ("a", "b", "c")
def test_a_role_counts_only_when_every_named_probe_passes() -> None:
evidence = _CONFIG["roles"]["evidence"]
outcomes = {n: "passed" for ns in evidence.values() for n in ns}
row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC)
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
outcomes[evidence["checker"][0]] = "failed"
row = gate.score_roles(_CONFIG["roles"], outcomes, _SRC)
assert (row.k, row.n, row.status) == (1, 2, gate.RED)
def test_row2_today_is_red_because_the_named_probes_do_not_exist_yet() -> None:
row = gate.score_roles(_CONFIG["roles"], {}, _SRC)
assert (row.k, row.n, row.status) == (0, 2, gate.RED)
assert all("missing" in x for x in row.exceptions)
def test_m4_row2_demands_byte_identical_artefacts_not_merely_present_ones() -> None:
"""M-4: byttes kravet til «finnes», står kontrakten igjen uten den ene setningen som skiller
en ekte likhet fra to filer med samme navn."""
assert "byte-identisk" in gate.BYTE_IDENTICAL_RULE
row = gate.score_roles(_CONFIG["roles"], {}, _SRC)
assert any("byte-identisk" in a for a in row.attests)
# ---------------------------------------------------------------------------------------------
# Rad 3 — po har ingen vei til Claude
# ---------------------------------------------------------------------------------------------
def _patterns() -> list[dict[str, str]]:
return list(_CONFIG["no_claude_path"]["patterns"])
def test_every_pattern_hits_its_own_known_positive_and_misses_its_known_negative() -> None:
"""Fravær er et måleresultat: en vakt som ikke kan treffe, måler ikke null — den måler
ingenting. Nevner: alle mønstrene i kontrakten."""
assert len(_patterns()) == _PATTERNS_TODAY
for spec in _patterns():
rx = re.compile(gate.decode(spec["pattern_b64"]))
assert rx.search(gate.decode(spec["known_positive_b64"])), spec["id"]
assert not rx.search(gate.decode(spec["known_negative_b64"])), spec["id"]
def test_m3_the_denominator_is_the_number_of_patterns_that_can_measure(tmp_path: Path) -> None:
"""M-3: fjernes et mønster, faller nevneren — og et mønster som ikke treffer sin egen
kjent-positive telles ikke som en vakt."""
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
cfg["roots"] = ["src"]
_write(tmp_path / "src" / "rent.py", "print('ingenting her')\n")
row = gate.score_no_claude_path(cfg, tmp_path)
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN)
cfg["patterns"][0]["pattern_b64"] = base64.b64encode(b"finnes-aldri-xyzzy").decode()
row = gate.score_no_claude_path(cfg, tmp_path)
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY - 1, _PATTERNS_TODAY, gate.RED)
assert any("kjent-positiv" in x for x in row.exceptions)
cfg["patterns"] = cfg["patterns"][1:]
row = gate.score_no_claude_path(cfg, tmp_path)
assert row.n == _PATTERNS_TODAY - 1
def test_a_planted_path_to_claude_turns_the_row_red_for_that_pattern(tmp_path: Path) -> None:
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
cfg["roots"] = ["src"]
for spec in cfg["patterns"]:
_write(tmp_path / "src" / f"{spec['id']}.py", gate.decode(spec["known_positive_b64"]) + "\n")
row = gate.score_no_claude_path(cfg, tmp_path)
assert (row.k, row.status) == (0, gate.RED)
assert len(row.exceptions) == _PATTERNS_TODAY
def test_markdown_counts_only_inside_fenced_blocks(tmp_path: Path) -> None:
"""«docs med kjørbare kommandoer»: en pakkenavn-omtale i brødtekst er ikke en vei til Claude;
den samme linja i en kodeblokk er en kommando noen kan kjøre."""
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
cfg["roots"] = ["docs"]
spec = next(s for s in cfg["patterns"] if s["id"] == "maf-anthropic")
line = gate.decode(spec["known_positive_b64"])
_write(tmp_path / "docs" / "prosa.md", f"- vurdert og forkastet: `{line}` (aldri tatt inn)\n")
assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN
_write(tmp_path / "docs" / "kjorbar.md", f"```bash\n{line}\n```\n")
row = gate.score_no_claude_path(cfg, tmp_path)
assert row.status == gate.RED
assert any("kjorbar.md" in x for x in row.exceptions)
def test_row3_on_the_real_published_surface_is_green_today() -> None:
row = gate.score_no_claude_path(_CONFIG["no_claude_path"], _REPO)
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN), (
row.exceptions
)
def test_the_surface_names_its_denominator_in_files_not_in_prose() -> None:
files = gate.surface_files(_REPO, _CONFIG["no_claude_path"]["roots"])
assert len(files) > 100, "en tom flate ville gitt 0 treff uten å måle noe"
assert any(f.name == "run.py" for f in files)
# ---------------------------------------------------------------------------------------------
# Rad 4 — ingen modellkall i verktøykasse-modus
# ---------------------------------------------------------------------------------------------
def test_row4_is_red_today_and_names_all_three_checks() -> None:
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
assert (row.k, row.n, row.status) == (0, 3, gate.RED)
assert len(row.exceptions) == 3
def test_row4_counts_only_checks_whose_probes_pass() -> None:
checks = _CONFIG["no_model_calls"]["checks"]
outcomes = {n: "passed" for ns in checks.values() for n in ns}
assert gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes).status == gate.GREEN
outcomes[checks["fullfører"][0]] = "skipped"
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes)
assert (row.k, row.status) == (2, gate.RED)
def test_row4_states_what_the_budget_guard_is_in_b_instead_of_pretending() -> None:
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
assert any("budsjett" in a.casefold() for a in row.attests)
assert "valgt" in gate.BUDGET_IN_B.casefold()
# ---------------------------------------------------------------------------------------------
# Rad 5 — Foundry-veien urørt
# ---------------------------------------------------------------------------------------------
def test_the_foundry_path_is_intact_in_the_source_today() -> None:
ok, missing = gate.foundry_intact(_CONFIG["foundry"], _SRC)
assert (ok, missing) == (True, ())
def test_the_foundry_check_fails_when_the_azure_profile_is_gone(tmp_path: Path) -> None:
src = tmp_path / "fakesrc"
_write(src / "backends.py", "class Profile:\n LOCAL = 'local'\n")
_write(src / "run.py", "def run_project(client_factory=None):\n return 1\n")
ok, missing = gate.foundry_intact(_CONFIG["foundry"], src)
assert ok is False
assert any("AZURE" in m for m in missing)
def test_row5_is_red_until_the_schema_comparison_probe_passes() -> None:
row = gate.score_foundry(_CONFIG["foundry"], {}, _SRC)
assert (row.k, row.n, row.status) == (1, 2, gate.RED)
nodeid = _CONFIG["foundry"]["evidence"]["samme-skjema"][0]
row = gate.score_foundry(_CONFIG["foundry"], {nodeid: "passed"}, _SRC)
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
# ---------------------------------------------------------------------------------------------
# Rad 6 — kjøreboka finnes
# ---------------------------------------------------------------------------------------------
def test_row6_is_not_measured_while_the_runbook_is_missing(tmp_path: Path) -> None:
row = gate.score_runbook(_CONFIG["runbook"], tmp_path)
assert (row.k, row.n, row.status) == (0, 2, gate.NOT_MEASURED)
assert row.failing is True
def test_row6_is_still_not_measured_without_the_operators_attestation(tmp_path: Path) -> None:
_write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n\nsteg 1 …\n")
row = gate.score_runbook(_CONFIG["runbook"], tmp_path)
assert (row.k, row.n, row.status) == (1, 2, gate.NOT_MEASURED)
def test_row6_goes_green_only_with_every_required_key(tmp_path: Path) -> None:
cfg = _CONFIG["runbook"]
_write(tmp_path / cfg["path"], "# kjørebok\n")
attest = tmp_path / cfg["attestation"]
_write(attest, "kjørebok: docs/kjoerebok-verktoeykassen.md\nkjørt: operatøren\n")
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
_write(attest, "kjørebok: x\nkjørt: operatøren\ndato: 2026-09-20\n")
row = gate.score_runbook(cfg, tmp_path)
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
def test_a_duplicated_key_in_the_attestation_is_red(tmp_path: Path) -> None:
cfg = _CONFIG["runbook"]
_write(tmp_path / cfg["path"], "# kjørebok\n")
_write(tmp_path / cfg["attestation"], "kjørebok: x\nkjørt: a\nkjørt: b\ndato: 2026-09-20\n")
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
def test_m5_the_gate_never_writes_the_attestation_itself(tmp_path: Path) -> None:
"""M-5: kan gaten skrive fila den leser, er rad 6 en sløyfe som bekrefter seg selv."""
_write(tmp_path / _CONFIG["runbook"]["path"], "# kjørebok\n")
attest = tmp_path / _CONFIG["runbook"]["attestation"]
rows = gate.evaluate(
config=_CONFIG, repo_root=tmp_path, src=_SRC, probe_runner=_probes()
)
assert not attest.exists(), "gaten skrev operatørens attestering — da beviser rad 6 ingenting"
assert _row(rows, "kjørebok").status == gate.NOT_MEASURED
assert any("aldri" in a for a in _row(rows, "kjørebok").attests)
# ---------------------------------------------------------------------------------------------
# Hele gaten
# ---------------------------------------------------------------------------------------------
def test_the_gate_is_red_today_and_every_row_is_red_for_a_measured_reason() -> None:
rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes())
assert gate.exit_code(rows) == 1
assert [r.key for r in rows] == [
"verktøykasse",
"roller",
"ingen-claude-vei",
"ingen-modellkall",
"foundry",
"kjørebok",
]
for row in rows:
assert row.status in {gate.GREEN, gate.RED, gate.NOT_MEASURED}
if row.status != gate.GREEN:
assert row.k is not None and row.n is not None, f"{row.key} mangler k av N"
assert row.exceptions, f"{row.key} er ikke grønn uten å si hvorfor"
def test_the_probe_runner_can_return_passed_for_a_test_that_exists() -> None:
"""Kjent-positiv for selve probe-mekanismen: «missing» over alt ville vært et måleresultat
uten nevner. Nevner: én nodeid som finnes i denne suiten."""
nodeid = "tests/test_b_gate.py::test_the_probe_runner_can_return_passed_for_a_test_that_exists"
assert v1_gate.run_probes([nodeid], _REPO)[nodeid] == "passed"
def test_render_names_every_row_the_budget_choice_and_the_runbook_rule() -> None:
rows = gate.evaluate(config=_CONFIG, repo_root=_REPO, src=_SRC, probe_runner=_probes())
text = gate.render(rows)
for row in rows:
assert row.title in text
assert gate.BUDGET_IN_B in text
assert gate.RUNBOOK_RULE in text
def test_the_command_runs_offline_and_exits_one() -> None:
proc = subprocess.run(
[sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--json"],
cwd=_REPO,
capture_output=True,
text=True,
)
assert proc.returncode == 1, proc.stderr
payload = json.loads(proc.stdout)
assert payload["exit"] == 1
assert len(payload["rows"]) == 6
def test_a_bad_attestation_path_is_usage_error_not_a_silent_green(tmp_path: Path) -> None:
proc = subprocess.run(
[sys.executable, "-m", "portfolio_optimiser.evals.b_gate", "--attest", str(tmp_path / "x")],
cwd=_REPO,
capture_output=True,
text=True,
)
assert proc.returncode == 2