test(b-gate): 31 arms against the gate's own denominators, 30 red on an assert about behaviour
The PM checkpoint on 207337c judged the gate DELVIS: row 1's M=13 is a curated list in the
gate's OWN b_gate.json (the run path has 41 po-calls, 7 of 10 outbox writers), rows 4, 5 and 6
have denominators with no source at all, 6 of 10 cheat-attacks got through, and the
never-Claude guard sees 433 of 512 published files.
This commit is the red half. Every arm fails on an ASSERT about behaviour, never at collection:
the four names that do not exist yet (ENTRY_KINDS, run_path_calls, registered_entry,
published_files) are stubbed here with DELIBERATELY wrong values — everything is a door, the
run path calls nothing, the surface is empty — so each arm measures the defect rather than the
absence of a symbol.
30 av 31 red on assert. The one that is green is the rc-0 control
(test_a_valid_attestation_is_the_only_thing_that_turns_row6_green): a valid attestation must
turn row 6 green both before and after, or the row refuses everything, which proves as little
as refusing nothing. All 34 pre-existing arms stay green — measured, not assumed.
The planted claude-invocations are base64 in the test file for the same reason the contract's
patterns are: tests/ is itself part of the surface row 3 scans, and a cleartext variant here
would register as its own finding.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
f78d57a7b7
commit
332eb5965b
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 576 additions and 7 deletions
|
|
@ -123,6 +123,37 @@ finnes er RØD, aldri grønn og aldri hoppet over.
|
|||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# STUBBER (rød-først). Navnene finnes her for at armene under skal felle på en ASSERT om ATFERD
|
||||
# og ikke ved innsamling; verdiene er MED VILJE gale og erstattes i reparasjons-commiten.
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
#: Inngangsartene gaten kan VERIFISERE. Tom her med vilje: stubben kjenner ingen.
|
||||
ENTRY_KINDS: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def run_path_calls(src: Path, spec: Mapping[str, Any]) -> dict[str, str]:
|
||||
"""STUB: skal lese kjørestiens faktiske kall av KILDEN. Returnerer ingenting ennå."""
|
||||
return {}
|
||||
|
||||
|
||||
def console_scripts(repo_root: Path) -> dict[str, str]:
|
||||
"""STUB: skal lese [project.scripts] fra pyproject.toml."""
|
||||
return {}
|
||||
|
||||
|
||||
def registered_entry(
|
||||
src: Path, repo_root: Path, package: str, entry: Mapping[str, Any]
|
||||
) -> tuple[bool, str]:
|
||||
"""STUB: godtar ALT som dør — nøyaktig feilen reparasjonen skal fjerne."""
|
||||
return True, ""
|
||||
|
||||
|
||||
def published_files(root: Path, manifest: Mapping[str, Any]) -> tuple[list[Path], str]:
|
||||
"""STUB: skal utlede den publiserte flaten av repo-manifestet."""
|
||||
return [], "stub"
|
||||
|
||||
|
||||
def load_config(path: Path = _DATA) -> dict[str, Any]:
|
||||
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
||||
return data
|
||||
|
|
@ -274,12 +305,26 @@ def measure_steps(steps: Sequence[Mapping[str, Any]], src: Path) -> list[Step]:
|
|||
return measured
|
||||
|
||||
|
||||
def score_toolbox(steps: Sequence[Mapping[str, Any]], src: Path) -> Row:
|
||||
def score_toolbox(
|
||||
steps: Sequence[Mapping[str, Any]],
|
||||
src: Path,
|
||||
run_path: Mapping[str, Any] | None = None,
|
||||
outcomes: Mapping[str, str] | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
) -> Row:
|
||||
measured = measure_steps(steps, src)
|
||||
in_path = [s for s in measured if s.resolved and s.called]
|
||||
n = len(in_path)
|
||||
undeclared = sorted(
|
||||
name
|
||||
for name in (run_path_calls(src, run_path) if run_path else {})
|
||||
if name not in {str(s["symbol"]) for s in steps}
|
||||
and name not in {str(h["symbol"]) for h in (run_path or {}).get("held_out", ())}
|
||||
)
|
||||
n = len(in_path) + len(undeclared)
|
||||
k = sum(1 for s in in_path if s.external)
|
||||
exceptions = tuple(f"{s.id}: {s.why}" for s in measured if not s.external)
|
||||
exceptions = tuple(f"{s.id}: {s.why}" for s in measured if not s.external) + tuple(
|
||||
f"udeklarert: {name} kalles i kjørestien uten å være et steg" for name in undeclared
|
||||
)
|
||||
return Row(
|
||||
"verktøykasse",
|
||||
"1 steg i kjørestien kallbare utenfra",
|
||||
|
|
@ -408,7 +453,12 @@ class Pattern:
|
|||
|
||||
def measure_patterns(config: Mapping[str, Any], root: Path) -> list[Pattern]:
|
||||
scanned: list[tuple[Path, list[tuple[int, str]]]] = []
|
||||
for path in surface_files(root, list(config["roots"])):
|
||||
surface = (
|
||||
surface_files(root, list(config["roots"]))
|
||||
if "roots" in config
|
||||
else published_files(root, config.get("manifest", {}))[0]
|
||||
)
|
||||
for path in surface:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
|
|
@ -439,7 +489,11 @@ def score_no_claude_path(config: Mapping[str, Any], root: Path) -> Row:
|
|||
measured = measure_patterns(config, root)
|
||||
n = len(measured)
|
||||
k = sum(1 for p in measured if p.valid and not p.hits)
|
||||
files = len(surface_files(root, list(config["roots"])))
|
||||
files = (
|
||||
len(surface_files(root, list(config["roots"])))
|
||||
if "roots" in config
|
||||
else len(published_files(root, config.get("manifest", {}))[0])
|
||||
)
|
||||
exceptions: list[str] = []
|
||||
for pattern in measured:
|
||||
if not pattern.valid:
|
||||
|
|
@ -463,7 +517,9 @@ def score_no_claude_path(config: Mapping[str, Any], root: Path) -> Row:
|
|||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def score_no_model_calls(config: Mapping[str, Any], outcomes: Mapping[str, str]) -> Row:
|
||||
def score_no_model_calls(
|
||||
config: Mapping[str, Any], outcomes: Mapping[str, str], src: Path | None = None
|
||||
) -> Row:
|
||||
checks = config["checks"]
|
||||
verdicts = {name: _probe_verdict(list(ids), outcomes) for name, ids in checks.items()}
|
||||
k = sum(1 for why in verdicts.values() if not why)
|
||||
|
|
@ -585,7 +641,12 @@ def read_runbook_attestation(path: Path, keys: Sequence[str]) -> RunbookAttestat
|
|||
return RunbookAttestation(True, True, "")
|
||||
|
||||
|
||||
def score_runbook(config: Mapping[str, Any], repo_root: Path, attest: Path | None = None) -> Row:
|
||||
def score_runbook(
|
||||
config: Mapping[str, Any],
|
||||
repo_root: Path,
|
||||
attest: Path | None = None,
|
||||
now: Any | None = None,
|
||||
) -> Row:
|
||||
runbook = repo_root / str(config["path"])
|
||||
has_runbook = runbook.is_file() and bool(runbook.read_text(encoding="utf-8").strip())
|
||||
attest_path = attest if attest is not None else repo_root / str(config["attestation"])
|
||||
|
|
|
|||
|
|
@ -20,10 +20,12 @@ from __future__ import annotations
|
|||
|
||||
import ast
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -34,6 +36,8 @@ from portfolio_optimiser.evals import v1_gate
|
|||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_SRC = _REPO / "src" / "portfolio_optimiser"
|
||||
_CONFIG = gate.load_config()
|
||||
#: Klokka rad 6 leses mot, injisert: den ENE sjekken som ser på tiden skal ikke flytte seg.
|
||||
_NOW = datetime(2026, 9, 19, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
#: Repoets egne tall ved skriving (19.09.2026), talt av armene under mot KILDEN. De står her for
|
||||
#: at en stille endring i kjørestien skal vise seg som en rød test og ikke som et nytt tall.
|
||||
|
|
@ -481,3 +485,507 @@ def test_a_bad_attestation_path_is_usage_error_not_a_silent_green(tmp_path: Path
|
|||
text=True,
|
||||
)
|
||||
assert proc.returncode == 2
|
||||
|
||||
|
||||
# =============================================================================================
|
||||
# REPARASJONEN 19.09.2026 (PM-sjekkpunktet på 207337c): nevnerne utledet av KILDEN, en dør som
|
||||
# må være REGISTRERT og bevist med atferd, hele den publiserte flaten, og en attestering som kan
|
||||
# felles. Hver arm under feller på en ASSERT om atferd, aldri ved innsamling.
|
||||
# =============================================================================================
|
||||
|
||||
#: po-funksjoner ``run.py::run_project`` faktisk kaller — talt av armens EGEN parser under.
|
||||
#: 41 = PMs 39 deterministiske + de to som krever en chatklient og holdes utenfor med grunn.
|
||||
_RUN_PATH_CALLS_TODAY = 41
|
||||
#: utboks-skrivere: definert i outbox.py / kalt i kjørestien. PMs tall, talt om igjen her.
|
||||
_WRITERS_DEFINED = 10
|
||||
_WRITERS_IN_RUN_PATH = 7
|
||||
#: publiserte filer i git-manifestet (uttrekk og arbeidstre gir SAMME tall — det var hele poenget
|
||||
#: med å slutte å telle filtreet: 435 i arbeidstreet var to gitignorerte .local.md-filer).
|
||||
_PUBLISHED_TODAY = 512
|
||||
_UNDECODABLE_TODAY = 1
|
||||
|
||||
|
||||
def _manifest() -> dict[str, Any]:
|
||||
manifest = _CONFIG["no_claude_path"].get("manifest")
|
||||
assert manifest is not None, "kontrakten utleder ikke flaten av repo-manifestet"
|
||||
return dict(manifest)
|
||||
|
||||
|
||||
def _run_path_spec() -> dict[str, Any]:
|
||||
spec = _CONFIG.get("run_path")
|
||||
assert spec is not None, "kontrakten navngir ingen kjørested — nevneren har da ingen kilde"
|
||||
return dict(spec)
|
||||
|
||||
|
||||
def _independently_counted_run_path() -> dict[str, str]:
|
||||
"""po-funksjonene ``run_project`` kaller, talt av TESTENS egen parser — ingen gate-kode rørt.
|
||||
|
||||
Regelen: et navn importert fra en ``portfolio_optimiser``-undermodul og kalt i scopet, eller
|
||||
et attributt på en modul importert som ``from portfolio_optimiser import X``. Klasser og
|
||||
unntak (stor forbokstav) er ikke steg; private hjelpere i run.py heller ikke."""
|
||||
tree = ast.parse((_SRC / "run.py").read_text(encoding="utf-8"))
|
||||
symbols: dict[str, str] = {}
|
||||
aliases: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.ImportFrom):
|
||||
continue
|
||||
module = node.module or ""
|
||||
if module == "portfolio_optimiser":
|
||||
aliases |= {a.asname or a.name for a in node.names}
|
||||
elif module.startswith("portfolio_optimiser."):
|
||||
for alias in node.names:
|
||||
symbols[alias.asname or alias.name] = module.split(".")[-1] + ".py"
|
||||
scope = next(
|
||||
n
|
||||
for n in ast.walk(tree)
|
||||
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name == "run_project"
|
||||
)
|
||||
found: dict[str, str] = {}
|
||||
for call in ast.walk(scope):
|
||||
if not isinstance(call, ast.Call):
|
||||
continue
|
||||
func = call.func
|
||||
if isinstance(func, ast.Name) and func.id in symbols and func.id[:1].islower():
|
||||
found[func.id] = symbols[func.id]
|
||||
elif (
|
||||
isinstance(func, ast.Attribute)
|
||||
and isinstance(func.value, ast.Name)
|
||||
and func.value.id in aliases
|
||||
and func.attr[:1].islower()
|
||||
):
|
||||
found[func.attr] = f"{func.value.id}.py"
|
||||
return found
|
||||
|
||||
|
||||
def test_the_run_path_denominator_is_read_from_the_source_not_from_the_contract() -> None:
|
||||
"""Rad 1s nevner kom fra gatens EGEN b_gate.json. Nå leses kjørestien av kilden, og testen
|
||||
teller den om igjen selv."""
|
||||
mine = _independently_counted_run_path()
|
||||
assert len(mine) == _RUN_PATH_CALLS_TODAY, sorted(mine)
|
||||
theirs = gate.run_path_calls(_SRC, _run_path_spec())
|
||||
assert set(theirs) == set(mine), sorted(set(mine) ^ set(theirs))
|
||||
|
||||
|
||||
def test_every_call_in_the_run_path_is_either_a_declared_step_or_held_out_with_a_reason() -> None:
|
||||
"""Den ene egenskapen en kuratert liste ikke har: ingenting kan utelates STILLE."""
|
||||
calls = set(_independently_counted_run_path())
|
||||
declared = {str(s["symbol"]) for s in _CONFIG["steps"]}
|
||||
held = {
|
||||
str(h["symbol"]): str(h.get("reason", "")) for h in _run_path_spec().get("held_out", ())
|
||||
}
|
||||
assert not (calls - declared - set(held)), sorted(calls - declared - set(held))
|
||||
assert all(held.values()), [s for s, why in held.items() if not why]
|
||||
assert len(calls & declared) + len(held) == _RUN_PATH_CALLS_TODAY
|
||||
|
||||
|
||||
def test_the_outbox_writers_the_run_path_uses_are_all_declared_steps() -> None:
|
||||
"""PMs 7 av 10, talt om igjen her: artefaktfamilien er ikke tre filer, den er sju."""
|
||||
outbox_tree = ast.parse((_SRC / "outbox.py").read_text(encoding="utf-8"))
|
||||
writers = {
|
||||
n.name
|
||||
for n in outbox_tree.body
|
||||
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name.startswith("write_")
|
||||
}
|
||||
assert len(writers) == _WRITERS_DEFINED
|
||||
in_path = writers & set(_independently_counted_run_path())
|
||||
assert len(in_path) == _WRITERS_IN_RUN_PATH, sorted(in_path)
|
||||
declared = {str(s["symbol"]) for s in _CONFIG["steps"]}
|
||||
assert not (in_path - declared), sorted(in_path - declared)
|
||||
|
||||
|
||||
def _mini_run_path(root: Path, *, extra_call: bool) -> tuple[Path, dict[str, Any]]:
|
||||
"""En konstruert kilde med en kjøresti som kaller ett eller to po-steg."""
|
||||
src = root / "minisrc"
|
||||
_write(src / "steg.py", "def alfa():\n return 1\n\n\ndef beta():\n return 2\n")
|
||||
body = " alfa()\n" + (" beta()\n" if extra_call else "")
|
||||
_write(
|
||||
src / "kjor.py",
|
||||
f"from minisrc.steg import alfa, beta\n\n\nasync def run_project():\n{body} return 0\n",
|
||||
)
|
||||
return src, {"module": "kjor.py", "scope": "run_project", "package": "minisrc"}
|
||||
|
||||
|
||||
def test_an_undeclared_call_in_the_run_path_raises_the_denominator_and_is_named(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""M-6: nevneren kan ikke krympe stille — et kall ingen erklærte teller som et steg UTEN dør."""
|
||||
src, spec = _mini_run_path(tmp_path, extra_call=True)
|
||||
steps = [
|
||||
{
|
||||
"id": "alfa",
|
||||
"label": "alfa",
|
||||
"module": "steg.py",
|
||||
"symbol": "alfa",
|
||||
"driver": {"module": "kjor.py", "scope": "run_project"},
|
||||
"entry": {"kind": "module-main", "module": "kjor.py", "scope": "main"},
|
||||
}
|
||||
]
|
||||
row = gate.score_toolbox(steps, src, spec, {}, tmp_path)
|
||||
assert row.n == 2, "beta kalles i kjørestien og er verken erklært eller holdt utenfor"
|
||||
assert any("beta" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_a_held_out_call_leaves_the_denominator_but_is_named_with_its_reason(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
src, spec = _mini_run_path(tmp_path, extra_call=True)
|
||||
spec["held_out"] = [{"symbol": "beta", "reason": "hjelper inne i alfa, ikke et eget steg"}]
|
||||
steps = [
|
||||
{
|
||||
"id": "alfa",
|
||||
"label": "alfa",
|
||||
"module": "steg.py",
|
||||
"symbol": "alfa",
|
||||
"driver": {"module": "kjor.py", "scope": "run_project"},
|
||||
"entry": {"kind": "module-main", "module": "kjor.py", "scope": "main"},
|
||||
}
|
||||
]
|
||||
row = gate.score_toolbox(steps, src, spec, {}, tmp_path)
|
||||
assert row.n == 1
|
||||
assert any("beta" in d and "hjelper inne i alfa" in d for d in row.diagnostics), row.diagnostics
|
||||
|
||||
|
||||
def test_removing_a_declared_step_does_not_shrink_the_denominator(tmp_path: Path) -> None:
|
||||
"""A2 fra sjekkpunktet: «utboks» fjernet fra steps ga «3 av 12» uten en klage."""
|
||||
src, spec = _mini_run_path(tmp_path, extra_call=True)
|
||||
row = gate.score_toolbox([], src, spec, {}, tmp_path)
|
||||
assert row.n == 2, "begge kallene står igjen i kjørestien selv om kontrakten glemte dem"
|
||||
assert len(row.exceptions) == 2
|
||||
|
||||
|
||||
# --- døren: registrert, av en art gaten kan verifisere, og bevist med atferd -------------------
|
||||
|
||||
|
||||
def _fake_pyproject(root: Path, scripts: dict[str, str]) -> None:
|
||||
lines = "\n".join(f'{name} = "{target}"' for name, target in scripts.items())
|
||||
_write(
|
||||
root / "pyproject.toml",
|
||||
f'[project]\nname = "fake"\nversion = "0"\n\n[project.scripts]\n{lines}\n\n[tool.x]\ny = 1\n',
|
||||
)
|
||||
|
||||
|
||||
def test_an_entry_kind_the_gate_cannot_verify_is_never_a_door(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
ok, why = gate.registered_entry(
|
||||
src, tmp_path, "fakesrc", {"kind": "magi", "module": "driver.py", "scope": "main"}
|
||||
)
|
||||
assert ok is False and "magi" in why
|
||||
assert "magi" not in gate.ENTRY_KINDS
|
||||
|
||||
|
||||
def test_a_console_script_is_a_door_only_when_the_manifest_registers_it(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
entry = {"kind": "console-script", "module": "driver.py", "scope": "main"}
|
||||
_fake_pyproject(tmp_path, {})
|
||||
ok, why = gate.registered_entry(src, tmp_path, "fakesrc", entry)
|
||||
assert ok is False and "pyproject" in why
|
||||
_fake_pyproject(tmp_path, {"fake-kommando": "fakesrc.driver:main"})
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry) == (True, "")
|
||||
|
||||
|
||||
def test_a_module_main_is_a_door_only_with_the_dunder_guard(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
entry = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry)[0] is False
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry) == (True, "")
|
||||
|
||||
|
||||
def test_a_subcommand_is_a_door_only_when_the_parser_registers_that_name(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
entry = {
|
||||
"kind": "subcommand",
|
||||
"module": "driver.py",
|
||||
"scope": "main",
|
||||
"command": "naviger-pakke",
|
||||
}
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry)[0] is False
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\ndef _parser(sub):\n sub.add_parser("naviger-pakke")\n',
|
||||
)
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry) == (True, "")
|
||||
|
||||
|
||||
def test_a_stub_main_is_not_a_door_without_a_registered_entry(tmp_path: Path) -> None:
|
||||
"""A1 fra sjekkpunktet, felt: en modul med bare `def main(): return <symbol>()` tok rad 1 fra
|
||||
3 til 4 av 13. Navn alene er aldri grønt."""
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
_fake_pyproject(tmp_path, {})
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "console-script", "module": "driver.py", "scope": "main"}
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("pyproject" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
||||
def test_a_registered_door_without_a_passing_behaviour_probe_does_not_count(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""«Kallet med en fixture gir det dokumenterte artefaktet» — et navn er ikke en atferd."""
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
steps[0]["probe"] = ["tests/test_noe.py::test_doeren_skriver_artefaktet"]
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("probe" in x for x in row.exceptions), row.exceptions
|
||||
passed = {"tests/test_noe.py::test_doeren_skriver_artefaktet": "passed"}
|
||||
row = gate.score_toolbox(steps, src, None, passed, tmp_path)
|
||||
assert (row.k, row.n, row.status) == (1, 1, gate.GREEN)
|
||||
|
||||
|
||||
def test_a_step_without_any_probe_is_never_a_door(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("ingen atferdsprobe" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
||||
def test_the_door_rule_claims_only_kinds_the_gate_can_verify() -> None:
|
||||
"""A8: EXTERNAL_DOOR påsto at en MCP-registrert dør telles; «kind» ble aldri lest."""
|
||||
assert gate.ENTRY_KINDS
|
||||
assert "mcp" not in gate.EXTERNAL_DOOR.casefold().replace("mcp ville", "")
|
||||
assert all(kind in gate.EXTERNAL_DOOR for kind in gate.ENTRY_KINDS)
|
||||
assert '"kind"' in Path(gate.__file__).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
# --- rad 3: hele den publiserte flaten, og de indirekte veiene ---------------------------------
|
||||
|
||||
|
||||
def test_the_published_surface_is_the_repo_manifest_and_not_a_handlist() -> None:
|
||||
"""A10: main.py, examples/, spikes/ og contexts/ er publisert, men lå utenfor roots."""
|
||||
files, source = gate.published_files(_REPO, _manifest())
|
||||
names = {str(p.relative_to(_REPO)) for p in files}
|
||||
assert "main.py" in names and "CLAUDE.md" in names and "llms.txt" in names
|
||||
assert any(n.startswith("examples/") for n in names)
|
||||
assert any(n.startswith("spikes/") for n in names)
|
||||
assert any(n.startswith("contexts/") for n in names)
|
||||
assert len(files) == _PUBLISHED_TODAY, len(files)
|
||||
assert "git" in source
|
||||
|
||||
|
||||
def test_the_surface_count_matches_an_independent_count_of_the_manifest() -> None:
|
||||
tracked = subprocess.run(
|
||||
["git", "ls-files", "-z"], cwd=_REPO, capture_output=True, text=True, check=True
|
||||
).stdout.split("\0")
|
||||
mine = {t for t in tracked if t}
|
||||
files, _ = gate.published_files(_REPO, _manifest())
|
||||
assert {str(p.relative_to(_REPO)) for p in files} == mine
|
||||
|
||||
|
||||
def test_a_surface_without_its_sentinel_is_not_measured_instead_of_green(tmp_path: Path) -> None:
|
||||
"""A4: «6 av 6 GRØNN over 0 publiserte filer». Et fravær uten nevner er ikke et bevis."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg.pop("roots", None)
|
||||
cfg["manifest"] = {"sentinel": "src/portfolio_optimiser/run.py"}
|
||||
row = gate.score_no_claude_path(cfg, tmp_path)
|
||||
assert row.status == gate.NOT_MEASURED, row.reason
|
||||
assert any("sentinel" in x for x in row.exceptions), row.exceptions
|
||||
assert "0 publiserte filer" in row.reason
|
||||
|
||||
|
||||
def test_the_row_names_which_manifest_it_read() -> None:
|
||||
row = gate.score_no_claude_path(_CONFIG["no_claude_path"], _REPO)
|
||||
assert str(_PUBLISHED_TODAY) in row.reason
|
||||
assert "git" in row.reason
|
||||
assert f"{_UNDECODABLE_TODAY} ulesbare" in row.reason
|
||||
|
||||
|
||||
def test_the_surface_falls_back_to_the_tree_when_there_is_no_git_metadata(tmp_path: Path) -> None:
|
||||
"""Et rent uttrekk (`git archive | tar -x`) har ingen .git — og ER nøyaktig det publiserte."""
|
||||
_write(tmp_path / "src" / "portfolio_optimiser" / "run.py", "x = 1\n")
|
||||
_write(tmp_path / "main.py", "y = 2\n")
|
||||
_write(tmp_path / "__pycache__" / "skrot.pyc", "nei\n")
|
||||
files, source = gate.published_files(tmp_path, {"sentinel": "src/portfolio_optimiser/run.py"})
|
||||
names = {str(p.relative_to(tmp_path)) for p in files}
|
||||
assert names == {"src/portfolio_optimiser/run.py", "main.py"}
|
||||
assert "uttrekk" in source
|
||||
|
||||
|
||||
#: PMs fem realistiske kall-varianter, base64-kodet av samme grunn som mønstrene i kontrakten:
|
||||
#: tests/ er selv en del av den publiserte flaten raden skanner, og en klartekst-variant her
|
||||
#: ville registrert seg som sitt eget funn.
|
||||
_INDIRECT_VARIANTS = (
|
||||
("direkte", "c3VicHJvY2Vzcy5ydW4oWyJjbGF1ZGUiLCAiLXAiLCAieCJdKQ=="),
|
||||
("absolutt-sti", "c3VicHJvY2Vzcy5ydW4oWyIvdXNyL2xvY2FsL2Jpbi9jbGF1ZGUiLCAiLXAiLCAieCJdKQ=="),
|
||||
("liste-i-variabel", "Y21kID0gWyJjbGF1ZGUiLCAiLXAiLCAieCJdCnN1YnByb2Nlc3MucnVuKGNtZCk="),
|
||||
(
|
||||
"konstant",
|
||||
"Q0xBVURFX0JJTiA9ICJjbGF1ZGUiCnN1YnByb2Nlc3MucnVuKFtDTEFVREVfQklOLCAiLXAiLCAieCJdKQ==",
|
||||
),
|
||||
("shell-streng", "c3VicHJvY2Vzcy5ydW4oc2hsZXguc3BsaXQoImNsYXVkZSAtcCB4Iikp"),
|
||||
)
|
||||
|
||||
|
||||
def test_every_indirect_way_of_starting_claude_is_caught(tmp_path: Path) -> None:
|
||||
"""A9: 4 av 5 realistiske varianter slapp forbi. Hver plantes som kjent-positiv, én om
|
||||
gangen, og hver skal felle raden."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg.pop("roots", None)
|
||||
cfg["manifest"] = {"sentinel": "src/run.py"}
|
||||
_write(tmp_path / "src" / "run.py", "x = 1\n")
|
||||
assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN, "kontroll: ren flate"
|
||||
for name, snippet in _INDIRECT_VARIANTS:
|
||||
planted = tmp_path / "src" / f"{name}.py"
|
||||
_write(planted, gate.decode(snippet) + "\n")
|
||||
row = gate.score_no_claude_path(cfg, tmp_path)
|
||||
assert row.status == gate.RED, name
|
||||
assert any(name in x for x in row.exceptions), (name, row.exceptions)
|
||||
planted.unlink()
|
||||
assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN
|
||||
|
||||
|
||||
def test_the_row_states_the_limit_its_patterns_cannot_reach() -> None:
|
||||
row = gate.score_no_claude_path(_CONFIG["no_claude_path"], _REPO)
|
||||
assert any("satt sammen" in a or "miljøvariabel" in a for a in row.attests), row.attests
|
||||
|
||||
|
||||
# --- rad 4, 5 og 6: hver nevner med en navngitt kilde -----------------------------------------
|
||||
|
||||
|
||||
def test_row4_counts_only_checks_whose_named_source_resolves(tmp_path: Path) -> None:
|
||||
"""M=3 var en konstant i gatens egen konfig. Nå er hver sjekk bundet til et symbol i kilden."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_model_calls"]))
|
||||
row = gate.score_no_model_calls(cfg, {}, _SRC)
|
||||
assert (row.k, row.n, row.status) == (0, 3, gate.RED)
|
||||
for name, check in cfg["checks"].items():
|
||||
assert isinstance(check, dict) and "source" in check, name
|
||||
tree = ast.parse((_SRC / check["source"]["module"]).read_text(encoding="utf-8"))
|
||||
names = {
|
||||
n.name for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
} | {
|
||||
t.id
|
||||
for n in tree.body
|
||||
if isinstance(n, ast.Assign)
|
||||
for t in n.targets
|
||||
if isinstance(t, ast.Name)
|
||||
}
|
||||
assert check["source"]["symbol"] in names, name
|
||||
src = tmp_path / "tom"
|
||||
src.mkdir()
|
||||
row = gate.score_no_model_calls(cfg, {}, src)
|
||||
assert row.n == 0, "en sjekk uten kilde i koden teller ikke med i nevneren"
|
||||
assert len(row.exceptions) == 3
|
||||
|
||||
|
||||
def test_row5_counts_each_structural_requirement_on_its_own(tmp_path: Path) -> None:
|
||||
"""M=2 blandet profil, fabrikk og injeksjonssøm til ÉN enhet. Nå telles hvert krav for seg,
|
||||
og hvert av dem er et symbol i kilden."""
|
||||
row = gate.score_foundry(_CONFIG["foundry"], {}, _SRC)
|
||||
members = len(_CONFIG["foundry"]["profile"]["members"])
|
||||
evidence = len(_CONFIG["foundry"]["evidence"])
|
||||
assert row.n == members + 2 + evidence
|
||||
assert (row.k, row.status) == (members + 2, gate.RED)
|
||||
nodeid = _CONFIG["foundry"]["evidence"]["samme-skjema"][0]
|
||||
row = gate.score_foundry(_CONFIG["foundry"], {nodeid: "passed"}, _SRC)
|
||||
assert row.status == gate.GREEN
|
||||
|
||||
|
||||
def test_row5_denominator_follows_the_named_members(tmp_path: Path) -> None:
|
||||
cfg = json.loads(json.dumps(_CONFIG["foundry"]))
|
||||
cfg["profile"]["members"].append("FINNES_IKKE")
|
||||
row = gate.score_foundry(cfg, {}, _SRC)
|
||||
assert row.n == len(cfg["profile"]["members"]) + 2 + len(cfg["evidence"])
|
||||
assert any("FINNES_IKKE" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_row6_denominator_comes_from_the_artefacts_the_contract_names(tmp_path: Path) -> None:
|
||||
artefacts = _CONFIG["runbook"].get("artefacts")
|
||||
assert artefacts is not None, "M=2 var en konstant i koden, ikke noe kontrakten navngir"
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path)
|
||||
assert row.n == len(artefacts)
|
||||
|
||||
|
||||
# --- rad 6: attesteringen kan felles ----------------------------------------------------------
|
||||
|
||||
|
||||
def _runbook(root: Path) -> str:
|
||||
text = "# kjørebok\n\nsteg 1: naviger pakken\n"
|
||||
_write(root / _CONFIG["runbook"]["path"], text)
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _attestation_lines(root: Path, digest: str, **over: str) -> None:
|
||||
fields = {
|
||||
"kjørebok": _CONFIG["runbook"]["path"],
|
||||
"sjekksum": digest,
|
||||
"kjørt": "operatøren",
|
||||
"dato": "2026-09-19",
|
||||
}
|
||||
fields.update(over)
|
||||
_write(
|
||||
root / _CONFIG["runbook"]["attestation"],
|
||||
"".join(f"{k}: {v}\n" for k, v in fields.items()),
|
||||
)
|
||||
|
||||
|
||||
def test_a_valid_attestation_is_the_only_thing_that_turns_row6_green(tmp_path: Path) -> None:
|
||||
"""rc-0-kontrollen FØRST: en gate som nekter alt er like ubrukelig som en som nekter noe."""
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest)
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW)
|
||||
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN), row.exceptions
|
||||
|
||||
|
||||
def test_an_attestation_that_names_another_runbook_is_refused(tmp_path: Path) -> None:
|
||||
"""A7: «kjørebok: x» ga 2 av 2 GRØNN."""
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest, **{"kjørebok": "x"})
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("kjørebok" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_an_attestation_whose_checksum_does_not_match_the_runbook_is_refused(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""En attestering av en ANNEN versjon av kjøreboka er ikke en attestering av denne."""
|
||||
_runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, "0" * 64)
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("sjekksum" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_an_attestation_dated_in_the_future_is_refused(tmp_path: Path) -> None:
|
||||
"""A7: framtidsdato 3026-01-01 ga GRØNN. Samme regel som v1-gaten, gjenbrukt."""
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest, dato="3026-01-01")
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("framtiden" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_an_attestation_without_a_real_date_is_refused(tmp_path: Path) -> None:
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest, dato="x")
|
||||
row = gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("ISO" in x or "dato" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_an_attestation_with_a_bom_is_still_read(tmp_path: Path) -> None:
|
||||
"""v1-gaten TÅLER BOM — en editor som skriver den er ikke operatørens feil."""
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest)
|
||||
path = tmp_path / _CONFIG["runbook"]["attestation"]
|
||||
path.write_bytes(b"\xef\xbb\xbf" + path.read_bytes())
|
||||
assert gate.score_runbook(_CONFIG["runbook"], tmp_path, now=_NOW).status == gate.GREEN
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue