fix(b-gate): every denominator answers to a source outside the gate, and a name is no longer a door [skip-docs]
Row 1's M was a curated list of 13 in the gate's OWN b_gate.json. It is now DERIVED: run_path_calls reads what run.py::run_project actually calls (41 po-functions, re-counted here against PM's 39 — the two that differ are generate_via_llm and fresh_workflow, held out because they need a chat client). A call that is neither a declared step nor named-with-a-reason counts in the denominator WITHOUT a door, so the number cannot shrink in silence: removing a step from the contract now leaves N unchanged and names the orphan. The four outbox writers the run path uses and nobody had declared (write_prepass, write_parse_failures, write_proposal_reviews, write_debate_tools) are steps now; 28 calls are held out, each with its reason printed under the row. A door must be REGISTERED and PROVEN. entry["kind"] is read (it was read 0 times before) and must be one of three kinds the gate has code to verify: console-script in pyproject, module-main with its own __main__ guard, subcommand registered in the module's argparse. On top of that every step needs a named probe that calls the door and reads the artefact. The MCP sentence is struck from EXTERNAL_DOOR: it claimed a capability with no code behind it. Row 3 now scans the repo manifest (git ls-files, or the tree itself in an extract), not a hand list of 11 roots: 512 published files instead of 433, so main.py, examples/, spikes/, contexts/, CLAUDE.md and llms.txt are inside the guard for the first time. Three new patterns catch the indirect invocations that walked past the old six — absolute path, list in a variable, constant, shell string — 5 of 5 of PM's variants are refused now, with 0 false positives measured over the whole surface. An empty surface is IKKE MAALT, not GREEN: the row demands a sentinel file and prints the file count and the manifest it read. Rows 4, 5 and 6 get sources for their denominators. Row 4 counts only checks whose named source symbol resolves in the code. Row 5 counts each structural requirement on its own (2 profile members + factory + seam + probe = 5) instead of collapsing three into one unit. Row 6's N comes from the artefacts the contract names, and the attestation is VALIDATED: it must name the contract's runbook, carry its sha256, say who ran it, and bear a real ISO date that is not in the future — v1_gate's own date rule, reused, BOM tolerated as there. Measured in a scratch clone (/tmp/claude-po/bgate-mut): 12 of 12 mutants felled, control 65 of 65. All six of PM's broken attacks reproduced as refused, with the rc-0 control green. No row got greener: 3 of 17 (was 3 of 13), 0 of 2, GREEN, 0 of 3, 4 of 5 (was 1 of 2), IKKE MAALT. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
332eb5965b
commit
59f35fde22
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 734 additions and 221 deletions
|
|
@ -41,10 +41,10 @@ _NOW = datetime(2026, 9, 19, 12, 0, tzinfo=timezone.utc)
|
|||
|
||||
#: Repoets egne tall ved skriving (19.09.2026), talt av armene under mot KILDEN. De står her for
|
||||
#: at en stille endring i kjørestien skal vise seg som en rød test og ikke som et nytt tall.
|
||||
_STEPS_TODAY = 13
|
||||
_STEPS_TODAY = 17
|
||||
_EXTERNAL_TODAY = 3
|
||||
_ROLES_TODAY = ("proposer", "checker")
|
||||
_PATTERNS_TODAY = 6
|
||||
_PATTERNS_TODAY = 9
|
||||
|
||||
|
||||
def _write(path: Path, text: str) -> None:
|
||||
|
|
@ -100,16 +100,18 @@ def _independently_counted_steps() -> tuple[int, int]:
|
|||
def test_the_step_denominator_equals_an_independent_count_of_the_source() -> None:
|
||||
resolved, called = _independently_counted_steps()
|
||||
assert resolved == called == _STEPS_TODAY, "kjørestien har endret seg — oppdater nevneren"
|
||||
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
|
||||
row = gate.score_toolbox(_CONFIG["steps"], _SRC, _CONFIG["run_path"], {}, _REPO)
|
||||
assert row.n == _STEPS_TODAY
|
||||
|
||||
|
||||
def test_row1_today_is_red_with_three_of_thirteen_steps_callable_from_outside() -> None:
|
||||
row = gate.score_toolbox(_CONFIG["steps"], _SRC)
|
||||
def test_row1_today_is_red_with_three_of_seventeen_steps_callable_from_outside() -> None:
|
||||
passing = {n: "passed" for step in _CONFIG["steps"] for n in step.get("probe", ())}
|
||||
row = gate.score_toolbox(_CONFIG["steps"], _SRC, _CONFIG["run_path"], passing, _REPO)
|
||||
assert (row.k, row.n, row.status) == (_EXTERNAL_TODAY, _STEPS_TODAY, gate.RED)
|
||||
named = {x.split(":")[0] for x in row.exceptions}
|
||||
assert "rundebinding" not in named and "gate" not in named
|
||||
assert "validering" in named and "utboks" in named
|
||||
assert "prepass-artefakt" in named and "parse-feil" in named
|
||||
|
||||
|
||||
def _fake_src(root: Path, *, chat_client: bool, symbol: str = "gjoer_noe") -> Path:
|
||||
|
|
@ -126,11 +128,17 @@ def _fake_src(root: Path, *, chat_client: bool, symbol: str = "gjoer_noe") -> Pa
|
|||
"def _hjelper():\n" + body + "\n\n"
|
||||
"def kjor():\n"
|
||||
f" _hjelper()\n return {symbol}()\n\n\n"
|
||||
"def main(argv=None):\n return kjor()\n",
|
||||
"def main(argv=None):\n return kjor()\n\n\n"
|
||||
'if __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
return src
|
||||
|
||||
|
||||
#: En dør er registrert OG bevist med atferd. De konstruerte kildene under bærer begge deler.
|
||||
_FAKE_PROBE = "tests/test_konstruert.py::test_doeren_skriver_artefaktet"
|
||||
_FAKE_PASSED = {_FAKE_PROBE: "passed"}
|
||||
|
||||
|
||||
def _one_step(symbol: str = "gjoer_noe") -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
|
|
@ -139,21 +147,22 @@ def _one_step(symbol: str = "gjoer_noe") -> list[dict[str, Any]]:
|
|||
"module": "steg.py",
|
||||
"symbol": symbol,
|
||||
"driver": {"module": "driver.py", "scope": "kjor"},
|
||||
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
||||
"entry": {"kind": "module-main", "module": "driver.py", "scope": "main"},
|
||||
"probe": [_FAKE_PROBE],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_a_step_reachable_from_a_cli_entry_without_a_chat_client_counts(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
row = gate.score_toolbox(_one_step(), src)
|
||||
row = gate.score_toolbox(_one_step(), src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n, row.status) == (1, 1, gate.GREEN)
|
||||
|
||||
|
||||
def test_m2_a_step_that_needs_a_chat_client_is_not_callable_from_outside(tmp_path: Path) -> None:
|
||||
"""M-2: fjernes chatklient-sjekken, hopper k fra 0 til 1 og denne armen feller mutanten."""
|
||||
src = _fake_src(tmp_path, chat_client=True)
|
||||
row = gate.score_toolbox(_one_step(), src)
|
||||
row = gate.score_toolbox(_one_step(), src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n, row.status) == (0, 1, gate.RED)
|
||||
assert any("chatklient" in x for x in row.exceptions)
|
||||
|
||||
|
|
@ -170,10 +179,10 @@ def test_m1_the_denominator_follows_the_source_and_is_not_the_config_length(
|
|||
"module": "steg.py",
|
||||
"symbol": "finnes_ikke",
|
||||
"driver": {"module": "driver.py", "scope": "kjor"},
|
||||
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
||||
"entry": {"kind": "module-main", "module": "driver.py", "scope": "main"},
|
||||
}
|
||||
]
|
||||
row = gate.score_toolbox(steps, src)
|
||||
row = gate.score_toolbox(steps, src, None, _FAKE_PASSED, tmp_path)
|
||||
assert row.n == 1, "et steg uten symbol i kilden er ikke et steg"
|
||||
assert any("borte" in x for x in row.exceptions)
|
||||
|
||||
|
|
@ -188,10 +197,10 @@ def test_a_step_the_driver_no_longer_calls_leaves_the_denominator(tmp_path: Path
|
|||
"module": "steg.py",
|
||||
"symbol": "ubrukt",
|
||||
"driver": {"module": "driver.py", "scope": "kjor"},
|
||||
"entry": {"kind": "cli", "module": "driver.py", "scope": "main"},
|
||||
"entry": {"kind": "module-main", "module": "driver.py", "scope": "main"},
|
||||
}
|
||||
]
|
||||
row = gate.score_toolbox(steps, src)
|
||||
row = gate.score_toolbox(steps, src, None, _FAKE_PASSED, tmp_path)
|
||||
assert row.n == 1
|
||||
|
||||
|
||||
|
|
@ -199,7 +208,7 @@ def test_an_entry_without_a_main_is_no_entry(tmp_path: Path) -> None:
|
|||
src = _fake_src(tmp_path, chat_client=False)
|
||||
text = (src / "driver.py").read_text(encoding="utf-8").replace("def main(", "def _main(")
|
||||
_write(src / "driver.py", text)
|
||||
row = gate.score_toolbox(_one_step(), src)
|
||||
row = gate.score_toolbox(_one_step(), src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("main" in x for x in row.exceptions)
|
||||
|
||||
|
|
@ -263,12 +272,18 @@ def test_every_pattern_hits_its_own_known_positive_and_misses_its_known_negative
|
|||
assert not rx.search(gate.decode(spec["known_negative_b64"])), spec["id"]
|
||||
|
||||
|
||||
def _clean_surface(tmp_path: Path) -> dict[str, Any]:
|
||||
"""En ren, konstruert flate med sin egen sentinel — nevneren er aldri 0 her."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg["manifest"] = {"sentinel": "src/rent.py", "skip_dirs": [".git", "__pycache__"]}
|
||||
_write(tmp_path / "src" / "rent.py", "print('ingenting her')\n")
|
||||
return cfg
|
||||
|
||||
|
||||
def test_m3_the_denominator_is_the_number_of_patterns_that_can_measure(tmp_path: Path) -> None:
|
||||
"""M-3: fjernes et mønster, faller nevneren — og et mønster som ikke treffer sin egen
|
||||
kjent-positive telles ikke som en vakt."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg["roots"] = ["src"]
|
||||
_write(tmp_path / "src" / "rent.py", "print('ingenting her')\n")
|
||||
cfg = _clean_surface(tmp_path)
|
||||
row = gate.score_no_claude_path(cfg, tmp_path)
|
||||
assert (row.k, row.n, row.status) == (_PATTERNS_TODAY, _PATTERNS_TODAY, gate.GREEN)
|
||||
|
||||
|
|
@ -283,8 +298,8 @@ def test_m3_the_denominator_is_the_number_of_patterns_that_can_measure(tmp_path:
|
|||
|
||||
|
||||
def test_a_planted_path_to_claude_turns_the_row_red_for_that_pattern(tmp_path: Path) -> None:
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg["roots"] = ["src"]
|
||||
cfg = _clean_surface(tmp_path)
|
||||
assert gate.score_no_claude_path(cfg, tmp_path).status == gate.GREEN, "rc-0-kontrollen først"
|
||||
for spec in cfg["patterns"]:
|
||||
_write(
|
||||
tmp_path / "src" / f"{spec['id']}.py", gate.decode(spec["known_positive_b64"]) + "\n"
|
||||
|
|
@ -297,8 +312,7 @@ def test_a_planted_path_to_claude_turns_the_row_red_for_that_pattern(tmp_path: P
|
|||
def test_markdown_counts_only_inside_fenced_blocks(tmp_path: Path) -> None:
|
||||
"""«docs med kjørbare kommandoer»: en pakkenavn-omtale i brødtekst er ikke en vei til Claude;
|
||||
den samme linja i en kodeblokk er en kommando noen kan kjøre."""
|
||||
cfg = json.loads(json.dumps(_CONFIG["no_claude_path"]))
|
||||
cfg["roots"] = ["docs"]
|
||||
cfg = _clean_surface(tmp_path)
|
||||
spec = next(s for s in cfg["patterns"] if s["id"] == "maf-anthropic")
|
||||
line = gate.decode(spec["known_positive_b64"])
|
||||
_write(tmp_path / "docs" / "prosa.md", f"- vurdert og forkastet: `{line}` (aldri tatt inn)\n")
|
||||
|
|
@ -317,7 +331,7 @@ def test_row3_on_the_real_published_surface_is_green_today() -> None:
|
|||
|
||||
|
||||
def test_the_surface_names_its_denominator_in_files_not_in_prose() -> None:
|
||||
files = gate.surface_files(_REPO, _CONFIG["no_claude_path"]["roots"])
|
||||
files, _ = gate.published_files(_REPO, _manifest())
|
||||
assert len(files) > 100, "en tom flate ville gitt 0 treff uten å måle noe"
|
||||
assert any(f.name == "run.py" for f in files)
|
||||
|
||||
|
|
@ -328,22 +342,23 @@ def test_the_surface_names_its_denominator_in_files_not_in_prose() -> None:
|
|||
|
||||
|
||||
def test_row4_is_red_today_and_names_all_three_checks() -> None:
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {}, _SRC)
|
||||
assert (row.k, row.n, row.status) == (0, 3, gate.RED)
|
||||
assert len(row.exceptions) == 3
|
||||
|
||||
|
||||
def test_row4_counts_only_checks_whose_probes_pass() -> None:
|
||||
checks = _CONFIG["no_model_calls"]["checks"]
|
||||
outcomes = {n: "passed" for ns in checks.values() for n in ns}
|
||||
assert gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes).status == gate.GREEN
|
||||
outcomes[checks["fullfører"][0]] = "skipped"
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes)
|
||||
outcomes = {n: "passed" for c in checks.values() for n in c["probe"]}
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes, _SRC)
|
||||
assert row.status == gate.GREEN
|
||||
outcomes[checks["fullfører"]["probe"][0]] = "skipped"
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], outcomes, _SRC)
|
||||
assert (row.k, row.status) == (2, gate.RED)
|
||||
|
||||
|
||||
def test_row4_states_what_the_budget_guard_is_in_b_instead_of_pretending() -> None:
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {})
|
||||
row = gate.score_no_model_calls(_CONFIG["no_model_calls"], {}, _SRC)
|
||||
assert any("budsjett" in a.casefold() for a in row.attests)
|
||||
assert "valgt" in gate.BUDGET_IN_B.casefold()
|
||||
|
||||
|
|
@ -368,11 +383,12 @@ def test_the_foundry_check_fails_when_the_azure_profile_is_gone(tmp_path: Path)
|
|||
|
||||
|
||||
def test_row5_is_red_until_the_schema_comparison_probe_passes() -> None:
|
||||
structural = len(_CONFIG["foundry"]["profile"]["members"]) + 2
|
||||
row = gate.score_foundry(_CONFIG["foundry"], {}, _SRC)
|
||||
assert (row.k, row.n, row.status) == (1, 2, gate.RED)
|
||||
assert (row.k, row.n, row.status) == (structural, structural + 1, gate.RED)
|
||||
nodeid = _CONFIG["foundry"]["evidence"]["samme-skjema"][0]
|
||||
row = gate.score_foundry(_CONFIG["foundry"], {nodeid: "passed"}, _SRC)
|
||||
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
|
||||
assert (row.k, row.n, row.status) == (structural + 1, structural + 1, gate.GREEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
|
@ -394,20 +410,26 @@ def test_row6_is_still_not_measured_without_the_operators_attestation(tmp_path:
|
|||
|
||||
def test_row6_goes_green_only_with_every_required_key(tmp_path: Path) -> None:
|
||||
cfg = _CONFIG["runbook"]
|
||||
_write(tmp_path / cfg["path"], "# kjørebok\n")
|
||||
digest = _runbook(tmp_path)
|
||||
attest = tmp_path / cfg["attestation"]
|
||||
_write(attest, "kjørebok: docs/kjoerebok-verktoeykassen.md\nkjørt: operatøren\n")
|
||||
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
|
||||
_write(attest, "kjørebok: x\nkjørt: operatøren\ndato: 2026-09-20\n")
|
||||
row = gate.score_runbook(cfg, tmp_path)
|
||||
_write(attest, f"kjørebok: {cfg['path']}\nsjekksum: {digest}\nkjørt: operatøren\n")
|
||||
row = gate.score_runbook(cfg, tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("dato" in x for x in row.exceptions)
|
||||
_attestation_lines(tmp_path, digest)
|
||||
row = gate.score_runbook(cfg, tmp_path, now=_NOW)
|
||||
assert (row.k, row.n, row.status) == (2, 2, gate.GREEN)
|
||||
|
||||
|
||||
def test_a_duplicated_key_in_the_attestation_is_red(tmp_path: Path) -> None:
|
||||
cfg = _CONFIG["runbook"]
|
||||
_write(tmp_path / cfg["path"], "# kjørebok\n")
|
||||
_write(tmp_path / cfg["attestation"], "kjørebok: x\nkjørt: a\nkjørt: b\ndato: 2026-09-20\n")
|
||||
assert gate.score_runbook(cfg, tmp_path).status == gate.NOT_MEASURED
|
||||
digest = _runbook(tmp_path)
|
||||
_attestation_lines(tmp_path, digest)
|
||||
path = tmp_path / cfg["attestation"]
|
||||
path.write_text(path.read_text(encoding="utf-8") + "kjørt: en annen\n", encoding="utf-8")
|
||||
row = gate.score_runbook(cfg, tmp_path, now=_NOW)
|
||||
assert row.status == gate.NOT_MEASURED
|
||||
assert any("to ganger" in x for x in row.exceptions)
|
||||
|
||||
|
||||
def test_m5_the_gate_never_writes_the_attestation_itself(tmp_path: Path) -> None:
|
||||
|
|
@ -686,13 +708,11 @@ def test_a_console_script_is_a_door_only_when_the_manifest_registers_it(tmp_path
|
|||
def test_a_module_main_is_a_door_only_with_the_dunder_guard(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
entry = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry)[0] is False
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
assert gate.registered_entry(src, tmp_path, "fakesrc", entry) == (True, "")
|
||||
stripped = (src / "driver.py").read_text(encoding="utf-8")
|
||||
_write(src / "driver.py", stripped[: stripped.index("if __name__")])
|
||||
ok, why = gate.registered_entry(src, tmp_path, "fakesrc", entry)
|
||||
assert ok is False and "__main__" in why
|
||||
|
||||
|
||||
def test_a_subcommand_is_a_door_only_when_the_parser_registers_that_name(tmp_path: Path) -> None:
|
||||
|
|
@ -719,7 +739,7 @@ def test_a_stub_main_is_not_a_door_without_a_registered_entry(tmp_path: Path) ->
|
|||
_fake_pyproject(tmp_path, {})
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "console-script", "module": "driver.py", "scope": "main"}
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
row = gate.score_toolbox(steps, src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("pyproject" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
|
@ -729,32 +749,21 @@ def test_a_registered_door_without_a_passing_behaviour_probe_does_not_count(
|
|||
) -> None:
|
||||
"""«Kallet med en fixture gir det dokumenterte artefaktet» — et navn er ikke en atferd."""
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
steps[0]["probe"] = ["tests/test_noe.py::test_doeren_skriver_artefaktet"]
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("probe" in x for x in row.exceptions), row.exceptions
|
||||
passed = {"tests/test_noe.py::test_doeren_skriver_artefaktet": "passed"}
|
||||
row = gate.score_toolbox(steps, src, None, passed, tmp_path)
|
||||
assert any("atferdsprobe" in x for x in row.exceptions), row.exceptions
|
||||
row = gate.score_toolbox(steps, src, None, {_FAKE_PROBE: "skipped"}, tmp_path)
|
||||
assert row.k == 0, "hoppet over er ikke bestått"
|
||||
row = gate.score_toolbox(steps, src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n, row.status) == (1, 1, gate.GREEN)
|
||||
|
||||
|
||||
def test_a_step_without_any_probe_is_never_a_door(tmp_path: Path) -> None:
|
||||
src = _fake_src(tmp_path, chat_client=False)
|
||||
_write(
|
||||
src / "driver.py",
|
||||
(src / "driver.py").read_text(encoding="utf-8")
|
||||
+ '\n\nif __name__ == "__main__":\n raise SystemExit(main())\n',
|
||||
)
|
||||
steps = _one_step()
|
||||
steps[0]["entry"] = {"kind": "module-main", "module": "driver.py", "scope": "main"}
|
||||
row = gate.score_toolbox(steps, src, None, {}, tmp_path)
|
||||
steps[0].pop("probe")
|
||||
row = gate.score_toolbox(steps, src, None, _FAKE_PASSED, tmp_path)
|
||||
assert (row.k, row.n) == (0, 1)
|
||||
assert any("ingen atferdsprobe" in x for x in row.exceptions), row.exceptions
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue