test(b-gate): 34 red tests for the gate that measures po as a toolbox Claude Code drives
Operator decision 19.09.2026: in development and test Claude Code LEADS and portfolio-optimiser is the toolbox. po never calls Claude; production stays on Foundry. This commit writes the measurement RED — the contract first, the capability later. Six rows, each with a denominator taken from the SOURCE and counted independently here: 1. toolbox complete — every deterministic step of the run path, anchored to (module, symbol) and to the scope that calls it; k = steps reachable from a CLI entry WITHOUT a chat client. 2. what the model delivered can be delivered from outside — denominator read off workflow._MAKER_CHECKER_ROLES, k = roles with passing named probes. 3. po has no path to Claude — six patterns, each carrying its own known-positive AND known-negative fixture. Pattern text is base64 in the config so the gate's own contract cannot register as a hit against the surface it scans. 4. no model calls in toolbox mode — three named probes. 5. the Foundry path untouched — profiles, factory and injection seam read from source, plus a schema-comparison probe. 6. the runbook exists — NOT MEASURED until the operator attests, and the gate never writes that attestation itself. Red now for the reason that matters: the tests fail to import a module that does not exist yet. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
86c61a343b
commit
0a784065d0
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
2 changed files with 795 additions and 0 deletions
317
src/portfolio_optimiser/evals/b_gate.json
Normal file
317
src/portfolio_optimiser/evals/b_gate.json
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
{
|
||||
"outcome": "En Claude Code-økt driver en hel analyse med po som verktøykasse, uten at po gjør ett eneste modellkall.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "pakke",
|
||||
"label": "last kunnskapspakken og naviger den",
|
||||
"module": "okf.py",
|
||||
"symbol": "navigate_bundle",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "kostnadsgrunnlag",
|
||||
"label": "slå opp kostnadsgrunnlaget",
|
||||
"module": "okf.py",
|
||||
"symbol": "derive_cost_baseline",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "kontekst",
|
||||
"label": "hent kontekst-utdrag til forslaget",
|
||||
"module": "datasource.py",
|
||||
"symbol": "retrieve_chunks",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "prepass",
|
||||
"label": "slipp inn prepass-nyttelasten",
|
||||
"module": "prepass.py",
|
||||
"symbol": "admit_payload",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "validering",
|
||||
"label": "valider forslaget, metodetak inkludert",
|
||||
"module": "validator.py",
|
||||
"symbol": "validate_proposal",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "evaluate_mandate_candidates"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "verdict-id",
|
||||
"label": "mint verdict_id fra IR-ens trekk",
|
||||
"module": "verdicts.py",
|
||||
"symbol": "verdict_key",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dom",
|
||||
"label": "fang fagpersonens dom som en Verdict",
|
||||
"module": "verdicts.py",
|
||||
"symbol": "capture_verdict",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "kjørekonfig",
|
||||
"label": "skriv kjøringens konfigurasjon",
|
||||
"module": "outbox.py",
|
||||
"symbol": "write_run_config",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "coverage",
|
||||
"label": "skriv coverage-raden per tilnærming",
|
||||
"module": "outbox.py",
|
||||
"symbol": "write_coverage",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "utboks",
|
||||
"label": "skriv utboksen",
|
||||
"module": "outbox.py",
|
||||
"symbol": "write_outbox",
|
||||
"driver": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "run.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rundebinding",
|
||||
"label": "bind en utboks til en rundekatalog",
|
||||
"module": "evals/round_builder.py",
|
||||
"symbol": "build_round",
|
||||
"driver": {
|
||||
"module": "evals/round_builder.py",
|
||||
"scope": "main"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "evals/round_builder.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rapport",
|
||||
"label": "skriv rundens rapport på norsk",
|
||||
"module": "evals/round_builder.py",
|
||||
"symbol": "build_report",
|
||||
"driver": {
|
||||
"module": "evals/round_builder.py",
|
||||
"scope": "build_round"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "evals/round_builder.py",
|
||||
"scope": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "gate",
|
||||
"label": "døm rundene med v1-gaten",
|
||||
"module": "evals/v1_gate.py",
|
||||
"symbol": "evaluate",
|
||||
"driver": {
|
||||
"module": "evals/v1_gate.py",
|
||||
"scope": "main"
|
||||
},
|
||||
"entry": {
|
||||
"kind": "cli",
|
||||
"module": "evals/v1_gate.py",
|
||||
"scope": "main"
|
||||
}
|
||||
}
|
||||
],
|
||||
"roles": {
|
||||
"source": {
|
||||
"module": "workflow.py",
|
||||
"symbol": "_MAKER_CHECKER_ROLES"
|
||||
},
|
||||
"evidence": {
|
||||
"proposer": [
|
||||
"tests/test_b_toolbox_probes.py::test_proposer_from_outside_mints_byte_identical_artefacts",
|
||||
"tests/test_b_toolbox_probes.py::test_proposer_from_outside_is_rejected_with_the_same_reason"
|
||||
],
|
||||
"checker": [
|
||||
"tests/test_b_toolbox_probes.py::test_checker_from_outside_mints_byte_identical_artefacts",
|
||||
"tests/test_b_toolbox_probes.py::test_checker_from_outside_is_rejected_with_the_same_reason"
|
||||
]
|
||||
}
|
||||
},
|
||||
"no_claude_path": {
|
||||
"roots": [
|
||||
"src",
|
||||
"scripts",
|
||||
"shared",
|
||||
"docs",
|
||||
"tests",
|
||||
"pyproject.toml",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"DEPLOY.md",
|
||||
"CONTRIBUTING.md",
|
||||
"SECURITY.md"
|
||||
],
|
||||
"patterns": [
|
||||
{
|
||||
"id": "prosess",
|
||||
"label": "prosess-start av claude fra kode",
|
||||
"pattern_b64": "KHN1YnByb2Nlc3NcLihydW58UG9wZW58Y2FsbHxjaGVja19vdXRwdXQpfG9zXC5zeXN0ZW18b3NcLmV4ZWNcdyspXHMqXChccypcWz9ccypbIiddY2xhdWRlWyInXQ==",
|
||||
"known_positive_b64": "c3VicHJvY2Vzcy5ydW4oWyJjbGF1ZGUiLCAiLXAiLCAiYW5hbHlzZXIgcHJvc2pla3RldCJdKQ==",
|
||||
"known_negative_b64": "c3VicHJvY2Vzcy5ydW4oWyJweXRob24iLCAiLW0iLCAicG9ydGZvbGlvX29wdGltaXNlci5ydW4iLCAiUDEiXSk="
|
||||
},
|
||||
{
|
||||
"id": "kommando",
|
||||
"label": "claude som kommando i en kjørbar linje",
|
||||
"pattern_b64": "KF58W1xzOyZ8YCQoXSljbGF1ZGVccysoLXBcYnwtLXByaW50XGJ8LS1wZXJtaXNzaW9uLW1vZGVcYnwtLWRhbmdlcm91c2x5LXNraXAtcGVybWlzc2lvbnNcYnwtLW1vZGVsXHMp",
|
||||
"known_positive_b64": "Y2xhdWRlIC1wICJrasO4ciBhbmFseXNlbiIgLS1tb2RlbCBvcHVz",
|
||||
"known_negative_b64": "cHl0aG9uIC1tIHBvcnRmb2xpb19vcHRpbWlzZXIucnVuIFAxIC0tcHJvZmlsZSBsb2NhbCAtLW1heC1yb3VuZHMgMg=="
|
||||
},
|
||||
{
|
||||
"id": "sdk-import",
|
||||
"label": "import av Claude Agent SDK",
|
||||
"pattern_b64": "XlxzKihmcm9tfGltcG9ydClccysoY2xhdWRlX2FnZW50X3Nka3xjbGF1ZGVfY29kZV9zZGspXGI=",
|
||||
"known_positive_b64": "aW1wb3J0IGNsYXVkZV9hZ2VudF9zZGs=",
|
||||
"known_negative_b64": "aW1wb3J0IGFnZW50X2ZyYW1ld29yaw=="
|
||||
},
|
||||
{
|
||||
"id": "maf-anthropic",
|
||||
"label": "MAF sin Anthropic-provider",
|
||||
"pattern_b64": "KGFnZW50X2ZyYW1ld29ya1wuYW50aHJvcGljfGFnZW50W18tXWZyYW1ld29ya1tfLV1jbGF1ZGV8QW50aHJvcGljQ2hhdENsaWVudCk=",
|
||||
"known_positive_b64": "ZnJvbSBhZ2VudF9mcmFtZXdvcmsuYW50aHJvcGljIGltcG9ydCBBbnRocm9waWNDaGF0Q2xpZW50",
|
||||
"known_negative_b64": "ZnJvbSBhZ2VudF9mcmFtZXdvcmsuYXp1cmUgaW1wb3J0IEF6dXJlQUlBZ2VudENsaWVudA=="
|
||||
},
|
||||
{
|
||||
"id": "oauth",
|
||||
"label": "lesing av Claude-abonnementets legitimasjon",
|
||||
"pattern_b64": "KENMQVVERV9DT0RFX09BVVRIX1RPS0VOfFwuY2xhdWRlL1wuY3JlZGVudGlhbHNcLmpzb258Y2xhdWRlXC5haS9vYXV0aCk=",
|
||||
"known_positive_b64": "dG9rZW4gPSBvcy5lbnZpcm9uWyJDTEFVREVfQ09ERV9PQVVUSF9UT0tFTiJd",
|
||||
"known_negative_b64": "dG9rZW4gPSBvcy5lbnZpcm9uWyJBWlVSRV9PUEVOQUlfQVBJX0tFWSJd"
|
||||
},
|
||||
{
|
||||
"id": "proxy",
|
||||
"label": "proxy mot Anthropics API",
|
||||
"pattern_b64": "KEFOVEhST1BJQ19CQVNFX1VSTHxBTlRIUk9QSUNfQVBJX0tFWXxhcGlcLmFudGhyb3BpY1wuY29tKQ==",
|
||||
"known_positive_b64": "b3MuZW52aXJvblsiQU5USFJPUElDX0JBU0VfVVJMIl0gPSBlbmRwb2ludA==",
|
||||
"known_negative_b64": "b3MuZW52aXJvblsiQVpVUkVfT1BFTkFJX0VORFBPSU5UIl0gPSBlbmRwb2ludA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
"no_model_calls": {
|
||||
"checks": {
|
||||
"fullfører": [
|
||||
"tests/test_b_toolbox_probes.py::test_the_whole_run_completes_with_the_chat_client_raising"
|
||||
],
|
||||
"rundekatalog": [
|
||||
"tests/test_b_toolbox_probes.py::test_the_toolbox_outbox_binds_to_a_scratch_round_directory"
|
||||
],
|
||||
"gate-form-ok": [
|
||||
"tests/test_b_toolbox_probes.py::test_v1_gate_reads_that_round_as_form_ok_not_proven"
|
||||
]
|
||||
}
|
||||
},
|
||||
"foundry": {
|
||||
"profiles": [
|
||||
"AZURE",
|
||||
"LOCAL"
|
||||
],
|
||||
"factory": {
|
||||
"module": "run.py",
|
||||
"symbol": "_default_factory"
|
||||
},
|
||||
"seam": {
|
||||
"module": "run.py",
|
||||
"scope": "run_project",
|
||||
"parameter": "client_factory"
|
||||
},
|
||||
"evidence": {
|
||||
"samme-skjema": [
|
||||
"tests/test_b_toolbox_probes.py::test_both_modes_write_the_same_artefact_family"
|
||||
]
|
||||
}
|
||||
},
|
||||
"runbook": {
|
||||
"path": "docs/kjoerebok-verktoeykassen.md",
|
||||
"attestation": "v1-rounds/kjoerebok-attestering.txt",
|
||||
"keys": [
|
||||
"kjørebok",
|
||||
"kjørt",
|
||||
"dato"
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue