open/portfolio-optimiser
1
0
Fork 0
Code Issues Releases 4 Activity

test(v1-gate): the v1 gate, written RED

Browse source 
One command says how far the repo is from v1, row by row, with an exit
code: rounds with a real domain expert 0/3, traced measurable change 0/3,
feedback types with a way in and an action 3/8 (1, 3, 7), round 3 report
kept - none, MAF points with a green type pointer 0/8 (list not approved),
validated without the approach's own declaration 10/10 in stress round 6,
and `named` 1/20 as a diagnosis that never moves the exit code.

The gate defines the contract (a fixed rounds directory, gitignored by
default), not the generator. Rows 3 and 6 run named tests with
--runxfail; the red probes are xfail(strict=True) so the suite stays green
while the gap is real. No product code changed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-17 15:31:48 +02:00
commit 83c94e4fb6
7 changed files with 1397 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

87
src/portfolio_optimiser/evals/v1_gate.json Normal file
View file

@ -0,0 +1,87 @@
{
"rounds_required": 3,
"keep_threshold": 0.8,
"ai_authored": ["docs/ekspert-svar.md"],
"feedback_types": {
"1": {
"label": "feil - rett en feil i forslaget",
"evidence": [
"tests/test_proposal_review_loop_loadbearing.py::test_t13_the_flag_answers_the_review_from_a_real_argv_and_the_answer_is_used"
]
},
"2": {
"label": "bort - ta en retning bort",
"evidence": ["tests/test_v1_probes.py::test_type_2_remove_a_direction_has_a_typed_door"]
},
"3": {
"label": "vinklinger - nye vinklinger",
"evidence": ["tests/test_mandate_cli.py::test_run_settles_against_the_mandate_afterwards"]
},
"4": {
"label": "lette paa krav",
"evidence": ["tests/test_v1_probes.py::test_type_4_relax_a_requirement_has_a_door"]
},
"5": {
"label": "konseptgrafer",
"evidence": ["tests/test_v1_probes.py::test_type_5_edit_the_concept_graph_has_a_door"]
},
"6": {
"label": "skills per analyse",
"evidence": ["tests/test_v1_probes.py::test_type_6_skills_per_analysis_has_a_door"]
},
"7": {
"label": "MCP - verktoey i debatten",
"evidence": [
"tests/test_mcp_run_loadbearing.py::test_configured_server_becomes_a_tool_the_agents_have",
"tests/test_b4_mcp_call_trace_loadbearing.py::test_a_called_mcp_tool_is_recorded_in_provenance"
]
},
"8": {
"label": "inline kontekst",
"evidence": ["tests/test_v1_probes.py::test_type_8_inline_context_has_a_door"]
}
},
"row6_evidence": [
"tests/test_v1_probes.py::test_row6_an_approach_that_declared_nothing_cannot_be_validated",
"tests/test_v1_probes.py::test_row6_a_run_level_declaration_does_not_stand_in_for_the_approach"
],
"maf_points": {
"approved": false,
"points": [
{"u_id": "U13", "construct": "MagenticPlanReviewResponse", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "_drive"}, "types": [1, 2]},
{"u_id": "U9", "construct": "ContextProvider", "package": "agent_framework",
"callsite": {"module": "verdicts.py", "scope": "ExpeLContextProvider"}, "types": [1, 8]},
{"u_id": "U12", "construct": "FileCheckpointStorage", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "checkpoint_storage"}, "types": [1]},
{"u_id": "U4", "construct": "MagenticBuilder", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "fresh_exploration_workflow"}, "types": [3]},
{"u_id": "U7", "construct": "tool", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "navigator_tools"}, "types": [4]},
{"u_id": "U11", "construct": "TextSpanRegion", "package": "agent_framework",
"callsite": {"module": "provenance.py", "scope": "ProvenanceStamp"}, "types": [5]},
{"u_id": "U5", "construct": "SkillsProvider", "package": "agent_framework",
"callsite": null, "types": [6]},
{"u_id": "U6", "construct": "MCPStdioTool", "package": "agent_framework",
"callsite": {"module": "mcp_tools.py", "scope": "build_mcp_tools"}, "types": [7]}
]
},
"stress_evidence": {
"label": "stressrunde 6",
"root": "scratchpad/p22-stress",
"runs": [
{"context": "contexts/gate-nordvik-2027", "outbox": "gate-nordvik-2027",
"run_id": "gate-nordvik-2027-08"},
{"context": "contexts/tunnel-hauglia-2027", "outbox": "tunnel-hauglia-2027",
"run_id": "tunnel-hauglia-2027-08"},
{"context": "contexts/fv412-dekkefornyelse-2027", "outbox": "fv412-dekkefornyelse-2027",
"run_id": "fv412-dekkefornyelse-2027-08"},
{"context": "contexts/kontrakt-sorasen-2027", "outbox": "kontrakt-sorasen-2027",
"run_id": "kontrakt-sorasen-2027-08"},
{"context": "contexts/dekke-og-kontrakt-lindaas-2027", "outbox": "lindaas",
"run_id": "lindaas-04-vegnormal-n200-2024", "bundle": "vegnormal-n200-2024"},
{"context": "contexts/dekke-og-kontrakt-lindaas-2027", "outbox": "lindaas",
"run_id": "lindaas-04-vegnormal-r761-2025", "bundle": "vegnormal-r761-2025"}
]
}
}