portfolio-optimiser/src/portfolio_optimiser/evals/v1_gate.json
Kjell Tore Guttormsen 9825b2677c test(v1-gate): harden the gate against a handwritten green
An independent review made rows 1, 2 and 4 green from a handwritten
directory in a minute, and 10 of 20 mutants survived the gate's tests.

Rounds now need a new point and their own ids, a timezone-aware given_at
in order, and a report the feedback was given on; every outcome must name
a run whose own coverage confirms (a)-(d), the feedback must fall between
the two runs, and a NOK change under 1 % is noise. Row 4 counts content
lines kept unchanged and in order, shows the expert's additions, and calls
a byte-identical copy untouched unless round 3 acknowledges it. Row 6
counts the runs' own proposals. Types 3 and 7 are proven through the real
flags with the action in the result (still 3 of 8). The contract numbers
and the evidence register are pinned to their source. Every run prints
that rows 1-2 cannot prove who wrote the feedback. A rounds directory
inside the repo that git would commit, and a missing stress or bundle
root, are usage errors.

The review's 20 mutants, re-run: 20 of 20 killed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-17 17:53:40 +02:00

92 lines
4.2 KiB
JSON

{
"rounds_required": 3,
"keep_threshold": 0.8,
"ai_authored": ["docs/ekspert-svar.md"],
"feedback_types": {
"1": {
"label": "feil - rett en feil i forslaget",
"evidence": [
"tests/test_proposal_review_loop_loadbearing.py::test_t13_the_flag_answers_the_review_from_a_real_argv_and_the_answer_is_used"
]
},
"2": {
"label": "bort - ta en retning bort",
"evidence": ["tests/test_v1_probes.py::test_type_2_remove_a_direction_has_a_typed_door"]
},
"3": {
"label": "vinklinger - nye vinklinger",
"evidence": [
"tests/test_v1_probes.py::test_type_3_a_commissioned_angle_is_evaluated_through_the_cli",
"tests/test_v1_probes.py::test_type_3_a_new_angle_changes_the_outcome"
]
},
"4": {
"label": "lette paa krav",
"evidence": ["tests/test_v1_probes.py::test_type_4_relax_a_requirement_has_a_door"]
},
"5": {
"label": "konseptgrafer",
"evidence": ["tests/test_v1_probes.py::test_type_5_edit_the_concept_graph_has_a_door"]
},
"6": {
"label": "skills per analyse",
"evidence": ["tests/test_v1_probes.py::test_type_6_skills_per_analysis_has_a_door"]
},
"7": {
"label": "MCP - verktoey i debatten",
"evidence": [
"tests/test_v1_probes.py::test_type_7_the_mcp_flag_puts_a_service_the_run_calls_into_the_result",
"tests/test_b4_mcp_call_trace_loadbearing.py::test_a_called_mcp_tool_is_recorded_in_provenance"
]
},
"8": {
"label": "inline kontekst",
"evidence": ["tests/test_v1_probes.py::test_type_8_inline_context_has_a_door"]
}
},
"row6_evidence": [
"tests/test_v1_probes.py::test_row6_an_approach_that_declared_nothing_cannot_be_validated",
"tests/test_v1_probes.py::test_row6_a_run_level_declaration_does_not_stand_in_for_the_approach"
],
"maf_points": {
"approved": true,
"approved_on": "2026-09-17",
"approved_by": "operatørgodkjent",
"points": [
{"u_id": "U13", "construct": "MagenticPlanReviewResponse", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "_drive"}, "types": [1, 2]},
{"u_id": "U9", "construct": "ContextProvider", "package": "agent_framework",
"callsite": {"module": "verdicts.py", "scope": "ExpeLContextProvider"}, "types": [1, 8]},
{"u_id": "U12", "construct": "FileCheckpointStorage", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "checkpoint_storage"}, "types": [1]},
{"u_id": "U4", "construct": "MagenticBuilder", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "fresh_exploration_workflow"}, "types": [3]},
{"u_id": "U7", "construct": "tool", "package": "agent_framework",
"callsite": {"module": "explore.py", "scope": "navigator_tools"}, "types": [4]},
{"u_id": "U11", "construct": "TextSpanRegion", "package": "agent_framework",
"callsite": {"module": "provenance.py", "scope": "ProvenanceStamp"}, "types": [5]},
{"u_id": "U5", "construct": "SkillsProvider", "package": "agent_framework",
"callsite": null, "types": [6]},
{"u_id": "U6", "construct": "MCPStdioTool", "package": "agent_framework",
"callsite": {"module": "mcp_tools.py", "scope": "build_mcp_tools"}, "types": [7]}
]
},
"stress_evidence": {
"label": "stressrunde 6",
"root": "scratchpad/p22-stress",
"runs": [
{"context": "contexts/gate-nordvik-2027", "outbox": "gate-nordvik-2027",
"run_id": "gate-nordvik-2027-08"},
{"context": "contexts/tunnel-hauglia-2027", "outbox": "tunnel-hauglia-2027",
"run_id": "tunnel-hauglia-2027-08"},
{"context": "contexts/fv412-dekkefornyelse-2027", "outbox": "fv412-dekkefornyelse-2027",
"run_id": "fv412-dekkefornyelse-2027-08"},
{"context": "contexts/kontrakt-sorasen-2027", "outbox": "kontrakt-sorasen-2027",
"run_id": "kontrakt-sorasen-2027-08"},
{"context": "contexts/dekke-og-kontrakt-lindaas-2027", "outbox": "lindaas",
"run_id": "lindaas-04-vegnormal-n200-2024", "bundle": "vegnormal-n200-2024"},
{"context": "contexts/dekke-og-kontrakt-lindaas-2027", "outbox": "lindaas",
"run_id": "lindaas-04-vegnormal-r761-2025", "bundle": "vegnormal-r761-2025"}
]
}
}