feat(p17b): a context set that spans TWO bases, and a judge told which one [skip-docs]
``contexts/dekke-og-kontrakt-lindaas-2027`` is the first set whose approaches route at more than one knowledge base: a1/a2 at n200-2024 (material requirements) and a3/a4 at r761-2025 (the rig, and the falsification arm). That is the whole reason it exists -- P17b measures that ONE commission can be run across several. ``bundle.txt`` grows a block per base; a set naming one base is one block, so the four pre-P17b files parse byte-identically. The reader now has ONE home (``stress.read_bundle_declarations``): it used to be a private copy in the P14 gate and a second, looser one inside ``stress.main``, and the multi-base form is exactly the change that would have let them drift. Rule U becomes the UNION of every declared base, and that is not a formality. MEASURED 15.09: ``enhetspris`` is absent from n200-2024 and carried by 70 of r761-2025's 2 756 concepts, so anchors admitted per base would have admitted a question the pass as a whole CAN ground. It was dropped from the fifth set's anchors for that reason. ``score_context_set(bundle_id=...)`` restricts the judgement to the approaches routed at THIS base. Without it, judging the n200 outbox reports the r761 approach as ``not_evaluated``/``absent`` -- a false finding, because that approach WAS evaluated, against the other base, under the other run_id. That defect is pinned by its own arm. The judge's CLI refuses to guess when a set declares several bases, with an rc-0 control on ``--bundle``. Arm (d) gained a second half: every DECLARED base must be named by some approach, because a base no approach names is never run. The P19/B2 fasit denominator moved 26 -> 32 and is asserted, not dropped: six new references, two of them bare ``prosessnr`` (12.11, 12.12), so B1's punctuation-and-digits form is now exercised by a fasit and not only by a known-positive. Suite 1774/5, golden byte-unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
5e4c497a84
commit
da0ccd0489
8 changed files with 496 additions and 57 deletions
|
|
@ -486,3 +486,146 @@ def test_k_the_cli_writes_the_verdict_file_and_prints_it(tmp_path: Path) -> None
|
|||
payload = json.loads(written.read_text(encoding="utf-8"))
|
||||
assert payload["ferdig"] is True
|
||||
assert json.loads(proc.stdout)["ferdig"] is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------
|
||||
# P17b DEL 2 — a context set spanning SEVERAL bases is judged ONE base at a time, and the judge
|
||||
# is told which. Without that restriction the other base's approach is reported
|
||||
# ``not_evaluated``/``absent``, which is a FALSE finding: that approach WAS evaluated, against the
|
||||
# other base, under the other ``run_id``.
|
||||
# --------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _two_base_context(root: Path) -> Path:
|
||||
ctx = root / "ctx2"
|
||||
(ctx / "docs").mkdir(parents=True)
|
||||
(ctx / "bundle.txt").write_text(
|
||||
"name: minibase\nbundle_id: minibase\nname: otherbase\nbundle_id: otherbase\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ctx / "mandate.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"objective": "o",
|
||||
"success_criteria": "s",
|
||||
"approaches": [
|
||||
{
|
||||
"id": "a1",
|
||||
"label": "Here",
|
||||
"affected_codes": ["CODE-1"],
|
||||
"claimed_saving_nok": 1000.0,
|
||||
"bundle_id": "minibase",
|
||||
},
|
||||
{
|
||||
"id": "a2",
|
||||
"label": "Over there",
|
||||
"affected_codes": ["CODE-2"],
|
||||
"claimed_saving_nok": 2000.0,
|
||||
"bundle_id": "otherbase",
|
||||
},
|
||||
],
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(ctx / "fasit.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"project_id": "proj",
|
||||
"must_cite": [
|
||||
{
|
||||
"approach_id": "a1",
|
||||
"rationale": "why",
|
||||
"concepts": [{"path": _GOOD, "title": _TITLE, "ref": _REF}],
|
||||
},
|
||||
{
|
||||
"approach_id": "a2",
|
||||
"rationale": "why",
|
||||
"concepts": [{"path": _OTHER, "title": "Other", "ref": "Krav 9.9.9-9"}],
|
||||
},
|
||||
],
|
||||
"must_refuse": [],
|
||||
"honesty": "synthetic",
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return ctx
|
||||
|
||||
|
||||
def test_a_multi_base_set_is_judged_one_base_at_a_time(tmp_path: Path) -> None:
|
||||
"""Told which base this outbox is for, the judge answers for THAT base's approaches only."""
|
||||
base = _minibase(tmp_path)
|
||||
ctx = _two_base_context(tmp_path)
|
||||
outbox = tmp_path / "out"
|
||||
_write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))
|
||||
|
||||
verdict = stress.score_context_set(ctx, outbox, "r1-minibase", base, bundle_id="minibase")
|
||||
|
||||
assert [row.approach_id for row in verdict.approaches] == ["a1"]
|
||||
assert verdict.bundle_id == "minibase"
|
||||
assert verdict.approaches[0].status == "validated"
|
||||
|
||||
|
||||
def test_without_the_restriction_the_other_bases_approach_is_falsely_reported_absent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""The defect the restriction removes, stated as a measurement rather than a worry.
|
||||
|
||||
This is the UNRESTRICTED call on the same outbox: ``a2`` has no artefact here — it was run
|
||||
against the other base, under the other ``run_id`` — and the judge reports it as an approach
|
||||
nobody evaluated, which is exactly the silence ``not_evaluated`` exists to remove.
|
||||
"""
|
||||
base = _minibase(tmp_path)
|
||||
ctx = _two_base_context(tmp_path)
|
||||
outbox = tmp_path / "out"
|
||||
_write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))
|
||||
|
||||
verdict = stress.score_context_set(ctx, outbox, "r1-minibase", base)
|
||||
|
||||
rows = {row.approach_id: row for row in verdict.approaches}
|
||||
assert set(rows) == {"a1", "a2"}
|
||||
assert rows["a2"].status == "not_evaluated"
|
||||
assert rows["a2"].not_evaluated_reason == "absent"
|
||||
|
||||
|
||||
def test_a_base_no_approach_is_routed_at_has_no_denominator(tmp_path: Path) -> None:
|
||||
base = _minibase(tmp_path)
|
||||
ctx = _two_base_context(tmp_path)
|
||||
outbox = tmp_path / "out"
|
||||
_write_outbox(outbox, "r1-minibase", approach_id="a1")
|
||||
|
||||
with pytest.raises(stress.EmptyMeasurement, match="routed at"):
|
||||
stress.score_context_set(ctx, outbox, "r1-minibase", base, bundle_id="thirdbase")
|
||||
|
||||
|
||||
def test_the_cli_refuses_to_guess_which_base_a_multi_base_outbox_is_for(
|
||||
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""Refused, never guessed: picking would score one base's run against another's fasit rows.
|
||||
|
||||
Paired with the rc-0 control below, so "rc 1" cannot be coming from the rest of the argv.
|
||||
"""
|
||||
_minibase(tmp_path)
|
||||
ctx = _two_base_context(tmp_path)
|
||||
outbox = tmp_path / "out"
|
||||
_write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))
|
||||
|
||||
argv = [
|
||||
str(ctx),
|
||||
"--outbox-dir",
|
||||
str(outbox),
|
||||
"--run-id",
|
||||
"r1-minibase",
|
||||
"--bundle-root",
|
||||
str(tmp_path),
|
||||
]
|
||||
assert stress.main(argv) == 1
|
||||
assert "--bundle" in capsys.readouterr().err
|
||||
|
||||
assert stress.main([*argv, "--bundle", "minibase"]) == 0, "control: naming the base works"
|
||||
|
||||
assert stress.main([*argv, "--bundle", "nowhere"]) == 1
|
||||
assert "nowhere" in capsys.readouterr().err
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue