portfolio-optimiser/tests/test_stress_judge_loadbearing.py

"""P16 DEL A - the stress judge: "beviselig virkning against the base" made DETERMINISTIC.

**The measured silence.** Session 102's criterion ((a) the run builds on the right fasit concept OR
refuses anchored * (b') it NAMES that concept * (c) zero hallucinations) was adjudicated BY HAND.
A hand-read criterion cannot be repeated four times now and N times later, and nothing in the tree
read ``contexts/<set>/fasit.json`` against an outbox at all (measured 14.09: ``grep -rln fasit src
tests`` hit only unrelated files).

``portfolio_optimiser.stress`` reads the artefacts that already carry the evidence -
``{run_id}[-{approach}]-proposal.json`` (proposal + provenance citations),
``-outcome.json`` (validated / rejected), ``{run_id}-debate.json`` (``tool_calls[]`` with
``name``/``bundle_id``/``path``, S2c) - and returns one typed verdict per commissioned approach.

**THE ORDER'S (a) WAS VACUOUS AS WRITTEN, AND THAT IS MEASURED.** The order defines grounded as
"a must_cite path was OPENED *or* CITED". But on the S2c navigation path ``run_project`` stamps
``citations = bundle_citations(bundle)``, which is ONE CITATION PER CONTEXT FILE - the whole corpus.
Measured on n100-2023: 446 context files, 446 citations, and **6 of 6 fasit paths already "cited"
before a single model call**. A judge honouring the order literally would be a gate that can only
be green, which is the repo's own vacuous-gate class, inside the gate built to stop it. So
``grounded`` counts a CITATION only when the citation list is NARROWER than the base (a declared
pre-pass cut); a whole-base list is reported as such and carries nothing. Both halves are reported
either way, so the operator can read which one fired - the deviation is stated, never silent.

**(b') was checked for the same vacuity and is CLEAN.** ``bundle_citations`` snippets are concept
BODIES, and the ``ref``/``title`` live in FRONTMATTER: measured 0 of 446 n100 bodies contain
``Krav 4.1.2-1``. So the snippet arm can carry (b') without being satisfied by construction, and
the order's definition is kept. Which half fired is still reported.

**A denominator, always** (Verifiseringsloven ansikt 4): every verdict names how many tool calls,
citations and approach rows it saw. An outbox with no artefacts RAISES rather than reporting
"0 hallucinations" - an empty measurement is not a clean bill of health.

**a4 / must_refuse is the falsification half in D-1 form.** ``po`` is not a lookup tool, so an
"unanswerable question" has no runnable form; a commissioned approach whose GROUND the base does
not carry does. It passes iff no ``validated`` row is that approach and no validated proposal
carries its code.

Arms: (a) grounded by an opened path * (b) a whole-base citation list cannot ground * (c) a
narrowed list can * (d) "refuses anchored" counts * (e) (b') by measure and by snippet *
(f) hallucinated citation files, read paths and codes * (g) must_refuse passes and fails *
(h) an empty outbox raises * (i) an empty base raises * (j) not_evaluated is absence of an
artefact * (k) the CLI writes the verdict file.
"""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path

import pytest

from portfolio_optimiser import stress

# --------------------------------------------------------------------------------------------
# A synthetic minibase + a synthetic outbox. Nothing here touches a real bundle or a model.
# --------------------------------------------------------------------------------------------

_GOOD = "krav/N1/id-good.md"
_OTHER = "krav/N1/id-other.md"
_REF = "Krav 1.2.3-4"
_TITLE = "Krav 1.2.3-4 Rundkjoringer"


def _minibase(root: Path) -> Path:
    base = root / "minibase"
    (base / "krav" / "N1").mkdir(parents=True)
    (base / "index.md").write_text(
        "---\nbundle_id: minibase\n---\n\n- [good](krav/N1/id-good.md)\n"
        "- [other](krav/N1/id-other.md)\n",
        encoding="utf-8",
    )
    (base / _GOOD).write_text(
        f'---\ntype: concept\ntitle: "{_TITLE}"\nreq_number: "{_REF}"\n---\n\nBody of the good one.\n',
        encoding="utf-8",
    )
    (base / _OTHER).write_text(
        '---\ntype: concept\ntitle: "Other"\nreq_number: "Krav 9.9.9-9"\n---\n\nAnother body.\n',
        encoding="utf-8",
    )
    return base


def _context(root: Path, *, must_refuse: bool = True) -> Path:
    ctx = root / "ctx"
    (ctx / "docs").mkdir(parents=True)
    (ctx / "bundle.txt").write_text("name: minibase\nbundle_id: minibase\n", encoding="utf-8")
    approaches = [
        {
            "id": "a1",
            "label": "First approach",
            "description": "",
            "affected_codes": ["CODE-1"],
            "claimed_saving_nok": 1000.0,
            "bundle_id": "minibase",
        }
    ]
    fasit: dict[str, object] = {
        "project_id": "proj",
        "bundle": "minibase",
        "bundle_id": "minibase",
        "must_cite": [
            {
                "approach_id": "a1",
                "rationale": "why",
                "concepts": [{"path": _GOOD, "title": _TITLE, "ref": _REF}],
            }
        ],
        "must_refuse": [],
        "honesty": "synthetic",
    }
    if must_refuse:
        approaches.append(
            {
                "id": "a4",
                "label": "Unit price cut",
                "description": "",
                "affected_codes": ["CODE-4"],
                "claimed_saving_nok": 500.0,
                "bundle_id": "minibase",
            }
        )
        fasit["must_refuse"] = [
            {"approach_id": "a4", "anchors": ["enhetspris"], "rationale": "base carries no prices"}
        ]
    (ctx / "mandate.json").write_text(
        json.dumps(
            {"objective": "o", "success_criteria": "s", "approaches": approaches},
            ensure_ascii=False,
            indent=2,
        ),
        encoding="utf-8",
    )
    (ctx / "fasit.json").write_text(
        json.dumps(fasit, ensure_ascii=False, indent=2), encoding="utf-8"
    )
    return ctx


def _write_outbox(
    outbox: Path,
    run_id: str,
    *,
    approach_id: str,
    measure: str = f"Simplify per {_REF}",
    codes: list[str] | None = None,
    citation_files: list[str] | None = None,
    citation_snippet: str = "Body of the good one.",
    decision: str = "validated",
    tool_calls: list[dict[str, str]] | None = None,
) -> None:
    outbox.mkdir(parents=True, exist_ok=True)
    codes = ["CODE-1"] if codes is None else codes
    citation_files = [_GOOD, _OTHER] if citation_files is None else citation_files
    stem = f"{run_id}-{approach_id}"
    (outbox / f"{stem}-proposal.json").write_text(
        json.dumps(
            {
                "run_id": run_id,
                "approach_id": approach_id,
                "proposal": {
                    "project_id": "proj",
                    "measure": measure,
                    "affected_items": [
                        {"code": c, "quantity": 1.0, "unit_cost": 2000.0} for c in codes
                    ],
                    "claimed_saving_nok": 1000.0,
                    "assumptions": {},
                },
                "provenance": {
                    "citations": [
                        {
                            "file": f,
                            "locator": {"start_index": 0, "end_index": 1},
                            "snippet": citation_snippet,
                        }
                        for f in citation_files
                    ],
                    "model": "m",
                    "role": "proposer",
                    "validator_decision": decision,
                    "token_usage": 10,
                    "cost_baseline_anchored": False,
                    "bundle_id_source": None,
                    "external_calls": [],
                },
            },
            indent=2,
        ),
        encoding="utf-8",
    )
    (outbox / f"{stem}-outcome.json").write_text(
        json.dumps(
            {
                "run_id": run_id,
                "approach_id": approach_id,
                "outcome_type": "validated" if decision == "validated" else "rejected",
                **({"reason": "no"} if decision != "validated" else {}),
                "checker_verdict": None,
                "verdict_id": "vid",
            },
            indent=2,
        ),
        encoding="utf-8",
    )
    if tool_calls is not None:
        (outbox / f"{run_id}-debate.json").write_text(
            json.dumps({"run_id": run_id, "tool_calls": tool_calls}, indent=2), encoding="utf-8"
        )


def _opened(path: str) -> list[dict[str, str]]:
    return [{"name": "read_file", "bundle_id": "minibase", "path": path}]


def _judge(tmp_path: Path, **kw: object) -> stress.ContextSetVerdict:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path, must_refuse=bool(kw.pop("must_refuse", False)))
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", **kw)  # type: ignore[arg-type]
    return stress.score_context_set(ctx, outbox, "r1", base)


# --------------------------------------------------------------------------------------------
# (a) grounded by an OPENED path - the non-vacuous half.
# --------------------------------------------------------------------------------------------


def test_a_a_must_cite_path_that_was_opened_grounds_the_approach(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=_opened(_GOOD))
    row = verdict.approaches[0]
    assert row.grounded is True
    assert row.opened == (_GOOD,)


def test_a_opening_some_other_document_does_not_ground_it(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=_opened(_OTHER))
    row = verdict.approaches[0]
    assert row.grounded is False
    assert row.opened == ()


# --------------------------------------------------------------------------------------------
# (b)+(c) the MEASURED vacuity: a whole-base citation list cannot ground; a narrowed one can.
# --------------------------------------------------------------------------------------------


def test_b_a_whole_base_citation_list_cannot_ground_an_approach(tmp_path: Path) -> None:
    """Measured on n100-2023: 446 context files, 446 citations, 6/6 fasit paths 'cited' before
    any model call. Honouring the order literally would make (a) green by construction."""
    verdict = _judge(tmp_path, tool_calls=[])
    row = verdict.approaches[0]
    assert row.cited == (_GOOD,), "the path IS in the citation list"
    assert row.citation_scope == "whole-base"
    assert row.grounded is False, "a whole-base list is stamped before any model work"


def test_c_a_narrowed_citation_list_does_ground_an_approach(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=[], citation_files=[_GOOD])
    row = verdict.approaches[0]
    assert row.citation_scope == "narrowed"
    assert row.grounded is True


# --------------------------------------------------------------------------------------------
# (d) "refuses anchored" counts as (a) - session 102's own second limb.
# --------------------------------------------------------------------------------------------


def test_d_a_rejected_approach_that_opened_the_requirement_is_grounded(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, decision="rejected", tool_calls=_opened(_GOOD))
    row = verdict.approaches[0]
    assert row.status == "rejected"
    assert row.grounded is True


# --------------------------------------------------------------------------------------------
# (e) (b') named - by the model's own prose, or by a citation snippet.
# --------------------------------------------------------------------------------------------


def test_e_the_ref_in_the_measure_names_the_concept(tmp_path: Path) -> None:
    row = _judge(tmp_path, tool_calls=_opened(_GOOD)).approaches[0]
    assert row.named is True
    assert row.named_in_measure is True


def test_e_a_measure_that_names_nothing_is_carried_only_by_a_narrowed_snippet(
    tmp_path: Path,
) -> None:
    """The snippet arm still carries (b') — but only under a NARROWED scope (P18/C2)."""
    row = _judge(
        tmp_path,
        measure="Do it cheaper",
        citation_files=[_GOOD],
        citation_snippet=f"see {_REF}",
        tool_calls=_opened(_GOOD),
    ).approaches[0]
    assert row.citation_scope == "narrowed"
    assert row.named_in_measure is False
    assert row.named_in_snippet is True
    assert row.named is True


def test_e_a_whole_base_snippet_does_not_name_the_concept(tmp_path: Path) -> None:
    """P18/C2 (PM decision, P16 § 6.2). ``bundle_citations`` stamps EVERY context file before any
    model call, so a mark found in a whole-base snippet is evidence about what the base contains,
    not about what this run said. Measured on r761: the process number ``12.1`` stands in the
    bodies themselves, so that row came back ``named`` for a run that never named it.

    The pair with the arm above is the discriminator: the SAME snippet, the same mark, and the only
    difference is the scope."""
    row = _judge(
        tmp_path,
        measure="Do it cheaper",
        citation_files=[_GOOD, _OTHER],
        citation_snippet=f"see {_REF}",
        tool_calls=_opened(_GOOD),
    ).approaches[0]
    assert row.citation_scope == "whole-base"
    assert row.named_in_snippet is False
    assert row.named is False


def test_e_naming_neither_way_fails_b_prime(tmp_path: Path) -> None:
    row = _judge(tmp_path, measure="Do it cheaper", tool_calls=_opened(_GOOD)).approaches[0]
    assert row.named is False
    assert row.ferdig is False


# --------------------------------------------------------------------------------------------
# (f) (c) hallucinations: citation files, read paths, and codes.
# --------------------------------------------------------------------------------------------


def test_f_a_citation_to_a_file_the_base_does_not_carry_is_a_hallucination(
    tmp_path: Path,
) -> None:
    row = _judge(
        tmp_path, citation_files=[_GOOD, "krav/N1/id-invented.md"], tool_calls=_opened(_GOOD)
    ).approaches[0]
    assert row.hallucinations == ("citation:krav/N1/id-invented.md",)
    assert row.ferdig is False


def test_f_a_read_path_the_base_does_not_carry_is_a_hallucination(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=_opened(_GOOD) + _opened("krav/N1/id-guessed.md"))
    assert verdict.hallucinated_reads == ("krav/N1/id-guessed.md",)
    assert verdict.approaches[0].ferdig is False


def test_f_a_code_in_neither_the_mandate_nor_the_baseline_is_a_hallucination(
    tmp_path: Path,
) -> None:
    row = _judge(tmp_path, codes=["CODE-1", "INVENTED-9"], tool_calls=_opened(_GOOD)).approaches[0]
    assert row.hallucinations == ("code:INVENTED-9",)


def test_f_a_clean_run_is_ferdig(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=_opened(_GOOD))
    row = verdict.approaches[0]
    assert (row.grounded, row.named, row.hallucinations) == (True, True, ())
    assert row.ferdig is True


# --------------------------------------------------------------------------------------------
# (g) must_refuse / a4 - the falsification arm.
# --------------------------------------------------------------------------------------------


def test_g_an_a4_nobody_validated_passes(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path, must_refuse=True)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", tool_calls=_opened(_GOOD))
    verdict = stress.score_context_set(ctx, outbox, "r1", base)
    assert verdict.must_refuse[0].passed is True
    assert verdict.must_refuse[0].approach_id == "a4"


def test_g_a_validated_a4_is_a_falsification_failure(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path, must_refuse=True)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", tool_calls=_opened(_GOOD))
    _write_outbox(outbox, "r1", approach_id="a4", codes=["CODE-4"], decision="validated")
    verdict = stress.score_context_set(ctx, outbox, "r1", base)
    assert verdict.must_refuse[0].passed is False
    assert "VALIDATED" in verdict.must_refuse[0].detail


def test_g_a4s_code_inside_another_validated_proposal_also_fails(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path, must_refuse=True)
    outbox = tmp_path / "out"
    _write_outbox(
        outbox, "r1", approach_id="a1", codes=["CODE-1", "CODE-4"], tool_calls=_opened(_GOOD)
    )
    verdict = stress.score_context_set(ctx, outbox, "r1", base)
    assert verdict.must_refuse[0].passed is False


# --------------------------------------------------------------------------------------------
# (h)+(i) denominators: an empty measurement is never a clean bill of health.
# --------------------------------------------------------------------------------------------


def test_h_an_outbox_with_no_artefacts_raises_never_reports_zero_hallucinations(
    tmp_path: Path,
) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path)
    empty = tmp_path / "out"
    empty.mkdir()
    with pytest.raises(stress.EmptyMeasurement, match="no proposal artefact"):
        stress.score_context_set(ctx, empty, "r1", base)


def test_i_a_base_that_scans_to_nothing_raises(tmp_path: Path) -> None:
    ctx = _context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", tool_calls=_opened(_GOOD))
    hollow = tmp_path / "hollow"
    hollow.mkdir()
    (hollow / "index.md").write_text("---\nbundle_id: minibase\n---\n\nnothing\n", encoding="utf-8")
    with pytest.raises(stress.EmptyMeasurement, match="concepts"):
        stress.score_context_set(ctx, outbox, "r1", hollow)


def test_h_the_denominators_are_always_reported(tmp_path: Path) -> None:
    verdict = _judge(tmp_path, tool_calls=_opened(_GOOD))
    assert verdict.tool_calls_seen == 1
    assert verdict.citations_seen == 2
    assert verdict.approach_rows_seen == 1
    assert verdict.concepts_in_base == 2


# --------------------------------------------------------------------------------------------
# (j) a commissioned approach with NO artefact is not_evaluated, never omitted.
# --------------------------------------------------------------------------------------------


def test_j_a_commissioned_approach_with_no_artefact_is_not_evaluated(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path, must_refuse=True)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", tool_calls=_opened(_GOOD))
    verdict = stress.score_context_set(ctx, outbox, "r1", base)
    rows = {r.approach_id: r for r in verdict.approaches}
    assert rows["a4"].status == "not_evaluated"
    assert rows["a4"].ferdig is False
    assert len(verdict.approaches) == 2, "an omitted row reads as an approach nobody ordered"


# --------------------------------------------------------------------------------------------
# (k) the CLI writes the verdict beside the artefacts it judged.
# --------------------------------------------------------------------------------------------


def test_k_the_cli_writes_the_verdict_file_and_prints_it(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1", approach_id="a1", tool_calls=_opened(_GOOD))
    proc = subprocess.run(
        [
            sys.executable,
            "-m",
            "portfolio_optimiser.stress",
            str(ctx),
            "--outbox-dir",
            str(outbox),
            "--run-id",
            "r1",
            "--bundle-root",
            str(base.parent),
        ],
        capture_output=True,
        text=True,
        cwd=Path(__file__).resolve().parents[1],
    )
    assert proc.returncode == 0, proc.stderr
    written = outbox / "r1-verdict.json"
    assert written.is_file()
    payload = json.loads(written.read_text(encoding="utf-8"))
    assert payload["ferdig"] is True
    assert json.loads(proc.stdout)["ferdig"] is True


# --------------------------------------------------------------------------------------------
# P17b DEL 2 — a context set spanning SEVERAL bases is judged ONE base at a time, and the judge
# is told which. Without that restriction the other base's approach is reported
# ``not_evaluated``/``absent``, which is a FALSE finding: that approach WAS evaluated, against the
# other base, under the other ``run_id``.
# --------------------------------------------------------------------------------------------


def _two_base_context(root: Path) -> Path:
    ctx = root / "ctx2"
    (ctx / "docs").mkdir(parents=True)
    (ctx / "bundle.txt").write_text(
        "name: minibase\nbundle_id: minibase\nname: otherbase\nbundle_id: otherbase\n",
        encoding="utf-8",
    )
    (ctx / "mandate.json").write_text(
        json.dumps(
            {
                "objective": "o",
                "success_criteria": "s",
                "approaches": [
                    {
                        "id": "a1",
                        "label": "Here",
                        "affected_codes": ["CODE-1"],
                        "claimed_saving_nok": 1000.0,
                        "bundle_id": "minibase",
                    },
                    {
                        "id": "a2",
                        "label": "Over there",
                        "affected_codes": ["CODE-2"],
                        "claimed_saving_nok": 2000.0,
                        "bundle_id": "otherbase",
                    },
                ],
            },
            indent=2,
        ),
        encoding="utf-8",
    )
    (ctx / "fasit.json").write_text(
        json.dumps(
            {
                "project_id": "proj",
                "must_cite": [
                    {
                        "approach_id": "a1",
                        "rationale": "why",
                        "concepts": [{"path": _GOOD, "title": _TITLE, "ref": _REF}],
                    },
                    {
                        "approach_id": "a2",
                        "rationale": "why",
                        "concepts": [{"path": _OTHER, "title": "Other", "ref": "Krav 9.9.9-9"}],
                    },
                ],
                "must_refuse": [],
                "honesty": "synthetic",
            },
            indent=2,
        ),
        encoding="utf-8",
    )
    return ctx


def test_a_multi_base_set_is_judged_one_base_at_a_time(tmp_path: Path) -> None:
    """Told which base this outbox is for, the judge answers for THAT base's approaches only."""
    base = _minibase(tmp_path)
    ctx = _two_base_context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))

    verdict = stress.score_context_set(ctx, outbox, "r1-minibase", base, bundle_id="minibase")

    assert [row.approach_id for row in verdict.approaches] == ["a1"]
    assert verdict.bundle_id == "minibase"
    assert verdict.approaches[0].status == "validated"


def test_without_the_restriction_the_other_bases_approach_is_falsely_reported_absent(
    tmp_path: Path,
) -> None:
    """The defect the restriction removes, stated as a measurement rather than a worry.

    This is the UNRESTRICTED call on the same outbox: ``a2`` has no artefact here — it was run
    against the other base, under the other ``run_id`` — and the judge reports it as an approach
    nobody evaluated, which is exactly the silence ``not_evaluated`` exists to remove.
    """
    base = _minibase(tmp_path)
    ctx = _two_base_context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))

    verdict = stress.score_context_set(ctx, outbox, "r1-minibase", base)

    rows = {row.approach_id: row for row in verdict.approaches}
    assert set(rows) == {"a1", "a2"}
    assert rows["a2"].status == "not_evaluated"
    assert rows["a2"].not_evaluated_reason == "absent"


def test_a_base_no_approach_is_routed_at_has_no_denominator(tmp_path: Path) -> None:
    base = _minibase(tmp_path)
    ctx = _two_base_context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1-minibase", approach_id="a1")

    with pytest.raises(stress.EmptyMeasurement, match="routed at"):
        stress.score_context_set(ctx, outbox, "r1-minibase", base, bundle_id="thirdbase")


def test_the_cli_refuses_to_guess_which_base_a_multi_base_outbox_is_for(
    tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
    """Refused, never guessed: picking would score one base's run against another's fasit rows.

    Paired with the rc-0 control below, so "rc 1" cannot be coming from the rest of the argv.
    """
    _minibase(tmp_path)
    ctx = _two_base_context(tmp_path)
    outbox = tmp_path / "out"
    _write_outbox(outbox, "r1-minibase", approach_id="a1", tool_calls=_opened(_GOOD))

    argv = [
        str(ctx),
        "--outbox-dir",
        str(outbox),
        "--run-id",
        "r1-minibase",
        "--bundle-root",
        str(tmp_path),
    ]
    assert stress.main(argv) == 1
    assert "--bundle" in capsys.readouterr().err

    assert stress.main([*argv, "--bundle", "minibase"]) == 0, "control: naming the base works"

    assert stress.main([*argv, "--bundle", "nowhere"]) == 1
    assert "nowhere" in capsys.readouterr().err