fix(adjudicate): a judgement that keeps nothing gets an artifact
Measured on the K3 corpus: 4 of 12 judgements produced no artifact, because the verdict was "none of these segments should be persisted" and the plan grammar refuses zero entries. That refusal is correct for the run path -- an empty plan replayed would silently persist nothing for a document that was dropped -- so the grammar is untouched and the recording tool is taught to record a rejection instead. Refusing to materialize and refusing to record are different acts. The rejection artifact is deliberately NOT replayable: parse_segmentation_plan still refuses it, and the suite asserts that rather than assuming it. The dwell time rides at the top level because there is no entry to carry it, and a ratified rejection with no time on it is as unfalsifiable as a ratified acceptance with none. Only the empty LIST takes the branch. A missing entries key, or one that is not a list, stays the grammar's to refuse: "the adjudicator kept nothing" and "this file is not a plan" must not collapse. K4a re-run after the change: propose, adjudicate, run the path twice into two bundles under SEGMENTED_OKF_V0_2, diff -r exit 0 with no output. 1034 -> 1041 tests.
This commit is contained in:
parent
16eeeb007e
commit
62b61927a4
2 changed files with 200 additions and 9 deletions
|
|
@ -34,6 +34,7 @@ from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from llm_ingestion_okf.errors import SegmentationError
|
||||||
from llm_ingestion_okf.segmentation import parse_segmentation_plan
|
from llm_ingestion_okf.segmentation import parse_segmentation_plan
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
||||||
|
|
@ -209,3 +210,136 @@ def test_a_missing_plan_exits_two_and_says_so(
|
||||||
)
|
)
|
||||||
assert code == 2
|
assert code == 2
|
||||||
assert "nothing.json" in capsys.readouterr().err
|
assert "nothing.json" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
|
# --- the empty plan: a judgement with nothing to keep -----------------------
|
||||||
|
#
|
||||||
|
# Measured on the K3 corpus: 4 of 12 judgements produced no artifact at all,
|
||||||
|
# because the adjudicator's verdict was "none of these segments should be
|
||||||
|
# persisted" and the parser refuses a plan with zero entries. That refusal is
|
||||||
|
# CORRECT for the run path -- an empty plan would silently persist nothing for a
|
||||||
|
# document that was dropped -- so the grammar is left alone and the recording
|
||||||
|
# tool is taught to record a rejection. The two are different acts: refusing to
|
||||||
|
# materialize is about a bundle, recording a judgement is about a person.
|
||||||
|
|
||||||
|
|
||||||
|
def empty_proposal(tmp_path: Path) -> Path:
|
||||||
|
"""A real proposal with its entries removed -- the plan-level fields stay
|
||||||
|
exactly as the proposer wrote them, so this is a rejection and not a stub."""
|
||||||
|
plan_path = proposal(tmp_path)
|
||||||
|
written = payload(plan_path)
|
||||||
|
written["entries"] = []
|
||||||
|
rejected = tmp_path / "rejected.json"
|
||||||
|
rejected.write_text(json.dumps(written, indent=2) + "\n", encoding="utf-8", newline="")
|
||||||
|
return rejected
|
||||||
|
|
||||||
|
|
||||||
|
def adjudicate_empty(
|
||||||
|
tmp_path: Path, plan_path: Path, out_name: str = "verdict.json"
|
||||||
|
) -> tuple[int, Path]:
|
||||||
|
verdict = tmp_path / out_name
|
||||||
|
code = okf_adjudicate.main(
|
||||||
|
[
|
||||||
|
"--plan",
|
||||||
|
str(plan_path),
|
||||||
|
"--out",
|
||||||
|
str(verdict),
|
||||||
|
"--adjudicator",
|
||||||
|
ADJUDICATOR,
|
||||||
|
"--adjudicated-at",
|
||||||
|
AT,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return code, verdict
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_judgement_over_an_empty_plan_gets_an_artifact(tmp_path: Path) -> None:
|
||||||
|
"""The defect this closes: the judgement happened and left no trace."""
|
||||||
|
code, verdict = adjudicate_empty(tmp_path, empty_proposal(tmp_path))
|
||||||
|
|
||||||
|
assert code == 0
|
||||||
|
assert verdict.is_file()
|
||||||
|
written = payload(verdict)
|
||||||
|
assert written["entries"] == []
|
||||||
|
assert written["adjudicated"] is True
|
||||||
|
assert written["adjudicated_by"] == ADJUDICATOR
|
||||||
|
assert written["adjudicated_at"] == AT
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_empty_verdict_carries_the_dwell_time_at_the_top(tmp_path: Path) -> None:
|
||||||
|
"""There is no entry to hang it on, and a ratified rejection with no time on
|
||||||
|
it is as unfalsifiable as a ratified acceptance with none."""
|
||||||
|
_, verdict = adjudicate_empty(tmp_path, empty_proposal(tmp_path))
|
||||||
|
written = payload(verdict)
|
||||||
|
|
||||||
|
assert isinstance(written["adjudication_dwell_s"], int)
|
||||||
|
assert not isinstance(written["adjudication_dwell_s"], bool)
|
||||||
|
assert written["adjudication_dwell_s"] > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_empty_verdict_is_not_replayable_by_the_run_path(tmp_path: Path) -> None:
|
||||||
|
"""The grammar is UNCHANGED. Recording a rejection and materializing from it
|
||||||
|
are different acts, and only the first one is now possible."""
|
||||||
|
_, verdict = adjudicate_empty(tmp_path, empty_proposal(tmp_path))
|
||||||
|
|
||||||
|
with pytest.raises(SegmentationError) as excinfo:
|
||||||
|
parse_segmentation_plan(payload(verdict))
|
||||||
|
|
||||||
|
assert excinfo.value.code == "segmentation_plan_invalid"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_rejected_proposal_survives_untouched(tmp_path: Path) -> None:
|
||||||
|
plan_path = empty_proposal(tmp_path)
|
||||||
|
before = plan_path.read_bytes()
|
||||||
|
|
||||||
|
adjudicate_empty(tmp_path, plan_path)
|
||||||
|
|
||||||
|
assert plan_path.read_bytes() == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_replaying_an_empty_verdict_produces_identical_bytes(tmp_path: Path) -> None:
|
||||||
|
"""K4a over the arm that had no artifact to compare before."""
|
||||||
|
plan_path = empty_proposal(tmp_path)
|
||||||
|
_, first = adjudicate_empty(tmp_path, plan_path, "first.json")
|
||||||
|
kept = first.read_bytes()
|
||||||
|
|
||||||
|
_, second = adjudicate_empty(tmp_path, plan_path, "second.json")
|
||||||
|
|
||||||
|
assert second.read_bytes() == kept
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_plan_missing_a_required_field_is_still_refused(
|
||||||
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||||
|
) -> None:
|
||||||
|
"""The empty branch is not a hole in the validation: a plan is still a plan,
|
||||||
|
and only its entry list is allowed to be empty."""
|
||||||
|
plan_path = empty_proposal(tmp_path)
|
||||||
|
written = payload(plan_path)
|
||||||
|
del written["source_sha256"]
|
||||||
|
plan_path.write_text(json.dumps(written), encoding="utf-8", newline="")
|
||||||
|
|
||||||
|
code, verdict = adjudicate_empty(tmp_path, plan_path)
|
||||||
|
|
||||||
|
assert code == 2
|
||||||
|
assert "source_sha256" in capsys.readouterr().err
|
||||||
|
assert not verdict.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_entries_value_that_is_not_a_list_is_still_refused(tmp_path: Path) -> None:
|
||||||
|
"""Empty is a judgement; the wrong TYPE is a malformed plan, and the two
|
||||||
|
must not collapse. The malformed one still meets the unchanged grammar.
|
||||||
|
|
||||||
|
It reaches the caller as a raised SegmentationError rather than as exit 2,
|
||||||
|
which is pre-existing behaviour for every malformed plan and is left alone
|
||||||
|
here rather than repaired inside a change about empty ones. Recorded as a
|
||||||
|
finding, not fixed."""
|
||||||
|
plan_path = empty_proposal(tmp_path)
|
||||||
|
written = payload(plan_path)
|
||||||
|
written["entries"] = "none"
|
||||||
|
plan_path.write_text(json.dumps(written), encoding="utf-8", newline="")
|
||||||
|
|
||||||
|
with pytest.raises(SegmentationError) as excinfo:
|
||||||
|
adjudicate_empty(tmp_path, plan_path)
|
||||||
|
|
||||||
|
assert excinfo.value.code == "segmentation_plan_invalid"
|
||||||
|
assert not (tmp_path / "verdict.json").exists()
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ from typing import Any
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||||
|
|
||||||
from llm_ingestion_okf.segmentation import parse_segmentation_plan # noqa: E402
|
from llm_ingestion_okf.segmentation import PLAN_FIELDS, parse_segmentation_plan # noqa: E402
|
||||||
|
|
||||||
#: This tool's identity, written into the artifact so an operator reading a
|
#: This tool's identity, written into the artifact so an operator reading a
|
||||||
#: verdict six months later can tell what produced it.
|
#: verdict six months later can tell what produced it.
|
||||||
|
|
@ -95,6 +95,56 @@ def run_model(model: str, prompt: str, *, timeout: int = 300) -> str:
|
||||||
return proc.stdout.strip()
|
return proc.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def is_rejection(payload: dict[str, Any]) -> bool:
|
||||||
|
"""A plan whose entry list is present and EMPTY.
|
||||||
|
|
||||||
|
Only the empty list. A missing `entries`, or one that is not a list at all,
|
||||||
|
is a malformed plan and stays the grammar's to refuse -- "the adjudicator
|
||||||
|
kept nothing" and "this file is not a plan" are different facts, and a
|
||||||
|
branch that accepted both would launder the second into the first.
|
||||||
|
"""
|
||||||
|
entries = payload.get("entries")
|
||||||
|
return isinstance(entries, list) and not entries
|
||||||
|
|
||||||
|
|
||||||
|
def build_rejection(
|
||||||
|
payload: dict[str, Any],
|
||||||
|
*,
|
||||||
|
adjudicator: str,
|
||||||
|
adjudicated_at: str,
|
||||||
|
dwell_s: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""The verdict for a plan the adjudicator kept nothing from.
|
||||||
|
|
||||||
|
The plan grammar refuses zero entries, and that refusal is CORRECT for the
|
||||||
|
run path: an empty plan replayed would silently persist nothing for a
|
||||||
|
document that was dropped. But refusing to MATERIALIZE and refusing to
|
||||||
|
RECORD are different acts. Measured on the K3 corpus: 4 of 12 judgements
|
||||||
|
left no artifact at all, because the judgement was "none of these segments
|
||||||
|
should be persisted" and there was nowhere to write it. A judgement that
|
||||||
|
leaves no trace cannot be counted, audited or disagreed with.
|
||||||
|
|
||||||
|
So the grammar is untouched and this artifact is deliberately NOT replayable
|
||||||
|
by the run path -- `parse_segmentation_plan` still refuses it, which the
|
||||||
|
suite asserts rather than assumes. The dwell time rides at the top level
|
||||||
|
because there is no entry to carry it, and a ratified rejection with no time
|
||||||
|
on it is exactly as unfalsifiable as a ratified acceptance with none.
|
||||||
|
"""
|
||||||
|
for key in PLAN_FIELDS:
|
||||||
|
if key not in payload:
|
||||||
|
raise AdjudicationError(
|
||||||
|
f"the plan is missing the required field {key!r} -- an empty entry "
|
||||||
|
"list is a judgement, but a plan is still a plan"
|
||||||
|
)
|
||||||
|
verdict = dict(payload)
|
||||||
|
verdict["entries"] = []
|
||||||
|
verdict["adjudicated"] = True
|
||||||
|
verdict["adjudicated_at"] = adjudicated_at
|
||||||
|
verdict["adjudicated_by"] = adjudicator
|
||||||
|
verdict["adjudication_dwell_s"] = dwell_s
|
||||||
|
return verdict
|
||||||
|
|
||||||
|
|
||||||
def build_verdict(
|
def build_verdict(
|
||||||
payload: dict[str, Any],
|
payload: dict[str, Any],
|
||||||
*,
|
*,
|
||||||
|
|
@ -140,10 +190,12 @@ def run(
|
||||||
payload = json.loads(plan_path.read_text(encoding="utf-8"))
|
payload = json.loads(plan_path.read_text(encoding="utf-8"))
|
||||||
except json.JSONDecodeError as exc:
|
except json.JSONDecodeError as exc:
|
||||||
raise AdjudicationError(f"{plan_path} is not readable JSON: {exc}") from exc
|
raise AdjudicationError(f"{plan_path} is not readable JSON: {exc}") from exc
|
||||||
# Parsed before anything is written: a proposal this library cannot read
|
rejection = is_rejection(payload)
|
||||||
# back is one no verdict can be recorded against, and finding that out
|
if not rejection:
|
||||||
# after writing would leave a verdict pointing at nothing.
|
# Parsed before anything is written: a proposal this library cannot read
|
||||||
parse_segmentation_plan(payload)
|
# back is one no verdict can be recorded against, and finding that out
|
||||||
|
# after writing would leave a verdict pointing at nothing.
|
||||||
|
parse_segmentation_plan(payload)
|
||||||
|
|
||||||
if model is not None:
|
if model is not None:
|
||||||
# Advisory only, and recorded rather than applied. The judgement stays
|
# Advisory only, and recorded rather than applied. The judgement stays
|
||||||
|
|
@ -152,10 +204,15 @@ def run(
|
||||||
# very number this command exists to produce.
|
# very number this command exists to produce.
|
||||||
run_model(model, "Summarise the proposed segmentation for review.")
|
run_model(model, "Summarise the proposed segmentation for review.")
|
||||||
|
|
||||||
verdict = build_verdict(
|
if rejection:
|
||||||
payload, adjudicator=adjudicator, adjudicated_at=adjudicated_at, dwell_s=dwell_s
|
verdict = build_rejection(
|
||||||
)
|
payload, adjudicator=adjudicator, adjudicated_at=adjudicated_at, dwell_s=dwell_s
|
||||||
parse_segmentation_plan(verdict)
|
)
|
||||||
|
else:
|
||||||
|
verdict = build_verdict(
|
||||||
|
payload, adjudicator=adjudicator, adjudicated_at=adjudicated_at, dwell_s=dwell_s
|
||||||
|
)
|
||||||
|
parse_segmentation_plan(verdict)
|
||||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
out_path.write_text(
|
out_path.write_text(
|
||||||
json.dumps(verdict, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline=""
|
json.dumps(verdict, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline=""
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue