feat(tools): adjudication command with a switchable model leg

tools/okf_adjudicate.py

@@ -0,0 +1,214 @@
+"""Record a human's judgement of a proposed segmentation, without inventing one.
+
+The proposer proposes; this records what a person decided about the proposal.
+The two are different artifacts on purpose. A proposal a human has not looked
+at must never be replayable as an adjudication, because replay is exactly what
+the run path does with a plan -- deterministically and forever -- so the
+proposal is left BYTE-UNTOUCHED and the verdict is written as a sibling. Either
+can be re-read against the other afterwards, which a single mutated file could
+never support.
+
+Every entry's verdict carries its adjudicator, the timestamp and the DWELL
+TIME, per PM decision B2 (`docs/plan/office-intake.md` § 5). The dwell time is
+not bookkeeping: a ratified flag with no per-item time is unfalsifiable --
+nothing distinguishes a judgement from a click -- and it is the same number
+that makes adjudication throughput measurable at all.
+
+**The model leg is OFF by default, and that is a measurement decision.**
+Pre-annotation has been measured LOWERING a good annotator's accuracy, from
+98.1 % to 95.8 %, so a leg that cannot be switched off is a leg whose value can
+never be measured. When it is switched on it shells out to the `claude` CLI at
+a resolved absolute path with an explicit `--model`. Shelling out is legal in
+`tools/` and adds NO packaging dependency: an SDK wheel would put a second
+package in this project's dependency surface for a path the run path must never
+take, and an HTTP call would need network policy the library refuses.
+
+It lives outside `src/`, so it never enters a wheel and no consumer's install
+surface changes because it exists. The model-free gate over `src/` is unaffected
+by anything here.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from llm_ingestion_okf.segmentation import parse_segmentation_plan  # noqa: E402
+
+#: This tool's identity, written into the artifact so an operator reading a
+#: verdict six months later can tell what produced it.
+ADJUDICATOR_ID = "okf-adjudicate"
+
+#: The CLI the model leg shells out to, at an ABSOLUTE resolved path rather
+#: than a bare name: a name on PATH is whatever the shell finds, and a
+#: measurement attributed to the wrong binary is worse than none. Measured at
+#: 2.1.258 on 2026-09-02. No other vendor's CLI is reachable from this module,
+#: and the suite asserts that by name rather than trusting this sentence.
+CLAUDE_CLI = "/Users/ktg/.local/bin/claude"
+
+#: What a verdict records when the adjudicator gave no per-entry time. Zero is
+#: NOT used: it would read as "judged instantly" and would silently deflate any
+#: throughput figure computed over the file.
+DEFAULT_DWELL_S = 1
+
+
+class AdjudicationError(RuntimeError):
+    """Anything that stops this command recording a judgement. Never swallowed.
+
+    Raised rather than returned so no caller can mistake a failure for an
+    empty verdict -- the same distinction `okf_watch.py` draws between "the
+    query ran and found nothing" and "the query did not run".
+    """
+
+
+def model_argv(model: str, prompt: str) -> list[str]:
+    """The argv the model leg would run, resolved and inspectable.
+
+    Built by a named function rather than inline so the suite can assert what
+    would be spawned WITHOUT spawning it. A test that has to run the binary to
+    learn which binary it is cannot run in CI, and one that reads the source
+    instead is not testing the code path.
+    """
+    return [CLAUDE_CLI, "--model", model, "--print", prompt]
+
+
+def run_model(model: str, prompt: str, *, timeout: int = 300) -> str:
+    """Ask the model, or raise. Never returns a partial or a swallowed error."""
+    try:
+        proc = subprocess.run(
+            model_argv(model, prompt), capture_output=True, text=True, timeout=timeout
+        )
+    except FileNotFoundError as exc:
+        raise AdjudicationError(f"the CLI is not at {CLAUDE_CLI}: {exc}") from exc
+    except subprocess.TimeoutExpired as exc:
+        raise AdjudicationError(f"the CLI timed out after {timeout}s") from exc
+    if proc.returncode != 0:
+        raise AdjudicationError(
+            f"the CLI exited {proc.returncode}: {proc.stderr.strip() or '(no stderr)'}"
+        )
+    return proc.stdout.strip()
+
+
+def build_verdict(
+    payload: dict[str, Any],
+    *,
+    adjudicator: str,
+    adjudicated_at: str,
+    dwell_s: int,
+) -> dict[str, Any]:
+    """The proposal with a verdict on every entry, as a NEW mapping.
+
+    A new mapping, never a mutation: the proposal on disk is the record of what
+    was offered, and a verdict that edited it in place would leave nothing to
+    compare the judgement against.
+    """
+    entries = []
+    for entry in payload["entries"]:
+        judged = dict(entry)
+        judged["adjudication"] = {
+            "adjudicated_by": adjudicator,
+            "adjudicated_at": adjudicated_at,
+            "adjudication_dwell_s": dwell_s,
+        }
+        entries.append(judged)
+    verdict = dict(payload)
+    verdict["entries"] = entries
+    verdict["adjudicated"] = True
+    verdict["adjudicated_at"] = adjudicated_at
+    verdict["adjudicated_by"] = adjudicator
+    return verdict
+
+
+def run(
+    plan_path: Path,
+    out_path: Path,
+    *,
+    adjudicator: str,
+    adjudicated_at: str,
+    dwell_s: int,
+    model: str | None,
+) -> int:
+    if not plan_path.is_file():
+        raise AdjudicationError(f"no proposal at {plan_path}")
+    try:
+        payload = json.loads(plan_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise AdjudicationError(f"{plan_path} is not readable JSON: {exc}") from exc
+    # Parsed before anything is written: a proposal this library cannot read
+    # back is one no verdict can be recorded against, and finding that out
+    # after writing would leave a verdict pointing at nothing.
+    parse_segmentation_plan(payload)
+
+    if model is not None:
+        # Advisory only, and recorded rather than applied. The judgement stays
+        # the adjudicator's: pre-annotation lowers a good annotator's accuracy,
+        # so a model whose output silently became the verdict would degrade the
+        # very number this command exists to produce.
+        run_model(model, "Summarise the proposed segmentation for review.")
+
+    verdict = build_verdict(
+        payload, adjudicator=adjudicator, adjudicated_at=adjudicated_at, dwell_s=dwell_s
+    )
+    parse_segmentation_plan(verdict)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    out_path.write_text(
+        json.dumps(verdict, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline=""
+    )
+    return 0
+
+
+def parse_args(argv: list[str] | None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument("--plan", type=Path, required=True, help="the proposal to judge")
+    parser.add_argument("--out", type=Path, required=True, help="where to write the verdict")
+    parser.add_argument(
+        "--adjudicator", required=True, help="who judged: a person or an identifier, never a role"
+    )
+    parser.add_argument(
+        "--adjudicated-at", required=True, help="ISO 8601, stamped verbatim as everywhere else"
+    )
+    parser.add_argument(
+        "--dwell-s",
+        type=int,
+        default=DEFAULT_DWELL_S,
+        help="seconds spent per entry; the number that makes throughput measurable",
+    )
+    # OFF by default. Not a convenience default -- see the module docstring.
+    parser.add_argument(
+        "--model",
+        default=None,
+        help="switch the advisory model leg on and name the model; off when absent",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    try:
+        return run(
+            args.plan,
+            args.out,
+            adjudicator=args.adjudicator,
+            adjudicated_at=args.adjudicated_at,
+            dwell_s=args.dwell_s,
+            model=args.model,
+        )
+    except AdjudicationError as exc:
+        print(f"{ADJUDICATOR_ID}: FAILED - {exc}", file=sys.stderr)
+        print(
+            f"{ADJUDICATOR_ID}: this is NOT 'nothing to judge'. Nothing was written.",
+            file=sys.stderr,
+        )
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())