feat(tools): segmentation proposer emitting adjudicable PROPOSED entries

tests/test_propose_segments.py

@@ -0,0 +1,190 @@
+"""The proposer: it proposes, a human adjudicates, and it never pretends otherwise.
+
+The research is explicit about why this tool exists and about what it may not
+claim. Topic 2 measured that the OKF reference agent splits on **what a thing
+is** -- four semantic gates that need a model this run path does not have --
+while a handful of MECHANICAL rules port today. Topic 1b measured that heading
+derivation is inert on most of the K2 corpus: 23 of 33 PDFs carry no outline at
+all and 95 % of the outline entries that exist are CAD export metadata, so the
+scoreable denominator is ONE document. A rule validated on n=1 is not validated.
+
+So the tool marks every decision `PROPOSED` and nothing else. The distinction
+is the whole design: a proposal a human has not looked at must never be
+replayable as an adjudication, because replay is exactly what the run path does
+with a plan, deterministically and forever.
+
+It lives outside `src/`, so it never enters a wheel (`tests/test_packaging.py`
+enforces that) and no consumer's install surface changes because it exists.
+"""
+
+from __future__ import annotations
+
+import json
+import socket
+import sys
+from pathlib import Path
+
+import pytest
+
+from llm_ingestion_okf.segmentation import parse_segmentation_plan
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
+
+import okf_propose_segments  # noqa: E402
+
+DOCUMENT = """# N500 Vegbygging
+
+Innledende tekst om vegbygging og dens omfang.
+
+## 3.1 Brannkonsept
+
+Krav til seksjonering av bygget over flere etasjer.
+
+## 3.2 Roemning
+
+To uavhengige roemningsveier fra hver branncelle.
+
+## Og
+
+## 3.3 Baereevne
+
+Hovedbaeresystemet skal ha R60 dokumentert baereevne.
+"""
+
+
+def write(tmp_path: Path, text: str = DOCUMENT, name: str = "n500.md") -> Path:
+    path = tmp_path / name
+    path.write_text(text, encoding="utf-8", newline="")
+    return path
+
+
+def propose(tmp_path: Path, **kwargs: object) -> dict:
+    source = kwargs.pop("source", None) or write(tmp_path)
+    out = tmp_path / "plan.json"
+    code = okf_propose_segments.main([str(source), "--out", str(out)])
+    assert code == 0, f"proposer exited {code}"
+    return json.loads(out.read_text(encoding="utf-8"))
+
+
+# --- the artifact is a valid plan -----------------------------------------
+
+
+def test_the_emitted_artifact_parses_as_a_segmentation_plan(tmp_path: Path) -> None:
+    plan = parse_segmentation_plan(propose(tmp_path))
+    assert plan.entries
+
+
+def test_it_carries_its_provenance(tmp_path: Path) -> None:
+    payload = propose(tmp_path)
+    assert payload["source_sha256"]
+    assert payload["extractor_id"]
+    assert payload["extractor_version"]
+    assert payload["proposed_by"].startswith("okf-propose-segments")
+
+
+def test_the_source_hash_is_the_hash_of_the_bytes_it_read(tmp_path: Path) -> None:
+    import hashlib
+
+    source = write(tmp_path)
+    payload = propose(tmp_path, source=source)
+    assert payload["source_sha256"] == hashlib.sha256(source.read_bytes()).hexdigest()
+
+
+# --- PROPOSED, never adjudicated ------------------------------------------
+
+
+def test_every_entry_is_marked_proposed(tmp_path: Path) -> None:
+    payload = propose(tmp_path)
+    assert payload["entries"]
+    for entry in payload["entries"]:
+        assert okf_propose_segments.PROPOSED_MARKER in entry["derived"]
+
+
+def test_no_entry_claims_to_be_adjudicated(tmp_path: Path) -> None:
+    payload = propose(tmp_path)
+    assert payload["adjudicated"] is False
+    for entry in payload["entries"]:
+        assert "adjudicated" not in entry
+
+
+def test_every_entry_names_the_rule_that_proposed_it(tmp_path: Path) -> None:
+    # Per-decision provenance. A proposal an operator cannot trace to a rule is
+    # one they can only accept or reject wholesale.
+    for entry in propose(tmp_path)["entries"]:
+        rules = [name for name in entry["derived"] if name.startswith("rule:")]
+        assert len(rules) == 1
+        assert rules[0] in okf_propose_segments.RULE_NAMES
+
+
+# --- the mechanical rules that actually ported -----------------------------
+
+
+def test_a_heading_of_only_stop_words_is_not_a_boundary(tmp_path: Path) -> None:
+    # "Og" is a stop word. Without the gate it would open a segment of its own.
+    titles = [entry["title"] for entry in propose(tmp_path)["entries"]]
+    assert "Og" not in titles
+    assert any("Brannkonsept" in title for title in titles)
+
+
+def test_a_heading_with_no_body_proposes_nothing(tmp_path: Path) -> None:
+    payload = propose(tmp_path, source=write(tmp_path, "# Tom\n\n## Ogsaa tom\n"))
+    assert payload["entries"] == []
+
+
+def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) -> None:
+    from llm_ingestion_okf.extract import extract_text
+
+    source = write(tmp_path)
+    payload = propose(tmp_path, source=source)
+    text = extract_text(source.name, source.read_bytes())
+    for entry in payload["entries"]:
+        start, end = entry["span"]
+        assert text[start:end].strip()
+
+
+def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
+    plan = parse_segmentation_plan(propose(tmp_path))
+    paths = [entry.path for entry in plan.entries]
+    assert len(paths) == len(set(paths))
+    assert any("/" in path for path in paths)
+
+
+# --- it does not reach the network ----------------------------------------
+
+
+def test_it_makes_no_network_call(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    def refuse(*args: object, **kwargs: object) -> None:
+        raise AssertionError("the proposer opened a socket")
+
+    monkeypatch.setattr(socket, "socket", refuse)
+    monkeypatch.setattr(socket, "create_connection", refuse)
+    parse_segmentation_plan(propose(tmp_path))
+
+
+def test_the_socket_guard_can_actually_fire(monkeypatch: pytest.MonkeyPatch) -> None:
+    # The known-positive control. Without it, a guard that patched the wrong
+    # name would make "no network call" a statement about nothing.
+    def refuse(*args: object, **kwargs: object) -> None:
+        raise AssertionError("the proposer opened a socket")
+
+    monkeypatch.setattr(socket, "socket", refuse)
+    with pytest.raises(AssertionError, match="opened a socket"):
+        socket.socket()
+
+
+# --- failure is loud ------------------------------------------------------
+
+
+def test_a_missing_source_exits_two_and_says_so(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    code = okf_propose_segments.main([str(tmp_path / "absent.md"), "--out", str(tmp_path / "o")])
+    assert code == 2
+    assert "okf-propose-segments" in capsys.readouterr().err
+
+
+def test_an_unknown_file_type_exits_two(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+    source = write(tmp_path, "body\n", name="thing.unknown")
+    code = okf_propose_segments.main([str(source), "--out", str(tmp_path / "o")])
+    assert code == 2
+    assert "okf-propose-segments" in capsys.readouterr().err