Kjell Tore Guttormsen
a5e129d413
Measured on the K2 corpus 2026-09-03: 39 documents proposed 618 entries under 601 distinct paths -- 17 paths were claimed by two documents each. Section numbering is document-local (`1 Innledning` is in most procurement documents), so this is structural, not unlucky. Every collision reaches Door B's gate, which refuses per DOCUMENT, so those documents would land as coded rejections rather than concepts and a corpus run could not be built at all. `--path-prefix` is an argument and not something the tool derives: the proposer sees ONE document and cannot know what else is in the bundle. It is reduced to the id grammar before anything is read, and a prefix that reduces to nothing is refused rather than silently producing the unscoped paths the caller asked to avoid. Without the flag every artifact already produced is byte-identical. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""The proposer: it proposes, a human adjudicates, and it never pretends otherwise.
|
|
|
|
The research is explicit about why this tool exists and about what it may not
|
|
claim. Topic 2 measured that the OKF reference agent splits on **what a thing
|
|
is** -- four semantic gates that need a model this run path does not have --
|
|
while a handful of MECHANICAL rules port today. Topic 1b measured that heading
|
|
derivation is inert on most of the K2 corpus: 23 of 33 PDFs carry no outline at
|
|
all and 95 % of the outline entries that exist are CAD export metadata, so the
|
|
scoreable denominator is ONE document. A rule validated on n=1 is not validated.
|
|
|
|
So the tool marks every decision `PROPOSED` and nothing else. The distinction
|
|
is the whole design: a proposal a human has not looked at must never be
|
|
replayable as an adjudication, because replay is exactly what the run path does
|
|
with a plan, deterministically and forever.
|
|
|
|
It lives outside `src/`, so it never enters a wheel (`tests/test_packaging.py`
|
|
enforces that) and no consumer's install surface changes because it exists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import socket
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from llm_ingestion_okf.segmentation import parse_segmentation_plan
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
|
|
|
import okf_propose_segments # noqa: E402
|
|
|
|
DOCUMENT = """# N500 Vegbygging
|
|
|
|
Innledende tekst om vegbygging og dens omfang.
|
|
|
|
## 3.1 Brannkonsept
|
|
|
|
Krav til seksjonering av bygget over flere etasjer.
|
|
|
|
## 3.2 Roemning
|
|
|
|
To uavhengige roemningsveier fra hver branncelle.
|
|
|
|
## Og
|
|
|
|
## 3.3 Baereevne
|
|
|
|
Hovedbaeresystemet skal ha R60 dokumentert baereevne.
|
|
"""
|
|
|
|
|
|
def write(tmp_path: Path, text: str = DOCUMENT, name: str = "n500.md") -> Path:
|
|
path = tmp_path / name
|
|
path.write_text(text, encoding="utf-8", newline="")
|
|
return path
|
|
|
|
|
|
def propose(tmp_path: Path, **kwargs: object) -> dict:
|
|
source = kwargs.pop("source", None) or write(tmp_path)
|
|
out = tmp_path / "plan.json"
|
|
code = okf_propose_segments.main([str(source), "--out", str(out)])
|
|
assert code == 0, f"proposer exited {code}"
|
|
return json.loads(out.read_text(encoding="utf-8"))
|
|
|
|
|
|
# --- the artifact is a valid plan -----------------------------------------
|
|
|
|
|
|
def test_the_emitted_artifact_parses_as_a_segmentation_plan(tmp_path: Path) -> None:
|
|
plan = parse_segmentation_plan(propose(tmp_path))
|
|
assert plan.entries
|
|
|
|
|
|
def test_it_carries_its_provenance(tmp_path: Path) -> None:
|
|
payload = propose(tmp_path)
|
|
assert payload["source_sha256"]
|
|
assert payload["extractor_id"]
|
|
assert payload["extractor_version"]
|
|
assert payload["proposed_by"].startswith("okf-propose-segments")
|
|
|
|
|
|
def test_the_source_hash_is_the_hash_of_the_bytes_it_read(tmp_path: Path) -> None:
|
|
import hashlib
|
|
|
|
source = write(tmp_path)
|
|
payload = propose(tmp_path, source=source)
|
|
assert payload["source_sha256"] == hashlib.sha256(source.read_bytes()).hexdigest()
|
|
|
|
|
|
# --- PROPOSED, never adjudicated ------------------------------------------
|
|
|
|
|
|
def test_every_entry_is_marked_proposed(tmp_path: Path) -> None:
|
|
payload = propose(tmp_path)
|
|
assert payload["entries"]
|
|
for entry in payload["entries"]:
|
|
assert okf_propose_segments.PROPOSED_MARKER in entry["derived"]
|
|
|
|
|
|
def test_no_entry_claims_to_be_adjudicated(tmp_path: Path) -> None:
|
|
payload = propose(tmp_path)
|
|
assert payload["adjudicated"] is False
|
|
for entry in payload["entries"]:
|
|
assert "adjudicated" not in entry
|
|
|
|
|
|
def test_every_entry_names_the_rule_that_proposed_it(tmp_path: Path) -> None:
|
|
# Per-decision provenance. A proposal an operator cannot trace to a rule is
|
|
# one they can only accept or reject wholesale.
|
|
for entry in propose(tmp_path)["entries"]:
|
|
rules = [name for name in entry["derived"] if name.startswith("rule:")]
|
|
assert len(rules) == 1
|
|
assert rules[0] in okf_propose_segments.RULE_NAMES
|
|
|
|
|
|
# --- the mechanical rules that actually ported -----------------------------
|
|
|
|
|
|
def test_a_heading_of_only_stop_words_is_not_a_boundary(tmp_path: Path) -> None:
|
|
# "Og" is a stop word. Without the gate it would open a segment of its own.
|
|
titles = [entry["title"] for entry in propose(tmp_path)["entries"]]
|
|
assert "Og" not in titles
|
|
assert any("Brannkonsept" in title for title in titles)
|
|
|
|
|
|
def test_a_heading_with_no_body_proposes_nothing(tmp_path: Path) -> None:
|
|
payload = propose(tmp_path, source=write(tmp_path, "# Tom\n\n## Ogsaa tom\n"))
|
|
assert payload["entries"] == []
|
|
|
|
|
|
def test_the_spans_it_proposes_are_slices_of_the_text_it_read(tmp_path: Path) -> None:
|
|
from llm_ingestion_okf.extract import extract_text
|
|
|
|
source = write(tmp_path)
|
|
payload = propose(tmp_path, source=source)
|
|
text = extract_text(source.name, source.read_bytes())
|
|
for entry in payload["entries"]:
|
|
start, end = entry["span"]
|
|
assert text[start:end].strip()
|
|
|
|
|
|
def test_every_entry_carries_an_anchor_quoting_its_own_span(tmp_path: Path) -> None:
|
|
"""Written at proposal time, when text and offsets are known to agree.
|
|
|
|
Reconstructed later it would quote whatever the extraction had already
|
|
become, which is precisely the drift the anchor exists to survive.
|
|
"""
|
|
from llm_ingestion_okf.extract import extract_text
|
|
|
|
source = write(tmp_path)
|
|
payload = propose(tmp_path, source=source)
|
|
text = extract_text(source.name, source.read_bytes())
|
|
assert payload["entries"]
|
|
for entry in payload["entries"]:
|
|
start, end = entry["span"]
|
|
anchor = entry["anchor"]
|
|
assert anchor["quote"] == text[start:end]
|
|
assert text[start - len(anchor["prefix"]) : start] == anchor["prefix"]
|
|
assert text[end : end + len(anchor["suffix"])] == anchor["suffix"]
|
|
|
|
|
|
def test_a_proposed_plan_survives_a_shift_of_the_text_it_was_made_against(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""The round trip the anchor exists for, measured end to end.
|
|
|
|
Prepend a paragraph -- exactly the benign edit that leaves every offset
|
|
short -- and every body must still be the text the proposer quoted. Before
|
|
anchors this cut each segment early and landed on real prose, with nothing
|
|
anywhere able to tell.
|
|
"""
|
|
from llm_ingestion_okf.extract import extract_text
|
|
from llm_ingestion_okf.segmentation import slice_segments
|
|
|
|
source = write(tmp_path)
|
|
plan = parse_segmentation_plan(propose(tmp_path, source=source))
|
|
text = extract_text(source.name, source.read_bytes())
|
|
quoted = [text[start:end] for start, end in (item.span for item in plan.entries)]
|
|
|
|
shifted = "Et nytt avsnitt foran alt annet.\n\n" + text
|
|
assert [body for _, body in slice_segments(shifted, plan)] == quoted
|
|
|
|
|
|
def test_the_paths_it_proposes_are_unique_and_hierarchical(tmp_path: Path) -> None:
|
|
plan = parse_segmentation_plan(propose(tmp_path))
|
|
paths = [entry.path for entry in plan.entries]
|
|
assert len(paths) == len(set(paths))
|
|
assert any("/" in path for path in paths)
|
|
|
|
|
|
# --- it does not reach the network ----------------------------------------
|
|
|
|
|
|
def test_it_makes_no_network_call(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def refuse(*args: object, **kwargs: object) -> None:
|
|
raise AssertionError("the proposer opened a socket")
|
|
|
|
monkeypatch.setattr(socket, "socket", refuse)
|
|
monkeypatch.setattr(socket, "create_connection", refuse)
|
|
parse_segmentation_plan(propose(tmp_path))
|
|
|
|
|
|
def test_the_socket_guard_can_actually_fire(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
# The known-positive control. Without it, a guard that patched the wrong
|
|
# name would make "no network call" a statement about nothing.
|
|
def refuse(*args: object, **kwargs: object) -> None:
|
|
raise AssertionError("the proposer opened a socket")
|
|
|
|
monkeypatch.setattr(socket, "socket", refuse)
|
|
with pytest.raises(AssertionError, match="opened a socket"):
|
|
socket.socket()
|
|
|
|
|
|
# --- failure is loud ------------------------------------------------------
|
|
|
|
|
|
def test_a_missing_source_exits_two_and_says_so(
|
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
|
) -> None:
|
|
code = okf_propose_segments.main([str(tmp_path / "absent.md"), "--out", str(tmp_path / "o")])
|
|
assert code == 2
|
|
assert "okf-propose-segments" in capsys.readouterr().err
|
|
|
|
|
|
def test_an_unknown_file_type_exits_two(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
|
|
source = write(tmp_path, "body\n", name="thing.unknown")
|
|
code = okf_propose_segments.main([str(source), "--out", str(tmp_path / "o")])
|
|
assert code == 2
|
|
assert "okf-propose-segments" in capsys.readouterr().err
|
|
|
|
|
|
# --- the per-document scope a multi-document corpus needs ------------------
|
|
|
|
|
|
def test_a_path_prefix_scopes_every_entry_under_one_directory(tmp_path: Path) -> None:
|
|
"""Measured on the K2 corpus, 2026-09-03: 39 documents proposed 618 entries
|
|
under 601 distinct paths -- **17 paths were claimed by two documents each**.
|
|
Every one of them would hit Door B's collision gate, and the gate refuses
|
|
per DOCUMENT, so those documents would land as coded rejections instead of
|
|
concepts. Section numbering is document-local (`1 Innledning` is in most of
|
|
them), so the collision is structural rather than unlucky.
|
|
|
|
The scope is an argument and not something this tool invents: the proposer
|
|
sees one document and has no way of knowing what else is in the bundle, so
|
|
the caller who does supplies the prefix.
|
|
"""
|
|
out = tmp_path / "plan.json"
|
|
assert (
|
|
okf_propose_segments.main(
|
|
[str(write(tmp_path)), "--out", str(out), "--path-prefix", "Del II Bilag 3.1"]
|
|
)
|
|
== 0
|
|
)
|
|
payload = json.loads(out.read_text(encoding="utf-8"))
|
|
assert payload["entries"]
|
|
for entry in payload["entries"]:
|
|
assert entry["path"].startswith("del-ii-bilag-3-1/")
|
|
# Still a valid plan: the prefix passes the same path grammar as any other
|
|
# component, rather than being spliced in behind the validator's back.
|
|
assert parse_segmentation_plan(payload).entries
|
|
|
|
|
|
def test_without_a_prefix_the_artifact_is_byte_identical(tmp_path: Path) -> None:
|
|
"""Additive. Every plan already produced stays exactly what it was."""
|
|
first = tmp_path / "a.json"
|
|
second = tmp_path / "b.json"
|
|
source = write(tmp_path)
|
|
okf_propose_segments.main([str(source), "--out", str(first)])
|
|
okf_propose_segments.main([str(source), "--out", str(second), "--path-prefix", ""])
|
|
assert first.read_bytes() == second.read_bytes()
|
|
|
|
|
|
def test_a_prefix_that_reduces_to_nothing_is_refused(tmp_path: Path) -> None:
|
|
"""A prefix of punctuation would otherwise become an empty component and
|
|
silently produce the unscoped paths the caller asked to avoid."""
|
|
code = okf_propose_segments.main(
|
|
[str(write(tmp_path)), "--out", str(tmp_path / "p.json"), "--path-prefix", "###"]
|
|
)
|
|
assert code == 2
|
|
assert not (tmp_path / "p.json").exists()
|