feat(tools): the proposer scopes a document's segments under a caller's prefix

Measured on the K2 corpus 2026-09-03: 39 documents proposed 618 entries
under 601 distinct paths -- 17 paths were claimed by two documents each.
Section numbering is document-local (`1 Innledning` is in most procurement
documents), so this is structural, not unlucky. Every collision reaches
Door B's gate, which refuses per DOCUMENT, so those documents would land as
coded rejections rather than concepts and a corpus run could not be built
at all.

`--path-prefix` is an argument and not something the tool derives: the
proposer sees ONE document and cannot know what else is in the bundle. It
is reduced to the id grammar before anything is read, and a prefix that
reduces to nothing is refused rather than silently producing the unscoped
paths the caller asked to avoid. Without the flag every artifact already
produced is byte-identical.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-03 04:02:40 +02:00
commit a5e129d413
2 changed files with 104 additions and 8 deletions

View file

@ -230,3 +230,54 @@ def test_an_unknown_file_type_exits_two(tmp_path: Path, capsys: pytest.CaptureFi
code = okf_propose_segments.main([str(source), "--out", str(tmp_path / "o")])
assert code == 2
assert "okf-propose-segments" in capsys.readouterr().err
# --- the per-document scope a multi-document corpus needs ------------------
def test_a_path_prefix_scopes_every_entry_under_one_directory(tmp_path: Path) -> None:
"""Measured on the K2 corpus, 2026-09-03: 39 documents proposed 618 entries
under 601 distinct paths -- **17 paths were claimed by two documents each**.
Every one of them would hit Door B's collision gate, and the gate refuses
per DOCUMENT, so those documents would land as coded rejections instead of
concepts. Section numbering is document-local (`1 Innledning` is in most of
them), so the collision is structural rather than unlucky.
The scope is an argument and not something this tool invents: the proposer
sees one document and has no way of knowing what else is in the bundle, so
the caller who does supplies the prefix.
"""
out = tmp_path / "plan.json"
assert (
okf_propose_segments.main(
[str(write(tmp_path)), "--out", str(out), "--path-prefix", "Del II Bilag 3.1"]
)
== 0
)
payload = json.loads(out.read_text(encoding="utf-8"))
assert payload["entries"]
for entry in payload["entries"]:
assert entry["path"].startswith("del-ii-bilag-3-1/")
# Still a valid plan: the prefix passes the same path grammar as any other
# component, rather than being spliced in behind the validator's back.
assert parse_segmentation_plan(payload).entries
def test_without_a_prefix_the_artifact_is_byte_identical(tmp_path: Path) -> None:
"""Additive. Every plan already produced stays exactly what it was."""
first = tmp_path / "a.json"
second = tmp_path / "b.json"
source = write(tmp_path)
okf_propose_segments.main([str(source), "--out", str(first)])
okf_propose_segments.main([str(source), "--out", str(second), "--path-prefix", ""])
assert first.read_bytes() == second.read_bytes()
def test_a_prefix_that_reduces_to_nothing_is_refused(tmp_path: Path) -> None:
"""A prefix of punctuation would otherwise become an empty component and
silently produce the unscoped paths the caller asked to avoid."""
code = okf_propose_segments.main(
[str(write(tmp_path)), "--out", str(tmp_path / "p.json"), "--path-prefix", "###"]
)
assert code == 2
assert not (tmp_path / "p.json").exists()