open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(fixtures): replace sector-specific example material with generic, fictitious examples — green

Browse source 
Every fixture, test document, tool example and document now uses an invented
kitchen-and-baking handbook series, written in this repository. The package's
behaviour is unchanged; src/ changes are comments and help text only.

- Generated fixtures are regenerated from their generators. Their structural
  counts are identical before and after: elements, images, rows, cells,
  headings, bookmarks and the witness inventory's per-document totals. The
  image-inbox and accounting documents are renamed kapittel-84-*.
- tools/okf_accounting_gate.py: the two options that named one real corpus
  each are replaced by a generic, repeatable --corpus PATH with no default.
  Row 5 compares the PDF pair alone. Gate verdict unchanged: RED rows 2, 3, 6.
- tools/okf_witness.py: the STS JSON reader for one publisher's delivery is
  removed, along with its three twins and five tests. The mutation harness
  loses W09.
- docs/: 13 dated reports that documented runs on a retired reference corpus
  are removed, and 40 are neutralized. Dead links are removed, and no new
  dangling path is introduced.
- The synthetic MCP-gate corpus and the residual probe words are neutral.

Valgt: keep the `okf quality --fasit` bar value (the measured fraction, one corpus) and
rewrite only its provenance, because the verdict stays unchanged and the
number names nothing.

Term check with the local list: 0 of 411 tracked files, 0 file names, 0 of
27 binary fixtures. Suite after git add: 2457 passed, 1 skipped. The base
tree had 2460 passed and 2 skipped; five tests went with the JSON reader and
four were added by the term check. ruff, ruff format and mypy --strict src/
are clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 13:54:57 +02:00
commit 9d1f4b14ed
174 changed files with 1889 additions and 6512 deletions
Download patch file Download diff file Expand all files Collapse all files

28
tests/test_segmented_collisions.py
View file

@ -54,7 +54,7 @@ def drop(inbox: Path, name: str, text: str = DOCUMENT) -> Path:
return path
def _extracted_text_sha256(source_bytes: bytes, filename: str = "n500.md") -> str:
def _extracted_text_sha256(source_bytes: bytes, filename: str = "q500.md") -> str:
return hashlib.sha256(extract_text(filename, source_bytes).encode("utf-8")).hexdigest()
@ -120,8 +120,8 @@ def tree(bundle: Path) -> dict[str, bytes]:
def test_two_documents_claiming_one_segment_path_are_both_refused(tmp_path: Path) -> None:
# Identical bytes, so ONE plan covers both documents and both expand onto
# the same paths. Refused together, before any gate call or write.
drop(tmp_path / "round", "n500.md")
drop(tmp_path / "round", "v720.md")
drop(tmp_path / "round", "q500.md")
drop(tmp_path / "round", "w720.md")
plan = build_plan(
DOCUMENT.encode("utf-8"), ("krav/3-1/brannkonsept.md", "krav/3-2/roemning.md")
)
@ -129,14 +129,14 @@ def test_two_documents_claiming_one_segment_path_are_both_refused(tmp_path: Path
result = run(tmp_path, plan=plan)
assert {entry.error.code for entry in result.failed} == {"inbox_slug_collision"}
assert {entry.source_file for entry in result.failed} == {"n500.md", "v720.md"}
assert {entry.source_file for entry in result.failed} == {"q500.md", "w720.md"}
assert tree(tmp_path / "bundle") == {}
assert result.persisted == ()
def test_the_collision_message_names_the_contested_path(tmp_path: Path) -> None:
drop(tmp_path / "round", "n500.md")
drop(tmp_path / "round", "v720.md")
drop(tmp_path / "round", "q500.md")
drop(tmp_path / "round", "w720.md")
plan = build_plan(DOCUMENT.encode("utf-8"), ("krav/3-1/brannkonsept.md",))
result = run(tmp_path, plan=plan)
assert all("krav/3-1/brannkonsept.md" in str(entry.error) for entry in result.failed)
@ -154,7 +154,7 @@ def test_a_flat_run_still_refuses_two_names_reducing_to_one_slug(tmp_path: Path)
def test_a_single_over_long_component_is_refused(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md")
source = drop(tmp_path / "round", "q500.md")
too_long = "a" * (NAME_MAX_BYTES + 1)
plan = build_plan(source.read_bytes(), (f"krav/{too_long}.md",))
result = run(tmp_path, plan=plan)
@ -166,7 +166,7 @@ def test_a_joined_path_over_name_max_with_legal_components_is_accepted(tmp_path:
# The check the joined measurement gets backwards. Every component here is
# well under NAME_MAX; the joined path is well over it, and the filesystem
# does not care -- NAME_MAX is a per-entry limit.
source = drop(tmp_path / "round", "n500.md")
source = drop(tmp_path / "round", "q500.md")
deep = "/".join(f"niva-{index}-{'x' * 40}" for index in range(6))
target = f"{deep}/krav.md"
assert len(target.encode("utf-8")) > NAME_MAX_BYTES
@ -183,7 +183,7 @@ def test_a_joined_path_over_name_max_with_legal_components_is_accepted(tmp_path:
def test_concepts_holds_one_entry_per_concept_and_persisted_one_per_source(
tmp_path: Path,
) -> None:
source = drop(tmp_path / "round", "n500.md")
source = drop(tmp_path / "round", "q500.md")
plan = build_plan(source.read_bytes(), ("krav/a.md", "krav/b.md", "krav/c.md"))
result = run(tmp_path, plan=plan)
@ -192,19 +192,19 @@ def test_concepts_holds_one_entry_per_concept_and_persisted_one_per_source(
# `persisted` keeps its existing meaning: one entry per SOURCE FILE. A
# consumer reading it sees exactly what it saw before segmentation existed.
assert len(result.persisted) == 1
assert result.persisted[0].source_file == "n500.md"
assert result.persisted[0].source_file == "q500.md"
def test_under_default_the_two_fields_agree(tmp_path: Path) -> None:
drop(tmp_path / "round", "n500.md")
drop(tmp_path / "round", "v720.md", "annet\n")
drop(tmp_path / "round", "q500.md")
drop(tmp_path / "round", "w720.md", "annet\n")
result = run(tmp_path, profile=DEFAULT)
assert len(result.persisted) == 2
assert result.concepts == result.persisted
def test_concepts_names_every_segment_path_it_wrote(tmp_path: Path) -> None:
source = drop(tmp_path / "round", "n500.md")
source = drop(tmp_path / "round", "q500.md")
plan = build_plan(source.read_bytes(), ("krav/3-1/a.md", "krav/3-2/b.md"))
result = run(tmp_path, plan=plan)
bundle = tmp_path / "bundle"
@ -212,4 +212,4 @@ def test_concepts_names_every_segment_path_it_wrote(tmp_path: Path) -> None:
"krav/3-1/a.md",
"krav/3-2/b.md",
}
assert all(entry.source_file == "n500.md" for entry in result.concepts)
assert all(entry.source_file == "q500.md" for entry in result.concepts)