open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(materialize): add §5 materialization with collision gate and replacement

Browse source 
TDD step 5: materialize_bundle takes three explicit inputs (manifest,
bundle dir, regex-validated ingested_at — no wall-clock default) and
stages every extraction in memory before the first disk mutation. The
§3 collision gate refuses to overwrite any file without the ingest
stamp, before any mutation; replacement removes exactly the stamped
set. Frontmatter is the seven §5 keys in spec order with whitespace
collapse; output is LF-only raw bytes with one trailing newline; the
provenance stamp hashes the same bytes that are parsed. Fresh index.md
gets bundle_summary plus idempotent links in extraction order (§6
creation path; preserve/removal rules come next). The §8 network gate
refuses http fail-fast without the per-run opt-in; the http transport
itself lands in step 7.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QeqhJpYQyghASjiJo5EhGg
This commit is contained in:
Kjell Tore Guttormsen 2026-07-16 20:02:35 +02:00
commit 0ff946696c
5 changed files with 593 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

335
tests/test_materialize.py Normal file
View file

@ -0,0 +1,335 @@
"""Materialization (ingest-spec §5): staging, collision gate, replacement.
Three explicit inputs — manifest, bundle dir, ingested_at (validated, no
wall-clock default). All extractions execute and render IN MEMORY before the
first disk mutation; the §3 collision gate runs before any mutation; only
ingest-stamped files are ever replaced. LF-only bytes, one trailing newline.
"""
from __future__ import annotations
import hashlib
import json
import logging
import sqlite3
from pathlib import Path
from typing import Any
import pytest
from llm_ingestion_okf.errors import IngestError, MaterializationError, NetworkGateError
from llm_ingestion_okf.materialize import IngestResult, materialize_bundle
INGESTED_AT = "2026-07-16T12:00:00Z"
def write_manifest(directory: Path, data: dict[str, Any]) -> Path:
directory.mkdir(parents=True, exist_ok=True)
path = directory / "manifest.json"
path.write_text(json.dumps(data), encoding="utf-8")
return path
def stamp_of(manifest_path: Path) -> str:
h16 = hashlib.sha256(manifest_path.read_bytes()).hexdigest()[:16]
return f"{manifest_path.stem}@{h16}"
def file_manifest_data(extractions: list[dict[str, Any]] | None = None) -> dict[str, Any]:
return {
"manifest_version": 1,
"source": {"type": "file", "id": "catalogue-1", "root": "data"},
"bundle_summary": "A test bundle.",
"extractions": extractions
or [
{
"id": "orders",
"title": "Orders",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 100,
}
],
}
@pytest.fixture
def file_setup(tmp_path: Path) -> tuple[Path, Path]:
"""A manifest + CSV catalogue in tmp/src, and an empty bundle target path."""
src = tmp_path / "src"
manifest_path = write_manifest(src, file_manifest_data())
(src / "data").mkdir()
(src / "data" / "orders.csv").write_text("a,b\n1,x\n2,y\n", encoding="utf-8", newline="")
return manifest_path, tmp_path / "bundle"
# --- error hierarchy ---
def test_materialization_error_is_ingest_error() -> None:
assert issubclass(MaterializationError, IngestError)
assert issubclass(NetworkGateError, IngestError)
# --- ingested_at: explicit, validated, no wall-clock default ---
@pytest.mark.parametrize(
"bad",
[
"2026-07-16",
"2026-07-16T12:00:00",
"2026-07-16T12:00:00+00:00",
"2026-07-16 12:00:00Z",
"16-07-2026T12:00:00Z",
"",
],
)
def test_bad_ingested_at_rejected_before_any_mutation(
file_setup: tuple[Path, Path], bad: str
) -> None:
manifest_path, bundle = file_setup
with pytest.raises(MaterializationError):
materialize_bundle(manifest_path, bundle, bad)
assert not bundle.exists()
def test_missing_manifest_raises_typed_error(tmp_path: Path) -> None:
with pytest.raises(IngestError):
materialize_bundle(tmp_path / "nope.json", tmp_path / "bundle", INGESTED_AT)
# --- file source: exact §5 bytes ---
def test_file_source_concept_file_exact_bytes(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
materialize_bundle(manifest_path, bundle, INGESTED_AT)
expected = (
"---\n"
"type: dataset\n"
"title: Orders\n"
"source_system: catalogue-1\n"
"source_query: orders.csv\n"
f"ingested_at: {INGESTED_AT}\n"
f"ingest_manifest: {stamp_of(manifest_path)}\n"
"generated: true\n"
"---\n"
"\n"
"| a | b |\n"
"| --- | --- |\n"
"| 1 | x |\n"
"| 2 | y |\n"
)
assert (bundle / "ingest-orders.md").read_bytes() == expected.encode("utf-8")
def test_result_lists_written_concept_files(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
result = materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert isinstance(result, IngestResult)
assert result.written == (bundle / "ingest-orders.md",)
def test_relative_root_resolves_against_manifest_dir(
tmp_path: Path, file_setup: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
"""Pinned decision: a relative root resolves against the manifest file's
directory — never the process cwd (reproducibility)."""
manifest_path, bundle = file_setup
elsewhere = tmp_path / "elsewhere"
elsewhere.mkdir()
monkeypatch.chdir(elsewhere)
result = materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert len(result.written) == 1
# --- sql source end-to-end ---
def test_sql_source_end_to_end(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
db = tmp_path / "fixture.db"
with sqlite3.connect(db) as conn:
conn.execute("CREATE TABLE t (id INTEGER, name TEXT)")
conn.executemany("INSERT INTO t VALUES (?, ?)", [(1, "alpha"), (2, None)])
monkeypatch.setenv("OKF_TEST_DB", str(db))
data = file_manifest_data(
[
{
"id": "things",
"title": "Things",
"query": "SELECT id,\n name FROM t ORDER BY id",
"okf_type": "dataset",
"max_rows": 10,
}
]
)
data["source"] = {"type": "sql", "id": "db-1", "connection_ref": "OKF_TEST_DB"}
manifest_path = write_manifest(tmp_path / "src", data)
bundle = tmp_path / "bundle"
materialize_bundle(manifest_path, bundle, INGESTED_AT)
content = (bundle / "ingest-things.md").read_text(encoding="utf-8")
# §5: whitespace runs (incl. newlines) in source_query collapse to single spaces.
assert "source_query: SELECT id, name FROM t ORDER BY id\n" in content
assert content.endswith("| 1 | alpha |\n| 2 | |\n")
# --- fresh index generation (§6, creation path) ---
def test_fresh_index_exact_bytes(tmp_path: Path) -> None:
src = tmp_path / "src"
data = file_manifest_data(
[
{
"id": "orders",
"title": "Orders",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 10,
},
{
"id": "refunds",
"title": "Refunds",
"query": "refunds.csv",
"okf_type": "dataset",
"max_rows": 10,
},
]
)
manifest_path = write_manifest(src, data)
(src / "data").mkdir()
(src / "data" / "orders.csv").write_text("a\n1\n", encoding="utf-8", newline="")
(src / "data" / "refunds.csv").write_text("b\n2\n", encoding="utf-8", newline="")
bundle = tmp_path / "bundle"
materialize_bundle(manifest_path, bundle, INGESTED_AT)
expected = "A test bundle.\n- [Orders](ingest-orders.md)\n- [Refunds](ingest-refunds.md)\n"
assert (bundle / "index.md").read_bytes() == expected.encode("utf-8")
# --- §3 collision gate: BEFORE any mutation ---
def test_collision_with_unstamped_file_fails_without_mutation(
file_setup: tuple[Path, Path],
) -> None:
manifest_path, bundle = file_setup
bundle.mkdir()
curated = bundle / "ingest-orders.md"
curated.write_text("hand-curated content, no stamp\n", encoding="utf-8")
stale = bundle / "ingest-stale.md"
stale.write_text(
"---\ntype: dataset\ngenerated: true\ningest_manifest: old@0000\n---\n\nold\n",
encoding="utf-8",
)
with pytest.raises(MaterializationError):
materialize_bundle(manifest_path, bundle, INGESTED_AT)
# Gate fired BEFORE any mutation: curated file untouched, stale stamped file still there.
assert curated.read_text(encoding="utf-8") == "hand-curated content, no stamp\n"
assert stale.is_file()
# --- §5 replacement: only ingest-stamped files are replaced ---
def test_replacement_removes_stale_stamped_files(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
bundle.mkdir()
stale = bundle / "ingest-stale.md"
stale.write_text(
"---\ntype: dataset\ngenerated: true\ningest_manifest: old@0000\n---\n\nold\n",
encoding="utf-8",
)
materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert not stale.exists()
assert (bundle / "ingest-orders.md").is_file()
def test_curated_files_survive_byte_identical(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
bundle.mkdir()
curated = bundle / "notes.md"
curated_bytes = b"---\ntype: note\n---\n\ncurated, no stamp\n"
curated.write_bytes(curated_bytes)
materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert curated.read_bytes() == curated_bytes
# --- in-memory staging: extraction failure leaves the disk untouched ---
def test_failed_extraction_means_zero_disk_mutation(tmp_path: Path) -> None:
src = tmp_path / "src"
data = file_manifest_data(
[
{
"id": "good",
"title": "Good",
"query": "good.csv",
"okf_type": "dataset",
"max_rows": 10,
},
{"id": "bad", "title": "Bad", "query": "bad.csv", "okf_type": "dataset", "max_rows": 1},
]
)
manifest_path = write_manifest(src, data)
(src / "data").mkdir()
(src / "data" / "good.csv").write_text("a\n1\n", encoding="utf-8", newline="")
(src / "data" / "bad.csv").write_text("a\n1\n2\n", encoding="utf-8", newline="")
bundle = tmp_path / "bundle"
with pytest.raises(IngestError):
materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert not bundle.exists()
# --- determinism and idempotence (§10) ---
def test_two_runs_into_fresh_dirs_are_byte_identical(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
other = bundle.parent / "bundle2"
materialize_bundle(manifest_path, bundle, INGESTED_AT)
materialize_bundle(manifest_path, other, INGESTED_AT)
files = sorted(path.name for path in bundle.iterdir())
assert files == sorted(path.name for path in other.iterdir())
for name in files:
assert (bundle / name).read_bytes() == (other / name).read_bytes()
def test_rerun_over_own_output_is_idempotent(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
materialize_bundle(manifest_path, bundle, INGESTED_AT)
before = {path.name: path.read_bytes() for path in bundle.iterdir()}
materialize_bundle(manifest_path, bundle, INGESTED_AT)
after = {path.name: path.read_bytes() for path in bundle.iterdir()}
assert after == before # incl. no duplicated index links
# --- §8: source calls are logged (which source, when, row count) ---
def test_source_call_logged(
file_setup: tuple[Path, Path], caplog: pytest.LogCaptureFixture
) -> None:
manifest_path, bundle = file_setup
with caplog.at_level(logging.INFO, logger="llm_ingestion_okf.materialize"):
materialize_bundle(manifest_path, bundle, INGESTED_AT)
joined = " ".join(record.getMessage() for record in caplog.records)
assert "catalogue-1" in joined
assert INGESTED_AT in joined
assert "rows=2" in joined
# --- §8 network gate: http refused fail-fast without the per-run opt-in ---
def test_http_without_opt_in_refused_fail_fast(tmp_path: Path) -> None:
"""Load-bearing (spec §11): the manifest cannot grant itself network access."""
data = file_manifest_data()
data["source"] = {"type": "http", "id": "api-1", "base_url": "https://example.test"}
data["extractions"][0]["query"] = "/orders"
manifest_path = write_manifest(tmp_path / "src", data)
bundle = tmp_path / "bundle"
with pytest.raises(NetworkGateError):
materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert not bundle.exists()