open/llm-ingestion-okf
1
0
Fork 0
Code Issues Pull requests Releases Activity

fix(materialize): scope §5 replacement to the running manifest's stem

Browse source 
The §3 ownership scan classified every ingest-stamped file as replaceable,
so materializing one manifest into a bundle deleted the ingest files another
manifest had stamped there — and removed their index links too.

Narrow ownership to files whose stamp names the running manifest by stem.
The stamp is `{stem}@{sha256[:16]}`; matching on the stem, not the whole
stamp, lets an edited manifest still reclaim the files a prior run of the
same manifest wrote (the stem is stable across content edits), while a
different manifest sharing the bundle keeps its own files. Implements the
trinn-e §10.2 decision. This does not close the operator-copy restriction
(a generated file copied into curated content), which stays documented in
ingest-spec.md:69-72, not enforced.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HBbjgS5A55RVavoyjJC4FX
This commit is contained in:
Kjell Tore Guttormsen 2026-07-23 07:00:27 +02:00
commit 1747238a83
2 changed files with 96 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/llm_ingestion_okf/materialize.py
View file

@ -81,12 +81,25 @@ def _parse_frontmatter(path: Path) -> dict[str, str]:
return frontmatter
def _is_ingest_owned(path: Path) -> bool:
def _is_ingest_owned(path: Path, manifest_stem: str) -> bool:
# §3/§5 ownership: the ingest stamp is `generated: true` AND an
# `ingest_manifest` reference. Promoted verdict files carry neither key,
# so they can never classify as ingest-owned.
#
# §10.2 per-manifest ownership: a file is THIS manifest's to replace only
# when the reference names it by stem. The stamp is `{stem}@{sha256[:16]}`;
# matching on the stem — not the whole stamp — lets an edited manifest (new
# content -> new sha -> new stamp) still reclaim the files a prior run of
# the same manifest wrote, while a DIFFERENT manifest sharing the bundle
# keeps its own. rsplit strips the trailing `@{sha}`, so a stem that itself
# contains `@` still compares correctly.
frontmatter = _parse_frontmatter(path)
return frontmatter.get("generated") == "true" and "ingest_manifest" in frontmatter
if frontmatter.get("generated") != "true":
return False
reference = frontmatter.get("ingest_manifest")
if reference is None:
return False
return reference.rsplit("@", 1)[0] == manifest_stem
def _render_concept_file(
@ -255,7 +268,7 @@ def materialize_bundle(
owned = {
path.name
for path in sorted(bundle.glob("*.md"))
if path.name != _INDEX_NAME and _is_ingest_owned(path)
if path.name != _INDEX_NAME and _is_ingest_owned(path, manifest_file.stem)
}
# §3 collision gate — BEFORE any mutation: a staged filename occupied by
# a file WITHOUT the stamp is curated content; never overwrite it.

81
tests/test_materialize.py
View file

@ -310,11 +310,14 @@ def test_collision_with_unstamped_file_fails_without_mutation(
def test_replacement_removes_stale_stamped_files(file_setup: tuple[Path, Path]) -> None:
# A prior run of THIS manifest (stem `manifest`, older content -> older
# sha) left ingest-stale.md, which the current run no longer produces.
# §5 replacement reclaims it because the stem still names this manifest.
manifest_path, bundle = file_setup
bundle.mkdir()
stale = bundle / "ingest-stale.md"
stale.write_text(
"---\ntype: dataset\ngenerated: true\ningest_manifest: old@0000\n---\n\nold\n",
"---\ntype: dataset\ngenerated: true\ningest_manifest: manifest@0000\n---\n\nold\n",
encoding="utf-8",
)
materialize_bundle(manifest_path, bundle, INGESTED_AT)
@ -322,6 +325,82 @@ def test_replacement_removes_stale_stamped_files(file_setup: tuple[Path, Path])
assert (bundle / "ingest-orders.md").is_file()
def test_replacement_leaves_another_manifests_stamped_file(file_setup: tuple[Path, Path]) -> None:
# The mirror of the case above: a stale file stamped by a DIFFERENT
# manifest (stem `other`) is NOT this manifest's to remove, so it survives.
manifest_path, bundle = file_setup
bundle.mkdir()
foreign = bundle / "ingest-foreign.md"
foreign.write_text(
"---\ntype: dataset\ngenerated: true\ningest_manifest: other@0000\n---\n\nold\n",
encoding="utf-8",
)
materialize_bundle(manifest_path, bundle, INGESTED_AT)
assert foreign.is_file()
assert (bundle / "ingest-orders.md").is_file()
def test_second_manifest_does_not_delete_first_manifests_stamped_file(tmp_path: Path) -> None:
# §10.2 per-manifest ownership: two manifests writing into ONE bundle each
# own only the files whose stamp names them by stem. Running manifest B
# must leave the ingest file that manifest A stamped in place — a narrower
# replacement blast radius than "remove every stamped file". (This does not
# close the operator-copy restriction, which stays documented in
# ingest-spec.md:69-72, not enforced.)
src = tmp_path / "src"
src.mkdir()
(src / "data").mkdir()
(src / "data" / "orders.csv").write_text("a,b\n1,x\n", encoding="utf-8", newline="")
alpha = src / "alpha.json"
alpha.write_text(
json.dumps(
file_manifest_data(
[
{
"id": "alpha-thing",
"title": "Alpha",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 10,
}
]
)
),
encoding="utf-8",
)
beta = src / "beta.json"
beta.write_text(
json.dumps(
file_manifest_data(
[
{
"id": "beta-thing",
"title": "Beta",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 10,
}
]
)
),
encoding="utf-8",
)
bundle = tmp_path / "bundle"
materialize_bundle(alpha, bundle, INGESTED_AT)
alpha_file = bundle / "ingest-alpha-thing.md"
assert alpha_file.is_file()
materialize_bundle(beta, bundle, INGESTED_AT)
# A's stamped file survives B's run, and B's own file is written alongside it.
assert alpha_file.is_file()
assert (bundle / "ingest-beta-thing.md").is_file()
# A's index link survives too — B only removes links to files it owns.
index = (bundle / "index.md").read_text(encoding="utf-8")
assert "- [Alpha](ingest-alpha-thing.md)" in index
assert "- [Beta](ingest-beta-thing.md)" in index
def test_curated_files_survive_byte_identical(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup
bundle.mkdir()