feat(okf-v0.2): D5 — the v0.2 golden fixture, with okf_version in root frontmatter
Placement settled by catalog's own reading of upstream at the pinned commit
3fcbb9f: SS8:509-510 and SS12:773-775 both put `okf_version` in a bundle-root
`index.md` frontmatter block, and SS12 calls it the only place frontmatter is
permitted in an index. Catalog's spec says the opposite about the same file;
that divergence is theirs against upstream, and we conform to upstream.
The value never touches a profile. `OKF_V0_2.index.root_frontmatter` names the
key; the caller supplies the value through a new keyword-only
`root_frontmatter_values` mapping. That keeps V4/V-A5 intact - `okf_version`'s
value tracks the upstream Google version and belongs to catalog (E1), so a
constant here would claim a decision we do not own and would have to be chased
on every upstream release. In the fixture the value is fixture DATA
(`okf-version.txt`), not a literal in our source.
Ordering comes from the policy, not the caller's mapping: a dict preserves
insertion order, so two callers passing the same keys would otherwise emit
different bytes. A key the policy does not name is refused fail-fast, before
any disk mutation. Omitting the argument emits no block at all - SS12 is a MAY
and none of upstream's four reference bundles declares the key.
The block is written only when the index is CREATED, so a re-run into an
existing bundle stays byte-identical (A-E5).
Raw-byte assertions rather than parsed ones, on the committed fixture as well
as on fresh runs: catalog measured that a quoted value fails their shape regex
with exit 1 and that a BOM hides the marker while still exiting 0.
`yaml.safe_load` returns "0.2" either way and strips a BOM first, so a parsed
assertion masks exactly those two defects. Asserting the frozen fixture catches
what a self-comparison cannot - regenerating from a broken emitter moves both
sides together.
A-E6 is now placement-explicit (promised catalog in 99cf987), and separates the
two byte properties: BOM-free is a property of the file, unquoted is a property
of CATALOG'S GATE and not of OKF v0.2 - upstream's own SS12 example is quoted,
so their gate rejects the spec's canonical form.
README gains the upstream-version section it was missing; CLAUDE.md gains the
mechanism behind "no profile hard-codes an upstream version": a profile names a
key, a caller owns its value.
550 -> 559 tests. test_profile_threading's `OKF_V0_2.index is DEFAULT.index`
assertion is replaced rather than deleted: object identity was a proxy for "the
shipped profiles differ in no NAME-bearing field", which is what makes the
synthetic test profile necessary, so the guard now asserts that directly.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013dgkSPjkLpACjMayd9R5jx
This commit is contained in:
parent
ed08ac15e9
commit
2504011010
14 changed files with 470 additions and 34 deletions
|
|
@ -14,9 +14,11 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
from llm_ingestion_okf.materialize import materialize_bundle
|
||||
from llm_ingestion_okf.profiles import OKF_V0_2
|
||||
|
||||
EXAMPLES = Path(__file__).parent.parent / "examples"
|
||||
INGESTED_AT_NAME = "ingested-at.txt"
|
||||
OKF_VERSION_NAME = "okf-version.txt"
|
||||
|
||||
|
||||
def fixture_backed_get(fixture_dir: Path) -> object:
|
||||
|
|
@ -42,11 +44,34 @@ def materialize_case(case_dir: Path, out_dir: Path, monkeypatch: pytest.MonkeyPa
|
|||
allow_network=True,
|
||||
http_get=fixture_backed_get(case_dir / "fixture"), # type: ignore[arg-type]
|
||||
)
|
||||
elif case_dir.name.endswith("okf-v0-2"):
|
||||
# The declared version is fixture DATA, read from the case, never a
|
||||
# constant in this file. Its value tracks the upstream Google version
|
||||
# and belongs to catalog (E1); a literal here would be this repo
|
||||
# claiming a decision it does not own, and the golden would then have
|
||||
# to be chased on every upstream release.
|
||||
materialize_bundle(
|
||||
case_dir / "manifest.json",
|
||||
out_dir,
|
||||
ingested_at,
|
||||
profile=OKF_V0_2,
|
||||
root_frontmatter_values={
|
||||
"okf_version": (case_dir / OKF_VERSION_NAME).read_text(encoding="utf-8").strip()
|
||||
},
|
||||
)
|
||||
else:
|
||||
materialize_bundle(case_dir / "manifest.json", out_dir, ingested_at)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["ingest-golden-file", "ingest-golden-sql", "ingest-golden-http"])
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"ingest-golden-file",
|
||||
"ingest-golden-sql",
|
||||
"ingest-golden-http",
|
||||
"ingest-golden-okf-v0-2",
|
||||
],
|
||||
)
|
||||
def test_golden_case_byte_for_byte(
|
||||
case: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -214,4 +214,14 @@ def test_synthetic_profile_is_not_a_shipped_profile() -> None:
|
|||
assert _SYNTHETIC.paths is not DEFAULT.paths
|
||||
assert _SYNTHETIC.index is not DEFAULT.index
|
||||
assert OKF_V0_2.paths is DEFAULT.paths
|
||||
assert OKF_V0_2.index is DEFAULT.index
|
||||
# `OKF_V0_2.index` stopped being DEFAULT's object at D5, which named
|
||||
# `okf_version` in its `root_frontmatter`. That is NOT the distinctness this
|
||||
# guard is about: every field that carries a NAME — the index filename and
|
||||
# the concept filenames — is still shared, so a threading bug that passed
|
||||
# the wrong profile would still emit identical bytes under either. Object
|
||||
# identity was only ever a proxy for that; assert the thing itself, or this
|
||||
# guard would go green on a change that leaves the instrument blind.
|
||||
assert OKF_V0_2.index.name == DEFAULT.index.name
|
||||
assert OKF_V0_2.index.link_template == DEFAULT.index.link_template
|
||||
assert _SYNTHETIC.index.name != DEFAULT.index.name
|
||||
assert _SYNTHETIC.paths.concept_suffix != DEFAULT.paths.concept_suffix
|
||||
|
|
|
|||
215
tests/test_root_frontmatter_emission.py
Normal file
215
tests/test_root_frontmatter_emission.py
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
"""D5's emission half: the root `index.md` frontmatter block.
|
||||
|
||||
Upstream §8 binds `index.md` to "no frontmatter, with one exception: a
|
||||
bundle-root `index.md` MAY carry an `okf_version` key (§12)", and §12 puts the
|
||||
declaration "in a bundle-root `index.md` frontmatter block (the only place
|
||||
frontmatter is permitted in an `index.md`)". Catalog verified both quotations
|
||||
against the pinned commit `3fcbb9f` on 2026-07-31 and reported the same
|
||||
reading, so placement is settled: frontmatter, not a body line.
|
||||
|
||||
The value never comes from a profile. V4/V-A5 gives `okf_version`'s *value* to
|
||||
catalog (decision E1) and leaves this library the narrower obligation — name
|
||||
the key, express any value. So the profile pins `root_frontmatter` and the
|
||||
caller supplies the mapping.
|
||||
|
||||
Two assertions here read RAW BYTES rather than a parsed value, deliberately.
|
||||
Catalog measured that a quoted value fails their shape regex with `exit 1`, and
|
||||
that a UTF-8 BOM makes the marker invisible to them while still exiting `0`. A
|
||||
parsed assertion masks exactly those two defects: `yaml.safe_load` returns the
|
||||
string `"0.2"` whether or not it was quoted, and strips a BOM before the
|
||||
caller ever sees it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf.errors import MaterializationError
|
||||
from llm_ingestion_okf.materialize import materialize_bundle
|
||||
from llm_ingestion_okf.profiles import DEFAULT, OKF_V0_2
|
||||
|
||||
INGESTED_AT = "2026-07-16T12:00:00Z"
|
||||
|
||||
|
||||
def build_case(tmp_path: Path) -> Path:
|
||||
"""A minimal file-source manifest; the frontmatter block is what is under
|
||||
test, so the extraction is kept as small as it can be."""
|
||||
case = tmp_path / "case"
|
||||
fixture = case / "fixture"
|
||||
fixture.mkdir(parents=True)
|
||||
(fixture / "rows.csv").write_text("id,label\n1,alpha\n", encoding="utf-8")
|
||||
manifest = {
|
||||
"manifest_version": 1,
|
||||
"source": {"type": "file", "id": "root-fm", "root": "fixture"},
|
||||
"bundle_summary": "Root frontmatter case.",
|
||||
"extractions": [
|
||||
{
|
||||
"id": "rows",
|
||||
"title": "Rows",
|
||||
"query": "rows.csv",
|
||||
"okf_type": "dataset",
|
||||
"max_rows": 10,
|
||||
}
|
||||
],
|
||||
}
|
||||
(case / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return case
|
||||
|
||||
|
||||
def test_root_index_carries_the_declared_key(tmp_path: Path) -> None:
|
||||
"""The block opens the file and the body follows it."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(
|
||||
case / "manifest.json",
|
||||
bundle,
|
||||
INGESTED_AT,
|
||||
profile=OKF_V0_2,
|
||||
root_frontmatter_values={"okf_version": "0.2"},
|
||||
)
|
||||
|
||||
assert (bundle / "index.md").read_bytes() == (
|
||||
b"---\nokf_version: 0.2\n---\n\nRoot frontmatter case.\n- [Rows](ingest-rows.md)\n"
|
||||
)
|
||||
|
||||
|
||||
def test_the_value_is_unquoted_in_the_raw_bytes(tmp_path: Path) -> None:
|
||||
"""Catalog's shape regex `/^\\d+(\\.\\d+)*$/` sees the quotes as part of the
|
||||
value and exits 1. Asserted on bytes: a parsed check passes either way."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(
|
||||
case / "manifest.json",
|
||||
bundle,
|
||||
INGESTED_AT,
|
||||
profile=OKF_V0_2,
|
||||
root_frontmatter_values={"okf_version": "0.2"},
|
||||
)
|
||||
|
||||
raw = (bundle / "index.md").read_bytes()
|
||||
assert b"okf_version: 0.2\n" in raw
|
||||
assert b'okf_version: "0.2"' not in raw
|
||||
assert b"okf_version: '0.2'" not in raw
|
||||
|
||||
|
||||
def test_the_file_carries_no_utf8_bom(tmp_path: Path) -> None:
|
||||
"""A BOM leaves the marker invisible to catalog's gate while still exiting
|
||||
0 — the failure that reports success, so it gets its own byte assertion."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(
|
||||
case / "manifest.json",
|
||||
bundle,
|
||||
INGESTED_AT,
|
||||
profile=OKF_V0_2,
|
||||
root_frontmatter_values={"okf_version": "0.2"},
|
||||
)
|
||||
|
||||
raw = (bundle / "index.md").read_bytes()
|
||||
assert not raw.startswith(b"\xef\xbb\xbf")
|
||||
assert raw.startswith(b"---\n")
|
||||
|
||||
|
||||
def test_a_key_the_policy_does_not_name_is_refused(tmp_path: Path) -> None:
|
||||
"""`DEFAULT` names no root-frontmatter key, so offering one is a caller
|
||||
error rather than something to write. Fail-fast: refused BEFORE any disk
|
||||
mutation, or a rejected run would still leave a half-written bundle."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
with pytest.raises(MaterializationError) as excinfo:
|
||||
materialize_bundle(
|
||||
case / "manifest.json",
|
||||
bundle,
|
||||
INGESTED_AT,
|
||||
profile=DEFAULT,
|
||||
root_frontmatter_values={"okf_version": "0.2"},
|
||||
)
|
||||
|
||||
assert excinfo.value.code == "index_root_frontmatter_unexpected"
|
||||
assert not bundle.exists()
|
||||
|
||||
|
||||
def test_omitting_the_values_emits_no_frontmatter(tmp_path: Path) -> None:
|
||||
"""§12 is a MAY, and none of upstream's four reference bundles declares the
|
||||
key at all (catalog grepped `okf/bundles` and `okf/samples`: zero hits). A
|
||||
profile that names the key must therefore still emit a bundle without it."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(case / "manifest.json", bundle, INGESTED_AT, profile=OKF_V0_2)
|
||||
|
||||
raw = (bundle / "index.md").read_bytes()
|
||||
assert not raw.startswith(b"---")
|
||||
assert b"okf_version" not in raw
|
||||
|
||||
|
||||
def test_key_order_follows_the_policy_not_the_mapping(tmp_path: Path) -> None:
|
||||
"""The policy pins the order; a caller's dict ordering must not reach the
|
||||
file, or two callers passing the same keys would emit different bytes."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
profile = replace(
|
||||
OKF_V0_2,
|
||||
index=replace(OKF_V0_2.index, root_frontmatter=("okf_version", "bundle_profile")),
|
||||
)
|
||||
|
||||
materialize_bundle(
|
||||
case / "manifest.json",
|
||||
bundle,
|
||||
INGESTED_AT,
|
||||
profile=profile,
|
||||
root_frontmatter_values={"bundle_profile": "okf-v0-2", "okf_version": "0.2"},
|
||||
)
|
||||
|
||||
raw = (bundle / "index.md").read_bytes()
|
||||
assert raw.startswith(b"---\nokf_version: 0.2\nbundle_profile: okf-v0-2\n---\n\n")
|
||||
|
||||
|
||||
def test_the_frozen_golden_declares_the_version_gate_safely() -> None:
|
||||
"""D5's raw-byte guard, asserted on the COMMITTED fixture rather than on a
|
||||
fresh run.
|
||||
|
||||
`test_golden.py` already compares a run against these bytes, but that only
|
||||
proves the code agrees with itself: regenerate the fixture from a quoted or
|
||||
BOM-carrying emitter and both sides move together, silently. This asserts
|
||||
the shape of what we froze and hand to catalog's gate — the one thing a
|
||||
self-comparison cannot catch.
|
||||
"""
|
||||
raw = (
|
||||
Path(__file__).parent.parent
|
||||
/ "examples"
|
||||
/ "ingest-golden-okf-v0-2"
|
||||
/ "expected-bundle"
|
||||
/ "index.md"
|
||||
).read_bytes()
|
||||
|
||||
assert raw.startswith(b"---\nokf_version: 0.2\n---\n\n")
|
||||
assert not raw.startswith(b"\xef\xbb\xbf") # a BOM exits 0 while hiding the marker
|
||||
assert b'"' not in raw.split(b"---\n", 2)[1] # a quoted value exits 1
|
||||
assert b"\r\n" not in raw # LF-only, like every other emitted file
|
||||
|
||||
|
||||
def test_reingest_in_place_is_byte_identical(tmp_path: Path) -> None:
|
||||
"""A-E5. The second run goes into the FIRST run's directory — the shape the
|
||||
pilot runs — so the frontmatter block must survive §6 index maintenance
|
||||
rather than being appended a second time or dropped."""
|
||||
case = build_case(tmp_path)
|
||||
bundle = tmp_path / "bundle"
|
||||
call = dict(
|
||||
profile=OKF_V0_2,
|
||||
root_frontmatter_values={"okf_version": "0.2"},
|
||||
)
|
||||
|
||||
materialize_bundle(case / "manifest.json", bundle, INGESTED_AT, **call) # type: ignore[arg-type]
|
||||
first = (bundle / "index.md").read_bytes()
|
||||
materialize_bundle(case / "manifest.json", bundle, INGESTED_AT, **call) # type: ignore[arg-type]
|
||||
|
||||
assert (bundle / "index.md").read_bytes() == first
|
||||
Loading…
Add table
Add a link
Reference in a new issue