Compare commits

..

No commits in common. "v0.3.2" and "v0.3.1" have entirely different histories.

10 changed files with 10 additions and 124 deletions

View file

@ -5,22 +5,6 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.3.2] — 2026-07-23
### Fixed
- **Frontmatter and index-label values are emitted verbatim; only
`source_query` is whitespace-collapsed.** Earlier releases collapsed every
whitespace run in every frontmatter value and index link label to a single
space. §5 of `ingest-spec.md` mandates that collapse for `source_query`
alone — where a multi-line SQL `SELECT` must render on one line — while
every other value is validated single-line at manifest load and passed
through unchanged: validation, not repair. A `title` carrying an internal
whitespace run now survives byte-for-byte at both the `title` frontmatter
and the index link label, instead of being silently altered. Output bytes
change only for values that contained a collapsible whitespace run; the
shipped golden fixtures and both consumers are unaffected.
## [0.3.1] — 2026-07-19 ## [0.3.1] — 2026-07-19
### Fixed ### Fixed

View file

@ -11,8 +11,7 @@ one boundary rule:
`ingest-{id}.md` → index generation; zero model calls). This repo `ingest-{id}.md` → index generation; zero model calls). This repo
IMPLEMENTS the spec; commons keeps authorship. Spec changes the library IMPLEMENTS the spec; commons keeps authorship. Spec changes the library
needs go via commons, never edited locally. The library ships the §11 needs go via commons, never edited locally. The library ships the §11
golden fixtures (byte-exact) for the three door-A source types golden fixtures (byte-exact) that are missing today.
(`ingest-golden-{file,sql,http}/`, shipped in `9dd86b1`).
- **Door B — bundle inbox:** converts dropped files to OKF concepts. All - **Door B — bundle inbox:** converts dropped files to OKF concepts. All
file-type→text extraction lives HERE (the guard is text-only). v1 core: file-type→text extraction lives HERE (the guard is text-only). v1 core:
`md`, `txt`, `csv`, `json`, `html` (stdlib). `pdf`/`docx`/`xlsx` only via `md`, `txt`, `csv`, `json`, `html` (stdlib). `pdf`/`docx`/`xlsx` only via

View file

@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project] [project]
name = "llm-ingestion-okf" name = "llm-ingestion-okf"
version = "0.3.2" version = "0.3.1"
description = "Shared OKF (Open Knowledge Format) ingestion library: spec-based connectors, bundle inbox, and external-bundle import, with security delegated to llm-ingestion-guard." description = "Shared OKF (Open Knowledge Format) ingestion library: spec-based connectors, bundle inbox, and external-bundle import, with security delegated to llm-ingestion-guard."
readme = "README.md" readme = "README.md"
license = "MIT" license = "MIT"

View file

@ -35,7 +35,7 @@ from .manifest import (
) )
from .materialize import IngestResult, materialize_bundle from .materialize import IngestResult, materialize_bundle
__version__ = "0.3.2" __version__ = "0.3.1"
__all__ = [ __all__ = [
"Extraction", "Extraction",

View file

@ -35,14 +35,7 @@ def safe_resolve(root: Path, relative: str) -> Path:
a naive startswith would not). a naive startswith would not).
""" """
real_root = os.path.realpath(root) real_root = os.path.realpath(root)
try: candidate = os.path.realpath(os.path.join(real_root, relative))
candidate = os.path.realpath(os.path.join(real_root, relative))
except ValueError as exc:
# An embedded NUL makes the path syscalls reject the string before
# any boundary check can run. SourceError promises a typed failure,
# so a target that cannot be resolved at all fails closed under the
# boundary code rather than leaking an untyped ValueError.
raise SourceError(f"path is not resolvable: {relative!r}", code="path_escape") from exc
try: try:
within = os.path.commonpath([real_root, candidate]) == real_root within = os.path.commonpath([real_root, candidate]) == real_root
except ValueError: except ValueError:

View file

@ -55,8 +55,7 @@ class SourceError(IngestError):
OSError and never silent truncation. OSError and never silent truncation.
Codes: Codes:
- `path_escape` a path resolves outside its root directory, or cannot - `path_escape` a path resolves outside its root directory
be resolved at all (e.g. an embedded NUL byte)
- `source_root_missing` the file-source root is not a directory - `source_root_missing` the file-source root is not a directory
- `source_file_missing` the query does not resolve to a file - `source_file_missing` the query does not resolve to a file
- `csv_no_header` the CSV has no header row - `csv_no_header` the CSV has no header row

View file

@ -49,22 +49,11 @@ class IngestResult:
stamp: str stamp: str
def _collapse_whitespace(value: str) -> str:
# §5 mandates whitespace-run collapse for ONE field only: `source_query`
# (ingest-spec.md:140-141), where a legitimately multi-line SQL SELECT
# must render on one line. Every other value is validated single-line at
# manifest load and emitted verbatim — validation, not repair — so
# operator-supplied bytes (e.g. a title's internal double space) survive.
return " ".join(value.split())
def _render_frontmatter(frontmatter: dict[str, str]) -> str: def _render_frontmatter(frontmatter: dict[str, str]) -> str:
# Line-oriented `key: value`, insertion order. Only `source_query` is # Line-oriented `key: value`; values are single-lined (whitespace runs,
# collapsed; all other values pass through verbatim. # including newlines, collapse to one space — §5) because parsing is
return "\n".join( # line-oriented and stops at the first `---` line. Insertion order.
f"{key}: {_collapse_whitespace(value) if key == 'source_query' else value}" return "\n".join(f"{key}: {' '.join(value.split())}" for key, value in frontmatter.items())
for key, value in frontmatter.items()
)
def _parse_frontmatter(path: Path) -> dict[str, str]: def _parse_frontmatter(path: Path) -> dict[str, str]:

View file

@ -157,17 +157,6 @@ def test_path_escape(tmp_path: Path) -> None:
assert code_of(excinfo) == "path_escape" assert code_of(excinfo) == "path_escape"
def test_path_escape_covers_embedded_null(tmp_path: Path) -> None:
# A NUL byte makes the OS path syscalls raise ValueError before any
# boundary check runs. SourceError promises "always typed, never a
# leaked OSError" — an untyped ValueError breaks that contract, so a
# malformed target must fail closed under the same code as any other
# target that cannot resolve inside the root.
with pytest.raises(SourceError) as excinfo:
safe_resolve(tmp_path, "bad\x00.md")
assert code_of(excinfo) == "path_escape"
def test_source_root_missing(tmp_path: Path) -> None: def test_source_root_missing(tmp_path: Path) -> None:
with pytest.raises(SourceError) as excinfo: with pytest.raises(SourceError) as excinfo:
read_csv(tmp_path / "nope", "a.csv", max_rows=1) read_csv(tmp_path / "nope", "a.csv", max_rows=1)

View file

@ -124,73 +124,6 @@ def test_file_source_concept_file_exact_bytes(file_setup: tuple[Path, Path]) ->
assert (bundle / "ingest-orders.md").read_bytes() == expected.encode("utf-8") assert (bundle / "ingest-orders.md").read_bytes() == expected.encode("utf-8")
def test_title_renders_identically_in_frontmatter_and_index(tmp_path: Path) -> None:
# §4: the title "becomes the `title` frontmatter AND the index link
# label" — one value, two sites, so the two MUST agree. The frontmatter
# renderer single-lines its values; the index label was written raw, so
# a title carrying a whitespace run diverged between the two.
src = tmp_path / "src"
manifest_path = write_manifest(
src,
file_manifest_data(
[
{
"id": "orders",
"title": "Energiforbruk 2024",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 100,
}
]
),
)
(src / "data").mkdir()
(src / "data" / "orders.csv").write_text("a,b\n1,x\n", encoding="utf-8", newline="")
bundle = tmp_path / "bundle"
materialize_bundle(manifest_path, bundle, ingested_at=INGESTED_AT)
concept = (bundle / "ingest-orders.md").read_text(encoding="utf-8")
index = (bundle / "index.md").read_text(encoding="utf-8")
frontmatter_title = next(
line.partition(":")[2].strip() for line in concept.splitlines() if line.startswith("title:")
)
index_label = index.partition("- [")[2].partition("]")[0]
assert frontmatter_title == index_label
def test_title_emitted_verbatim_not_collapsed(tmp_path: Path) -> None:
# §5 mandates whitespace-run collapse ONLY for `source_query`
# (ingest-spec.md:140-141). `title` is validated single-line at manifest
# load and emitted verbatim — validation, not repair — so an internal
# whitespace run survives at BOTH the frontmatter and the index-label site.
src = tmp_path / "src"
manifest_path = write_manifest(
src,
file_manifest_data(
[
{
"id": "orders",
"title": "Energiforbruk 2024",
"query": "orders.csv",
"okf_type": "dataset",
"max_rows": 100,
}
]
),
)
(src / "data").mkdir()
(src / "data" / "orders.csv").write_text("a,b\n1,x\n", encoding="utf-8", newline="")
bundle = tmp_path / "bundle"
materialize_bundle(manifest_path, bundle, ingested_at=INGESTED_AT)
concept = (bundle / "ingest-orders.md").read_text(encoding="utf-8")
index = (bundle / "index.md").read_text(encoding="utf-8")
assert "title: Energiforbruk 2024\n" in concept
assert "- [Energiforbruk 2024](ingest-orders.md)" in index
def test_result_lists_written_concept_files(file_setup: tuple[Path, Path]) -> None: def test_result_lists_written_concept_files(file_setup: tuple[Path, Path]) -> None:
manifest_path, bundle = file_setup manifest_path, bundle = file_setup
result = materialize_bundle(manifest_path, bundle, INGESTED_AT) result = materialize_bundle(manifest_path, bundle, INGESTED_AT)

2
uv.lock generated
View file

@ -166,7 +166,7 @@ wheels = [
[[package]] [[package]]
name = "llm-ingestion-okf" name = "llm-ingestion-okf"
version = "0.3.2" version = "0.3.1"
source = { editable = "." } source = { editable = "." }
[package.dev-dependencies] [package.dev-dependencies]