Compare commits
5 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f14c075a65 | |||
| af35978fb0 | |||
| c011534e9f | |||
| a7e1bc88d8 | |||
| 64e3187639 |
10 changed files with 124 additions and 10 deletions
16
CHANGELOG.md
16
CHANGELOG.md
|
|
@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [0.3.2] — 2026-07-23
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Frontmatter and index-label values are emitted verbatim; only
|
||||
`source_query` is whitespace-collapsed.** Earlier releases collapsed every
|
||||
whitespace run in every frontmatter value and index link label to a single
|
||||
space. §5 of `ingest-spec.md` mandates that collapse for `source_query`
|
||||
alone — where a multi-line SQL `SELECT` must render on one line — while
|
||||
every other value is validated single-line at manifest load and passed
|
||||
through unchanged: validation, not repair. A `title` carrying an internal
|
||||
whitespace run now survives byte-for-byte at both the `title` frontmatter
|
||||
and the index link label, instead of being silently altered. Output bytes
|
||||
change only for values that contained a collapsible whitespace run; the
|
||||
shipped golden fixtures and both consumers are unaffected.
|
||||
|
||||
## [0.3.1] — 2026-07-19
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ one boundary rule:
|
|||
`ingest-{id}.md` → index generation; zero model calls). This repo
|
||||
IMPLEMENTS the spec; commons keeps authorship. Spec changes the library
|
||||
needs go via commons, never edited locally. The library ships the §11
|
||||
golden fixtures (byte-exact) that are missing today.
|
||||
golden fixtures (byte-exact) for the three door-A source types
|
||||
(`ingest-golden-{file,sql,http}/`, shipped in `9dd86b1`).
|
||||
- **Door B — bundle inbox:** converts dropped files to OKF concepts. All
|
||||
file-type→text extraction lives HERE (the guard is text-only). v1 core:
|
||||
`md`, `txt`, `csv`, `json`, `html` (stdlib). `pdf`/`docx`/`xlsx` only via
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "llm-ingestion-okf"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
description = "Shared OKF (Open Knowledge Format) ingestion library: spec-based connectors, bundle inbox, and external-bundle import, with security delegated to llm-ingestion-guard."
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from .manifest import (
|
|||
)
|
||||
from .materialize import IngestResult, materialize_bundle
|
||||
|
||||
__version__ = "0.3.1"
|
||||
__version__ = "0.3.2"
|
||||
|
||||
__all__ = [
|
||||
"Extraction",
|
||||
|
|
|
|||
|
|
@ -35,7 +35,14 @@ def safe_resolve(root: Path, relative: str) -> Path:
|
|||
a naive startswith would not).
|
||||
"""
|
||||
real_root = os.path.realpath(root)
|
||||
candidate = os.path.realpath(os.path.join(real_root, relative))
|
||||
try:
|
||||
candidate = os.path.realpath(os.path.join(real_root, relative))
|
||||
except ValueError as exc:
|
||||
# An embedded NUL makes the path syscalls reject the string before
|
||||
# any boundary check can run. SourceError promises a typed failure,
|
||||
# so a target that cannot be resolved at all fails closed under the
|
||||
# boundary code rather than leaking an untyped ValueError.
|
||||
raise SourceError(f"path is not resolvable: {relative!r}", code="path_escape") from exc
|
||||
try:
|
||||
within = os.path.commonpath([real_root, candidate]) == real_root
|
||||
except ValueError:
|
||||
|
|
|
|||
|
|
@ -55,7 +55,8 @@ class SourceError(IngestError):
|
|||
OSError and never silent truncation.
|
||||
|
||||
Codes:
|
||||
- `path_escape` — a path resolves outside its root directory
|
||||
- `path_escape` — a path resolves outside its root directory, or cannot
|
||||
be resolved at all (e.g. an embedded NUL byte)
|
||||
- `source_root_missing` — the file-source root is not a directory
|
||||
- `source_file_missing` — the query does not resolve to a file
|
||||
- `csv_no_header` — the CSV has no header row
|
||||
|
|
|
|||
|
|
@ -49,11 +49,22 @@ class IngestResult:
|
|||
stamp: str
|
||||
|
||||
|
||||
def _collapse_whitespace(value: str) -> str:
|
||||
# §5 mandates whitespace-run collapse for ONE field only: `source_query`
|
||||
# (ingest-spec.md:140-141), where a legitimately multi-line SQL SELECT
|
||||
# must render on one line. Every other value is validated single-line at
|
||||
# manifest load and emitted verbatim — validation, not repair — so
|
||||
# operator-supplied bytes (e.g. a title's internal double space) survive.
|
||||
return " ".join(value.split())
|
||||
|
||||
|
||||
def _render_frontmatter(frontmatter: dict[str, str]) -> str:
|
||||
# Line-oriented `key: value`; values are single-lined (whitespace runs,
|
||||
# including newlines, collapse to one space — §5) because parsing is
|
||||
# line-oriented and stops at the first `---` line. Insertion order.
|
||||
return "\n".join(f"{key}: {' '.join(value.split())}" for key, value in frontmatter.items())
|
||||
# Line-oriented `key: value`, insertion order. Only `source_query` is
|
||||
# collapsed; all other values pass through verbatim.
|
||||
return "\n".join(
|
||||
f"{key}: {_collapse_whitespace(value) if key == 'source_query' else value}"
|
||||
for key, value in frontmatter.items()
|
||||
)
|
||||
|
||||
|
||||
def _parse_frontmatter(path: Path) -> dict[str, str]:
|
||||
|
|
|
|||
|
|
@ -157,6 +157,17 @@ def test_path_escape(tmp_path: Path) -> None:
|
|||
assert code_of(excinfo) == "path_escape"
|
||||
|
||||
|
||||
def test_path_escape_covers_embedded_null(tmp_path: Path) -> None:
|
||||
# A NUL byte makes the OS path syscalls raise ValueError before any
|
||||
# boundary check runs. SourceError promises "always typed, never a
|
||||
# leaked OSError" — an untyped ValueError breaks that contract, so a
|
||||
# malformed target must fail closed under the same code as any other
|
||||
# target that cannot resolve inside the root.
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
safe_resolve(tmp_path, "bad\x00.md")
|
||||
assert code_of(excinfo) == "path_escape"
|
||||
|
||||
|
||||
def test_source_root_missing(tmp_path: Path) -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path / "nope", "a.csv", max_rows=1)
|
||||
|
|
|
|||
|
|
@ -124,6 +124,73 @@ def test_file_source_concept_file_exact_bytes(file_setup: tuple[Path, Path]) ->
|
|||
assert (bundle / "ingest-orders.md").read_bytes() == expected.encode("utf-8")
|
||||
|
||||
|
||||
def test_title_renders_identically_in_frontmatter_and_index(tmp_path: Path) -> None:
|
||||
# §4: the title "becomes the `title` frontmatter AND the index link
|
||||
# label" — one value, two sites, so the two MUST agree. The frontmatter
|
||||
# renderer single-lines its values; the index label was written raw, so
|
||||
# a title carrying a whitespace run diverged between the two.
|
||||
src = tmp_path / "src"
|
||||
manifest_path = write_manifest(
|
||||
src,
|
||||
file_manifest_data(
|
||||
[
|
||||
{
|
||||
"id": "orders",
|
||||
"title": "Energiforbruk 2024",
|
||||
"query": "orders.csv",
|
||||
"okf_type": "dataset",
|
||||
"max_rows": 100,
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
(src / "data").mkdir()
|
||||
(src / "data" / "orders.csv").write_text("a,b\n1,x\n", encoding="utf-8", newline="")
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(manifest_path, bundle, ingested_at=INGESTED_AT)
|
||||
|
||||
concept = (bundle / "ingest-orders.md").read_text(encoding="utf-8")
|
||||
index = (bundle / "index.md").read_text(encoding="utf-8")
|
||||
frontmatter_title = next(
|
||||
line.partition(":")[2].strip() for line in concept.splitlines() if line.startswith("title:")
|
||||
)
|
||||
index_label = index.partition("- [")[2].partition("]")[0]
|
||||
assert frontmatter_title == index_label
|
||||
|
||||
|
||||
def test_title_emitted_verbatim_not_collapsed(tmp_path: Path) -> None:
|
||||
# §5 mandates whitespace-run collapse ONLY for `source_query`
|
||||
# (ingest-spec.md:140-141). `title` is validated single-line at manifest
|
||||
# load and emitted verbatim — validation, not repair — so an internal
|
||||
# whitespace run survives at BOTH the frontmatter and the index-label site.
|
||||
src = tmp_path / "src"
|
||||
manifest_path = write_manifest(
|
||||
src,
|
||||
file_manifest_data(
|
||||
[
|
||||
{
|
||||
"id": "orders",
|
||||
"title": "Energiforbruk 2024",
|
||||
"query": "orders.csv",
|
||||
"okf_type": "dataset",
|
||||
"max_rows": 100,
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
(src / "data").mkdir()
|
||||
(src / "data" / "orders.csv").write_text("a,b\n1,x\n", encoding="utf-8", newline="")
|
||||
bundle = tmp_path / "bundle"
|
||||
|
||||
materialize_bundle(manifest_path, bundle, ingested_at=INGESTED_AT)
|
||||
|
||||
concept = (bundle / "ingest-orders.md").read_text(encoding="utf-8")
|
||||
index = (bundle / "index.md").read_text(encoding="utf-8")
|
||||
assert "title: Energiforbruk 2024\n" in concept
|
||||
assert "- [Energiforbruk 2024](ingest-orders.md)" in index
|
||||
|
||||
|
||||
def test_result_lists_written_concept_files(file_setup: tuple[Path, Path]) -> None:
|
||||
manifest_path, bundle = file_setup
|
||||
result = materialize_bundle(manifest_path, bundle, INGESTED_AT)
|
||||
|
|
|
|||
2
uv.lock
generated
2
uv.lock
generated
|
|
@ -166,7 +166,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "llm-ingestion-okf"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
source = { editable = "." }
|
||||
|
||||
[package.dev-dependencies]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue