Compare commits
2 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 83928e73c9 | |||
| 9d296ca686 |
9 changed files with 471 additions and 46 deletions
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "llm-ingestion-okf"
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
description = "Shared OKF (Open Knowledge Format) ingestion library: spec-based connectors, bundle inbox, and external-bundle import, with security delegated to llm-ingestion-guard."
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from .manifest import (
|
|||
)
|
||||
from .materialize import IngestResult, materialize_bundle
|
||||
|
||||
__version__ = "0.2.0"
|
||||
__version__ = "0.3.0"
|
||||
|
||||
__all__ = [
|
||||
"Extraction",
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ def safe_resolve(root: Path, relative: str) -> Path:
|
|||
# Different drives / mixed absolute-relative -> not within.
|
||||
within = False
|
||||
if not within:
|
||||
raise SourceError(f"path escapes its root directory: {relative!r}")
|
||||
raise SourceError(f"path escapes its root directory: {relative!r}", code="path_escape")
|
||||
return Path(candidate)
|
||||
|
||||
|
||||
|
|
@ -56,25 +56,32 @@ def read_csv(root: str | Path, query: str, *, max_rows: int) -> tuple[list[str],
|
|||
"""
|
||||
root_path = Path(root)
|
||||
if not root_path.is_dir():
|
||||
raise SourceError(f"file-source root is not a directory: {root_path}")
|
||||
raise SourceError(
|
||||
f"file-source root is not a directory: {root_path}", code="source_root_missing"
|
||||
)
|
||||
resolved = safe_resolve(root_path, query)
|
||||
if not resolved.is_file():
|
||||
raise SourceError(f"extraction query does not resolve to a file: {query!r}")
|
||||
raise SourceError(
|
||||
f"extraction query does not resolve to a file: {query!r}",
|
||||
code="source_file_missing",
|
||||
)
|
||||
with resolved.open(encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.reader(handle)
|
||||
header = next(reader, None)
|
||||
if header is None:
|
||||
raise SourceError(f"CSV has no header row: {query!r}")
|
||||
raise SourceError(f"CSV has no header row: {query!r}", code="csv_no_header")
|
||||
rows: list[list[str]] = []
|
||||
for row in reader:
|
||||
if len(rows) >= max_rows:
|
||||
raise SourceError(
|
||||
f"extraction {query!r} exceeds max_rows={max_rows} "
|
||||
"(error, never silent truncation — spec §8)"
|
||||
"(error, never silent truncation — spec §8)",
|
||||
code="max_rows_exceeded",
|
||||
)
|
||||
if len(row) != len(header):
|
||||
raise SourceError(
|
||||
f"ragged CSV row in {query!r}: expected {len(header)} cells, got {len(row)}"
|
||||
f"ragged CSV row in {query!r}: expected {len(header)} cells, got {len(row)}",
|
||||
code="csv_ragged_row",
|
||||
)
|
||||
rows.append(row)
|
||||
return header, rows
|
||||
|
|
@ -98,12 +105,14 @@ def read_sql(
|
|||
if not dsn:
|
||||
raise SourceError(
|
||||
f"sql source connection_ref {connection_ref!r} is not set in the environment "
|
||||
"(paths/credentials resolve at run time, never from the manifest — §4)"
|
||||
"(paths/credentials resolve at run time, never from the manifest — §4)",
|
||||
code="connection_ref_unset",
|
||||
)
|
||||
db_file = Path(dsn)
|
||||
if not db_file.is_file():
|
||||
raise SourceError(
|
||||
f"sql connection_ref {connection_ref!r} points at a missing database file: {db_file}"
|
||||
f"sql connection_ref {connection_ref!r} points at a missing database file: {db_file}",
|
||||
code="database_missing",
|
||||
)
|
||||
# quote keeps '/' but encodes spaces/'?'/'#' so an odd path can't corrupt
|
||||
# the URI; file:{abs path}?mode=ro is sqlite's documented read-only open.
|
||||
|
|
@ -112,18 +121,23 @@ def read_sql(
|
|||
with closing(sqlite3.connect(uri, uri=True)) as conn:
|
||||
cursor = conn.execute(query)
|
||||
if cursor.description is None: # a SELECT always has columns; defensive
|
||||
raise SourceError(f"sql extraction returned no columns: {query!r}")
|
||||
raise SourceError(
|
||||
f"sql extraction returned no columns: {query!r}", code="sql_no_columns"
|
||||
)
|
||||
header = [column[0] for column in cursor.description]
|
||||
rows: list[list[str]] = []
|
||||
for row in cursor:
|
||||
if len(rows) >= max_rows:
|
||||
raise SourceError(
|
||||
f"extraction {query!r} exceeds max_rows={max_rows} "
|
||||
"(error, never silent truncation — spec §8)"
|
||||
"(error, never silent truncation — spec §8)",
|
||||
code="max_rows_exceeded",
|
||||
)
|
||||
rows.append([sql_value_to_text(value) for value in row])
|
||||
except (sqlite3.Error, sqlite3.Warning) as exc:
|
||||
raise SourceError(f"sql extraction failed for query {query!r}: {exc}") from exc
|
||||
raise SourceError(
|
||||
f"sql extraction failed for query {query!r}: {exc}", code="sql_failed"
|
||||
) from exc
|
||||
return header, rows
|
||||
|
||||
|
||||
|
|
@ -142,7 +156,7 @@ def urllib_get(url: str, credential: str | None) -> str:
|
|||
raw: bytes = response.read()
|
||||
return raw.decode("utf-8")
|
||||
except (URLError, OSError, UnicodeDecodeError) as exc:
|
||||
raise SourceError(f"http GET failed for {url!r}: {exc}") from exc
|
||||
raise SourceError(f"http GET failed for {url!r}: {exc}", code="http_transport") from exc
|
||||
|
||||
|
||||
def read_http(
|
||||
|
|
@ -174,7 +188,8 @@ def read_http(
|
|||
if not credential:
|
||||
raise SourceError(
|
||||
f"http source credential_ref {credential_ref!r} is not set in the environment "
|
||||
"(credentials resolve at run time, never from the manifest — §4)"
|
||||
"(credentials resolve at run time, never from the manifest — §4)",
|
||||
code="credential_ref_unset",
|
||||
)
|
||||
url = base_url.rstrip("/") + "/" + query.lstrip("/")
|
||||
normalized = get(url, credential).replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
|
@ -183,7 +198,8 @@ def read_http(
|
|||
raise SourceError(
|
||||
f"http extraction {query!r} contains a code-fence marker line (```) — "
|
||||
"cannot be safely embedded in a fenced code block; fail-fast, never "
|
||||
"silent corruption"
|
||||
"silent corruption",
|
||||
code="fence_marker_in_body",
|
||||
)
|
||||
newline_count = normalized.count("\n")
|
||||
line_count = (
|
||||
|
|
@ -192,6 +208,7 @@ def read_http(
|
|||
if line_count > max_rows:
|
||||
raise SourceError(
|
||||
f"http extraction {query!r} exceeds max_rows={max_rows} "
|
||||
"(error, never silent truncation — spec §8)"
|
||||
"(error, never silent truncation — spec §8)",
|
||||
code="max_rows_exceeded",
|
||||
)
|
||||
return normalized
|
||||
|
|
|
|||
|
|
@ -1,18 +1,50 @@
|
|||
"""Typed error hierarchy rooted in IngestError."""
|
||||
"""Typed error hierarchy rooted in IngestError.
|
||||
|
||||
STABILITY CONTRACT: `IngestError.code` is the machine-readable API for
|
||||
distinguishing sub-causes — consumers assert on it, never on message
|
||||
wording. Codes listed in the class docstrings below are stable across
|
||||
releases; message text is NOT stable and may improve freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class IngestError(Exception):
|
||||
"""Base class for every error raised by this library."""
|
||||
"""Base class for every error raised by this library.
|
||||
|
||||
Carries a stable, machine-readable `code` naming the sub-cause (see the
|
||||
subclass docstrings for the registry). Errors constructed without an
|
||||
explicit code carry `"unspecified"`.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, *, code: str = "unspecified") -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
class ManifestError(IngestError):
|
||||
"""The manifest failed fail-fast schema validation (ingest-spec §4)."""
|
||||
"""The manifest failed fail-fast schema validation (ingest-spec §4).
|
||||
|
||||
Codes:
|
||||
- `manifest_unreadable` — the manifest file cannot be read
|
||||
- `manifest_invalid_json` — the bytes are not valid UTF-8 JSON
|
||||
- `manifest_version_unsupported` — manifest_version is not the integer 1
|
||||
- `manifest_schema` — a generic shape violation (wrong type, missing or
|
||||
unknown field, empty string, bad identifier, multi-line title,
|
||||
non-positive max_rows, empty extractions list)
|
||||
- `source_type_unknown` — source.type is not 'file', 'sql', or 'http'
|
||||
- `credential_embedded` — source.base_url embeds userinfo credentials
|
||||
- `extraction_id_duplicate` — two extractions share an id
|
||||
- `okf_type_reserved` — an extraction claims the reserved 'verdict' layer
|
||||
"""
|
||||
|
||||
|
||||
class RenderError(IngestError):
|
||||
"""A value cannot be rendered under the §5 body rules (never silent coercion)."""
|
||||
"""A value cannot be rendered under the §5 body rules (never silent coercion).
|
||||
|
||||
Codes:
|
||||
- `unsupported_cell_type` — a SQL cell is not integer/float/text/NULL
|
||||
"""
|
||||
|
||||
|
||||
class SourceError(IngestError):
|
||||
|
|
@ -21,14 +53,31 @@ class SourceError(IngestError):
|
|||
Covers fail-closed path-boundary violations, missing/malformed source
|
||||
content, and max_rows cap violations — always typed, never a leaked
|
||||
OSError and never silent truncation.
|
||||
|
||||
Codes:
|
||||
- `path_escape` — a path resolves outside its root directory
|
||||
- `source_root_missing` — the file-source root is not a directory
|
||||
- `source_file_missing` — the query does not resolve to a file
|
||||
- `csv_no_header` — the CSV has no header row
|
||||
- `csv_ragged_row` — a CSV row's cell count differs from the header's
|
||||
- `max_rows_exceeded` — an extraction exceeds max_rows (any source type)
|
||||
- `connection_ref_unset` — the sql connection_ref env var is not set
|
||||
- `database_missing` — the sql connection_ref points at a missing file
|
||||
- `sql_no_columns` — the sql statement produced no result columns
|
||||
- `sql_failed` — the sql statement failed at the database
|
||||
- `credential_ref_unset` — the http credential_ref env var is not set
|
||||
- `http_transport` — the http GET failed at transport or decode
|
||||
- `fence_marker_in_body` — an http body line is a code-fence marker
|
||||
"""
|
||||
|
||||
|
||||
class MaterializationError(IngestError):
|
||||
"""Materialization refused or failed (ingest-spec §5).
|
||||
|
||||
Covers an invalid ingested_at argument and the §3 collision gate (a
|
||||
generated name occupied by a file without the ingest stamp).
|
||||
Codes:
|
||||
- `ingested_at_invalid` — ingested_at is not ISO-8601 UTC with a Z suffix
|
||||
- `collision_unstamped` — the §3 collision gate: a generated name is
|
||||
occupied by a file without the ingest stamp
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -37,4 +86,7 @@ class NetworkGateError(IngestError):
|
|||
|
||||
Local-only default, no silent egress: the flag is a run argument — the
|
||||
manifest cannot grant itself network access.
|
||||
|
||||
Codes:
|
||||
- `network_opt_in_missing` — an http source without allow_network=True
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -77,7 +77,9 @@ def load_manifest(path: Path) -> Manifest:
|
|||
try:
|
||||
raw = path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise ManifestError(f"cannot read manifest {path}: {exc}") from exc
|
||||
raise ManifestError(
|
||||
f"cannot read manifest {path}: {exc}", code="manifest_unreadable"
|
||||
) from exc
|
||||
return load_manifest_bytes(raw)
|
||||
|
||||
|
||||
|
|
@ -91,7 +93,9 @@ def load_manifest_bytes(raw: bytes) -> Manifest:
|
|||
try:
|
||||
data: object = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError) as exc:
|
||||
raise ManifestError(f"manifest is not valid UTF-8 JSON: {exc}") from exc
|
||||
raise ManifestError(
|
||||
f"manifest is not valid UTF-8 JSON: {exc}", code="manifest_invalid_json"
|
||||
) from exc
|
||||
return _validate_manifest(data)
|
||||
|
||||
|
||||
|
|
@ -101,7 +105,10 @@ def _validate_manifest(data: object) -> Manifest:
|
|||
|
||||
version = obj["manifest_version"]
|
||||
if not _is_int(version) or version != 1:
|
||||
raise ManifestError(f"manifest_version must be the integer 1, got {version!r}")
|
||||
raise ManifestError(
|
||||
f"manifest_version must be the integer 1, got {version!r}",
|
||||
code="manifest_version_unsupported",
|
||||
)
|
||||
|
||||
source = _validate_source(obj["source"])
|
||||
bundle_summary = _require_str(obj["bundle_summary"], "bundle_summary", allow_empty=True)
|
||||
|
|
@ -139,25 +146,33 @@ def _validate_source(data: object) -> Source:
|
|||
base_url=_validate_base_url(obj["base_url"]),
|
||||
credential_ref=credential_ref,
|
||||
)
|
||||
raise ManifestError(f"source.type must be one of 'file', 'sql', 'http', got {source_type!r}")
|
||||
raise ManifestError(
|
||||
f"source.type must be one of 'file', 'sql', 'http', got {source_type!r}",
|
||||
code="source_type_unknown",
|
||||
)
|
||||
|
||||
|
||||
def _validate_base_url(value: object) -> str:
|
||||
base_url = _require_str(value, "source.base_url")
|
||||
# Credentials never live in the manifest (spec §4): no userinfo in the URL.
|
||||
if urllib.parse.urlsplit(base_url).username is not None:
|
||||
raise ManifestError("source.base_url must not embed credentials; use credential_ref")
|
||||
raise ManifestError(
|
||||
"source.base_url must not embed credentials; use credential_ref",
|
||||
code="credential_embedded",
|
||||
)
|
||||
return base_url
|
||||
|
||||
|
||||
def _validate_extractions(data: object) -> tuple[Extraction, ...]:
|
||||
if not isinstance(data, list) or not data:
|
||||
raise ManifestError("extractions must be a non-empty list")
|
||||
raise ManifestError("extractions must be a non-empty list", code="manifest_schema")
|
||||
extractions = tuple(_validate_extraction(entry, index) for index, entry in enumerate(data))
|
||||
seen: set[str] = set()
|
||||
for extraction in extractions:
|
||||
if extraction.id in seen:
|
||||
raise ManifestError(f"duplicate extraction id {extraction.id!r}")
|
||||
raise ManifestError(
|
||||
f"duplicate extraction id {extraction.id!r}", code="extraction_id_duplicate"
|
||||
)
|
||||
seen.add(extraction.id)
|
||||
return extractions
|
||||
|
||||
|
|
@ -169,17 +184,22 @@ def _validate_extraction(data: object, index: int) -> Extraction:
|
|||
|
||||
title = _require_str(obj["title"], f"{label}.title")
|
||||
if "\n" in title or "\r" in title:
|
||||
raise ManifestError(f"{label}.title must be single-line")
|
||||
raise ManifestError(f"{label}.title must be single-line", code="manifest_schema")
|
||||
|
||||
okf_type = _require_str(obj["okf_type"], f"{label}.okf_type")
|
||||
# The verdict layer is RESERVED (spec §3): the promotion gate is the only
|
||||
# path into it — enforced here, fail-fast, before any source call.
|
||||
if okf_type.lower() == _RESERVED_OKF_TYPE:
|
||||
raise ManifestError(f"{label}.okf_type must not be 'verdict' (reserved layer)")
|
||||
raise ManifestError(
|
||||
f"{label}.okf_type must not be 'verdict' (reserved layer)", code="okf_type_reserved"
|
||||
)
|
||||
|
||||
max_rows = obj["max_rows"]
|
||||
if not _is_int(max_rows) or max_rows < 1:
|
||||
raise ManifestError(f"{label}.max_rows must be a positive integer, got {max_rows!r}")
|
||||
raise ManifestError(
|
||||
f"{label}.max_rows must be a positive integer, got {max_rows!r}",
|
||||
code="manifest_schema",
|
||||
)
|
||||
|
||||
return Extraction(
|
||||
id=_validate_id(obj["id"], f"{label}.id"),
|
||||
|
|
@ -193,13 +213,17 @@ def _validate_extraction(data: object, index: int) -> Extraction:
|
|||
def _validate_id(value: object, label: str) -> str:
|
||||
identifier = _require_str(value, label)
|
||||
if not _ID_PATTERN.match(identifier):
|
||||
raise ManifestError(f"{label} must match [a-z0-9][a-z0-9-]*, got {identifier!r}")
|
||||
raise ManifestError(
|
||||
f"{label} must match [a-z0-9][a-z0-9-]*, got {identifier!r}", code="manifest_schema"
|
||||
)
|
||||
return identifier
|
||||
|
||||
|
||||
def _require_object(data: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(data, dict):
|
||||
raise ManifestError(f"{label} must be a JSON object, got {type(data).__name__}")
|
||||
raise ManifestError(
|
||||
f"{label} must be a JSON object, got {type(data).__name__}", code="manifest_schema"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
|
|
@ -208,17 +232,24 @@ def _require_keys(
|
|||
) -> None:
|
||||
missing = required - obj.keys()
|
||||
if missing:
|
||||
raise ManifestError(f"{label} is missing required field(s): {', '.join(sorted(missing))}")
|
||||
raise ManifestError(
|
||||
f"{label} is missing required field(s): {', '.join(sorted(missing))}",
|
||||
code="manifest_schema",
|
||||
)
|
||||
unknown = obj.keys() - required - (optional or set())
|
||||
if unknown:
|
||||
raise ManifestError(f"{label} has unknown field(s): {', '.join(sorted(unknown))}")
|
||||
raise ManifestError(
|
||||
f"{label} has unknown field(s): {', '.join(sorted(unknown))}", code="manifest_schema"
|
||||
)
|
||||
|
||||
|
||||
def _require_str(value: object, label: str, *, allow_empty: bool = False) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise ManifestError(f"{label} must be a string, got {type(value).__name__}")
|
||||
raise ManifestError(
|
||||
f"{label} must be a string, got {type(value).__name__}", code="manifest_schema"
|
||||
)
|
||||
if not value and not allow_empty:
|
||||
raise ManifestError(f"{label} must be a non-empty string")
|
||||
raise ManifestError(f"{label} must be a non-empty string", code="manifest_schema")
|
||||
return value
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -166,13 +166,16 @@ def materialize_bundle(
|
|||
if not _INGESTED_AT_RE.match(ingested_at):
|
||||
raise MaterializationError(
|
||||
"ingested_at must be ISO-8601 UTC with a Z suffix "
|
||||
f"(e.g. 2026-07-03T12:00:00Z), got {ingested_at!r}"
|
||||
f"(e.g. 2026-07-03T12:00:00Z), got {ingested_at!r}",
|
||||
code="ingested_at_invalid",
|
||||
)
|
||||
manifest_file = Path(manifest_path)
|
||||
try:
|
||||
raw = manifest_file.read_bytes()
|
||||
except OSError as exc:
|
||||
raise ManifestError(f"cannot read manifest {manifest_file}: {exc}") from exc
|
||||
raise ManifestError(
|
||||
f"cannot read manifest {manifest_file}: {exc}", code="manifest_unreadable"
|
||||
) from exc
|
||||
# §5 provenance stamp over the exact bytes that are parsed below.
|
||||
stamp = f"{manifest_file.stem}@{hashlib.sha256(raw).hexdigest()[:16]}"
|
||||
manifest = load_manifest_bytes(raw)
|
||||
|
|
@ -183,7 +186,8 @@ def materialize_bundle(
|
|||
raise NetworkGateError(
|
||||
"http source requires the per-run network opt-in "
|
||||
"(materialize_bundle(..., allow_network=True)) — the manifest cannot "
|
||||
"grant itself network access (spec §8, local-only default)"
|
||||
"grant itself network access (spec §8, local-only default)",
|
||||
code="network_opt_in_missing",
|
||||
)
|
||||
|
||||
# Pinned decision (file): a relative root resolves against the manifest
|
||||
|
|
@ -248,7 +252,8 @@ def materialize_bundle(
|
|||
if (bundle / name).is_file() and name not in owned:
|
||||
raise MaterializationError(
|
||||
f"generated filename {name!r} collides with an existing file that does "
|
||||
"not carry the ingest stamp — refusing to overwrite curated content (§3)"
|
||||
"not carry the ingest stamp — refusing to overwrite curated content (§3)",
|
||||
code="collision_unstamped",
|
||||
)
|
||||
|
||||
# §5 replacement: remove every stamped file, then write the new set.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,10 @@ def sql_value_to_text(value: object) -> str:
|
|||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
raise RenderError(f"unsupported SQL cell type bool ({value!r}) — never silent coercion")
|
||||
raise RenderError(
|
||||
f"unsupported SQL cell type bool ({value!r}) — never silent coercion",
|
||||
code="unsupported_cell_type",
|
||||
)
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
|
|
@ -31,7 +34,8 @@ def sql_value_to_text(value: object) -> str:
|
|||
return value
|
||||
raise RenderError(
|
||||
f"unsupported SQL cell type {type(value).__name__} — spec §5 allows "
|
||||
"integer/float/text/NULL only; never silent coercion"
|
||||
"integer/float/text/NULL only; never silent coercion",
|
||||
code="unsupported_cell_type",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
316
tests/test_error_codes.py
Normal file
316
tests/test_error_codes.py
Normal file
|
|
@ -0,0 +1,316 @@
|
|||
"""Stable error codes: `IngestError.code` is the machine-readable API.
|
||||
|
||||
Every raise site carries a documented, stable code so consumers can assert
|
||||
on sub-causes without binding message wording (messages may improve freely;
|
||||
codes may not change). One test per code — this file IS the registry's
|
||||
conformance suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
import llm_ingestion_okf.connectors as connectors
|
||||
from llm_ingestion_okf.connectors import read_csv, read_http, read_sql, safe_resolve
|
||||
from llm_ingestion_okf.errors import (
|
||||
IngestError,
|
||||
ManifestError,
|
||||
MaterializationError,
|
||||
NetworkGateError,
|
||||
RenderError,
|
||||
SourceError,
|
||||
)
|
||||
from llm_ingestion_okf.manifest import load_manifest, load_manifest_bytes
|
||||
from llm_ingestion_okf.materialize import materialize_bundle
|
||||
from llm_ingestion_okf.render import sql_value_to_text
|
||||
|
||||
INGESTED_AT = "2026-07-17T12:00:00Z"
|
||||
|
||||
|
||||
def manifest_data(**overrides: Any) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {
|
||||
"manifest_version": 1,
|
||||
"source": {"type": "file", "id": "src-1", "root": "data"},
|
||||
"bundle_summary": "A bundle.",
|
||||
"extractions": [
|
||||
{
|
||||
"id": "orders",
|
||||
"title": "Orders",
|
||||
"query": "orders.csv",
|
||||
"okf_type": "dataset",
|
||||
"max_rows": 100,
|
||||
}
|
||||
],
|
||||
}
|
||||
data.update(overrides)
|
||||
return data
|
||||
|
||||
|
||||
def load(data: dict[str, Any]) -> None:
|
||||
load_manifest_bytes(json.dumps(data).encode("utf-8"))
|
||||
|
||||
|
||||
def code_of(excinfo: pytest.ExceptionInfo[IngestError]) -> str:
|
||||
return excinfo.value.code
|
||||
|
||||
|
||||
# --- the attribute itself ---
|
||||
|
||||
|
||||
def test_bare_ingest_error_defaults_to_unspecified() -> None:
|
||||
assert IngestError("something").code == "unspecified"
|
||||
|
||||
|
||||
def test_code_does_not_change_the_message() -> None:
|
||||
exc = SourceError("the message", code="max_rows_exceeded")
|
||||
assert str(exc) == "the message"
|
||||
assert exc.code == "max_rows_exceeded"
|
||||
|
||||
|
||||
# --- ManifestError codes ---
|
||||
|
||||
|
||||
def test_manifest_unreadable(tmp_path: Path) -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load_manifest(tmp_path / "nope.json")
|
||||
assert code_of(excinfo) == "manifest_unreadable"
|
||||
|
||||
|
||||
def test_manifest_unreadable_via_materialize(tmp_path: Path) -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
materialize_bundle(tmp_path / "nope.json", tmp_path / "bundle", INGESTED_AT)
|
||||
assert code_of(excinfo) == "manifest_unreadable"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b"\xff\xfe", b"not json"])
|
||||
def test_manifest_invalid_json(raw: bytes) -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load_manifest_bytes(raw)
|
||||
assert code_of(excinfo) == "manifest_invalid_json"
|
||||
|
||||
|
||||
def test_manifest_version_unsupported() -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load(manifest_data(manifest_version=2))
|
||||
assert code_of(excinfo) == "manifest_version_unsupported"
|
||||
|
||||
|
||||
def test_source_type_unknown() -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load(manifest_data(source={"type": "ftp", "id": "x"}))
|
||||
assert code_of(excinfo) == "source_type_unknown"
|
||||
|
||||
|
||||
def test_credential_embedded_in_base_url() -> None:
|
||||
source = {"type": "http", "id": "api-1", "base_url": "https://user:pw@example.test"}
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load(manifest_data(source=source))
|
||||
assert code_of(excinfo) == "credential_embedded"
|
||||
|
||||
|
||||
def test_extraction_id_duplicate() -> None:
|
||||
extraction = manifest_data()["extractions"][0]
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load(manifest_data(extractions=[extraction, dict(extraction)]))
|
||||
assert code_of(excinfo) == "extraction_id_duplicate"
|
||||
|
||||
|
||||
def test_okf_type_reserved() -> None:
|
||||
extraction = dict(manifest_data()["extractions"][0], okf_type="verdict")
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load(manifest_data(extractions=[extraction]))
|
||||
assert code_of(excinfo) == "okf_type_reserved"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
"not an object",
|
||||
manifest_data(bundle_summary=None),
|
||||
{k: v for k, v in manifest_data().items() if k != "source"},
|
||||
manifest_data(surprise="unknown field"),
|
||||
manifest_data(extractions=[]),
|
||||
manifest_data(source={"type": "file", "id": "UPPER", "root": "data"}),
|
||||
manifest_data(extractions=[dict(manifest_data()["extractions"][0], title="two\nlines")]),
|
||||
manifest_data(extractions=[dict(manifest_data()["extractions"][0], max_rows=0)]),
|
||||
manifest_data(extractions=[dict(manifest_data()["extractions"][0], query="")]),
|
||||
],
|
||||
)
|
||||
def test_manifest_schema_violations_share_one_code(data: Any) -> None:
|
||||
with pytest.raises(ManifestError) as excinfo:
|
||||
load_manifest_bytes(json.dumps(data).encode("utf-8"))
|
||||
assert code_of(excinfo) == "manifest_schema"
|
||||
|
||||
|
||||
# --- SourceError codes ---
|
||||
|
||||
|
||||
def test_path_escape(tmp_path: Path) -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
safe_resolve(tmp_path, "../outside")
|
||||
assert code_of(excinfo) == "path_escape"
|
||||
|
||||
|
||||
def test_source_root_missing(tmp_path: Path) -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path / "nope", "a.csv", max_rows=1)
|
||||
assert code_of(excinfo) == "source_root_missing"
|
||||
|
||||
|
||||
def test_source_file_missing(tmp_path: Path) -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path, "nope.csv", max_rows=1)
|
||||
assert code_of(excinfo) == "source_file_missing"
|
||||
|
||||
|
||||
def test_csv_no_header(tmp_path: Path) -> None:
|
||||
(tmp_path / "empty.csv").write_text("", encoding="utf-8")
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path, "empty.csv", max_rows=1)
|
||||
assert code_of(excinfo) == "csv_no_header"
|
||||
|
||||
|
||||
def test_csv_ragged_row(tmp_path: Path) -> None:
|
||||
(tmp_path / "ragged.csv").write_text("a,b\n1\n", encoding="utf-8", newline="")
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path, "ragged.csv", max_rows=10)
|
||||
assert code_of(excinfo) == "csv_ragged_row"
|
||||
|
||||
|
||||
def test_max_rows_exceeded_csv(tmp_path: Path) -> None:
|
||||
(tmp_path / "big.csv").write_text("a\n1\n2\n", encoding="utf-8", newline="")
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_csv(tmp_path, "big.csv", max_rows=1)
|
||||
assert code_of(excinfo) == "max_rows_exceeded"
|
||||
|
||||
|
||||
def test_max_rows_exceeded_sql(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = tmp_path / "fixture.db"
|
||||
with sqlite3.connect(db) as conn:
|
||||
conn.execute("CREATE TABLE t (id INTEGER)")
|
||||
conn.executemany("INSERT INTO t VALUES (?)", [(1,), (2,)])
|
||||
monkeypatch.setenv("OKF_TEST_DB", str(db))
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_sql("OKF_TEST_DB", "SELECT id FROM t", max_rows=1)
|
||||
assert code_of(excinfo) == "max_rows_exceeded"
|
||||
|
||||
|
||||
def test_max_rows_exceeded_http() -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_http("https://example.test", "/x", max_rows=1, get=lambda url, cred: "a\nb\n")
|
||||
assert code_of(excinfo) == "max_rows_exceeded"
|
||||
|
||||
|
||||
def test_connection_ref_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OKF_TEST_DB", raising=False)
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_sql("OKF_TEST_DB", "SELECT 1", max_rows=1)
|
||||
assert code_of(excinfo) == "connection_ref_unset"
|
||||
|
||||
|
||||
def test_database_missing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OKF_TEST_DB", str(tmp_path / "nope.db"))
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_sql("OKF_TEST_DB", "SELECT 1", max_rows=1)
|
||||
assert code_of(excinfo) == "database_missing"
|
||||
|
||||
|
||||
def test_sql_no_columns(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = tmp_path / "fixture.db"
|
||||
sqlite3.connect(db).close()
|
||||
monkeypatch.setenv("OKF_TEST_DB", str(db))
|
||||
# BEGIN succeeds read-only and yields no result columns.
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_sql("OKF_TEST_DB", "BEGIN", max_rows=1)
|
||||
assert code_of(excinfo) == "sql_no_columns"
|
||||
|
||||
|
||||
def test_sql_failed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = tmp_path / "fixture.db"
|
||||
sqlite3.connect(db).close()
|
||||
monkeypatch.setenv("OKF_TEST_DB", str(db))
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_sql("OKF_TEST_DB", "SELECT nope FROM missing", max_rows=1)
|
||||
assert code_of(excinfo) == "sql_failed"
|
||||
|
||||
|
||||
def test_credential_ref_unset(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OKF_TEST_TOKEN", raising=False)
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_http(
|
||||
"https://example.test",
|
||||
"/x",
|
||||
max_rows=1,
|
||||
credential_ref="OKF_TEST_TOKEN",
|
||||
get=lambda url, cred: "",
|
||||
)
|
||||
assert code_of(excinfo) == "credential_ref_unset"
|
||||
|
||||
|
||||
def test_http_transport(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def refuse(request: Any) -> Any:
|
||||
raise urllib.error.URLError("refused")
|
||||
|
||||
monkeypatch.setattr(connectors, "urlopen", refuse)
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
connectors.urllib_get("https://example.test/x", None)
|
||||
assert code_of(excinfo) == "http_transport"
|
||||
|
||||
|
||||
def test_fence_marker_in_body() -> None:
|
||||
with pytest.raises(SourceError) as excinfo:
|
||||
read_http("https://example.test", "/x", max_rows=10, get=lambda url, cred: "```\n")
|
||||
assert code_of(excinfo) == "fence_marker_in_body"
|
||||
|
||||
|
||||
# --- RenderError codes ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [True, b"bytes"])
|
||||
def test_unsupported_cell_type(value: object) -> None:
|
||||
with pytest.raises(RenderError) as excinfo:
|
||||
sql_value_to_text(value)
|
||||
assert code_of(excinfo) == "unsupported_cell_type"
|
||||
|
||||
|
||||
# --- MaterializationError codes ---
|
||||
|
||||
|
||||
def test_ingested_at_invalid(tmp_path: Path) -> None:
|
||||
with pytest.raises(MaterializationError) as excinfo:
|
||||
materialize_bundle(tmp_path / "m.json", tmp_path / "bundle", "2026-07-17")
|
||||
assert code_of(excinfo) == "ingested_at_invalid"
|
||||
|
||||
|
||||
def test_collision_unstamped(tmp_path: Path) -> None:
|
||||
src = tmp_path / "src"
|
||||
src.mkdir()
|
||||
(src / "manifest.json").write_text(json.dumps(manifest_data()), encoding="utf-8")
|
||||
(src / "data").mkdir()
|
||||
(src / "data" / "orders.csv").write_text("a\n1\n", encoding="utf-8", newline="")
|
||||
bundle = tmp_path / "bundle"
|
||||
bundle.mkdir()
|
||||
(bundle / "ingest-orders.md").write_text("curated, no stamp\n", encoding="utf-8")
|
||||
with pytest.raises(MaterializationError) as excinfo:
|
||||
materialize_bundle(src / "manifest.json", bundle, INGESTED_AT)
|
||||
assert code_of(excinfo) == "collision_unstamped"
|
||||
|
||||
|
||||
# --- NetworkGateError codes ---
|
||||
|
||||
|
||||
def test_network_opt_in_missing(tmp_path: Path) -> None:
|
||||
src = tmp_path / "src"
|
||||
src.mkdir()
|
||||
data = manifest_data(source={"type": "http", "id": "api-1", "base_url": "https://x.test"})
|
||||
data["extractions"][0]["query"] = "/orders"
|
||||
(src / "manifest.json").write_text(json.dumps(data), encoding="utf-8")
|
||||
with pytest.raises(NetworkGateError) as excinfo:
|
||||
materialize_bundle(src / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
assert code_of(excinfo) == "network_opt_in_missing"
|
||||
2
uv.lock
generated
2
uv.lock
generated
|
|
@ -166,7 +166,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "llm-ingestion-okf"
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
source = { editable = "." }
|
||||
|
||||
[package.dev-dependencies]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue