feat(connectors): add the file connector with fail-closed path boundary
TDD step 3: read_csv executes a file-source extraction — boundary-checked path resolution via commonpath on canonical paths (rejects .. traversal, absolute paths, symlink escapes, and prefix-collision siblings), utf-8-sig decoding, streaming max_rows cap as a typed error (never silent truncation), ragged-row rejection, and verbatim cells (escaping stays in the §5 renderers). New SourceError in the typed hierarchy. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QeqhJpYQyghASjiJo5EhGg
This commit is contained in:
parent
cda4a6a497
commit
af1849f0b3
3 changed files with 230 additions and 0 deletions
67
src/llm_ingestion_okf/connectors.py
Normal file
67
src/llm_ingestion_okf/connectors.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""Connectors: execute manifest extractions against sources (ingest-spec §4).
|
||||
|
||||
Pure readers — no logging, no writes (materialization owns the §8 log).
|
||||
Input-shape failures raise SourceError fail-fast, never a leaked OSError.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .errors import SourceError
|
||||
|
||||
|
||||
def _safe_resolve(root: Path, relative: str) -> Path:
|
||||
"""Resolve `relative` against `root`, fail-closed (the OKF path rule).
|
||||
|
||||
Raises SourceError if the resolved path escapes `root` — `..` traversal,
|
||||
an absolute query path, a symlink escape, or a prefix-collision sibling
|
||||
(`/a/data-evil` vs `/a/data`; commonpath on canonical paths catches what
|
||||
a naive startswith would not).
|
||||
"""
|
||||
real_root = os.path.realpath(root)
|
||||
candidate = os.path.realpath(os.path.join(real_root, relative))
|
||||
try:
|
||||
within = os.path.commonpath([real_root, candidate]) == real_root
|
||||
except ValueError:
|
||||
# Different drives / mixed absolute-relative -> not within.
|
||||
within = False
|
||||
if not within:
|
||||
raise SourceError(f"extraction query escapes the source root: {relative!r}")
|
||||
return Path(candidate)
|
||||
|
||||
|
||||
def read_csv(root: str | Path, query: str, *, max_rows: int) -> tuple[list[str], list[list[str]]]:
|
||||
"""Execute a `file`-source extraction: read the CSV at `query` inside `root`.
|
||||
|
||||
Boundary-checked fail-closed path resolution; utf-8-sig so a BOM never
|
||||
leaks into the first header cell; streaming max_rows cap — exceeding it
|
||||
is an ERROR the moment it happens, never silent truncation (§8);
|
||||
ragged rows rejected, cells returned verbatim (escaping is §5 rendering).
|
||||
"""
|
||||
root_path = Path(root)
|
||||
if not root_path.is_dir():
|
||||
raise SourceError(f"file-source root is not a directory: {root_path}")
|
||||
resolved = _safe_resolve(root_path, query)
|
||||
if not resolved.is_file():
|
||||
raise SourceError(f"extraction query does not resolve to a file: {query!r}")
|
||||
with resolved.open(encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.reader(handle)
|
||||
header = next(reader, None)
|
||||
if header is None:
|
||||
raise SourceError(f"CSV has no header row: {query!r}")
|
||||
rows: list[list[str]] = []
|
||||
for row in reader:
|
||||
if len(rows) >= max_rows:
|
||||
raise SourceError(
|
||||
f"extraction {query!r} exceeds max_rows={max_rows} "
|
||||
"(error, never silent truncation — spec §8)"
|
||||
)
|
||||
if len(row) != len(header):
|
||||
raise SourceError(
|
||||
f"ragged CSV row in {query!r}: expected {len(header)} cells, got {len(row)}"
|
||||
)
|
||||
rows.append(row)
|
||||
return header, rows
|
||||
|
|
@ -13,3 +13,12 @@ class ManifestError(IngestError):
|
|||
|
||||
class RenderError(IngestError):
|
||||
"""A value cannot be rendered under the §5 body rules (never silent coercion)."""
|
||||
|
||||
|
||||
class SourceError(IngestError):
|
||||
"""An extraction failed against its source (ingest-spec §4, §8).
|
||||
|
||||
Covers fail-closed path-boundary violations, missing/malformed source
|
||||
content, and max_rows cap violations — always typed, never a leaked
|
||||
OSError and never silent truncation.
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue