feat(ingest): CSV connector with boundary check, row cap and escaped table body (I2)

src/portfolio_optimiser/ingest.py

@@ -29,6 +29,7 @@ cells; empty/ragged CSV and missing ``root``/``query`` fail fast as ``IngestErro
 
 from __future__ import annotations
 
+import csv
 import hashlib
 import json
 from pathlib import Path
@@ -37,6 +38,8 @@ from urllib.parse import urlsplit
 
 from pydantic import BaseModel, Field, field_validator, model_validator
 
+from portfolio_optimiser.retrieval import safe_resolve
+
 _ID_PATTERN = r"^[a-z0-9][a-z0-9-]*$"
 
 
@@ -145,3 +148,61 @@ def load_manifest(path: str | Path) -> tuple[ManifestV1, str]:
     stamp = f"{manifest_path.stem}@{hashlib.sha256(raw).hexdigest()[:16]}"
     manifest = ManifestV1.model_validate(json.loads(raw.decode("utf-8")))
     return manifest, stamp
+
+
+def read_csv(root: str | Path, query: str, *, max_rows: int) -> tuple[list[str], list[list[str]]]:
+    """Execute a ``file``-source extraction: read the CSV at ``query`` inside ``root`` (§4).
+
+    Fail-closed boundary check via ``safe_resolve`` (a query can never read outside the
+    catalogue); ``utf-8-sig`` so a BOM never leaks into the first header cell (pinned
+    decision); streaming ``max_rows`` cap — exceeding it is an ERROR the moment it happens,
+    never a silent truncation (§8). Input-shape failures (missing root/file, empty CSV,
+    ragged row) raise ``IngestError`` fail-fast — never a leaked ``FileNotFoundError`` and
+    never silent coercion. Pure: no logging, no writes (materialization owns the §8 log)."""
+    root_path = Path(root)
+    if not root_path.is_dir():
+        raise IngestError(f"file-source root is not a directory: {root_path}")
+    resolved = Path(safe_resolve(str(root_path), query))
+    if not resolved.is_file():
+        raise IngestError(f"extraction query does not resolve to a file: {query!r}")
+    with resolved.open(encoding="utf-8-sig", newline="") as handle:
+        reader = csv.reader(handle)
+        header = next(reader, None)
+        if header is None:
+            raise IngestError(f"CSV has no header row: {query!r}")
+        rows: list[list[str]] = []
+        for row in reader:
+            if len(rows) >= max_rows:
+                raise IngestError(
+                    f"extraction {query!r} exceeds max_rows={max_rows} (error, never "
+                    "silent truncation — ingest spec §8)"
+                )
+            if len(row) != len(header):
+                raise IngestError(
+                    f"ragged CSV row in {query!r}: expected {len(header)} cells, "
+                    f"got {len(row)}"
+                )
+            rows.append(row)
+    return header, rows
+
+
+def _escape_cell(value: str) -> str:
+    # §5 escape order is load-bearing: `\` FIRST (else the backslash introduced by
+    # pipe-escaping gets double-escaped), then `|`, then newlines → single space with
+    # CRLF treated as ONE unit.
+    value = value.replace("\\", "\\\\").replace("|", "\\|")
+    return value.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")
+
+
+def render_table(header: list[str], rows: list[list[str]]) -> str:
+    """Render extraction rows as the §5 markdown-table body (LF-only, one trailing newline).
+
+    Cells are text-verbatim after escaping (pinned decision: on the ``file``/CSV path every
+    cell is a string; §5's integer/float/NULL clauses bite typed ``sql`` values in I4).
+    Header cells are escaped identically to data cells (pinned decision)."""
+
+    def line(cells: list[str]) -> str:
+        return "| " + " | ".join(_escape_cell(cell) for cell in cells) + " |"
+
+    separator = "| " + " | ".join("---" for _ in header) + " |"
+    return "\n".join([line(header), separator, *(line(row) for row in rows)]) + "\n"

tests/test_ingest_materialize.py

@@ -0,0 +1,124 @@
+"""I2 Steps 2–4 tests — CSV connector, deterministic materialization, index semantics.
+
+Covers the §4 connector rules (fail-closed ``root`` boundary via ``safe_resolve``, streaming
+``max_rows`` cap — error, never silent truncation §8), the §5 rendering rules (text-verbatim
+cells, escape order ``\\`` BEFORE ``|``, newlines → single space with CRLF as ONE unit,
+header cells escaped identically to data cells, LF-only + exactly one trailing newline), the
+§5/§7 provenance frontmatter (exact key order, verbatim ``ingested_at``, ``{stem}@{hash16}``
+stamp), and the §3/§6 bundle semantics (bundle_summary index, managed-link removal + in-place
+label refresh, curated-collision gate, §10 idempotence). Golden regression lives separately
+in tests/test_ingest_golden.py; the detach-RED seam quartet in tests/test_ingest_loadbearing.py.
+"""
+
+import copy
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from portfolio_optimiser.ingest import IngestError, read_csv, render_table
+from portfolio_optimiser.retrieval import PathSecurityError
+
+# --- local CSV-catalogue builder (conftest's LLM fixtures are irrelevant to ingest) ---
+
+
+def _catalogue(tmp_path: Path, files: dict[str, bytes]) -> Path:
+    root = tmp_path / "catalogue"
+    root.mkdir()
+    for name, content in files.items():
+        (root / name).write_bytes(content)
+    return root
+
+
+def _read(root: Path, query: str = "data.csv", max_rows: int = 100) -> Any:
+    return read_csv(root, query, max_rows=max_rows)
+
+
+# --- Step 2: connector + renderer ---
+
+
+def test_table_rendering_header_separator_rows_in_source_order(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a,b\n3,4\n1,2\n"})
+    header, rows = _read(root)
+    assert render_table(header, rows) == "| a | b |\n| --- | --- |\n| 3 | 4 |\n| 1 | 2 |\n"
+
+
+def test_cell_escaping_backslash(tmp_path: Path) -> None:
+    assert render_table(["h"], [["a\\b"]]).splitlines()[2] == "| a\\\\b |"
+
+
+def test_cell_escaping_pipe(tmp_path: Path) -> None:
+    assert render_table(["h"], [["a|b"]]).splitlines()[2] == "| a\\|b |"
+
+
+def test_cell_escaping_embedded_newline_to_single_space(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b'h\n"x\ny"\n'})
+    header, rows = _read(root)
+    assert rows == [["x\ny"]]  # csv preserves the quoted newline...
+    assert render_table(header, rows).splitlines()[2] == "| x y |"  # ...rendering collapses it
+
+
+def test_escape_order_backslash_before_pipe(tmp_path: Path) -> None:
+    # Input cell `\|`: escaping `\`→`\\` FIRST then `|`→`\|` yields `\\\|`; the reverse
+    # order would double-escape the introduced backslash.
+    assert render_table(["h"], [["\\|"]]).splitlines()[2] == "| \\\\\\| |"
+
+
+def test_crlf_embedded_newline_is_one_space(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b'h\n"x\r\ny"\n'})
+    header, rows = _read(root)
+    assert render_table(header, rows).splitlines()[2] == "| x y |"  # ONE space, not two
+
+
+def test_header_cells_escaped_like_data_cells(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b'"h|1","h\n2"\nv1,v2\n'})
+    header, rows = _read(root)
+    assert render_table(header, rows).splitlines()[0] == "| h\\|1 | h 2 |"
+
+
+def test_header_only_csv_renders_no_data_rows(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a,b\n"})
+    header, rows = _read(root)
+    assert render_table(header, rows) == "| a | b |\n| --- | --- |\n"
+
+
+def test_empty_csv_raises(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b""})
+    with pytest.raises(IngestError):
+        _read(root)
+
+
+def test_ragged_row_raises(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a,b\n1\n"})
+    with pytest.raises(IngestError):
+        _read(root)
+
+
+def test_max_rows_cap_is_an_error_not_truncation(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a\n1\n2\n3\n"})
+    with pytest.raises(IngestError):
+        _read(root, max_rows=2)
+
+
+def test_path_escape_raises(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a\n"})
+    (tmp_path / "outside.csv").write_bytes(b"a\n1\n")
+    with pytest.raises(PathSecurityError):
+        _read(root, query="../outside.csv")
+
+
+def test_missing_root_raises_ingest_error(tmp_path: Path) -> None:
+    with pytest.raises(IngestError):
+        read_csv(tmp_path / "no-such-root", "data.csv", max_rows=10)
+
+
+def test_missing_query_file_raises_ingest_error(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"a\n"})
+    with pytest.raises(IngestError):
+        _read(root, query="absent.csv")
+
+
+def test_bom_never_leaks_into_first_header_cell(tmp_path: Path) -> None:
+    root = _catalogue(tmp_path, {"data.csv": b"\xef\xbb\xbfa,b\n1,2\n"})
+    header, _rows = _read(root)
+    assert header == ["a", "b"]