refactor(ingest): adopt shared llm-ingestion-okf v0.3.1 behind a thin adapter
Door A (manifest -> connector -> deterministic materialization -> index) is no longer implemented here. src/portfolio_optimiser/ingest.py becomes a thin consumer seam over the shared library, git-pinned to v0.3.1 on the same Forgejo channel portfolio-optimiser-claude uses. Net -626/+385; ingest.py 599 -> 145 lines. shared/ingest-spec.md remains the normative spec: the library implements it, it does not replace it. Spec changes continue to go via commons. Acceptance criterion met and proven: all three golden bundles (file/sql/http) are byte-exact before and after, including the idempotence re-run. examples/ and shared/ carry ZERO modifications -- the fasit was not adjusted to fit. The rejection set was verified equivalent, not assumed: all 22 malformations the repo's pydantic models refused are refused by the library, with typed codes (okf_type_reserved, credential_embedded, extraction_id_duplicate, ...). Test rebinding (invariants preserved, vehicle changed): the library has zero runtime dependencies by design, so pydantic is unavailable to it. ManifestV1.model_validate(dict) -> load_manifest_bytes(bytes); ValidationError -> ManifestError; model_fields -> dataclasses.fields; PathSecurityError -> SourceError(path_escape); ValueError -> MaterializationError(ingested_at_invalid). Tests now also pin the refusal `code`, the library's documented stability contract -- a sharper assertion than "some validation error was raised". Two accepted behavioural deltas, recorded rather than silently dropped: - Title whitespace is stored verbatim instead of collapsed at validation, so the frontmatter title and the index label are no longer guaranteed identical for irregular whitespace. Both behaviours are spec-conformant (the spec is SILENT; the old one was a repo-local pinned decision). Queued as a commons-amendment candidate so both stacks pin the same answer. Goldens unaffected. - The section 8 audit log moves to logger llm_ingestion_okf.materialize. Nothing in the repo consumed the old channel. Also: the `type` discriminator is no longer a dataclass field, so the spec cross-check asserts it explicitly -- without that line the swap would have silently narrowed the test. New tests/test_ingest_library_seam.py pins the seam itself: the restated section 5 stamp formula against the stamp the library actually writes (the one place the adapter does not purely delegate, since v0.3.1 exposes no stamp helper), the local-only allow_network default, the list[Path] unwrapping, and a guard that the adapter never regrows local Door A machinery. All four verified RED when detached, as were both golden regressions under a byte-level render mutation. Door A is UNGATED: it calls no guard before writing to disk. Gating untrusted content remains the caller's responsibility (guard wiring still planned). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B4jNN186eVqfe1x5DnTU6r
This commit is contained in:
parent
7ec60618b0
commit
0a11af74a4
9 changed files with 385 additions and 626 deletions
|
|
@ -19,9 +19,10 @@ from typing import Any
|
|||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf import MaterializationError, SourceError
|
||||
|
||||
from portfolio_optimiser import okf
|
||||
from portfolio_optimiser.ingest import IngestError, materialize, read_csv, render_table
|
||||
from portfolio_optimiser.retrieval import PathSecurityError
|
||||
|
||||
_INGESTED_AT = "2026-07-03T12:00:00Z"
|
||||
|
||||
|
|
@ -109,8 +110,11 @@ def test_max_rows_cap_is_an_error_not_truncation(tmp_path: Path) -> None:
|
|||
def test_path_escape_raises(tmp_path: Path) -> None:
|
||||
root = _catalogue(tmp_path, {"data.csv": b"a\n"})
|
||||
(tmp_path / "outside.csv").write_bytes(b"a\n1\n")
|
||||
with pytest.raises(PathSecurityError):
|
||||
# Fail-closed containment is unchanged; the library raises its own typed refusal
|
||||
# (SourceError, code="path_escape") where the repo-local seam raised PathSecurityError.
|
||||
with pytest.raises(SourceError) as exc:
|
||||
_read(root, query="../outside.csv")
|
||||
assert exc.value.code == "path_escape"
|
||||
|
||||
|
||||
def test_missing_root_raises_ingest_error(tmp_path: Path) -> None:
|
||||
|
|
@ -206,13 +210,17 @@ def test_generated_file_is_lf_only_with_one_trailing_newline(tmp_path: Path) ->
|
|||
assert data.endswith(b"\n") and not data.endswith(b"\n\n")
|
||||
|
||||
|
||||
def test_invalid_ingested_at_raises_value_error(tmp_path: Path) -> None:
|
||||
def test_invalid_ingested_at_is_refused(tmp_path: Path) -> None:
|
||||
manifest_path, bundle_dir = _project(tmp_path)
|
||||
# §5 format is ISO-8601 UTC with a Z suffix — validated by regex (datetime.fromisoformat
|
||||
# rejects 'Z' on Python 3.10, the repo floor).
|
||||
# rejects 'Z' on Python 3.10, the repo floor). The refusal is unchanged; its type moved
|
||||
# from ValueError to the library's MaterializationError.
|
||||
for bad in ("2026-07-03 12:00:00", "2026-07-03T12:00:00+00:00", "2026-07-03", ""):
|
||||
with pytest.raises(ValueError):
|
||||
with pytest.raises(MaterializationError) as exc:
|
||||
materialize(manifest_path, bundle_dir, ingested_at=bad)
|
||||
assert exc.value.code == "ingested_at_invalid"
|
||||
# The refusal is fail-fast: nothing was written before the format was checked.
|
||||
assert not bundle_dir.exists()
|
||||
|
||||
|
||||
def test_materialize_creates_nonexistent_nested_bundle_dir(tmp_path: Path) -> None:
|
||||
|
|
@ -226,11 +234,16 @@ def test_source_call_logged_with_id_timestamp_rowcount(
|
|||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
manifest_path, bundle_dir = _project(tmp_path)
|
||||
with caplog.at_level(logging.INFO, logger="portfolio_optimiser.ingest"):
|
||||
# The §8 audit log is still emitted per source call, but the CHANNEL moved with the
|
||||
# implementation: "portfolio_optimiser.ingest" -> "llm_ingestion_okf.materialize"
|
||||
# (accepted 2026-07-20; nothing in the repo consumed the old logger name).
|
||||
with caplog.at_level(logging.INFO, logger="llm_ingestion_okf.materialize"):
|
||||
materialize(manifest_path, bundle_dir, ingested_at=_INGESTED_AT)
|
||||
# §8: which source, when (the deterministic ingested_at argument), row count.
|
||||
joined = " ".join(record.getMessage() for record in caplog.records)
|
||||
assert "prosjekt-arkiv" in joined and _INGESTED_AT in joined and "rows=2" in joined
|
||||
# §8 also bounds what may be logged: never cell contents.
|
||||
assert "led-retrofit" not in joined
|
||||
|
||||
|
||||
def test_two_runs_with_identical_inputs_are_byte_identical(tmp_path: Path) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue