portfolio-optimiser/src/portfolio_optimiser/ingest.py

"""Door A ingest — the consumer seam over the shared ``llm-ingestion-okf`` library.

An addition IN FRONT of the loop (``shared/ingest-spec.md`` §1): a deterministic step that
couples the framework to a real data source and materializes the extract as an OKF knowledge
bundle, which the existing 8-step loop then consumes UNCHANGED. Zero model calls; no network
for the ``file``/``sql`` source types.

Since the library adoption (2026-07-20) the implementation IS the shared
``llm-ingestion-okf`` library (git-pinned to ``v0.3.1``). The spec in ``shared/ingest-spec.md``
remains the normative source — the library implements it, it does not replace it — and the
repo-local goldens under ``examples/`` remain the fasit (verified byte-exact across all three
source types, before and after the swap).

This module is the ONE place the repo touches the library for Door A. It is deliberately thin:
it re-exports the library's typed surface and keeps the historical ``materialize`` signature
(keyword-only ``ingested_at``, ``list[Path]`` return) so callers and tests bind to a stable
repo-local name rather than to the library's evolving one.

**Gating is the CALL SITE's responsibility (library README, "What is gated today: nothing").**
Door A calls no guard function before writing to disk — ``materialize_bundle`` writes what it
is given. The repo's own local-only posture still holds at this seam via ``allow_network``
(default ``False``): an ``http`` source is refused fail-fast at the library's network gate
unless a run explicitly opts in — the manifest can never grant itself network (§8, no silent
egress). Untrusted-content scanning remains the separate, still-planned ``llm-ingestion-guard``
wiring (see ``docs/plan/2026-07-16-llm-ingestion-guard-inclusion.md``); adopting this library
does NOT provide it.

MAF-free (D7-portable), like the rest of the context seam: the library has zero runtime
dependencies and imports no ``agent_framework`` / ``mcp``. Guarded by
``tests/test_ingest_loadbearing.py::test_ingest_module_is_maf_free_and_context_layer_pure``.
"""

from __future__ import annotations

import hashlib
from pathlib import Path

from llm_ingestion_okf import (
    Extraction,
    FileSource,
    HttpSource,
    IngestError,
    IngestResult,
    Manifest,
    ManifestError,
    MaterializationError,
    NetworkGateError,
    RenderError,
    SourceError,
    SqlSource,
    materialize_bundle,
)
from llm_ingestion_okf import (
    load_manifest as _load_manifest,
)
from llm_ingestion_okf.connectors import (
    HttpGet,
    read_csv,
    read_http,
    read_sql,
)
from llm_ingestion_okf.manifest import generated_filename
from llm_ingestion_okf.render import render_fenced_block, render_table

# Two historical PRIVATE names the repo's existing tests bind to, re-exported so those bindings
# survive the adoption unchanged. `_urllib_get` backs an identity assertion that the default http
# transport is the real socket path rather than a stub (tests/test_ingest_http.py);
# `_sql_value_to_text` is exercised directly for the §5 bool/BLOB refusals
# (tests/test_ingest_sql.py). Not in `__all__` — they are not part of this module's contract.
from llm_ingestion_okf.connectors import urllib_get as _urllib_get  # noqa: F401
from llm_ingestion_okf.render import sql_value_to_text as _sql_value_to_text  # noqa: F401

#: The repo's historical name for the §4 top-level model. The library calls it ``Manifest`` and
#: carries the version in the ``manifest_version`` field (still ``1``); the alias keeps the
#: repo-local name stable for callers that bound to it before the library adoption.
ManifestV1 = Manifest

__all__ = [
    "Extraction",
    "FileSource",
    "HttpGet",
    "HttpSource",
    "IngestError",
    "IngestResult",
    "Manifest",
    "ManifestError",
    "ManifestV1",
    "MaterializationError",
    "NetworkGateError",
    "RenderError",
    "SourceError",
    "SqlSource",
    "generated_filename",
    "load_manifest",
    "materialize",
    "materialize_bundle",
    "read_csv",
    "read_http",
    "read_sql",
    "render_fenced_block",
    "render_table",
]


def load_manifest(path: str | Path) -> tuple[Manifest, str]:
    """Validate a manifest fail-fast, BEFORE any source access (§4, §9).

    Returns the validated manifest and the §5 provenance stamp ``{stem}@{hash16}`` — the first
    16 hex chars of SHA-256 over the manifest file's RAW bytes, so every generated file points
    at the exact manifest version that produced it. Raises ``ManifestError`` without touching
    any source.

    The two-value return is the repo-local shape: the library returns the manifest alone and
    mints the stamp INSIDE ``materialize_bundle``, exposing no stamp helper (v0.3.1). The
    formula is therefore restated here, which is the one place this adapter is not purely
    delegating — so it is pinned by
    ``tests/test_ingest_library_seam.py::test_adapter_stamp_equals_library_stamp``, which
    compares this value against the ``ingest_manifest`` the library actually writes. That test
    goes RED if either side's formula drifts."""
    manifest_path = Path(path)
    manifest = _load_manifest(manifest_path)
    stamp = f"{manifest_path.stem}@{hashlib.sha256(manifest_path.read_bytes()).hexdigest()[:16]}"
    return manifest, stamp


def materialize(
    manifest_path: str | Path,
    bundle_dir: str | Path,
    *,
    ingested_at: str,
    allow_network: bool = False,
    http_get: HttpGet | None = None,
) -> list[Path]:
    """Materialize a manifest's extractions into an OKF bundle (§5), returning the generated
    concept-file paths in extraction order.

    The historical repo signature over the library's ``materialize_bundle``: ``ingested_at`` is
    a REQUIRED keyword (no wall-clock default, stamped verbatim; mirrors the promotion gate's
    timestamp rule), and the return is the ``written`` paths as a list rather than the
    library's ``IngestResult``. ``allow_network`` (§8) is the per-run network opt-in — an
    ``http`` source is refused fail-fast unless it is set, so the manifest itself can never
    grant network access. ``http_get`` optionally injects the transport seam; both are ignored
    for ``file``/``sql`` sources. Call ``materialize_bundle`` directly when the stamp is
    wanted alongside the paths."""
    return list(
        materialize_bundle(
            Path(manifest_path),
            Path(bundle_dir),
            ingested_at,
            allow_network=allow_network,
            http_get=http_get,
        ).written
    )