portfolio-optimiser/tests/test_ingest_mcp.py

"""S2.2 — the MCP connector as a TRANSPORT inside the ``http`` source family (§4).

Commons settled this on 2026-08-01: MCP is **not** a fourth source family. ``shared/ingest-spec.md``
§4 says an MCP-based connector "is an extension of this family and MUST honour the same extraction,
materialization, and gate contracts". The reference plan
(``docs/plan/2026-07-10-sesjonsplan-fase2-6.md`` lines 193-208) predates that ruling and still
describes a fourth family ``type: "mcp"`` with ``server_ref``/``tool`` manifest fields — that part
is STALE and deliberately not followed here.

**Why the discriminator lives in ``base_url``, not in new manifest fields.** The shared library
rejects unknown manifest keys fail-fast (``llm_ingestion_okf.manifest._require_keys``): an ``http``
source admits exactly ``{type, id, base_url}`` plus optional ``credential_ref``. We are a PULL-ONLY
consumer pinned at ``v0.3.1``, so adding ``server_ref``/``tool`` as fields would require a spec
amendment plus a library release. It is not needed: the library's own URL join already composes
``base_url`` + ``/`` + ``query``, so ``mcp+stdio://<server_ref>`` + ``<tool>`` reproduces exactly
the two-part structure the stale plan wanted — with zero schema change. Field names were explicitly
left to us ("en implementasjonsbeslutning HOS OSS"); the MUST sits on the CONTRACTS, not the names.

**The §8 network gate is inherited, not re-implemented.** Because an MCP source IS ``type: "http"``,
the library refuses it fail-fast unless a run passes ``allow_network=True`` — measured below to fire
BEFORE any transport call. That is the whole point of staying inside the family: the gate cannot be
forgotten, because we never got the chance to write it.

Every test here injects a canned ``call_tool`` — NO subprocess, NO socket, NO server. Mirrors the
transport-seam discipline of ``tests/test_ingest_http.py``.
"""

from __future__ import annotations

import inspect
import json
from pathlib import Path

import pytest

from portfolio_optimiser.ingest import (
    IngestError,
    NetworkGateError,
    materialize,
)
from portfolio_optimiser.ingest_mcp import (
    MCP_SCHEME,
    mcp_get,
    parse_mcp_url,
)


def _make_call_tool(body: str, *, recorder: list[tuple[str, str, str | None]] | None = None):
    """A canned MCP tool call: records ``(server_ref, tool, credential)``, returns a fixed body.

    Local to this file (mirrors ``_make_get``/``_make_db`` locality; never conftest, which is
    MAF/LLM-only)."""

    def call_tool(server_ref: str, tool: str, credential: str | None) -> str:
        if recorder is not None:
            recorder.append((server_ref, tool, credential))
        return body

    return call_tool


def _write_manifest(tmp_path: Path, base_url: str, query: str = "retrieve_cost_docs") -> Path:
    manifest = {
        "manifest_version": 1,
        "source": {"type": "http", "id": "docs", "base_url": base_url},
        "bundle_summary": "MCP-sourced cost documentation.",
        "extractions": [
            {
                "id": "cost-docs",
                "title": "Cost documentation",
                "query": query,
                "okf_type": "concept",
                "max_rows": 50,
            }
        ],
    }
    path = tmp_path / "manifest.json"
    path.write_text(json.dumps(manifest), encoding="utf-8")
    return path


# --- the transport discriminator (§4) ------------------------------------------------------------


def test_mcp_url_splits_into_server_ref_and_tool() -> None:
    """The library joins ``base_url`` + ``/`` + ``query``; we split that back into the two parts
    the stale plan wanted as separate manifest fields."""
    assert parse_mcp_url("mcp+stdio://docs-server/retrieve_cost_docs") == (
        "docs-server",
        "retrieve_cost_docs",
    )


def test_mcp_url_preserves_case_of_server_ref() -> None:
    """``server_ref`` names an environment variable (mirroring ``connection_ref``), and env-var
    names are CASE-SENSITIVE. ``urlsplit().hostname`` lowercases the host — so parsing MUST be
    string-based, not urlsplit-based. Without this, ``PORTFOLIO_DOCS_MCP`` silently becomes
    ``portfolio_docs_mcp`` and the lookup fails for a reason nobody can see."""
    server_ref, tool = parse_mcp_url("mcp+stdio://PORTFOLIO_DOCS_MCP/retrieve_cost_docs")
    assert server_ref == "PORTFOLIO_DOCS_MCP"
    assert tool == "retrieve_cost_docs"


@pytest.mark.parametrize(
    "url",
    [
        "https://example.test/api/retrieve",
        "http://example.test/retrieve",
        "mcp://docs-server/tool",  # near-miss: right family, wrong transport token
        "mcp+stdio://docs-server",  # server but no tool
        "mcp+stdio:///retrieve",  # tool but no server
    ],
)
def test_mcp_transport_refuses_urls_it_does_not_own(url: str) -> None:
    """The discriminator GATES, it does not merely label. An MCP transport pointed at a plain
    ``https://`` manifest must refuse fail-fast rather than quietly serve it — otherwise the
    manifest's declared transport and the transport actually used can silently disagree, and the
    bundle's provenance would claim something untrue."""
    with pytest.raises(IngestError):
        parse_mcp_url(url)


def test_mcp_get_refuses_non_mcp_url_through_the_transport_seam() -> None:
    """Same gate, reached the way the library reaches it — and the tool is never called."""
    calls: list[tuple[str, str, str | None]] = []
    get = mcp_get(_make_call_tool("a,b\n1,2", recorder=calls))
    with pytest.raises(IngestError):
        get("https://example.test/retrieve", None)
    assert calls == [], "transport must refuse BEFORE invoking the MCP tool"


# --- HttpGet compatibility (the seam contract) ---------------------------------------------------


def test_mcp_get_is_shape_compatible_with_the_library_transport_seam() -> None:
    """``mcp_get`` must return something the library can use as its ``http_get``: a callable of
    ``(url, credential) -> str``. If this drifts, the library binds it and fails at call time
    instead of import time."""
    get = mcp_get(_make_call_tool("x"))
    params = list(inspect.signature(get).parameters)
    assert len(params) == 2, f"HttpGet takes (url, credential), got {params}"
    assert isinstance(get("mcp+stdio://s/t", None), str)


def test_mcp_get_returns_the_tool_body_verbatim() -> None:
    """§5 verbatim-body rule: the transport transports, it does not reformat."""
    body = "region,saving\nnorth,1200\nsouth,900"
    get = mcp_get(_make_call_tool(body))
    assert get("mcp+stdio://docs-server/retrieve_cost_docs", None) == body


def test_mcp_get_forwards_credential_without_placing_it_in_the_url() -> None:
    """Credentials resolve at run time and never live in the manifest (§4). The library resolves
    ``credential_ref`` from the environment and hands the VALUE to the transport; the MCP tool call
    receives it out-of-band, never joined into the URL."""
    calls: list[tuple[str, str, str | None]] = []
    get = mcp_get(_make_call_tool("a,b", recorder=calls))
    get("mcp+stdio://docs-server/retrieve_cost_docs", "secret-token")
    assert calls == [("docs-server", "retrieve_cost_docs", "secret-token")]


def test_mcp_scheme_constant_matches_what_the_parser_accepts() -> None:
    """One source of truth for the transport token — a drifting constant would make the
    manifest-authoring docs and the parser disagree."""
    server_ref, tool = parse_mcp_url(f"{MCP_SCHEME}://docs-server/retrieve_cost_docs")
    assert (server_ref, tool) == ("docs-server", "retrieve_cost_docs")


# --- end-to-end through the real materialization path (§5) ---------------------------------------


def test_mcp_source_materializes_a_bundle_through_the_http_family(tmp_path: Path) -> None:
    """The whole point of staying inside ``type: "http"``: materialization, rendering and stamping
    are the EXISTING code paths, reached with an MCP transport injected. Nothing bespoke."""
    manifest_path = _write_manifest(tmp_path, "mcp+stdio://docs-server")
    bundle_dir = tmp_path / "bundle"
    calls: list[tuple[str, str, str | None]] = []
    get = mcp_get(_make_call_tool("region,saving\nnorth,1200", recorder=calls))

    written = materialize(
        manifest_path,
        bundle_dir,
        ingested_at="2026-08-02T00:00:00Z",
        allow_network=True,
        http_get=get,
    )

    assert calls == [("docs-server", "retrieve_cost_docs", None)]
    assert [p.name for p in written] == ["ingest-cost-docs.md"]
    text = written[0].read_text(encoding="utf-8")
    assert "north,1200" in text, "extracted body must reach the concept file"
    assert "generated: true" in text, "§7 honesty stamp must survive the MCP path"


def test_mcp_source_is_refused_without_the_per_run_network_optin(tmp_path: Path) -> None:
    """§8, INHERITED for free: an MCP source is a non-local transport, so the network gate applies
    and fires BEFORE the transport is touched. The manifest cannot grant itself network access.

    This is the strongest argument for the http-family decision: had MCP become a fourth family,
    this gate would have been ours to write, and ours to forget."""
    manifest_path = _write_manifest(tmp_path, "mcp+stdio://docs-server")
    calls: list[tuple[str, str, str | None]] = []
    get = mcp_get(_make_call_tool("a,b", recorder=calls))

    with pytest.raises(NetworkGateError):
        materialize(
            manifest_path,
            tmp_path / "bundle",
            ingested_at="2026-08-02T00:00:00Z",
            http_get=get,  # allow_network defaults to False
        )

    assert calls == [], "the gate must fire BEFORE any MCP tool call"


def test_mcp_materialization_is_byte_deterministic(tmp_path: Path) -> None:
    """Ingest is deterministic end to end (§11): same manifest + same ``ingested_at`` + same body
    ⇒ byte-identical output. ``ingested_at`` is stamped verbatim, never wall-clock."""
    outputs = []
    for run in ("a", "b"):
        root = tmp_path / run
        root.mkdir()
        manifest_path = _write_manifest(root, "mcp+stdio://docs-server")
        written = materialize(
            manifest_path,
            root / "bundle",
            ingested_at="2026-08-02T00:00:00Z",
            allow_network=True,
            http_get=mcp_get(_make_call_tool("region,saving\nnorth,1200")),
        )
        outputs.append(written[0].read_bytes())
    assert outputs[0] == outputs[1]