feat(ingest): adopt llm-ingestion-okf as Door A implementation (first consumer)
Replace the local 391-line ingest implementation with a thin adapter over the shared llm-ingestion-okf library (git-pinned dae0bd1a via Forgejo, tool.uv.sources). The materialize() signature is preserved; error types are now the library's typed hierarchy rooted in IngestError, re-exported from the consumer seam. - tests/test_ingest_adoption.py: new load-bearing seam tests (delegation, offline invariant — allow_network is never passed, error contract), detach-proven red twice. - Golden suites (file + sql) pass UNCHANGED — byte-exact behaviour proven against the repo-local fixtures. - 6 test files migrated to the library error hierarchy; escaping/typed-cell unit tests dropped (byte-bound by the ingest-edge.md golden, unit-owned by the library's own 189-test suite). Provenance stamp now asserted independently from the §5 rule. - mypy override follow_untyped_imports for llm_ingestion_okf (no py.typed upstream yet — reported as a finding). Suite: 386 passed; ruff + format + mypy --strict clean; shared/, examples/, runs/s10/ and run_s10.py byte-untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
80a2fa1a77
commit
5732d13369
9 changed files with 307 additions and 495 deletions
|
|
@ -6,9 +6,17 @@ readme = "README.md"
|
|||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"claude-agent-sdk>=0.2.111,<0.3",
|
||||
"llm-ingestion-okf",
|
||||
"pydantic>=2",
|
||||
]
|
||||
|
||||
# Distribution channel (2026-07-16 decision, mirrors the library's B2 answer):
|
||||
# git pin against the public Forgejo repo — reproducible for every consumer of
|
||||
# this public repo, uv.lock pins the exact commit. Bump the rev to the release
|
||||
# tag once the library tags one.
|
||||
[tool.uv.sources]
|
||||
llm-ingestion-okf = { git = "https://git.fromaitochitta.com/open/llm-ingestion-okf.git", rev = "dae0bd1a2898b69f436a877538d67afd71e48ad8" }
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8",
|
||||
|
|
@ -30,5 +38,11 @@ target-version = "py310"
|
|||
[tool.mypy]
|
||||
strict = true
|
||||
|
||||
# llm-ingestion-okf ships full inline type hints but (as of 0.1.0 @ dae0bd1a)
|
||||
# no py.typed marker; follow its inline types instead of degrading to Any.
|
||||
[[tool.mypy.overrides]]
|
||||
module = "llm_ingestion_okf.*"
|
||||
follow_untyped_imports = true
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ from portfolio_optimiser_claude.experience import (
|
|||
mint_verdict_id,
|
||||
)
|
||||
|
||||
# R-6 id grammar — mirrors ingest.py's _ID_RE: lowercase alphanumerics and
|
||||
# R-6 id grammar — mirrors the ingest-spec §4 id grammar (enforced by the
|
||||
# llm-ingestion-okf library on the ingest side): lowercase alphanumerics and
|
||||
# hyphens only, so a verdict id can NEVER traverse paths (no dots, no
|
||||
# separators). Minted ids (16 hex chars, §4.2) always match.
|
||||
_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||||
|
|
|
|||
|
|
@ -1,391 +1,63 @@
|
|||
"""Deterministic ingest layer: manifest → file/CSV connector → OKF bundle (ingest-spec).
|
||||
"""Door A ingest — the consumer seam over the shared llm-ingestion-okf library.
|
||||
|
||||
An addition IN FRONT of the loop (ingest-spec §1): a deterministic step that couples the
|
||||
framework to a real data source and materializes the extract as an OKF knowledge bundle,
|
||||
which the existing 8-step loop then consumes UNCHANGED. Zero model calls; no network.
|
||||
|
||||
D7 scope (I3–I5, mirror of MAF I2/I4): the ``file`` source type (a local CSV catalogue) and
|
||||
the ``sql`` source type (a local SQL database). The ``http`` source type is a later, OPTIONAL
|
||||
extension point (§1) — a manifest naming it is rejected fail-fast at validation, never
|
||||
silently accepted.
|
||||
|
||||
Contract discipline mirrors ``contracts.py``: the manifest is schema-validated fail-fast
|
||||
BEFORE any source call (§4), and queries are declarative configuration, never evaluated as
|
||||
code. The provenance frontmatter layer (§7) is ITS OWN contract — separate from the method
|
||||
spec's §9 proposal provenance, never mixed.
|
||||
Since the first-consumer adoption (2026-07-16) the implementation IS the shared
|
||||
``llm-ingestion-okf`` library (byte-compatible with the reference implementation; the
|
||||
repo-local goldens under ``examples/`` remain the fasit). This module is the ONE place the
|
||||
repo touches the library for Door A: it re-exports the library's typed surface and keeps
|
||||
the historical ``materialize`` signature. The offline invariant lives at this seam — the
|
||||
per-run network opt-in (``allow_network``) is NEVER passed, so an ``http`` source is
|
||||
refused fail-fast at the library's network gate (§8, local-only default).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||||
_VERDICT_TYPE = "verdict"
|
||||
_INGEST_PREFIX = "ingest-"
|
||||
_STAMP_HASH_LEN = 16
|
||||
# An index cross-link owned by ingest: ``- [label](ingest-{id}.md)`` (§6). Only these
|
||||
# are managed on re-materialization; curated and promoted links are preserved verbatim.
|
||||
_INGEST_LINK_RE = re.compile(r"\]\((ingest-[a-z0-9][a-z0-9-]*\.md)\)")
|
||||
_CROSSLINK_RE = re.compile(r"\]\(([^)]+\.md)\)")
|
||||
|
||||
# Frontmatter keys, in the exact §5 order.
|
||||
_FRONTMATTER_ORDER = (
|
||||
"type",
|
||||
"title",
|
||||
"source_system",
|
||||
"source_query",
|
||||
"ingested_at",
|
||||
"ingest_manifest",
|
||||
"generated",
|
||||
from llm_ingestion_okf import (
|
||||
Extraction,
|
||||
FileSource,
|
||||
HttpSource,
|
||||
IngestError,
|
||||
IngestResult,
|
||||
Manifest,
|
||||
ManifestError,
|
||||
MaterializationError,
|
||||
NetworkGateError,
|
||||
RenderError,
|
||||
SourceError,
|
||||
SqlSource,
|
||||
load_manifest,
|
||||
materialize_bundle,
|
||||
)
|
||||
|
||||
|
||||
class FileSource(BaseModel):
|
||||
"""A local file catalogue (``source.type == "file"``, ingest-spec §4)."""
|
||||
|
||||
type: Literal["file"]
|
||||
id: str
|
||||
root: str
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _id_grammar(cls, value: str) -> str:
|
||||
if not _ID_RE.match(value):
|
||||
raise ValueError(f"source.id {value!r} must match [a-z0-9][a-z0-9-]*")
|
||||
return value
|
||||
|
||||
|
||||
class SqlSource(BaseModel):
|
||||
"""A local SQL database (``source.type == "sql"``, ingest-spec §4).
|
||||
|
||||
``connection_ref`` is the NAME of a runtime-resolved environment variable whose value is
|
||||
the database path (or connection string) — the secret/location never lives in the manifest
|
||||
(§4, §8), so the manifest stays versionable and shareable without credentials.
|
||||
"""
|
||||
|
||||
type: Literal["sql"]
|
||||
id: str
|
||||
connection_ref: str
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _id_grammar(cls, value: str) -> str:
|
||||
if not _ID_RE.match(value):
|
||||
raise ValueError(f"source.id {value!r} must match [a-z0-9][a-z0-9-]*")
|
||||
return value
|
||||
|
||||
|
||||
class Extraction(BaseModel):
|
||||
"""One extraction description (ingest-spec §4). ``max_rows`` is a required cap (§8)."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
query: str
|
||||
okf_type: str
|
||||
max_rows: int = Field(gt=0)
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _id_grammar(cls, value: str) -> str:
|
||||
if not _ID_RE.match(value):
|
||||
raise ValueError(f"extraction.id {value!r} must match [a-z0-9][a-z0-9-]*")
|
||||
return value
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def _title_single_line(cls, value: str) -> str:
|
||||
if "\n" in value or "\r" in value:
|
||||
raise ValueError("extraction.title must be single-line")
|
||||
return value
|
||||
|
||||
@field_validator("okf_type")
|
||||
@classmethod
|
||||
def _okf_type_not_verdict(cls, value: str) -> str:
|
||||
# §3 verdict-layer reservation: the promotion gate is the ONLY path into the
|
||||
# verdict layer — an ingest mapping to type: verdict would inject machine-made
|
||||
# "approved" verdicts around the gate. Case-insensitive, fail-fast.
|
||||
if value.lower() == _VERDICT_TYPE:
|
||||
raise ValueError("okf_type MUST NOT be 'verdict' (verdict layer is reserved, §3)")
|
||||
return value
|
||||
|
||||
|
||||
class ManifestContract(BaseModel):
|
||||
"""The ingest manifest (§4) — schema-validated fail-fast before any source call."""
|
||||
|
||||
manifest_version: Literal[1]
|
||||
source: Annotated[FileSource | SqlSource, Field(discriminator="type")]
|
||||
bundle_summary: str
|
||||
extractions: list[Extraction] = Field(min_length=1)
|
||||
|
||||
@field_validator("extractions")
|
||||
@classmethod
|
||||
def _unique_extraction_ids(cls, value: list[Extraction]) -> list[Extraction]:
|
||||
ids = [e.id for e in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError("extraction ids must be unique within the manifest")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoadedManifest:
|
||||
"""A validated manifest plus its provenance stamp and on-disk location."""
|
||||
|
||||
contract: ManifestContract
|
||||
stamp: str # ``{manifest stem}@{sha256(raw bytes)[:16]}`` (§5)
|
||||
path: Path
|
||||
|
||||
|
||||
def load_manifest(manifest_path: Path) -> LoadedManifest:
|
||||
"""Validate the manifest fail-fast (§4) and compute its provenance stamp (§5).
|
||||
|
||||
Raises ``pydantic.ValidationError`` on a malformed manifest — never starts a run on a
|
||||
bad contract. The stamp is over the manifest's RAW bytes, so any edit re-stamps.
|
||||
"""
|
||||
raw = manifest_path.read_bytes()
|
||||
contract = ManifestContract(**json.loads(raw))
|
||||
digest = hashlib.sha256(raw).hexdigest()[:_STAMP_HASH_LEN]
|
||||
stamp = f"{manifest_path.stem}@{digest}"
|
||||
return LoadedManifest(contract=contract, stamp=stamp, path=manifest_path)
|
||||
|
||||
|
||||
def _escape_cell(value: str) -> str:
|
||||
"""Render a cell value: text verbatim with §5 escaping.
|
||||
|
||||
Backslash FIRST (so the pipe escape it introduces is not re-escaped), then pipe,
|
||||
then any newline collapsed to a single space. For the ``file`` source every cell is
|
||||
a ``str`` — the integer/float/NULL typing rules (§5) belong to the ``sql`` source.
|
||||
"""
|
||||
escaped = value.replace("\\", "\\\\").replace("|", "\\|")
|
||||
escaped = escaped.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")
|
||||
return escaped
|
||||
|
||||
|
||||
def _render_table(rows: list[list[str]]) -> str:
|
||||
"""Header row, separator, data rows — GitHub-flavoured markdown table (§5)."""
|
||||
header, *data = rows
|
||||
lines = [
|
||||
"| " + " | ".join(_escape_cell(c) for c in header) + " |",
|
||||
"| " + " | ".join("---" for _ in header) + " |",
|
||||
]
|
||||
for row in data:
|
||||
lines.append("| " + " | ".join(_escape_cell(c) for c in row) + " |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _read_csv(csv_path: Path, max_rows: int) -> list[list[str]]:
|
||||
"""Read a CSV (first row = header). Enforce ``max_rows`` fail-fast on DATA rows (§8)."""
|
||||
with csv_path.open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.reader(handle))
|
||||
data_rows = len(rows) - 1 if rows else 0
|
||||
if data_rows > max_rows:
|
||||
raise ValueError(
|
||||
f"extraction exceeds max_rows: {data_rows} > {max_rows} for {csv_path.name}"
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _render_sql_cell(value: object) -> str:
|
||||
"""Type a SQL cell to its §5 string form (BEFORE §5 escaping via ``_escape_cell``).
|
||||
|
||||
``None`` (SQL NULL) → the empty string; ``int`` → plain decimal; ``float`` → its shortest
|
||||
round-trip decimal form (Python ``repr``); ``str`` → verbatim. Any other value type (a
|
||||
BLOB, say) MUST fail — never a silent coercion (§5).
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, float):
|
||||
return repr(value)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
raise ValueError(f"SQL cell of type {type(value).__name__} is not a supported value type (§5)")
|
||||
|
||||
|
||||
def _resolve_connection_ref(connection_ref: str) -> str:
|
||||
"""Resolve the ``connection_ref`` env var to the database location at run time (§4, §8).
|
||||
|
||||
Locations/credentials never live in the manifest — the reference is resolved from the
|
||||
environment. An unset reference fails fast (never a silent empty connection).
|
||||
"""
|
||||
dsn = os.environ.get(connection_ref)
|
||||
if not dsn:
|
||||
raise ValueError(f"connection_ref {connection_ref!r} is not set in the environment")
|
||||
return dsn
|
||||
|
||||
|
||||
def _read_sql(dsn: str, query: str, max_rows: int) -> list[list[str]]:
|
||||
"""Run one read-only SELECT against a local sqlite database (ingest-spec §4, §5, §8).
|
||||
|
||||
The connection is opened read-only (§4: the connector SHOULD enforce read-only access), and
|
||||
sqlite executes exactly ONE statement per ``execute`` call — so the single-statement rule is
|
||||
enforced by the driver. ``max_rows`` is enforced fail-fast on the returned rows (§8).
|
||||
Returns the column-name header row followed by typed-then-stringified data rows, in source
|
||||
order (§5); a stable order is the manifest query's responsibility via ORDER BY (§4).
|
||||
"""
|
||||
uri = Path(dsn).resolve().as_uri() + "?mode=ro"
|
||||
connection = sqlite3.connect(uri, uri=True)
|
||||
try:
|
||||
cursor = connection.execute(query)
|
||||
header = [description[0] for description in cursor.description]
|
||||
data = cursor.fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
if len(data) > max_rows:
|
||||
raise ValueError(f"extraction exceeds max_rows: {len(data)} > {max_rows}")
|
||||
rows: list[list[str]] = [header]
|
||||
for record in data:
|
||||
rows.append([_render_sql_cell(value) for value in record])
|
||||
return rows
|
||||
|
||||
|
||||
def _read_extraction(
|
||||
source: FileSource | SqlSource, manifest_dir: Path, extraction: Extraction
|
||||
) -> list[list[str]]:
|
||||
"""Dispatch to the connector for ``source.type`` — header + data rows in source order (§5).
|
||||
|
||||
``file``: resolve the query path within ``root`` (boundary-checked fail-closed, §4) and read
|
||||
the CSV. ``sql``: resolve ``connection_ref`` from the environment (§4, §8) and run the
|
||||
read-only SELECT. Both enforce ``max_rows`` fail-fast (§8).
|
||||
"""
|
||||
if source.type == "file":
|
||||
csv_path = _resolve_within(manifest_dir / source.root, extraction.query)
|
||||
return _read_csv(csv_path, extraction.max_rows)
|
||||
dsn = _resolve_connection_ref(source.connection_ref)
|
||||
return _read_sql(dsn, extraction.query, extraction.max_rows)
|
||||
|
||||
|
||||
def _resolve_within(root: Path, relative: str) -> Path:
|
||||
"""Resolve ``relative`` under ``root``, boundary-checked fail-closed (the OKF path rule)."""
|
||||
resolved = (root / relative).resolve()
|
||||
if not resolved.is_relative_to(root.resolve()):
|
||||
raise ValueError(f"extraction path {relative!r} escapes the source root")
|
||||
return resolved
|
||||
|
||||
|
||||
def _frontmatter_block(values: dict[str, str]) -> str:
|
||||
return "---\n" + "".join(f"{k}: {values[k]}\n" for k in _FRONTMATTER_ORDER) + "---"
|
||||
|
||||
|
||||
def _concept_file(
|
||||
extraction: Extraction, source_id: str, ingested_at: str, stamp: str, body: str
|
||||
) -> str:
|
||||
frontmatter = _frontmatter_block(
|
||||
{
|
||||
"type": extraction.okf_type,
|
||||
"title": extraction.title,
|
||||
"source_system": source_id,
|
||||
"source_query": _escape_ws(extraction.query),
|
||||
"ingested_at": ingested_at,
|
||||
"ingest_manifest": stamp,
|
||||
"generated": "true",
|
||||
}
|
||||
)
|
||||
return f"{frontmatter}\n\n{body}\n"
|
||||
|
||||
|
||||
def _escape_ws(value: str) -> str:
|
||||
"""Collapse whitespace runs (incl. newlines) to single spaces (§5) — single-line values."""
|
||||
return " ".join(value.split())
|
||||
|
||||
|
||||
def _is_ingest_stamped(md_path: Path) -> bool:
|
||||
"""True iff the file carries the ingest stamp: ``generated: true`` + an ``ingest_manifest``."""
|
||||
frontmatter: dict[str, str] = {}
|
||||
lines = md_path.read_text(encoding="utf-8").splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return False
|
||||
for line in lines[1:]:
|
||||
if line.strip() == "---":
|
||||
break
|
||||
key, sep, val = line.partition(":")
|
||||
if sep:
|
||||
frontmatter[key.strip()] = val.strip()
|
||||
return frontmatter.get("generated") == "true" and "ingest_manifest" in frontmatter
|
||||
|
||||
|
||||
def _update_index(bundle_dir: Path, bundle_summary: str, extractions: list[Extraction]) -> None:
|
||||
"""Create or update ``index.md`` (§6).
|
||||
|
||||
A fresh index carries ``bundle_summary`` as its body; an existing index keeps every
|
||||
line it does not manage byte for byte. Ingest cross-links (targets ``ingest-*.md``)
|
||||
are the ONLY managed lines: stale ones are dropped and the current set re-appended in
|
||||
extraction order, idempotently by target — curated and promoted links are preserved
|
||||
(so a promoted verdict's index link survives re-ingest, load-bearing §11).
|
||||
"""
|
||||
index_path = bundle_dir / "index.md"
|
||||
if index_path.is_file():
|
||||
kept = [
|
||||
line
|
||||
for line in index_path.read_text(encoding="utf-8").splitlines()
|
||||
if not _INGEST_LINK_RE.search(line)
|
||||
]
|
||||
else:
|
||||
kept = [bundle_summary]
|
||||
present = {t for line in kept for t in _CROSSLINK_RE.findall(line)}
|
||||
lines = list(kept)
|
||||
for extraction in extractions:
|
||||
target = f"{_INGEST_PREFIX}{extraction.id}.md"
|
||||
if target not in present:
|
||||
lines.append(f"- [{extraction.title}]({target})")
|
||||
present.add(target)
|
||||
index_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
__all__ = [
|
||||
"Extraction",
|
||||
"FileSource",
|
||||
"HttpSource",
|
||||
"IngestError",
|
||||
"IngestResult",
|
||||
"Manifest",
|
||||
"ManifestError",
|
||||
"MaterializationError",
|
||||
"NetworkGateError",
|
||||
"RenderError",
|
||||
"SourceError",
|
||||
"SqlSource",
|
||||
"load_manifest",
|
||||
"materialize",
|
||||
"materialize_bundle",
|
||||
]
|
||||
|
||||
|
||||
def materialize(manifest_path: Path, bundle_dir: Path, ingested_at: str) -> list[Path]:
|
||||
"""Materialize the manifest's extractions into ``bundle_dir`` (deterministic, §5).
|
||||
|
||||
``ingested_at`` is an EXPLICIT required argument (no wall-clock default, §5) — the ISO
|
||||
timestamp stamped verbatim into every generated file, which is what makes extractions
|
||||
bit-deterministic. Returns the generated concept-file paths in extraction order.
|
||||
|
||||
Replacement semantics (§3, §5): first every ingest-stamped file is removed, then the
|
||||
new set is written (a name colliding with a NON-stamped file fails — curated content is
|
||||
never overwritten), then the index is updated. Curated and promoted files always survive.
|
||||
``ingested_at`` is an EXPLICIT required argument (no wall-clock default, §5). Returns
|
||||
the generated concept-file paths in extraction order. Delegates to the library with
|
||||
its local-only defaults in force — no network opt-in, no transport injection.
|
||||
"""
|
||||
loaded = load_manifest(manifest_path)
|
||||
source = loaded.contract.source
|
||||
|
||||
bundle_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 1. Remove exactly the files carrying the ingest stamp (§5) — never curated/promoted.
|
||||
for md_path in sorted(bundle_dir.glob("*.md")):
|
||||
if md_path.name != "index.md" and _is_ingest_stamped(md_path):
|
||||
md_path.unlink()
|
||||
|
||||
# 2. Write the new set.
|
||||
generated: list[Path] = []
|
||||
for extraction in loaded.contract.extractions:
|
||||
rows = _read_extraction(source, manifest_path.parent, extraction)
|
||||
_logger.info(
|
||||
"ingest source=%s extraction=%s rows=%d",
|
||||
source.id,
|
||||
extraction.id,
|
||||
max(len(rows) - 1, 0),
|
||||
)
|
||||
target = bundle_dir / f"{_INGEST_PREFIX}{extraction.id}.md"
|
||||
if target.exists():
|
||||
# Post-removal, an existing target is a NON-stamped collision (§3): never overwrite.
|
||||
raise ValueError(f"generated name {target.name} collides with a non-ingest file")
|
||||
target.write_text(
|
||||
_concept_file(extraction, source.id, ingested_at, loaded.stamp, _render_table(rows)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
generated.append(target)
|
||||
|
||||
# 3. Update the index (§6).
|
||||
_update_index(bundle_dir, loaded.contract.bundle_summary, loaded.contract.extractions)
|
||||
return generated
|
||||
return list(materialize_bundle(manifest_path, bundle_dir, ingested_at).written)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
"""Ingest unit + fail-fast contract tests (ingest-spec §4, §5, §8).
|
||||
|
||||
The manifest is schema-validated fail-fast BEFORE any source call (§4, the startup-contract
|
||||
discipline). These tests pin the malformed-manifest rejections, the §5 cell-escaping rules,
|
||||
and the §8 size cap / fail-closed path resolution / no-overwrite collision.
|
||||
discipline). These tests pin the malformed-manifest rejections, the §8 size cap, and the
|
||||
fail-closed path resolution / no-overwrite collision — all through the consumer seam
|
||||
(``portfolio_optimiser_claude.ingest``, backed by llm-ingestion-okf since the adoption).
|
||||
|
||||
The §5 cell-escaping rules are NOT unit-tested here anymore: every escaping case
|
||||
(backslash, pipe, backslash-then-pipe order, newline collapse, verbatim text) is bound
|
||||
byte-for-byte by the ``ingest-edge.md`` golden (test_ingest_golden.py) and unit-owned by
|
||||
the library's own suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -11,11 +17,11 @@ import json
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude.ingest import (
|
||||
ManifestContract,
|
||||
_escape_cell,
|
||||
ManifestError,
|
||||
MaterializationError,
|
||||
SourceError,
|
||||
load_manifest,
|
||||
materialize,
|
||||
)
|
||||
|
|
@ -44,18 +50,29 @@ def _write_case(tmp_path: Path, manifest: dict, csvs: dict[str, str]) -> Path:
|
|||
return case
|
||||
|
||||
|
||||
def _load(tmp_path: Path, manifest: dict):
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return load_manifest(path)
|
||||
|
||||
|
||||
class TestManifestValidation:
|
||||
"""§4: a malformed manifest never starts a run (fail-fast)."""
|
||||
|
||||
def test_valid_manifest_loads_and_stamps(self) -> None:
|
||||
contract = ManifestContract(**_valid())
|
||||
assert contract.manifest_version == 1
|
||||
assert contract.source.id == "arkiv"
|
||||
def test_valid_manifest_loads(self, tmp_path: Path) -> None:
|
||||
manifest = _load(tmp_path, _valid())
|
||||
assert manifest.manifest_version == 1
|
||||
assert manifest.source.id == "arkiv"
|
||||
|
||||
def test_stamp_is_stem_at_sha256_16(self, tmp_path: Path) -> None:
|
||||
# The §5 provenance stamp (``{stem}@{sha256(raw)[:16]}``) is asserted from the
|
||||
# materialized frontmatter — the stamp is a property of the run, not the manifest.
|
||||
case = _write_case(tmp_path, _valid(), {"e.csv": "a\n1\n"})
|
||||
loaded = load_manifest(case / "manifest.json")
|
||||
stem, _, digest = loaded.stamp.partition("@")
|
||||
bundle = tmp_path / "bundle"
|
||||
materialize(case / "manifest.json", bundle, INGESTED_AT)
|
||||
frontmatter = (bundle / "ingest-e.md").read_text(encoding="utf-8").splitlines()
|
||||
(stamp_line,) = [ln for ln in frontmatter if ln.startswith("ingest_manifest: ")]
|
||||
stem, _, digest = stamp_line.removeprefix("ingest_manifest: ").partition("@")
|
||||
assert stem == "manifest"
|
||||
assert len(digest) == 16 and all(c in "0123456789abcdef" for c in digest)
|
||||
|
||||
|
|
@ -68,7 +85,7 @@ class TestManifestValidation:
|
|||
lambda m: m.pop("bundle_summary"),
|
||||
lambda m: m.__setitem__("extractions", []),
|
||||
lambda m: m["source"].__setitem__("id", "Bad_Id"),
|
||||
lambda m: m["source"].__setitem__("type", "http"), # optional/unimplemented (§1)
|
||||
lambda m: m["source"].__setitem__("type", "http"), # http requires base_url (§4)
|
||||
lambda m: m["source"].__setitem__("type", "unknown"), # bad discriminator
|
||||
lambda m: m["extractions"][0].__setitem__("id", "Bad Id"),
|
||||
lambda m: m["extractions"][0].__setitem__("title", "two\nlines"),
|
||||
|
|
@ -76,35 +93,17 @@ class TestManifestValidation:
|
|||
lambda m: m["extractions"][0].__setitem__("max_rows", -1),
|
||||
],
|
||||
)
|
||||
def test_malformed_manifest_is_rejected(self, mutate) -> None:
|
||||
def test_malformed_manifest_is_rejected(self, tmp_path: Path, mutate) -> None:
|
||||
manifest = _valid()
|
||||
mutate(manifest)
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
with pytest.raises(ManifestError):
|
||||
_load(tmp_path, manifest)
|
||||
|
||||
def test_duplicate_extraction_ids_are_rejected(self) -> None:
|
||||
def test_duplicate_extraction_ids_are_rejected(self, tmp_path: Path) -> None:
|
||||
manifest = _valid()
|
||||
manifest["extractions"].append(dict(manifest["extractions"][0]))
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
|
||||
|
||||
class TestCellEscaping:
|
||||
"""§5: text verbatim with backslash → \\\\, pipe → \\|, newline → single space."""
|
||||
|
||||
def test_backslash_then_pipe_order(self) -> None:
|
||||
assert _escape_cell("\\|") == "\\\\\\|"
|
||||
|
||||
def test_pipe_escaped(self) -> None:
|
||||
assert _escape_cell("a|b") == "a\\|b"
|
||||
|
||||
def test_newline_becomes_single_space(self) -> None:
|
||||
assert _escape_cell("x\ny") == "x y"
|
||||
assert _escape_cell("x\r\ny") == "x y"
|
||||
|
||||
def test_plain_text_verbatim(self) -> None:
|
||||
assert _escape_cell("007") == "007"
|
||||
assert _escape_cell("1.50") == "1.50"
|
||||
with pytest.raises(ManifestError):
|
||||
_load(tmp_path, manifest)
|
||||
|
||||
|
||||
class TestSecurityFrame:
|
||||
|
|
@ -114,7 +113,7 @@ class TestSecurityFrame:
|
|||
manifest = _valid()
|
||||
manifest["extractions"][0]["max_rows"] = 1
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n2\n"}) # 2 data rows > 1
|
||||
with pytest.raises(ValueError, match="max_rows"):
|
||||
with pytest.raises(SourceError, match="max_rows"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_query_escaping_root_is_refused(self, tmp_path: Path) -> None:
|
||||
|
|
@ -122,7 +121,7 @@ class TestSecurityFrame:
|
|||
manifest["extractions"][0]["query"] = "../secret.csv"
|
||||
case = _write_case(tmp_path, manifest, {"e.csv": "col\n1\n"})
|
||||
(case / "secret.csv").write_text("col\nx\n", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="escapes"):
|
||||
with pytest.raises(SourceError, match="escapes"):
|
||||
materialize(case / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_collision_with_non_ingest_file_fails(self, tmp_path: Path) -> None:
|
||||
|
|
@ -133,7 +132,7 @@ class TestSecurityFrame:
|
|||
(bundle / "ingest-e.md").write_text(
|
||||
"---\ntype: reference\ntitle: hand\n---\n\nCurated.\n", encoding="utf-8"
|
||||
)
|
||||
with pytest.raises(ValueError, match="collides"):
|
||||
with pytest.raises(MaterializationError, match="collides"):
|
||||
materialize(case / "manifest.json", bundle, INGESTED_AT)
|
||||
# The curated file is untouched — never overwritten.
|
||||
assert "Curated." in (bundle / "ingest-e.md").read_text(encoding="utf-8")
|
||||
|
|
|
|||
101
tests/test_ingest_adoption.py
Normal file
101
tests/test_ingest_adoption.py
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
"""Adoption seam: Door A ingest is the shared llm-ingestion-okf library (§11).
|
||||
|
||||
This repo's local ingest implementation was replaced by the shared library
|
||||
(first consumer adoption). These load-bearing tests bind the adapter seam:
|
||||
|
||||
- Delegation — ``ingest.materialize`` IS a call into the library (RED if a
|
||||
local reimplementation sneaks back in).
|
||||
- Offline invariant — the adapter NEVER passes the per-run network opt-in:
|
||||
an http-source manifest is refused at the library's network gate (RED if
|
||||
the adapter starts granting network access).
|
||||
- Error contract — the consumer-facing error types ARE the library's typed
|
||||
hierarchy rooted in ``IngestError`` (RED if the seam re-wraps or forks).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
import llm_ingestion_okf
|
||||
from portfolio_optimiser_claude import ingest
|
||||
|
||||
INGESTED_AT = "2026-07-16T12:00:00Z"
|
||||
|
||||
|
||||
def _http_manifest() -> dict[str, Any]:
|
||||
return {
|
||||
"manifest_version": 1,
|
||||
"source": {"type": "http", "id": "api", "base_url": "https://example.invalid"},
|
||||
"bundle_summary": "s",
|
||||
"extractions": [
|
||||
{"id": "e", "title": "T", "query": "rows", "okf_type": "dataset", "max_rows": 1}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestDelegation:
|
||||
"""Seam: the consumer entry points are the library's (RED if detached)."""
|
||||
|
||||
def test_load_manifest_is_the_library_entry_point(self) -> None:
|
||||
assert ingest.load_manifest is llm_ingestion_okf.load_manifest
|
||||
|
||||
def test_materialize_delegates_with_the_offline_default(
|
||||
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
calls: dict[str, Any] = {}
|
||||
|
||||
def fake(
|
||||
manifest_path: Path, bundle_dir: Path, ingested_at: str, **kwargs: Any
|
||||
) -> llm_ingestion_okf.IngestResult:
|
||||
calls["args"] = (manifest_path, bundle_dir, ingested_at)
|
||||
calls["kwargs"] = kwargs
|
||||
return llm_ingestion_okf.IngestResult(written=(tmp_path / "ingest-e.md",))
|
||||
|
||||
monkeypatch.setattr(ingest, "materialize_bundle", fake)
|
||||
out = ingest.materialize(tmp_path / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
assert out == [tmp_path / "ingest-e.md"] # IngestResult.written → list, order kept
|
||||
assert calls["args"] == (tmp_path / "manifest.json", tmp_path / "bundle", INGESTED_AT)
|
||||
# The offline invariant at the seam: allow_network/http_get are NEVER
|
||||
# passed — the library's local-only default stays in force.
|
||||
assert calls["kwargs"] == {}
|
||||
|
||||
|
||||
class TestOfflineInvariant:
|
||||
"""Seam: the adapter cannot grant network access (RED if it opts in)."""
|
||||
|
||||
def test_http_source_is_refused_at_the_network_gate(self, tmp_path: Path) -> None:
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps(_http_manifest()), encoding="utf-8")
|
||||
bundle = tmp_path / "bundle"
|
||||
with pytest.raises(ingest.NetworkGateError):
|
||||
ingest.materialize(manifest, bundle, INGESTED_AT)
|
||||
assert not bundle.exists() or not any(bundle.iterdir()) # gate fires before any write
|
||||
|
||||
|
||||
class TestErrorContract:
|
||||
"""Seam: consumer-facing errors ARE the library hierarchy (RED if forked)."""
|
||||
|
||||
def test_error_types_are_the_library_types(self) -> None:
|
||||
for name in (
|
||||
"IngestError",
|
||||
"ManifestError",
|
||||
"MaterializationError",
|
||||
"NetworkGateError",
|
||||
"RenderError",
|
||||
"SourceError",
|
||||
):
|
||||
assert getattr(ingest, name) is getattr(llm_ingestion_okf, name)
|
||||
|
||||
def test_every_error_roots_in_ingest_error(self) -> None:
|
||||
for exc in (
|
||||
ingest.ManifestError,
|
||||
ingest.MaterializationError,
|
||||
ingest.NetworkGateError,
|
||||
ingest.RenderError,
|
||||
ingest.SourceError,
|
||||
):
|
||||
assert issubclass(exc, ingest.IngestError)
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
"""Load-bearing ingest seams (ingest-spec §11) — D7 mirror of MAF I2's set.
|
||||
|
||||
Each test must go RED when its seam is detached (the method-spec §11 regime): a
|
||||
grønn-men-død test is the failure mode the rule exists for. The seams mirrored here:
|
||||
grønn-men-død test is the failure mode the rule exists for. The seams mirrored here,
|
||||
proven through the consumer seam (``portfolio_optimiser_claude.ingest``, backed by
|
||||
llm-ingestion-okf since the adoption):
|
||||
|
||||
- Provenance stamping — a generated file carries the §7 provenance layer, in order.
|
||||
- Navigability — the generated bundle is consumable by the UNCHANGED ``okf`` navigation
|
||||
|
|
@ -14,15 +16,15 @@ grønn-men-død test is the failure mode the rule exists for. The seams mirrored
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude import okf
|
||||
from portfolio_optimiser_claude.ingest import ManifestContract, load_manifest, materialize
|
||||
from portfolio_optimiser_claude.ingest import ManifestError, load_manifest, materialize
|
||||
|
||||
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-file"
|
||||
INGESTED_AT = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
|
||||
|
|
@ -30,6 +32,13 @@ INGESTED_AT = (GOLDEN / "ingested-at.txt").read_text(encoding="utf-8").strip()
|
|||
_PROVENANCE_KEYS = ("source_system", "source_query", "ingested_at", "ingest_manifest", "generated")
|
||||
|
||||
|
||||
def _expected_stamp(manifest_path: Path) -> str:
|
||||
# The §5 stamp rule, recomputed INDEPENDENTLY of the implementation:
|
||||
# ``{manifest stem}@{sha256(raw bytes)[:16]}`` — RED if the stamping detaches.
|
||||
digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest()[:16]
|
||||
return f"{manifest_path.stem}@{digest}"
|
||||
|
||||
|
||||
def _materialized(tmp_path: Path) -> Path:
|
||||
bundle = tmp_path / "bundle"
|
||||
bundle.mkdir()
|
||||
|
|
@ -48,8 +57,7 @@ class TestProvenanceStamping:
|
|||
assert concept.frontmatter["source_system"] == "prosjekt-arkiv"
|
||||
assert concept.frontmatter["ingested_at"] == INGESTED_AT
|
||||
assert concept.frontmatter["generated"] == "true"
|
||||
stamp = load_manifest(GOLDEN / "manifest.json").stamp
|
||||
assert concept.frontmatter["ingest_manifest"] == stamp
|
||||
assert concept.frontmatter["ingest_manifest"] == _expected_stamp(GOLDEN / "manifest.json")
|
||||
|
||||
def test_provenance_keys_are_in_the_spec_order(self, tmp_path: Path) -> None:
|
||||
# §5: exactly these keys, in exactly this order — the chain is a contract.
|
||||
|
|
@ -103,20 +111,24 @@ class TestVerdictReservation:
|
|||
],
|
||||
}
|
||||
|
||||
def test_verdict_okf_type_is_rejected(self) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**self._manifest("verdict"))
|
||||
def _write(self, tmp_path: Path, okf_type: str) -> Path:
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps(self._manifest(okf_type)), encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
def test_verdict_reservation_is_case_insensitive(self) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**self._manifest("Verdict"))
|
||||
def test_verdict_okf_type_is_rejected(self, tmp_path: Path) -> None:
|
||||
with pytest.raises(ManifestError):
|
||||
load_manifest(self._write(tmp_path, "verdict"))
|
||||
|
||||
def test_verdict_reservation_is_case_insensitive(self, tmp_path: Path) -> None:
|
||||
with pytest.raises(ManifestError):
|
||||
load_manifest(self._write(tmp_path, "Verdict"))
|
||||
|
||||
def test_rejection_is_fail_fast_before_any_source_call(self, tmp_path: Path) -> None:
|
||||
# A verdict manifest never touches the source: no bundle is written.
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps(self._manifest("verdict")), encoding="utf-8")
|
||||
manifest = self._write(tmp_path, "verdict")
|
||||
bundle = tmp_path / "bundle"
|
||||
with pytest.raises(ValidationError):
|
||||
with pytest.raises(ManifestError):
|
||||
materialize(manifest, bundle, INGESTED_AT)
|
||||
assert not bundle.exists() or not any(bundle.iterdir())
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
"""Ingest SQL unit + fail-fast contract tests (ingest-spec §4, §5, §8).
|
||||
|
||||
Pins the ``SqlSource`` manifest contract, the §5 typed-cell rendering (int/float/NULL/other),
|
||||
the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access enforcement.
|
||||
Everything is local: a tmp sqlite fixture, no network, no credentials.
|
||||
Pins the ``sql`` manifest contract, the §5 typed-cell fail-fast (a BLOB is never silently
|
||||
coerced), the runtime ``connection_ref`` resolution, the §8 size cap, and read-only access
|
||||
enforcement. Everything is local: a tmp sqlite fixture, no network, no credentials.
|
||||
|
||||
Since the adoption, every seam is proven THROUGH the consumer entry points
|
||||
(``load_manifest``/``materialize``) — the connector internals are unit-owned by the
|
||||
llm-ingestion-okf suite. The §5 typed-cell happy paths stay bound here in the repo by the
|
||||
sql golden (int/float/text) and test_ingest_sql_loadbearing.py (NULL/REAL).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -12,17 +17,18 @@ import sqlite3
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude.ingest import (
|
||||
ManifestContract,
|
||||
ManifestError,
|
||||
RenderError,
|
||||
SourceError,
|
||||
SqlSource,
|
||||
_read_sql,
|
||||
_render_sql_cell,
|
||||
_resolve_connection_ref,
|
||||
load_manifest,
|
||||
materialize,
|
||||
)
|
||||
|
||||
INGESTED_AT = "2026-07-04T12:00:00Z"
|
||||
|
||||
|
||||
def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path:
|
||||
db = tmp_path / "src.sqlite"
|
||||
|
|
@ -34,75 +40,65 @@ def _db(tmp_path: Path, rows: list[tuple[object, ...]]) -> Path:
|
|||
return db
|
||||
|
||||
|
||||
def _sql_manifest() -> dict:
|
||||
def _sql_manifest(query: str = "SELECT a FROM t ORDER BY a") -> dict:
|
||||
return {
|
||||
"manifest_version": 1,
|
||||
"source": {"type": "sql", "id": "db", "connection_ref": "SRC_DSN"},
|
||||
"bundle_summary": "s",
|
||||
"extractions": [
|
||||
{
|
||||
"id": "e",
|
||||
"title": "T",
|
||||
"query": "SELECT a FROM t ORDER BY a",
|
||||
"okf_type": "dataset",
|
||||
"max_rows": 5,
|
||||
}
|
||||
{"id": "e", "title": "T", "query": query, "okf_type": "dataset", "max_rows": 5}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _write_manifest(tmp_path: Path, manifest: dict) -> Path:
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
class TestSqlSourceContract:
|
||||
"""§4: the sql source is a valid discriminated variant; the reference is required."""
|
||||
|
||||
def test_valid_sql_source_loads(self) -> None:
|
||||
contract = ManifestContract(**_sql_manifest())
|
||||
assert isinstance(contract.source, SqlSource)
|
||||
assert contract.source.connection_ref == "SRC_DSN"
|
||||
def test_valid_sql_source_loads(self, tmp_path: Path) -> None:
|
||||
manifest = load_manifest(_write_manifest(tmp_path, _sql_manifest()))
|
||||
assert isinstance(manifest.source, SqlSource)
|
||||
assert manifest.source.connection_ref == "SRC_DSN"
|
||||
|
||||
def test_connection_ref_is_required(self) -> None:
|
||||
def test_connection_ref_is_required(self, tmp_path: Path) -> None:
|
||||
manifest = _sql_manifest()
|
||||
del manifest["source"]["connection_ref"]
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
with pytest.raises(ManifestError):
|
||||
load_manifest(_write_manifest(tmp_path, manifest))
|
||||
|
||||
def test_sql_source_id_grammar_enforced(self) -> None:
|
||||
def test_sql_source_id_grammar_enforced(self, tmp_path: Path) -> None:
|
||||
manifest = _sql_manifest()
|
||||
manifest["source"]["id"] = "Bad Id"
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**manifest)
|
||||
with pytest.raises(ManifestError):
|
||||
load_manifest(_write_manifest(tmp_path, manifest))
|
||||
|
||||
|
||||
class TestTypedCellRendering:
|
||||
"""§5: None→'', int→decimal, float→shortest round-trip, str→verbatim, other→fail."""
|
||||
class TestTypedCellFailFast:
|
||||
"""§5: an unsupported cell type (a BLOB, say) MUST fail — never a silent coercion."""
|
||||
|
||||
def test_null_is_empty_string(self) -> None:
|
||||
assert _render_sql_cell(None) == ""
|
||||
|
||||
def test_int_is_plain_decimal(self) -> None:
|
||||
assert _render_sql_cell(3) == "3"
|
||||
assert _render_sql_cell(0) == "0"
|
||||
|
||||
def test_float_is_shortest_round_trip(self) -> None:
|
||||
assert _render_sql_cell(1200.5) == "1200.5"
|
||||
assert _render_sql_cell(89.9) == "89.9"
|
||||
assert _render_sql_cell(4200.0) == "4200.0" # a whole-valued REAL keeps its .0
|
||||
|
||||
def test_str_is_returned_raw(self) -> None:
|
||||
# _render_sql_cell returns the RAW string; §5 escaping is _escape_cell's job.
|
||||
assert _render_sql_cell("a|b\\c") == "a|b\\c"
|
||||
|
||||
def test_other_value_type_fails(self) -> None:
|
||||
with pytest.raises(ValueError, match="not a supported value type"):
|
||||
_render_sql_cell(b"\x00\x01") # a BLOB — never a silent coercion
|
||||
def test_blob_cell_fails_typed(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = _db(tmp_path, [(1, 1.0, "x", b"\x00\x01")])
|
||||
manifest_path = _write_manifest(tmp_path, _sql_manifest("SELECT d FROM t"))
|
||||
monkeypatch.setenv("SRC_DSN", str(db))
|
||||
with pytest.raises(RenderError, match="unsupported SQL cell type"):
|
||||
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
|
||||
class TestConnectorRuntime:
|
||||
"""§8: reference resolution, size cap, and read-only access — all fail-fast."""
|
||||
|
||||
def test_unset_connection_ref_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_unset_connection_ref_fails_fast(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("SRC_DSN", raising=False)
|
||||
with pytest.raises(ValueError, match="not set in the environment"):
|
||||
_resolve_connection_ref("SRC_DSN")
|
||||
manifest_path = _write_manifest(tmp_path, _sql_manifest())
|
||||
with pytest.raises(SourceError, match="not set in the environment"):
|
||||
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_max_rows_cap_enforced_fail_fast(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -110,14 +106,16 @@ class TestConnectorRuntime:
|
|||
db = _db(tmp_path, [(1, 1.0, "x", None), (2, 2.0, "y", None)])
|
||||
manifest = _sql_manifest()
|
||||
manifest["extractions"][0]["max_rows"] = 1 # 2 rows > 1
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
manifest_path = _write_manifest(tmp_path, manifest)
|
||||
monkeypatch.setenv("SRC_DSN", str(db))
|
||||
with pytest.raises(ValueError, match="max_rows"):
|
||||
materialize(manifest_path, tmp_path / "bundle", "2026-07-04T12:00:00Z")
|
||||
with pytest.raises(SourceError, match="max_rows"):
|
||||
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
|
||||
|
||||
def test_connection_is_read_only(self, tmp_path: Path) -> None:
|
||||
def test_connection_is_read_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
db = _db(tmp_path, [(1, 1.0, "x", None)])
|
||||
# The connector opens read-only (§4 SHOULD): a write statement is refused.
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
_read_sql(str(db), "UPDATE t SET a = 9", 5)
|
||||
# The connector opens read-only (§4 SHOULD): a write statement is refused at the
|
||||
# database and surfaces as a typed SourceError.
|
||||
manifest_path = _write_manifest(tmp_path, _sql_manifest("UPDATE t SET a = 9"))
|
||||
monkeypatch.setenv("SRC_DSN", str(db))
|
||||
with pytest.raises(SourceError, match="readonly"):
|
||||
materialize(manifest_path, tmp_path / "bundle", INGESTED_AT)
|
||||
|
|
|
|||
|
|
@ -17,14 +17,14 @@ sqlite fixture — no network, no credentials):
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from portfolio_optimiser_claude import okf
|
||||
from portfolio_optimiser_claude.ingest import ManifestContract, load_manifest, materialize
|
||||
from portfolio_optimiser_claude.ingest import ManifestError, load_manifest, materialize
|
||||
|
||||
GOLDEN = Path(__file__).resolve().parents[1] / "examples" / "ingest-golden-sql"
|
||||
CONNECTION_REF = "PORTEFOLJE_SQL_DSN"
|
||||
|
|
@ -34,6 +34,13 @@ _FIXTURE = GOLDEN / "fixture" / "portefolje.sqlite"
|
|||
_PROVENANCE_KEYS = ("source_system", "source_query", "ingested_at", "ingest_manifest", "generated")
|
||||
|
||||
|
||||
def _expected_stamp(manifest_path: Path) -> str:
|
||||
# The §5 stamp rule, recomputed INDEPENDENTLY of the implementation:
|
||||
# ``{manifest stem}@{sha256(raw bytes)[:16]}`` — RED if the stamping detaches.
|
||||
digest = hashlib.sha256(manifest_path.read_bytes()).hexdigest()[:16]
|
||||
return f"{manifest_path.stem}@{digest}"
|
||||
|
||||
|
||||
def _materialized(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
monkeypatch.setenv(CONNECTION_REF, str(_FIXTURE))
|
||||
bundle = tmp_path / "bundle"
|
||||
|
|
@ -75,8 +82,7 @@ class TestProvenanceStamping:
|
|||
assert concept.frontmatter["source_system"] == "portefolje-db"
|
||||
assert concept.frontmatter["ingested_at"] == INGESTED_AT
|
||||
assert concept.frontmatter["generated"] == "true"
|
||||
stamp = load_manifest(GOLDEN / "manifest.json").stamp
|
||||
assert concept.frontmatter["ingest_manifest"] == stamp
|
||||
assert concept.frontmatter["ingest_manifest"] == _expected_stamp(GOLDEN / "manifest.json")
|
||||
|
||||
|
||||
class TestNavigability:
|
||||
|
|
@ -112,21 +118,23 @@ class TestVerdictReservation:
|
|||
],
|
||||
}
|
||||
|
||||
def test_verdict_okf_type_is_rejected(self) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
ManifestContract(**self._sql_manifest("verdict"))
|
||||
def test_verdict_okf_type_is_rejected(self, tmp_path: Path) -> None:
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps(self._sql_manifest("verdict")), encoding="utf-8")
|
||||
with pytest.raises(ManifestError):
|
||||
load_manifest(manifest)
|
||||
|
||||
def test_rejection_is_fail_fast_before_the_db_is_opened(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# With the connection_ref env var UNSET, a run that got past validation would fail with a
|
||||
# plain ValueError trying to open the db. Rejection is a ValidationError at validation —
|
||||
# so the db is never opened (RED if the reservation stops firing first).
|
||||
# With the connection_ref env var UNSET, a run that got past validation would fail
|
||||
# as a SourceError resolving the reference. Rejection is a ManifestError at
|
||||
# validation — so the db is never opened (RED if the reservation stops firing first).
|
||||
monkeypatch.delenv(CONNECTION_REF, raising=False)
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps(self._sql_manifest("verdict")), encoding="utf-8")
|
||||
bundle = tmp_path / "bundle"
|
||||
with pytest.raises(ValidationError):
|
||||
with pytest.raises(ManifestError):
|
||||
materialize(manifest, bundle, INGESTED_AT)
|
||||
assert not bundle.exists() or not any(bundle.iterdir())
|
||||
|
||||
|
|
|
|||
7
uv.lock
generated
7
uv.lock
generated
|
|
@ -463,6 +463,11 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/06/e6/42a475bfca683b0cd5366f6dd06580062b7e567bb8534d225c877c2f14f3/librt-0.12.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bca1472acbd473eff61059b4409f802c5a1bcb4cd0344d06f939df9c4c125d40", size = 104282, upload-time = "2026-06-30T16:14:09.29Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llm-ingestion-okf"
|
||||
version = "0.1.0"
|
||||
source = { git = "https://git.fromaitochitta.com/open/llm-ingestion-okf.git?rev=dae0bd1a2898b69f436a877538d67afd71e48ad8#dae0bd1a2898b69f436a877538d67afd71e48ad8" }
|
||||
|
||||
[[package]]
|
||||
name = "mcp"
|
||||
version = "1.28.1"
|
||||
|
|
@ -589,6 +594,7 @@ version = "0.1.0"
|
|||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "claude-agent-sdk" },
|
||||
{ name = "llm-ingestion-okf" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
|
||||
|
|
@ -602,6 +608,7 @@ dev = [
|
|||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "claude-agent-sdk", specifier = ">=0.2.111,<0.3" },
|
||||
{ name = "llm-ingestion-okf", git = "https://git.fromaitochitta.com/open/llm-ingestion-okf.git?rev=dae0bd1a2898b69f436a877538d67afd71e48ad8" },
|
||||
{ name = "pydantic", specifier = ">=2" },
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue