Kjell Tore Guttormsen
1260fac154
`okf project` built a bundle two rules behind `okf build`. `cli.build`'s signature defaulted `keep_table_heading` and `sheet_section_rows` to `False` while argparse defaulted both to `True`, and `project.create` calls `build()` as a function. Measured on a five-document folder: 15 concepts / 30 files where `okf build` wrote 26 / 52, the whole difference in the priced sheet -- the document a question about price has to reach. The invariant test could not see it: it compared `project.create` against the same function, and its two fixture documents had neither a table nor a sheet. Both gaps are tests now, and the two paths are byte-equal on that folder (`diff -rq`, 0 differences). README opens with what / one install line / two commands / the three shapes of request; the phase-status paragraph moved down, nothing deleted. One tag is pinned everywhere: README pinned v0.4.0 on its install lines and v0.6.0 below, llms.txt pinned v0.4.0, so an agent reading from the top installed a tag without `okf project`. The skill states three modes -- question, hypothesis (per premise, `confirmed` / `refuted` / `undecidable-from-bundle`), and a task producing a document (source per claim in the artefact, an ungrounded paragraph written and marked rather than dropped, the cut declared inside the document). The five markings are untouched. Generated skills state relative paths in the project layout: `okf consume .okf/<id>` and `okf check --skill .claude/skills/<id>-consume/SKILL.md`, runnable from where `okf project` tells the reader to start `claude`. Two absolute paths to zero, measured with a query shown capable of finding first -- O5's published "4 -> 0" used `grep -c "^/"` against paths indented by two spaces. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
197 lines
10 KiB
TOML
197 lines
10 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "llm-ingestion-okf"
|
|
version = "0.7.0"
|
|
description = "Shared OKF (Open Knowledge Format) ingestion library: spec-based connectors, bundle inbox, and external-bundle import, with security delegated to llm-ingestion-guard."
|
|
readme = "README.md"
|
|
license = "MIT"
|
|
requires-python = ">=3.10"
|
|
authors = [{ name = "Kjell Tore Guttormsen" }]
|
|
classifiers = [
|
|
"Development Status :: 3 - Alpha",
|
|
"Intended Audience :: Developers",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.10",
|
|
]
|
|
# Exactly one runtime dependency, ever: the security boundary. Everything
|
|
# else is stdlib. The version range is the real pin — it resolves normally
|
|
# against a package index, and is satisfied today by the git+https tag
|
|
# install documented in the README (a direct reference is an install-time
|
|
# channel, not a dependency declaration).
|
|
#
|
|
# Floor 1.2, not the 1.0.0 freeze: this library needs the flow-mapping
|
|
# frontmatter support (`generated: { by: x, at: y }`) that landed in the
|
|
# guard's 1.2.0, without which Door C fail-secures every concept carrying
|
|
# it. Ceiling <2.0, not a narrower minor: the guard's own 1.0.0 release
|
|
# promises no exported name is removed, renamed or given a different
|
|
# meaning short of a 2.0.0 — calibration (severities, dispositions) is
|
|
# explicitly free to move within 1.x under that same promise, so a tighter
|
|
# ceiling here would claim a stability guarantee the guard does not need to
|
|
# keep and we do not need to demand.
|
|
dependencies = ["llm-ingestion-guard>=1.2,<2.0"]
|
|
|
|
# The installed command. `okf build <folder> --bundle <dir>` is the packaged
|
|
# form of a path that was two unpackaged scripts under `tools/` and nine flags
|
|
# -- reachable only from a clone, which is not where a consumer stands.
|
|
[project.scripts]
|
|
okf = "llm_ingestion_okf.cli:main"
|
|
|
|
[project.optional-dependencies]
|
|
# Binary file-type extraction parsers. OPT-IN ONLY: this extra pulls binary
|
|
# wheels (pillow, pypdfium2) and a transitive tree that core must never have —
|
|
# the "exactly one runtime dependency" rule above covers the default install,
|
|
# and this extra is outside it by construction.
|
|
#
|
|
# The extra names the parsers it actually ships, so a consumer installing it
|
|
# gets what the error message promised and nothing else. It ships two: a `pdf`
|
|
# reader, and a converter that reaches the office types.
|
|
#
|
|
# WHY pdfplumber, and why the floor is not free (measured 2026-08-21,
|
|
# docs/2026-08-21-g2-pdf-extraction-measurement.md): on a real Vegnormalene
|
|
# requirement table pdfplumber keeps 4 of 4 rows with label and value on the
|
|
# same line; pypdf, pdfminer.six and pymupdf each keep 0 of 4, emitting all
|
|
# labels then all values, which a downstream reader can only re-pair by
|
|
# guessing. In a `krav` document that is a wrong answer that looks right.
|
|
# pymupdf is additionally out on LICENSE (AGPL-3.0 or commercial) — this
|
|
# package is MIT and an extra must not hand a consumer copyleft they did not
|
|
# choose.
|
|
#
|
|
# PARSER VERSION IS PART OF THE OUTPUT CONTRACT. pdfplumber pins
|
|
# `pdfminer.six==20260107` exactly, and pdfminer.six ships date-stamped
|
|
# releases with no stability contract. Extraction is deterministic WITHIN a
|
|
# parser version (measured, 5 configurations) and NOT guaranteed across one.
|
|
# `tests/test_extract.py` holds that promise against a committed fixture, so
|
|
# widening this range makes a test go red instead of letting extracted text
|
|
# drift silently. See tests/fixtures/README.md.
|
|
#
|
|
# WHY THE CONVERTER BINARY IS VENDORED RATHER THAN FOUND ON PATH. The `xlsx`
|
|
# and `pptx` readers exist only from pandoc 3.8.3. Debian 12 ships 2.17.1.1
|
|
# and Ubuntu 24.04 ships 3.1.3, so a PATH binary cannot deliver two of the
|
|
# five office formats on current stable distributions -- and a library whose
|
|
# output depends on which pandoc a host happens to carry is not deterministic
|
|
# in the sense the rest of this package means it.
|
|
#
|
|
# `pypandoc-binary` carries the binary inside the wheel (7 platform wheels at
|
|
# 1.17, including macosx x86_64/arm64, manylinux and musllinux x86_64/aarch64,
|
|
# and win_amd64 -- measured on the PyPI JSON API 2026-09-02). The pin is
|
|
# EXACT, not a range, because the binary's version is part of the output
|
|
# contract in the same way pdfminer.six's is: extraction is deterministic
|
|
# within a converter version and not across one.
|
|
#
|
|
# This does not widen the runtime dependency surface. The rule above governs
|
|
# `project.dependencies`, which still names the guard alone; the extra is
|
|
# outside it by construction, and the test below now pins its contents so a
|
|
# third entry cannot arrive unexamined.
|
|
extract = ["pdfplumber>=0.11.10,<0.12", "pypandoc-binary==1.17"]
|
|
|
|
# The OCR engine for `--ocr`, and NEVER a runtime dependency. It is a separate
|
|
# group from `extract` rather than three more entries in it, because it buys
|
|
# something categorically different: `extract` decides which file types can be
|
|
# read at all, while this one only changes how a PDF page is read when the
|
|
# page's own text never arrived. A consumer who installs `[extract]` gets every
|
|
# file type; a consumer who never meets a scanned document should never carry
|
|
# an inference runtime.
|
|
#
|
|
# WHY rapidocr ON onnxruntime, and why not the obvious alternative. Docling was
|
|
# measured first and is OUT on a platform fact, not a preference: it needs
|
|
# torch, and torch stopped publishing macOS x86_64 wheels after 2.2.2, with no
|
|
# `transformers` version inside Docling's own window that works against that
|
|
# one (4 tried, 2026-09-08). rapidocr on onnxruntime installs and runs on this
|
|
# machine, and it carries its ONNX models inside its own wheel, so `--ocr`
|
|
# needs no network at run time -- which matters here more than usual, since
|
|
# this library's network gate is an explicit per-run opt-in and an engine that
|
|
# downloaded a model on first use would walk straight through it.
|
|
#
|
|
# `pypdfium2` is named although `[extract]` already reaches it through
|
|
# pdfplumber: the OCR path RENDERS a page before reading it, and the renderer
|
|
# is a dependency of that path rather than a happy accident of another one.
|
|
#
|
|
# The pins are ranges rather than exact versions, and that is a weaker promise
|
|
# than `[extract]` makes on purpose: OCR output is a model's reading of an
|
|
# image, so it is deterministic within one model version and NOT across one,
|
|
# and no range can make it otherwise. A bundle built with `--ocr` is
|
|
# reproducible against the versions it was built with, which is stated in the
|
|
# report rather than implied by a pin.
|
|
ocr = ["rapidocr>=3.9,<4", "onnxruntime>=1.20,<2", "pypdfium2>=4,<6"]
|
|
|
|
[dependency-groups]
|
|
dev = ["pytest>=8", "mypy>=1.14", "ruff>=0.9"]
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/llm_ingestion_okf"]
|
|
|
|
# Two AUTHORED files the packaged commands cannot run without, carried into the
|
|
# wheel from where they are edited rather than committed a second time under
|
|
# `src/`. A duplicate would drift, and both of these are checked against
|
|
# literals in the code: a template whose blocks are pinned by a test, and a
|
|
# known-positive artefact whose byte count is a constant in `consume.py`.
|
|
#
|
|
# `okf skill` instantiates the template; `okf consume` measures the contract
|
|
# document as its section 7.4 known-positive and refuses without it. Before
|
|
# 2026-09-08 neither command was installable, so neither file had to travel.
|
|
[tool.hatch.build.targets.wheel.force-include]
|
|
"skills/okf-consume-template/SKILL.md" = "llm_ingestion_okf/_data/okf-consume-template.md"
|
|
"docs/consumption-contract.md" = "llm_ingestion_okf/_data/consumption-contract.md"
|
|
|
|
[tool.ruff]
|
|
line-length = 100
|
|
target-version = "py310"
|
|
|
|
[tool.mypy]
|
|
strict = true
|
|
python_version = "3.10"
|
|
|
|
# llm-ingestion-guard ships no py.typed marker, so its symbols arrive as Any.
|
|
# The adapter coerces every value it carries across the seam to a concrete
|
|
# type, which is what keeps --strict meaningful on this side of it.
|
|
[[tool.mypy.overrides]]
|
|
module = ["llm_ingestion_guard", "llm_ingestion_guard.*"]
|
|
ignore_missing_imports = true
|
|
|
|
# `pypandoc` ships no py.typed marker either. Only `_pandoc.py` imports it, and
|
|
# every value it hands back is coerced to `str`/`Path` there before it reaches
|
|
# the rest of the package -- the same discipline as the guard adapter above.
|
|
[[tool.mypy.overrides]]
|
|
module = ["pypandoc", "pypandoc.*"]
|
|
ignore_missing_imports = true
|
|
|
|
# `rapidocr` ships no py.typed marker either, and it is behind an OPTIONAL
|
|
# group -- so on a machine without that group installed the import does not
|
|
# resolve at all. Only `_ocr_reader` imports it, and the only value that
|
|
# crosses back is coerced to `str` there, the same discipline as the two
|
|
# overrides above.
|
|
[[tool.mypy.overrides]]
|
|
module = ["rapidocr", "rapidocr.*"]
|
|
ignore_missing_imports = true
|
|
|
|
# Install CHANNEL for the guard, which is not on a package index yet. It is
|
|
# uv-specific, and it reaches further than a dev-only setting: a consumer
|
|
# installing this package from git WITH UV picks the guard up from this tag
|
|
# automatically, because uv reads this file when it builds from the source
|
|
# tree. Measured against an empty cache 2026-07-25, 2026-08-20, and
|
|
# 2026-08-21 on uv 0.9.8. The 08-21 run also measured the TRANSITIVE form: a
|
|
# separate consumer project naming only this package still resolves the guard
|
|
# from the entry below, because this package reaches it as a git source.
|
|
#
|
|
# That source is the whole reach. A wheel carries Requires-Dist and nothing
|
|
# else, so this entry cannot survive an index install — and while the guard is
|
|
# off-index, removing it would break the one-command uv path the README
|
|
# documents.
|
|
#
|
|
# pip does not read it at all: it resolves [project.dependencies] alone and
|
|
# fails with "No matching distribution found for llm-ingestion-guard" until
|
|
# the guard is installed from its own tag first (README; measured 2026-08-21,
|
|
# both the failure and the two-command recovery).
|
|
#
|
|
# Either way the range above stays the pin, and the pin is per-tree: a wheel
|
|
# built from THIS tree carries `Requires-Dist: llm-ingestion-guard<2.0,>=1.2`,
|
|
# measured 2026-08-23 against the built wheel. The `<0.4,>=0.3` this comment
|
|
# carried before was the `v0.3.4` tag's range — still true of that tag, never
|
|
# true of this tree. Reading a range off one and installing it against the
|
|
# other is the one combination that fails.
|
|
[tool.uv.sources]
|
|
llm-ingestion-guard = { git = "https://git.fromaitochitta.com/open/llm-ingestion-pipeline-security.git", tag = "v1.3.0" }
|