Kjell Tore Guttormsen
0772dafb70
A received OKF bundle MAY legitimately carry index.md (directory listing, read first under progressive disclosure) and log.md (update history) at any level (spec §3.1/§6/§7). import_bundle previously hard-rejected those basenames in the T4 path gate, so a conformant third-party bundle was over-blocked in full (FAIL_SECURE) — and because the reject fired before scan_concept, index.md's body (the highest-priority injection surface) was never scanned. import_bundle now defaults allow_reserved=True: reserved basenames are scanned as structural files (path-safety checks — traversal / absolute / backslash / .md — still apply). The shadow-reject (an *upload* masquerading as index.md) is preserved: the front-end passes allow_reserved=False so a materialized upload landing on a reserved basename is still refused. That front-end opt-in was required to keep the shadow-reject once the default flipped (not in the plan's Filer set; traced from the code). - okf.py: validate_concept_path/_validate_concept/import_bundle gain the keyword; validate_concept_path default stays False (strict standalone). - tests: +3 (legit index/log admit; injection in index.md body caught; okf_version frontmatter admits). Per-concept-iteration test switched to a traversal vector; mode-b showcase's index.md surface reframed from reserved-name-reject to index.md-body-scan. - README honest-limits + CLAUDE.md context note the mode-b/upload distinction. Suite: 341 -> 344 passed. Core invariant intact (dependencies=[]).
362 lines
15 KiB
Python
362 lines
15 KiB
Python
"""OKF inbox front-end — stage 1 of the two-stage upload showcase (PLAN §247).
|
|
|
|
Reads the files a human actually drops into an inbox and *materializes* them into
|
|
an OKF bundle ``{concept_path: document_text}`` with provenance, so the stage-2
|
|
guard (:func:`llm_ingestion_guard.okf.import_bundle`) can validate every concept.
|
|
This stage owns the container/format threats; the guard owns the text/structural
|
|
threats.
|
|
|
|
**Placement.** This lives in the test tree, not ``src/``: the extraction parsers
|
|
are showcase/dev-scoped (``python-docx``/``python-pptx`` in the ``dev`` extra,
|
|
never core ``dependencies``), and the shippable core stays stdlib-only. It is an
|
|
in-repo demonstration a consumer reads and adapts, not v1 shipped code.
|
|
|
|
Slice 2a covers the text formats — ``.txt`` and ``.md`` (stdlib only). ``.zip``
|
|
(zip-slip / zip-bomb), ``.csv`` (formula injection), folders, ``.docx`` and
|
|
``.pptx`` land in later slices.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import stat
|
|
import zipfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from llm_ingestion_guard.okf import import_bundle, Origin, Channel, BundleResult
|
|
from llm_ingestion_guard.disposition import Disposition
|
|
|
|
|
|
# Aggregate disposition -> inbox verdict. Fail-secure by default: a front-end
|
|
# refusal (a container threat the guard never sees) is itself a REJECT.
|
|
_VERDICT = {
|
|
Disposition.WARN: "ADMIT",
|
|
Disposition.QUARANTINE_REVIEW: "HOLD",
|
|
Disposition.FAIL_SECURE: "REJECT",
|
|
}
|
|
|
|
# Where materialized uploads live inside the bundle. An upload named ``index.*``
|
|
# thus lands on the reserved ``uploads/index.md`` and is refused by the path gate.
|
|
_MATERIALIZE_PREFIX = "uploads"
|
|
|
|
# The text formats this slice reads directly (no parser dependency).
|
|
_TEXT_SUFFIXES = {".txt", ".md"}
|
|
|
|
# CSV cells leading with any of these parse as a formula in a spreadsheet — the
|
|
# CSV-injection / DDE vector (RCE when a human opens the file). Leading whitespace
|
|
# does not defuse it, so it is stripped before the check. Numeric cells that lead
|
|
# with '-'/'+' are the accepted false-positive (README honest-limits).
|
|
_FORMULA_LEADS = ("=", "+", "-", "@")
|
|
|
|
# Zip self-safety caps (OWASP LLM10). Bounded so a decompression bomb is refused
|
|
# before its uncompressed bytes are read into memory. Defaults are generous for a
|
|
# document inbox; tests pass small caps to exercise the gate.
|
|
MAX_ENTRY_BYTES = 5_000_000 # per uncompressed entry
|
|
MAX_TOTAL_BYTES = 25_000_000 # per archive, summed across entries
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Provenance:
|
|
"""Where one materialized concept came from — the audit record (brief §6)."""
|
|
|
|
concept_path: str
|
|
source_name: str
|
|
source_type: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InboxExtract:
|
|
"""The stage-1 output: the OKF bundle, its provenance, and front-end refusals.
|
|
|
|
``rejected`` holds ``(source_name, reason)`` for drops the front-end refuses
|
|
outright (a container threat the guard never gets to see, e.g. a zip bomb).
|
|
Empty for the text-format slice.
|
|
"""
|
|
|
|
bundle: dict
|
|
provenance: tuple
|
|
rejected: tuple
|
|
|
|
|
|
def _materialize_path(rel_name: str) -> str:
|
|
"""Assign an OKF concept path to a dropped file: ``<prefix>/<rel>.md``.
|
|
|
|
The relative name is preserved verbatim, including any ``..`` — a zip-slip
|
|
entry (``../../evil.md``) thus lands on a traversal concept path that the
|
|
stage-2 path gate (T4) rejects, rather than being silently normalized away.
|
|
"""
|
|
rel = Path(rel_name).with_suffix(".md").as_posix()
|
|
return f"{_MATERIALIZE_PREFIX}/{rel}"
|
|
|
|
|
|
def _is_symlink_entry(info: zipfile.ZipInfo) -> bool:
|
|
"""True if a zip entry encodes a Unix symlink (mode bits in external_attr)."""
|
|
return stat.S_ISLNK(info.external_attr >> 16)
|
|
|
|
|
|
def _materialize_text(rel_name, text, source_type, bundle, provenance):
|
|
"""Add one text concept to the bundle under the materialize prefix."""
|
|
concept_path = _materialize_path(rel_name)
|
|
bundle[concept_path] = text
|
|
provenance.append(Provenance(concept_path, rel_name, source_type))
|
|
|
|
|
|
def _is_formula_cell(cell: str) -> bool:
|
|
stripped = cell.lstrip("\t\r\n ")
|
|
return bool(stripped) and stripped[0] in _FORMULA_LEADS
|
|
|
|
|
|
def _ingest_csv(rel_name, text, bundle, provenance, rejected):
|
|
"""Materialize CSV cell text as a concept and flag formula-injection cells.
|
|
|
|
The raw cell text becomes the concept body so a prompt-injection *phrase* in
|
|
a cell is caught by the stage-2 scan; formula-lead cells are a spreadsheet
|
|
threat the guard would not recognize, so the front-end refuses them here.
|
|
"""
|
|
formula_cells = [
|
|
cell for row in csv.reader(io.StringIO(text)) for cell in row if _is_formula_cell(cell)
|
|
]
|
|
if formula_cells:
|
|
rejected.append(
|
|
(rel_name, f"CSV formula-injection lead in {len(formula_cells)} cell(s): {formula_cells[0][:24]!r}")
|
|
)
|
|
_materialize_text(rel_name, text, "csv", bundle, provenance)
|
|
|
|
|
|
def _extract_docx_text(fs_path) -> str:
|
|
"""Extract text from a ``.docx``, including the regions a human reviewing the
|
|
document in Word does not see: hidden/vanish runs (still runs, so ``paragraph
|
|
.text`` includes them), core metadata properties, and review comments.
|
|
|
|
``python-docx`` is imported lazily — it is a dev/showcase-scoped parser, not a
|
|
core dependency; a consumer would guard the import behind their own extra.
|
|
"""
|
|
from docx import Document
|
|
|
|
doc = Document(str(fs_path))
|
|
parts = [para.text for para in doc.paragraphs if para.text]
|
|
|
|
# Table cells live outside doc.paragraphs — iterate them explicitly.
|
|
for table in doc.tables:
|
|
for row in table.rows:
|
|
for cell in row.cells:
|
|
if cell.text:
|
|
parts.append(cell.text)
|
|
|
|
cp = doc.core_properties
|
|
for attr in ("title", "subject", "keywords", "comments", "category", "author"):
|
|
value = getattr(cp, attr, None)
|
|
if value:
|
|
parts.append(str(value))
|
|
|
|
for comment in doc.comments:
|
|
if comment.text:
|
|
parts.append(comment.text)
|
|
|
|
return "\n".join(parts)
|
|
|
|
|
|
def _shape_alt_text(shape) -> str:
|
|
"""Read a shape's alt-text (cNvPr@descr). python-pptx 1.0.2 has no stable
|
|
public accessor across shape types, so read it off the XML directly."""
|
|
for element in shape._element.iter():
|
|
if element.tag.endswith("}cNvPr"):
|
|
return element.get("descr") or ""
|
|
return ""
|
|
|
|
|
|
def _extract_pptx_text(fs_path) -> str:
|
|
"""Extract text from a ``.pptx``, including the regions an audience watching
|
|
the slides does not see: speaker notes, off-slide (off-canvas) text boxes, and
|
|
image/shape alt-text. ``python-pptx`` is imported lazily (dev/showcase-scoped).
|
|
"""
|
|
from pptx import Presentation
|
|
from pptx.enum.shapes import MSO_SHAPE_TYPE
|
|
|
|
def walk(shapes):
|
|
# Flatten grouped shapes: add_group_shape moves a shape inside the group,
|
|
# so only recursion reaches its text/alt-text.
|
|
for shape in shapes:
|
|
if shape.shape_type == MSO_SHAPE_TYPE.GROUP:
|
|
yield from walk(shape.shapes)
|
|
else:
|
|
yield shape
|
|
|
|
prs = Presentation(str(fs_path))
|
|
parts: list = []
|
|
for slide in prs.slides:
|
|
for shape in walk(slide.shapes):
|
|
if shape.has_text_frame and shape.text_frame.text:
|
|
parts.append(shape.text_frame.text) # incl. off-slide boxes
|
|
alt = _shape_alt_text(shape)
|
|
if alt:
|
|
parts.append(alt)
|
|
if slide.has_notes_slide:
|
|
notes = slide.notes_slide.notes_text_frame.text
|
|
if notes:
|
|
parts.append(notes)
|
|
return "\n".join(parts)
|
|
|
|
|
|
def _extract_xlsx(fs_path):
|
|
"""Extract text from an ``.xlsx``, including the regions a human reading the
|
|
workbook in Excel does not see: cells on a *hidden* sheet (still worksheets, so
|
|
iterated) and cell comments. Returns ``(text, formula_cells)`` — the text is
|
|
materialized as the concept body (so a hidden-sheet / comment injection rides
|
|
into the stage-2 scan) and ``formula_cells`` holds the formula-lead cells the
|
|
front-end refuses (a spreadsheet threat, RCE/DDE when a human opens the file).
|
|
|
|
``openpyxl`` is imported lazily — a dev/showcase-scoped parser, not a core
|
|
dependency. It reads formulas as their string (default ``data_only=False``).
|
|
"""
|
|
from openpyxl import load_workbook
|
|
|
|
wb = load_workbook(str(fs_path))
|
|
parts: list = []
|
|
formula_cells: list = []
|
|
for ws in wb.worksheets: # includes hidden / very-hidden sheets
|
|
for row in ws.iter_rows():
|
|
for cell in row:
|
|
value = cell.value
|
|
if value is not None and value != "":
|
|
parts.append(str(value))
|
|
# Only a genuine text/formula cell can carry the injection lead;
|
|
# a numeric cell is typed (int/float) by openpyxl, so a negative
|
|
# number never trips the gate — unlike CSV, where every cell is
|
|
# text and a leading '-'/'+' is the accepted false-positive.
|
|
if isinstance(value, str) and _is_formula_cell(value):
|
|
formula_cells.append(value)
|
|
comment = cell.comment
|
|
if comment is not None and comment.text:
|
|
parts.append(comment.text)
|
|
return "\n".join(parts), formula_cells
|
|
|
|
|
|
def _ingest_xlsx(rel_name, fs_path, bundle, provenance, rejected):
|
|
"""Materialize an ``.xlsx`` (all sheets incl. hidden, + cell comments) as a
|
|
concept and flag formula-injection cells — mirrors :func:`_ingest_csv`."""
|
|
text, formula_cells = _extract_xlsx(fs_path)
|
|
if formula_cells:
|
|
rejected.append(
|
|
(rel_name, f"XLSX formula-injection lead in {len(formula_cells)} cell(s): {formula_cells[0][:24]!r}")
|
|
)
|
|
_materialize_text(rel_name, text, "xlsx", bundle, provenance)
|
|
|
|
|
|
def _ingest_regular_file(fs_path, rel_name, bundle, provenance, rejected, *, strict):
|
|
"""Dispatch one on-disk file by suffix. ``strict`` raises on an unsupported
|
|
suffix (a top-level drop); a folder walk passes ``strict=False`` to skip it."""
|
|
suffix = Path(rel_name).suffix.lower()
|
|
if suffix == ".csv":
|
|
text = Path(fs_path).read_text(encoding="utf-8", errors="replace")
|
|
_ingest_csv(rel_name, text, bundle, provenance, rejected)
|
|
elif suffix == ".docx":
|
|
_materialize_text(rel_name, _extract_docx_text(fs_path), "docx", bundle, provenance)
|
|
elif suffix == ".pptx":
|
|
_materialize_text(rel_name, _extract_pptx_text(fs_path), "pptx", bundle, provenance)
|
|
elif suffix == ".xlsx":
|
|
_ingest_xlsx(rel_name, fs_path, bundle, provenance, rejected)
|
|
elif suffix in _TEXT_SUFFIXES:
|
|
text = Path(fs_path).read_text(encoding="utf-8", errors="replace")
|
|
_materialize_text(rel_name, text, suffix.lstrip("."), bundle, provenance)
|
|
elif strict:
|
|
raise ValueError(f"unsupported upload format in this slice: {Path(rel_name).name!r}")
|
|
|
|
|
|
def _extract_folder(root, bundle, provenance, rejected):
|
|
"""Walk a dropped folder, materializing its text/CSV members (relative paths
|
|
preserved, so a reserved-name member trips the guard's path gate)."""
|
|
root = Path(root)
|
|
for fs_path in sorted(root.rglob("*")):
|
|
if fs_path.is_symlink():
|
|
rejected.append((str(fs_path.relative_to(root)), "symlink refused (container threat)"))
|
|
continue
|
|
if not fs_path.is_file():
|
|
continue
|
|
rel_name = fs_path.relative_to(root).as_posix()
|
|
_ingest_regular_file(fs_path, rel_name, bundle, provenance, rejected, strict=False)
|
|
|
|
|
|
def _extract_zip(path, bundle, provenance, rejected, max_entry_bytes, max_total_bytes):
|
|
"""Read a ``.zip`` in memory, materializing its text entries; refuse bombs,
|
|
symlinks and oversize entries at the front-end (container threats)."""
|
|
total = 0
|
|
with zipfile.ZipFile(path) as zf:
|
|
for info in zf.infolist():
|
|
name = info.filename
|
|
if name.endswith("/"):
|
|
continue # directory entry — no content
|
|
if _is_symlink_entry(info):
|
|
rejected.append((name, "symlink entry refused (container threat)"))
|
|
continue
|
|
# Fast reject on the declared uncompressed size (a bomb, before reading).
|
|
if info.file_size > max_entry_bytes:
|
|
rejected.append((name, f"entry exceeds {max_entry_bytes}-byte cap (declared {info.file_size})"))
|
|
continue
|
|
if total + info.file_size > max_total_bytes:
|
|
rejected.append((name, f"archive exceeds {max_total_bytes}-byte total cap"))
|
|
continue
|
|
if Path(name).suffix.lower() not in _TEXT_SUFFIXES:
|
|
continue # only text concepts are materialized in this slice
|
|
# Bounded read defends against a header that lies about file_size.
|
|
with zf.open(info) as f:
|
|
data = f.read(max_entry_bytes + 1)
|
|
if len(data) > max_entry_bytes:
|
|
rejected.append((name, f"entry expands past {max_entry_bytes}-byte cap on read"))
|
|
continue
|
|
total += len(data)
|
|
concept_path = _materialize_path(name)
|
|
bundle[concept_path] = data.decode("utf-8", errors="replace")
|
|
provenance.append(Provenance(concept_path, name, "zip"))
|
|
|
|
|
|
def extract_inbox(
|
|
paths,
|
|
*,
|
|
max_entry_bytes: int = MAX_ENTRY_BYTES,
|
|
max_total_bytes: int = MAX_TOTAL_BYTES,
|
|
) -> InboxExtract:
|
|
"""Read dropped files and materialize them into an OKF bundle + provenance.
|
|
|
|
``paths`` is an iterable of file/folder paths. Each ``.txt`` / ``.md`` becomes
|
|
one concept (a ``.md`` keeps its OKF frontmatter verbatim); a ``.csv`` is
|
|
materialized and its formula-lead cells refused; a ``.zip`` is read in memory
|
|
and its text entries materialized, with bomb/symlink/oversize entries refused;
|
|
a folder is walked member-by-member. Refusals land in ``InboxExtract.rejected``.
|
|
"""
|
|
bundle: dict = {}
|
|
provenance: list = []
|
|
rejected: list = []
|
|
for path in paths:
|
|
path = Path(path)
|
|
if path.is_dir():
|
|
_extract_folder(path, bundle, provenance, rejected)
|
|
elif path.suffix.lower() == ".zip":
|
|
_extract_zip(path, bundle, provenance, rejected, max_entry_bytes, max_total_bytes)
|
|
else:
|
|
_ingest_regular_file(path, path.name, bundle, provenance, rejected, strict=True)
|
|
return InboxExtract(bundle, tuple(provenance), tuple(rejected))
|
|
|
|
|
|
def receive(
|
|
paths,
|
|
*,
|
|
max_entry_bytes: int = MAX_ENTRY_BYTES,
|
|
max_total_bytes: int = MAX_TOTAL_BYTES,
|
|
) -> tuple[InboxExtract, BundleResult, str]:
|
|
"""The full two-stage inbox: extract & materialize, then guard, then verdict.
|
|
|
|
Returns ``(extracted, guard_result, verdict)``. A front-end refusal forces a
|
|
REJECT regardless of the guard's aggregate — the guard never saw that drop.
|
|
"""
|
|
extracted = extract_inbox(paths, max_entry_bytes=max_entry_bytes, max_total_bytes=max_total_bytes)
|
|
# allow_reserved=False: these are individually-materialized *uploads*, so an
|
|
# upload landing on the reserved index.md/log.md is a shadow of the directory
|
|
# listing and is refused (T4). A received third-party bundle, by contrast,
|
|
# carries those as legitimate structural files (import_bundle's default).
|
|
result = import_bundle(
|
|
extracted.bundle, origin=Origin.EXTERNAL, channel=Channel.AUTOMATIC, allow_reserved=False
|
|
)
|
|
verdict = "REJECT" if extracted.rejected else _VERDICT[result.disposition]
|
|
return extracted, result, verdict
|