open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

feat(frontmatter): --frontmatter KEY=VALUE stamps a key on every concept of a run

Browse source 
K3-19 b. `okf build --frontmatter KEY=VALUE`, repeatable, split on the FIRST
'=' (`cli.frontmatter_from_flags`) because a publisher's address carries '='
itself. The value is written verbatim on ONE line: the block form of a
`sources` list is invisible to this package's line-oriented readers, so the
flow form is the only one that survives them. Also reachable as
`build(frontmatter=...)`, `measure(concept_frontmatter_values=...)`,
`process_inbox(concept_frontmatter_values=...)` and
`render_inbox_concept(concept_frontmatter_values=...)`, keyword-only with
defaults, so every existing call site is source-compatible.

Precedence: a stated value beats what the document declares, which beats the
file name. A run may ADD any key and REPLACE only `sources` and
`description` -- the two with a derived layer below the flag. Every other key
the door writes is refused by `inbox.validate_concept_frontmatter` before a
proposal is written (`run_frontmatter_invalid`): measured from the bytes, owned
by another argument, the ownership stamp a later run reads back (including
Door A's `ingest_manifest`, which would make that door claim a Door B file),
or a derived facet whose `derived` marker would go on naming a replaced value.
A value that would not read back as stated -- empty, multi-line, or padded,
since `parse_frontmatter` strips -- is refused too.

SPEC SS 4.1 "Extensions" lets a producer add any key and SS 11 forbids a
consumer to reject one. Without the flag nothing moves: a test holds the
flagged tree to the plain one minus exactly the stated line.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-11 03:14:46 +02:00
commit 912b85026b
5 changed files with 171 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

99
src/llm_ingestion_okf/inbox.py
View file

@ -21,6 +21,7 @@ from __future__ import annotations
import hashlib
import os
import re
import unicodedata
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
@ -114,6 +115,7 @@ def render_inbox_concept(
units: SourceUnits | None = None,
span: tuple[int, int] | None = None,
source_title: str | None = None,
concept_frontmatter_values: Mapping[str, str] | None = None,
) -> str:
"""Frame extracted text as an inbox concept file with its provenance layer.
@ -131,6 +133,10 @@ def render_inbox_concept(
`source_title` is what the document calls itself, for the `sources`
entry's `title`; `None` keeps the file name there, as before it existed.
`concept_frontmatter_values` are keys the CALLER states for every concept
of a run, validated by :func:`validate_concept_frontmatter` and applied
LAST, so a stated `sources` or `description` replaces the derived one.
`segment` and `bundle_id` carry the 1-to-N identity layer and are read ONLY
when the profile declares the segmentation capability. A concept the plan
does not cover keeps today's rule verbatim, and the four shipped profiles
@ -238,9 +244,95 @@ def render_inbox_concept(
title=source_title,
)
)
if concept_frontmatter_values:
# LAST, and the position is the precedence: a value the caller states
# for the run beats what the document declares, which beats the file
# name. Validation keeps it to the keys that have a derived layer to
# replace -- everything else this door writes is refused.
frontmatter.update(
validate_concept_frontmatter(concept_frontmatter_values, profile=profile)
)
return f"---\n{profile.frontmatter.emit(frontmatter)}\n---\n\n{_normalize_body(text)}"
#: The two keys a run may REPLACE rather than only add. Each has a layer below
#: the flag -- the document's own title or the file name for `sources`, the
#: document's own first spec point for `description` -- so stating one for the
#: run is choosing a layer, which is what the precedence exists for.
RUN_FRONTMATTER_OVERRIDES = frozenset({"sources", "description"})
# A key the line-oriented readers here recover exactly: no `:`, no space, no
# leading `-` that a YAML reader would take for a sequence entry.
_RUN_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
def _door_keys(profile: BundleProfile) -> frozenset[str]:
"""Every key this door writes itself under `profile`, less the overrides.
Refused as run values because each is something a run cannot restate
without lying: measured from the bytes (the hash, the offsets, the
locators), owned by another argument (`type`, `ingested_at`, the bundle
id), the ownership stamp a later run reads back (`generated`, and Door A's
`ingest_manifest`, which would make that door claim this one's file), or a
derived facet whose `derived` marker would go on naming a value the run had
replaced.
"""
keys = set(DEFAULT.frontmatter.order)
keys.update(("parent", "adjudicated_by", "adjudicated_at", "adjudication_dwell_s"))
if profile.index.facets is not None:
keys.update(profile.index.facets.keys)
if profile.segmentation is not None:
policy = profile.segmentation
keys.update((policy.bundle_id_key, policy.segment_id_key, policy.offset_key))
if policy.adjudication_key is not None:
keys.add(policy.adjudication_key)
if profile.provenance is not None:
address = profile.provenance
keys.update(
(
address.sources_key,
address.pages_key,
address.sheet_key,
address.rows_key,
address.lines_key,
)
)
return frozenset(keys - RUN_FRONTMATTER_OVERRIDES)
def validate_concept_frontmatter(
values: Mapping[str, str], *, profile: BundleProfile
) -> dict[str, str]:
"""Refuse a run-stated key or value that would not read back as stated.
SPEC SS 4.1 lets a producer add any key and SS 11 forbids a consumer to
reject one, so the limits here are this package's own and each is a
reader it has to survive: the value is written verbatim on ONE line
because every reader here is line-oriented, which rules out a line break
and -- since `parse_frontmatter` strips -- surrounding whitespace.
"""
written = _door_keys(profile)
for key, value in values.items():
if not _RUN_KEY.match(key):
raise MaterializationError(
f"frontmatter key {key!r} is not a plain key ([A-Za-z_][A-Za-z0-9_-]*)",
code="run_frontmatter_invalid",
)
if key in written:
raise MaterializationError(
f"frontmatter key {key!r} is written by this door itself; a run may "
f"add keys and replace only {sorted(RUN_FRONTMATTER_OVERRIDES)}",
code="run_frontmatter_invalid",
)
if not value or value != value.strip() or "\n" in value or "\r" in value:
raise MaterializationError(
f"frontmatter value for {key!r} must be one non-empty line with no "
f"surrounding whitespace, got {value!r}",
code="run_frontmatter_invalid",
)
return dict(values)
# The characters that would end a YAML flow mapping early, so a path carrying
# one would produce a `sources` list that parses as something other than what
# was written. The guard refuses a quoted scalar inside a flow mapping (1.3.0,
@ -678,6 +770,7 @@ def _render_segments(
source_file: str,
units: SourceUnits | None,
source_title: str | None = None,
concept_frontmatter_values: Mapping[str, str] | None = None,
) -> BlockedFile | None:
"""Render every segment, or refuse the WHOLE document.
@ -745,6 +838,7 @@ def _render_segments(
bundle_id=bundle_id,
units=units,
source_title=source_title,
concept_frontmatter_values=concept_frontmatter_values,
),
decision.reasons,
)
@ -804,6 +898,7 @@ def process_inbox(
pdf_headings: bool = False,
heading_reserve: Callable[[str], bool] | None = None,
ocr: bool = False,
concept_frontmatter_values: Mapping[str, str] | None = None,
) -> InboxResult:
"""Convert every file dropped in `inbox_dir` into an OKF concept.
@ -821,6 +916,8 @@ def process_inbox(
a reserved `okf_type`, and a missing inbox directory.
"""
validate_ingested_at(ingested_at)
# Wrong for every file at once, so refused for the run before any is read.
run_values = validate_concept_frontmatter(concept_frontmatter_values or {}, profile=profile)
# Rendered HERE, before anything is read or written, and the result carried
# to the index write at the bottom. `_render_root_frontmatter` refuses a key
# the policy does not name, and a refusal must leave no bundle behind --
@ -1077,6 +1174,7 @@ def process_inbox(
source_file=source_name(path),
units=units,
source_title=source_title,
concept_frontmatter_values=run_values,
)
if blocked is not None:
if blocked.disposition == _DISPOSITION_QUARANTINE:
@ -1130,6 +1228,7 @@ def process_inbox(
# unit boundary after it.
span=(0, len(text)),
source_title=source_title,
concept_frontmatter_values=run_values,
),
decision.reasons,
)