open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

feat(identity): an STS document's doc-number names its directory and its title the address

Browse source 
K3-19 a. `extract.declared_identity` reads what a NISO-STS document states
about itself -- exactly one <std-ident> (<doc-number>, <year>) and exactly one
<title-wrap> (<full>, else <main>) -- and returns None for every other row,
for XML that is not STS, for an unparseable file, and for a document that
states neither. A value stated more than once is not read: an adopted
standard carries one <std-ident> per issuing body, and picking one is a guess.

`okf build` names a document's directory from its <doc-number> through the id
grammar, replacing only the file's stem. A declared name another document in
the run also claims falls back to the file name for both, said on stderr: the
existing collision gate would refuse both with "rename one", and a name read
from inside a document is not one a rename can change.

`sources[0].title` becomes <doc-number> + <year>, then the <title-wrap>
title, then the file name -- the first that survives the gate and can be
written into the flow mapping verbatim. Measured on R761, <full> carries a
comma, which ends a flow mapping, so it is never the title there; it is never
cleaned up either. `resource` stays the inbox-relative file.

Every other row, and every profile without an address, is untouched: the
identity is asked for only where `sources` is written.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-11 03:08:30 +02:00
commit ee8d5b5776
6 changed files with 230 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

79
src/llm_ingestion_okf/inbox.py
View file

@ -27,7 +27,7 @@ from dataclasses import dataclass, replace
from pathlib import Path, PurePosixPath
from .errors import IngestError, MaterializationError, SegmentationError, SourceError
from .extract import SourceUnits, extract_text, source_units
from .extract import DeclaredIdentity, SourceUnits, declared_identity, extract_text, source_units
from .materialize import (
_render_root_frontmatter,
check_filename_length,
@ -113,6 +113,7 @@ def render_inbox_concept(
bundle_id: str | None = None,
units: SourceUnits | None = None,
span: tuple[int, int] | None = None,
source_title: str | None = None,
) -> str:
"""Frame extracted text as an inbox concept file with its provenance layer.
@ -127,6 +128,9 @@ def render_inbox_concept(
the text arriving here is the SANITIZED text and its length is not
necessarily the extracted text's.
`source_title` is what the document calls itself, for the `sources`
entry's `title`; `None` keeps the file name there, as before it existed.
`segment` and `bundle_id` carry the 1-to-N identity layer and are read ONLY
when the profile declares the segmentation capability. A concept the plan
does not cover keeps today's rule verbatim, and the four shipped profiles
@ -231,6 +235,7 @@ def render_inbox_concept(
source_file=source_file,
units=units,
span=located,
title=source_title,
)
)
return f"---\n{profile.frontmatter.emit(frontmatter)}\n---\n\n{_normalize_body(text)}"
@ -249,6 +254,7 @@ def _provenance_frontmatter(
source_file: str,
units: SourceUnits | None,
span: tuple[int, int] | None,
title: str | None = None,
) -> dict[str, str]:
"""The address, and the locator when one is available.
@ -265,11 +271,14 @@ def _provenance_frontmatter(
"than mangled",
code="inbox_source_file_unaddressable",
)
values = {
policy.sources_key: (
f"[{{ resource: {source_file}, title: {PurePosixPath(source_file).name} }}]"
if title is not None and not _flow_expressible(title):
raise MaterializationError(
f"source title {title!r} cannot be written into the `sources` flow "
"mapping verbatim; refused rather than mangled",
code="inbox_source_title_unaddressable",
)
}
shown = title if title is not None else PurePosixPath(source_file).name
values = {policy.sources_key: f"[{{ resource: {source_file}, title: {shown} }}]"}
if units is None or span is None:
return values
first, last = units.covering(*span)
@ -288,6 +297,54 @@ def _provenance_frontmatter(
return values
def _flow_expressible(value: str) -> bool:
"""Whether `value` survives as a plain scalar inside a flow mapping."""
return bool(value) and not any(char in value for char in f"{_FLOW_TERMINATORS}\n\r")
def _screened(gate: Gate, value: str | None) -> str | None:
"""A value read from the DOCUMENT and persisted outside its screened body.
The body goes through the gate before anything is written; a frontmatter
value taken from the same bytes would otherwise be the one route around
it. Kept only on the gate's non-blocking floor, as the SANITIZED text, and
dropped rather than refused otherwise: the body carrying the same words is
judged on its own, and a document is never lost over an optional key.
"""
if value is None:
return None
decision = gate(value)
if decision.disposition != _DISPOSITION_PERSIST:
return None
return " ".join(decision.sanitized_text.split()) or None
def _declared_sources_title(identity: DeclaredIdentity | None, gate: Gate) -> str | None:
"""The `sources` title a document declares, or `None` for the file name.
`<doc-number>` + `<year>` first, then the `<title-wrap>` title: measured on
the one STS document this row has, the `<full>` title carries a COMMA,
which ends a flow mapping, and the guard refuses the quoted scalar that
could have carried it. A declared value that cannot be written verbatim
falls to the next layer -- never cleaned up, because a title with its comma
removed is a title the document does not carry.
"""
if identity is None:
return None
candidates: list[str] = []
if identity.doc_number is not None:
candidates.append(
f"{identity.doc_number} {identity.year}" if identity.year else identity.doc_number
)
if identity.title is not None:
candidates.append(identity.title)
for candidate in candidates:
kept = _screened(gate, candidate)
if kept is not None and _flow_expressible(kept):
return kept
return None
# --- the guard seam -------------------------------------------------------
# The guard's non-blocking floor. `Disposition` is a `str, Enum` in
@ -620,6 +677,7 @@ def _render_segments(
bundle_id: str,
source_file: str,
units: SourceUnits | None,
source_title: str | None = None,
) -> BlockedFile | None:
"""Render every segment, or refuse the WHOLE document.
@ -686,6 +744,7 @@ def _render_segments(
segment=entry,
bundle_id=bundle_id,
units=units,
source_title=source_title,
),
decision.reasons,
)
@ -996,6 +1055,14 @@ def process_inbox(
if profile.provenance is not None
else None
)
# What the document says it is, for the address's title. Asked only
# where an address is written, so the four profiles without one do
# not parse anything they would never emit.
source_title = (
_declared_sources_title(declared_identity(source_name(path), source_bytes), gate)
if profile.provenance is not None
else None
)
covering = _plan_covering(plans, source_bytes)
if covering is not None:
blocked = _render_segments(
@ -1009,6 +1076,7 @@ def process_inbox(
bundle_id=(root_frontmatter_values or {})[_bundle_id_key(profile)],
source_file=source_name(path),
units=units,
source_title=source_title,
)
if blocked is not None:
if blocked.disposition == _DISPOSITION_QUARANTINE:
@ -1061,6 +1129,7 @@ def process_inbox(
# gate that removed a character would shift every
# unit boundary after it.
span=(0, len(text)),
source_title=source_title,
),
decision.reasons,
)