open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(extract,cli): typography as a PDF heading source and OCR behind an optional group, both off

Browse source 
A PDF carries no notion of a heading -- a heading in a PDF is a typographic
fact -- so the text stream `pdfplumber` hands the segment proposer has already
thrown away the only evidence there was. The `docx` path never had that problem:
the converter emits ATX headings and `_ATX` cuts on them. Two readers close the
gap, and both are OFF.

`--pdf-headings font` infers a heading from the conjunction this repository
already measured (size above the document's character-weighted body median AND
a bold font name, recall 1.000 / precision 0.846) and emits it as ATX in the
SAME markdown the office path produces, so `_ATX` applies unchanged and no
PDF-only heading grammar exists.

It stays off BY MEASUREMENT, and the measurement is the point of the round:
against the operator's unit worksheet it takes `pdf` from 2 of 8 to 0 of 8,
losing two exact matches. The mechanism of the loss is stated rather than
guessed -- on those documents the outline rule already recovers the document's
own numbered chapters, so a second heading source can only add. Whole-corpus
screen: 25 of 32 `pdf` change, 0 of 5 `docx`, 0 of 2 `xlsx`. The default bundle
is byte-identical before and after this commit (`diff -r`, exit 0).

`--ocr` reads a page as an image when its own text never arrived: empty, or
`(cid:N)` placeholder codes at or above a threshold READ OFF a measured
distribution -- 834 pages over 32 files, 818 at exactly 0.0 and 16 at 0.93 or
above, nothing in between. On the one corpus document with the failure: 95.07 %
cid to 0 %, 44 to 2561 words of four or more letters, 17 to 18 pages with text.
Its engine is an optional dependency group and never a runtime dependency; a
packaging test pins both halves, and without the group every affected file is a
coded rejection (`extractor_ocr_group_missing`) rather than a crash.

Also corrects two stale published facts found while measuring: the README still
said two segmentation rules were on by default after `f6fea13` made it three,
and CLAUDE.md's K2 digest named the round-3 default. The current default is
492 concepts / 944 files, `bdefa679...`.

Report: docs/2026-09-08-k3-runde4-pdf-skrift-og-ocr.md

Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-08 23:10:47 +02:00
commit 53d5c74c96
15 changed files with 1394 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

66
src/llm_ingestion_okf/cli.py
View file

@ -136,6 +136,20 @@ DEFAULT_OUTLINE_RUN = 3
DEFAULT_TABLE_GRID = True
DEFAULT_UNIT_FOLD = True
#: Round 4's two PDF READERS, both off. They are not arms and they are not in
#: the paragraph above: an arm changes how the proposer cuts a text, and these
#: two change what the text SAYS -- so they sit before every arm rather than
#: beside them, and a measurement of an arm is a measurement of it reading
#: whatever these produced.
#:
#: `DEFAULT_PDF_HEADINGS` is off BY MEASUREMENT and the measurement is in
#: `docs/2026-09-08-k3-runde4-pdf-skrift-og-ocr.md`, not by caution.
#: `DEFAULT_OCR` is off BY CONSTRUCTION and can never move: the engine is an
#: optional dependency group, so a default that needed it would make the
#: package's ordinary install fail on the first scanned page.
DEFAULT_PDF_HEADINGS = False
DEFAULT_OCR = False
#: The timestamp written when the caller passes none, for the ingest stamp and
#: the proposal stamp alike. ONE constant: two independently-defaulted literals
#: drift, and the drift shows up only as two bundles differing in a field
@ -156,6 +170,8 @@ def _propose_plans(
keep_table_heading: bool = False,
sheet_section_rows: bool = False,
drop_wrapped_outline: bool = False,
pdf_headings: bool = False,
ocr: bool = False,
) -> tuple[int, int, int]:
"""Propose a plan per dropped file. Returns (written, nothing, failed).
@ -186,6 +202,8 @@ def _propose_plans(
keep_table_heading=keep_table_heading,
sheet_section_rows=sheet_section_rows,
drop_wrapped_outline=drop_wrapped_outline,
pdf_headings=pdf_headings,
ocr=ocr,
)
except ProposerError as exc:
print(f"{CLI_ID}: {relative.as_posix()}: {exc}", file=sys.stderr)
@ -215,6 +233,8 @@ def build(
keep_table_heading: bool = False,
sheet_section_rows: bool = False,
drop_wrapped_outline: bool = False,
pdf_headings: bool = DEFAULT_PDF_HEADINGS,
ocr: bool = DEFAULT_OCR,
) -> CorpusReport:
"""Folder in, bundle out. The whole command, minus argument parsing.
@ -233,7 +253,14 @@ def build(
if proposed_at is None:
proposed_at = ingested_at
if not segments:
report = measure(inbox, bundle, ingested_at=ingested_at, profile=STRUCTURED_V1)
report = measure(
inbox,
bundle,
ingested_at=ingested_at,
profile=STRUCTURED_V1,
pdf_headings=pdf_headings,
ocr=ocr,
)
_write_log(bundle, report, profile=STRUCTURED_V1)
return report
@ -264,6 +291,8 @@ def build(
keep_table_heading=keep_table_heading,
sheet_section_rows=sheet_section_rows,
drop_wrapped_outline=drop_wrapped_outline,
pdf_headings=pdf_headings,
ocr=ocr,
)
print(
f"{CLI_ID}: proposed {written} plan(s); {nothing} document(s) with no boundary; "
@ -278,6 +307,8 @@ def build(
plans=plans,
profile=SEGMENTED_OKF_V0_2,
root_frontmatter_values={"okf_version": okf_version, "bundle_id": bundle_id},
pdf_headings=pdf_headings,
ocr=ocr,
)
_write_log(bundle, report, profile=SEGMENTED_OKF_V0_2)
return report
@ -474,6 +505,37 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
"and the risk-table rows the operator wrote off"
),
)
build_parser.add_argument(
"--pdf-headings",
choices=("none", "font"),
default="font" if DEFAULT_PDF_HEADINGS else "none",
help=(
"how a PDF's headings are recovered before any arm reads the text. "
"none (the default) is today's extraction: a PDF carries no heading "
"markup and none is inferred. font infers one from TYPOGRAPHY -- a "
"line whose dominant size is above the document's body median AND "
"whose dominant font name says bold -- and emits it as an ATX "
"heading in the same markdown the office path produces, so the "
"existing heading rule reads it and no PDF-only grammar exists. A "
"CONJUNCTION, measured: adding weight as a disjunct took precision "
"from 0.786 to 0.524"
),
)
build_parser.add_argument(
"--ocr",
action="store_true",
default=DEFAULT_OCR,
help=(
"read a PDF page as an IMAGE when its own text never arrived -- the "
"page extracts empty, or as a page of (cid:N) placeholder codes. "
"Requires the optional 'ocr' group (pip install "
"'llm-ingestion-okf[extract,ocr]'); without it every affected file "
"is a coded rejection, never a crash. Never a default and never "
"able to become one: the engine is an optional dependency. OCR text "
"is a reading of a picture, so it is deterministic within one model "
"version and not across one"
),
)
build_parser.add_argument("--report", type=Path, default=None, help="also write the report")
return parser.parse_args(argv)
@ -503,6 +565,8 @@ def main(argv: list[str] | None = None) -> int:
keep_table_heading=args.keep_table_heading,
sheet_section_rows=args.sheet_section_rows,
drop_wrapped_outline=args.drop_wrapped_outline,
pdf_headings=args.pdf_headings == "font",
ocr=args.ocr,
)
except (IngestError, OSError, ValueError) as exc:
print(f"{CLI_ID}: FAILED - {exc}", file=sys.stderr)

4
src/llm_ingestion_okf/corpus.py
View file

@ -280,6 +280,8 @@ def measure(
plans: Mapping[str, SegmentationPlan] | None = None,
profile: BundleProfile = STRUCTURED_V1,
root_frontmatter_values: Mapping[str, str] | None = None,
pdf_headings: bool = False,
ocr: bool = False,
) -> CorpusReport:
"""Run the corpus through the door and count what happened.
@ -301,6 +303,8 @@ def measure(
profile=profile,
root_frontmatter_values=root_frontmatter_values,
segmentations=plans,
pdf_headings=pdf_headings,
ocr=ocr,
)
elapsed = time.monotonic() - started

5
src/llm_ingestion_okf/errors.py
View file

@ -103,6 +103,11 @@ class ExtractionError(IngestError):
- `extractor_empty_conversion` — the converter returned no text; refused
rather than persisted as an empty concept, for the same reason as
`extractor_empty_pdf`
- `extractor_ocr_group_missing` — a PDF page was to be read with OCR but
the optional `ocr` group is not installed. DISTINCT from
`extractor_extra_missing`: the `[extract]` extra can be fully installed
and the document parsed, with only the OCR engine absent, and one code
for both would send an operator to reinstall what they already have
"""

280
src/llm_ingestion_okf/extract.py
View file

@ -23,10 +23,12 @@ registry's. No guard call and no model call anywhere in this module.
from __future__ import annotations
import collections
import csv
import functools
import io
import re
import statistics
import tempfile
import warnings
import zipfile
@ -237,6 +239,182 @@ def _extra_missing(suffix: str) -> ExtractionError:
)
def _ocr_group_missing() -> ExtractionError:
"""The one rejection for `--ocr` without the optional `ocr` group.
A DIFFERENT code from `extractor_extra_missing`, because it is a different
fact and a different remedy: the `[extract]` extra can be fully installed
-- the document parsed, the pages counted -- and the OCR engine still be
absent. One error naming both would send an operator to reinstall
something they already have.
"""
return ExtractionError(
"reading a PDF page with OCR requires the optional 'ocr' group "
"(pip install 'llm-ingestion-okf[extract,ocr]'), which ships rapidocr "
"on onnxruntime; it is not installed",
code="extractor_ocr_group_missing",
)
#: The literal placeholder `pdfminer.six` (behind `pdfplumber`) emits for a
#: glyph whose font carries no usable ToUnicode mapping. The text is present on
#: the page and unreadable in the extraction -- a failure that looks like
#: success, which is why it needs a measurement rather than an exception.
_CID_CODE = re.compile(r"\(cid:\d+\)")
#: The share of a page's extracted characters that must be `(cid:N)` codes
#: before `--ocr` reads the page as an image instead.
#:
#: MEASURED, not chosen: `docs/2026-09-08-k3-runde4-pdf-skrift-og-ocr.md`
#: reports the per-page distribution over the K2 corpus, and it is bimodal
#: with nothing in between -- one document's pages sit near 1.0 and every other
#: page in the corpus sits at 0.0. Any value in that gap selects the same
#: pages, which is what makes 0.10 defensible and also what makes it
#: uninformative about a corpus that has intermediate pages. Stated rather than
#: implied: this threshold is bounded by the corpus, not by a property of the
#: format.
OCR_CID_SHARE = 0.10
#: The resolution a page is rendered at before it is read as an image.
#: 200 dpi is what the round-4 measurement was taken at; the engine's own
#: preprocessing rescales from there, so this is a floor on how much of the
#: page's detail reaches it rather than a tuning knob. It is part of the output
#: contract in the same way the parser version is: OCR text is deterministic
#: within one resolution and one model version, and across neither.
OCR_DPI = 200
def cid_share(text: str) -> float:
"""The share of `text` made of `(cid:N)` placeholder codes, 0.0 for empty.
Module level and importable: `tools/okf_cid_measure.py` answers the same
question at DOCUMENT level, and two definitions of one metric drift.
"""
if not text:
return 0.0
return sum(len(match.group(0)) for match in _CID_CODE.finditer(text)) / len(text)
def needs_ocr(text: str) -> bool:
"""Whether a page's extracted text is unusable enough to read the image.
TWO conditions, because there are two ways a page's text never arrives and
they look nothing alike: a page with no text layer extracts as the empty
string, and a page whose fonts carry no ToUnicode mapping extracts as a
full page of `(cid:N)`. A trigger written for one of them would leave the
other exactly where it was.
"""
return not text.strip() or cid_share(text) >= OCR_CID_SHARE
def _ocr_reader() -> Callable[[object], list[str]]:
"""The OCR engine, or the typed refusal. The import IS the gate.
Same shape as `_extract_pdf`'s probe and for the same reason: membership in
a suffix set cannot tell whether a package is importable, and this group is
the one a consumer is most likely not to have.
"""
try:
import rapidocr
except ImportError as exc:
raise _ocr_group_missing() from exc
if rapidocr is None: # pragma: no cover - the sys.modules probe in tests
raise _ocr_group_missing()
engine = rapidocr.RapidOCR()
def read(image: object) -> list[str]:
result = engine(image)
# `txts` is None when the detector found nothing at all, which is a
# legitimate answer for a blank page and not an error.
return [str(line) for line in (getattr(result, "txts", None) or ())]
return read
#: Bold as a PDF says it: in the font's NAME (`Helvetica-Bold`,
#: `ABCDEF+Arial-BoldMT`). There is no weight attribute on a character, so the
#: name is the only place a text extractor can read it.
_PDF_BOLD_MARKER = "bold"
#: The deepest ATX level the emitted markdown may use. `_ATX` in `propose.py`
#: reads one to six hashes, and a document with seven distinct heading sizes
#: would otherwise emit a line the proposer reads as body.
_PDF_MAX_HEADING_LEVEL = 6
def _dominant(values: list[str]) -> str:
"""The most frequent value, ties broken by first occurrence.
`Counter.most_common(1)` reduces to `max` over the items in insertion
order, so the tie-break is document order and the result is deterministic
for identical bytes -- which is the property everything downstream is
pinned to.
"""
return collections.Counter(values).most_common(1)[0][0]
def _typography(line: dict[str, object]) -> tuple[float, str] | None:
"""One line's dominant font size and font name, or `None` if it is blank.
Blank characters are excluded from both: a space carries a size and a font
like any other character, and a heading padded with body-sized spaces would
read as body.
"""
chars = [char for char in line["chars"] if str(char["text"]).strip()] # type: ignore[attr-defined]
if not chars:
return None
sizes = [f"{float(char['size']):.1f}" for char in chars]
fonts = [str(char["fontname"]) for char in chars]
return float(_dominant(sizes)), _dominant(fonts)
def _heading_levels(lines: list[tuple[str, float, str]]) -> dict[float, int]:
"""Which font sizes are headings in this document, and at what ATX level.
The rule is the CONJUNCTION this repository already measured: larger than
the body AND bold. `docs/2026-09-07-k3-arm-d.md`'s predecessor measured
size-and-bold from poppler at recall 1.000 / precision 0.846, and measured
that adding weight as a DISJUNCT made precision worse (0.786 -> 0.524). A
disjunction here would mark every emphasised phrase in the body.
The body size is the CHARACTER-weighted median over the whole document, not
the page: a title page is 100 % heading by line count, and a per-page
median would compare it with itself and mark nothing. Weighted by
characters rather than lines for the same reason in miniature -- a document
front-loaded with short lines has a line median that no paragraph shares.
The ATX LEVEL is the size's rank among the heading sizes, largest first, so
a document's own typographic hierarchy survives into the markdown instead
of flattening to one level. Deeper than six is clamped, because `_ATX`
reads six.
"""
weighted: list[float] = []
for text, size, _ in lines:
weighted.extend([size] * len(text.replace(" ", "")))
if not weighted:
return {}
body = statistics.median(weighted)
sizes = {size for _, size, font in lines if size > body and _PDF_BOLD_MARKER in font.lower()}
return {
size: min(rank, _PDF_MAX_HEADING_LEVEL)
for rank, size in enumerate(sorted(sizes, reverse=True), start=1)
}
def _mark_headings(lines: list[tuple[str, float, str]], levels: dict[float, int]) -> str:
"""One page's lines as markdown, the heading sizes carrying their hashes.
BOLD is checked again here rather than folded into the size map: a document
can set a caption in the same size as a heading without setting it bold,
and a map keyed on size alone would promote it.
"""
out: list[str] = []
for text, size, font in lines:
level = levels.get(size) if _PDF_BOLD_MARKER in font.lower() else None
out.append(f"{'#' * level} {text}" if level is not None and text else text)
return "\n".join(out)
# How `_extract_pdf` joins its pages, named because the locator below has to
# reproduce the exact same arithmetic to turn a character offset back into a
# page number. Two constants that must agree, written once.
@ -244,7 +422,9 @@ _PDF_PAGE_SEPARATOR = "\n\n"
@functools.lru_cache(maxsize=1)
def _pdf_pages(data: bytes) -> tuple[tuple[int, str], ...]:
def _pdf_pages(
data: bytes, headings: bool = False, ocr: bool = False
) -> tuple[tuple[int, str], ...]:
"""Every page that produced text, as `(page number, text)`, in page order.
The page NUMBER is 1-based and comes from the document, so a page that
@ -253,20 +433,69 @@ def _pdf_pages(data: bytes) -> tuple[tuple[int, str], ...]:
produced text" and "page 3", and the whole reason a locator is worth
writing down.
Memoised on the bytes with room for exactly one document: extraction and
location are two calls about the same file, back to back, and parsing it
twice would double the PDF cost of every corpus run for nothing. Anything
larger would hold whole documents in memory for no gain, since the caller
never returns to an earlier file.
Memoised on the bytes AND on the two options, with room for exactly one
entry: extraction and location are two calls about the same file with the
same options, back to back, and parsing it twice would double the PDF cost
of every corpus run for nothing. The options are part of the key because
two renderings of one document are two different strings, and a locator
built against the wrong one points at the wrong place with full confidence.
`headings` and `ocr` are INDEPENDENT and compose. With both off this is the
path every byte-pinned golden was measured on, unchanged: the default
branch still calls `page.extract_text()` rather than reassembling the page
from its lines. Measured, the two agree on 11 of 11 pages of a real tender
PDF -- but "agree on the document I tried" is not a contract, so the
default does not depend on it.
"""
try:
import pdfplumber
except ImportError as exc:
raise _extra_missing(".pdf") from exc
read = _ocr_reader() if ocr else None
try:
with pdfplumber.open(io.BytesIO(data)) as pdf:
pages = [(page.extract_text() or "").rstrip() for page in pdf.pages]
# PASS ONE. Nothing is emitted here, because the heading rule needs
# a fact about the WHOLE document -- the body's size -- and a page
# cannot supply it. A title page is 100 % heading, and a per-page
# median would compare it with itself and mark nothing.
recovered: list[str | list[tuple[str, float, str]]] = []
for page in pdf.pages:
flat = (page.extract_text() or "").rstrip()
if read is not None and needs_ocr(flat):
# The page's own text is unusable, so it is replaced
# WHOLESALE rather than merged with: a page of `(cid:N)`
# has nothing worth keeping, and interleaving two readings
# of one page would put a guess and a fact in one paragraph
# with no way to tell them apart. An OCR'd page carries no
# typography either -- the engine reports text, not fonts --
# so it is a finished string and never a heading candidate.
recovered.append("\n".join(read(page.to_image(resolution=OCR_DPI).original)))
elif not headings:
recovered.append(flat)
else:
recovered.append(
[
(str(line["text"]), *found)
for line in page.extract_text_lines()
if (found := _typography(line)) is not None
]
)
levels = _heading_levels(
[
line
for page_lines in recovered
if not isinstance(page_lines, str)
for line in page_lines
]
)
# PASS TWO.
pages = [
page_lines
if isinstance(page_lines, str)
else _mark_headings(page_lines, levels).rstrip()
for page_lines in recovered
]
except ExtractionError:
raise
except Exception as exc: # noqa: BLE001 - third-party parser, wrapped never leaked
@ -276,7 +505,7 @@ def _pdf_pages(data: bytes) -> tuple[tuple[int, str], ...]:
return tuple((number, page) for number, page in enumerate(pages, start=1) if page)
def _extract_pdf(data: bytes) -> str:
def _extract_pdf(data: bytes, *, headings: bool = False, ocr: bool = False) -> str:
"""`pdf`: page text via `pdfplumber`, in page order, pages separated by a
blank line.
@ -292,12 +521,13 @@ def _extract_pdf(data: bytes) -> str:
and pymupdf each emit all labels then all values. Re-pairing those is
guesswork, and in a requirements document a wrong pairing looks right.
"""
pages = _pdf_pages(data)
pages = _pdf_pages(data, headings, ocr)
text = _PDF_PAGE_SEPARATOR.join(page for _, page in pages)
if not text:
raise ExtractionError(
"the PDF yielded no text on any page; a scanned or image-only "
"document needs OCR, which this registry does not do",
"document needs OCR, which this registry does only behind the "
"optional 'ocr' group and only when asked",
code="extractor_empty_pdf",
)
# After the parse, not before: a run that produced no text has nothing to
@ -563,11 +793,11 @@ def _line_units(text: str) -> SourceUnits:
return SourceUnits("lines", tuple(starts), tuple(range(1, len(starts) + 1)))
def _pdf_units(data: bytes) -> SourceUnits:
def _pdf_units(data: bytes, headings: bool, ocr: bool) -> SourceUnits:
starts: list[int] = []
numbers: list[int] = []
offset = 0
for number, page in _pdf_pages(data):
for number, page in _pdf_pages(data, headings, ocr):
starts.append(offset)
numbers.append(number)
offset += len(page) + len(_PDF_PAGE_SEPARATOR)
@ -614,7 +844,9 @@ def _spreadsheet_units(text: str) -> SourceUnits | None:
return SourceUnits("rows", tuple(starts), tuple(numbers), tuple(scopes))
def source_units(filename: str, data: bytes, text: str) -> SourceUnits | None:
def source_units(
filename: str, data: bytes, text: str, *, pdf_headings: bool = False, ocr: bool = False
) -> SourceUnits | None:
"""The unit table for one dropped file, or `None` when it has none.
`text` must be what `extract_text` returned for these exact bytes: the
@ -627,7 +859,7 @@ def source_units(filename: str, data: bytes, text: str) -> SourceUnits | None:
"""
suffix = Path(filename).suffix.lower()
if suffix == ".pdf":
return _pdf_units(data)
return _pdf_units(data, pdf_headings, ocr)
if suffix == ".xlsx":
return _spreadsheet_units(text)
if suffix in _CORE_EXTRACTORS or suffix in _PANDOC_FORMATS:
@ -636,7 +868,12 @@ def source_units(filename: str, data: bytes, text: str) -> SourceUnits | None:
def extract_text(
filename: str, data: bytes, *, renderer: Callable[[str], str] | None = None
filename: str,
data: bytes,
*,
renderer: Callable[[str], str] | None = None,
pdf_headings: bool = False,
ocr: bool = False,
) -> str:
"""Convert one dropped file's bytes to OKF concept text, dispatched by type.
@ -657,11 +894,22 @@ def extract_text(
The default is identity, which is what keeps every existing byte-pinned
golden byte-pinned.
`pdf_headings` and `ocr` are PDF-only and both default to off. They are
branched on here rather than expressed as two more registry rows because
the registry's contract is `bytes -> str`: a row per option combination
would be four rows for one reader, and a reader chosen by a suffix lookup
that also has to consult two flags is not a lookup. A non-PDF caller
passing either argument gets today's behaviour, silently, which is correct
-- the options describe a reader, not a policy for the run.
"""
suffix = Path(filename).suffix.lower()
extractor = _CORE_EXTRACTORS.get(suffix) or _OPTIONAL_EXTRACTORS.get(suffix)
if extractor is not None:
text = extractor(data)
if suffix == ".pdf" and (pdf_headings or ocr):
text = _extract_pdf(data, headings=pdf_headings, ocr=ocr)
else:
text = extractor(data)
return renderer(text) if renderer is not None else text
if suffix in _UNPARSED_OPTIONAL_EXTENSIONS:
raise _extra_missing(suffix)

12
src/llm_ingestion_okf/inbox.py
View file

@ -742,6 +742,8 @@ def process_inbox(
root_frontmatter_values: Mapping[str, str] | None = None,
segmentation: SegmentationPlan | None = None,
segmentations: Mapping[str, SegmentationPlan] | None = None,
pdf_headings: bool = False,
ocr: bool = False,
) -> InboxResult:
"""Convert every file dropped in `inbox_dir` into an OKF concept.
@ -957,6 +959,8 @@ def process_inbox(
source_name(path),
source_bytes,
renderer=_resolve_renderer(profile, path.name),
pdf_headings=pdf_headings,
ocr=ocr,
)
# Computed from the SAME text the plan's offsets index, so the
# locator and the offset can never disagree about which rendering
@ -964,7 +968,13 @@ def process_inbox(
# building a unit table nobody writes would re-parse every PDF for
# a key that is never emitted.
units = (
source_units(source_name(path), source_bytes, text)
source_units(
source_name(path),
source_bytes,
text,
pdf_headings=pdf_headings,
ocr=ocr,
)
if profile.provenance is not None
else None
)

10
src/llm_ingestion_okf/propose.py
View file

@ -1118,6 +1118,8 @@ def run(
keep_table_heading: bool = False,
sheet_section_rows: bool = False,
drop_wrapped_outline: bool = False,
pdf_headings: bool = False,
ocr: bool = False,
) -> int:
if max_segment_chars < 0:
raise ProposerError(
@ -1155,7 +1157,13 @@ def run(
except OSError as exc:
raise ProposerError(f"cannot read {source}: {exc}") from exc
try:
text = extract_text(source.name, source_bytes)
# The two READER options, not arms: they change what the extraction
# says, and every arm below reads whatever it says. Passed here as well
# as to the run path because the plan's `text_sha256` indexes this
# exact string -- a plan proposed against one rendering and replayed
# against another is refused by `assert_plan_applies`, which is the
# right outcome and a confusing one to debug.
text = extract_text(source.name, source_bytes, pdf_headings=pdf_headings, ocr=ocr)
except IngestError as exc:
raise ProposerError(f"cannot extract text from {source.name}: {exc}") from exc