feat(propose): a section the source DECLARES takes the declared-structure route

`extract.xml_outline` reports the marks the XML reader wrote itself -- no
bridge, no tolerance constant, no unresolved bucket, because the reader
appended the line it is naming. `find_candidates` gains `outline_rule` so the
route it already had for a PDF bookmark tree can carry a second reader's name,
and `build_plan` chooses it by the ROW (`DECLARED_STRUCTURE_IDS`), never by the
text: the same markdown from a `.md` file is still a guess and still keeps
`rule:heading`.

`rule:xml-section` is orphan-exempt for the reason the bookmark arm is -- the
check asks whether a guess was a heading, and a container section is not a
false positive.

MEASURED on R761 at SHIPPED DEFAULTS, no flag: 23 concepts -> 2761, and
2761 of 2761 declared sections became a concept with the directory and the
title the source states (0 unmatched, 0 concepts matching no declaration).
`a)`-points as concepts 0 of 4954, table blocks 10 of 10.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-10 07:00:37 +02:00
commit ee12f644ca
2 changed files with 103 additions and 11 deletions

View file

@ -108,6 +108,16 @@ _PANDOC_FORMATS: dict[str, str] = {
# format from ONE publisher, and the file boundaries and `<h1>`s are a # format from ONE publisher, and the file boundaries and `<h1>`s are a
# generator's cut of that document, not 828 documents anyone wrote. # generator's cut of that document, not 828 documents anyone wrote.
# #
# `.xml` JOINED THE TABLE 2026-09-11, as `measured`, and the class was read off
# the definitions above rather than inherited: the one file is a publisher's own
# NISO-STS delivery of R761, written for their purposes years before any lookup
# of ours, and its 2 761 titled `<sec>` are a fasit nobody here authored. The
# honesty limit that travels with it and does NOT move when the build reaches
# the reader's ceiling: the denominator is ONE file, ONE publisher, ONE schema.
# `.xml` as a file type is far wider than NISO-STS, and a document in any other
# schema keeps its text in document order and gets no structure at all -- which
# is measured on fixtures, not on a corpus.
#
# `.pdf` JOINED THE TABLE 2026-09-10, as `measured`, and it enters on the # `.pdf` JOINED THE TABLE 2026-09-10, as `measured`, and it enters on the
# strongest evidence of any row here: eight real corpus PDFs with a fasit the # strongest evidence of any row here: eight real corpus PDFs with a fasit the
# operator hand-counted document by document, plus a 701-page process code # operator hand-counted document by document, plus a 701-page process code
@ -467,6 +477,14 @@ class _XmlTextExtractor:
self._lines: list[str] = [] self._lines: list[str] = []
self._current: list[str] = [] self._current: list[str] = []
self._prefix = "" self._prefix = ""
# The declared structure, recorded WHERE it is written rather than
# recovered from the finished string. The PDF arm has to bridge from
# (page, `/XYZ` top) onto a line index and was wrong on 1 840 of 2 762
# nodes under the naive rule; here the reader appended the line, so the
# index is not a guess and carries no tolerance. Empty for a document
# that is not STS -- that is "this schema declares no section", and it
# must not collapse into "this document has no structure to state".
self.marks: list[OutlineMark] = []
def _break(self, prefix: str = "") -> None: def _break(self, prefix: str = "") -> None:
"""Close the line being accumulated and open the next one. """Close the line being accumulated and open the next one.
@ -543,7 +561,11 @@ class _XmlTextExtractor:
level = min(depth, _ATX_MAX_LEVEL) level = min(depth, _ATX_MAX_LEVEL)
parts = [self._text_of(label)] if label is not None else [] parts = [self._text_of(label)] if label is not None else []
parts.append(self._text_of(title)) parts.append(self._text_of(title))
self._emit("#" * level + " " + " ".join(part for part in parts if part)) heading = " ".join(part for part in parts if part)
self._emit("#" * level + " " + heading)
self.marks.append(
OutlineMark(line=len(self._lines) - 1, level=level, title=heading)
)
skip = {id(title)} | ({id(label)} if label is not None else set()) skip = {id(title)} | ({id(label)} if label is not None else set())
elif label is not None: elif label is not None:
# NEVER a heading. The label goes in FRONT of the body line the # NEVER a heading. The label goes in FRONT of the body line the
@ -573,7 +595,7 @@ class _XmlTextExtractor:
return "\n".join(self._lines) return "\n".join(self._lines)
def _extract_xml(data: bytes) -> str: def _xml_document(data: bytes) -> tuple[str, tuple[OutlineMark, ...]]:
"""`xml`: NISO-STS structure as markdown, any other schema as its text. """`xml`: NISO-STS structure as markdown, any other schema as its text.
A DTD IS REFUSED RATHER THAN PARSED, and that is a guarantee about this A DTD IS REFUSED RATHER THAN PARSED, and that is a guarantee about this
@ -604,7 +626,37 @@ def _extract_xml(data: bytes) -> str:
f"the XML parser failed on this file: {exc}", code="extractor_xml_parse_error" f"the XML parser failed on this file: {exc}", code="extractor_xml_parse_error"
) from exc ) from exc
sts = _local_name(root.tag) == _STS_ROOT or next(root.iter("sec"), None) is not None sts = _local_name(root.tag) == _STS_ROOT or next(root.iter("sec"), None) is not None
return _XmlTextExtractor(sts=sts).text(root) reader = _XmlTextExtractor(sts=sts)
return reader.text(root), tuple(reader.marks)
def _extract_xml(data: bytes) -> str:
return _xml_document(data)[0]
def xml_outline(name: str, data: bytes) -> tuple[OutlineMark, ...]:
"""`xml`: the sections the document DECLARES, as marks on the extracted text.
The counterpart of `pdf_outline`, and the difference between them is the
whole point of the row. A bookmark states a page and a y position, so that
arm has to BRIDGE onto a line and reports what did not bridge; an STS
`<sec><title>` is written into the output by this reader, so the line index
is the one it appended at -- nothing is recovered, nothing is unresolved,
and there is no tolerance constant to choose.
RE-READS the bytes rather than returning both from one call, for the same
reason `pdf_outline` does: `extract_text` has one signature that every
caller and every registry entry is keyed to, and a second return value
would change it for eight rows to serve one. The parse is stdlib and the
document is read twice; measured on a 2.4 MB NISO-STS file, that is the
smaller cost by a wide margin.
Empty for every schema that is not STS. That is a statement about the
document -- it declares no section -- and `find_candidates` reads an empty
list as "leave every rule untouched", never as a route.
"""
del name # the registry decides which reader runs; kept for `pdf_outline`'s shape
return _xml_document(data)[1]
def _local_name(tag: str) -> str: def _local_name(tag: str) -> str:

View file

@ -62,7 +62,7 @@ from pathlib import Path
from typing import Any from typing import Any
from .errors import IngestError from .errors import IngestError
from .extract import OutlineMark, extract_text, strip_converter_attribute from .extract import OutlineMark, extract_text, strip_converter_attribute, xml_outline
from .extract import pdf_outline as extract_pdf_outline from .extract import pdf_outline as extract_pdf_outline
from .materialize import reduce_to_id_grammar from .materialize import reduce_to_id_grammar
from .segmentation import observed_extractor_version from .segmentation import observed_extractor_version
@ -142,6 +142,26 @@ RULE_BOLD_TITLE = "rule:bold-title"
#: a structure index the file already carries. A reader who cannot tell the two #: a structure index the file already carries. A reader who cannot tell the two
#: apart in an artifact cannot tell a recovered heading from a declared one. #: apart in an artifact cannot tell a recovered heading from a declared one.
RULE_PDF_OUTLINE = "rule:pdf-outline" RULE_PDF_OUTLINE = "rule:pdf-outline"
#: A section the SOURCE FORMAT declares as an element, read by a reader in this
#: package. NOT `RULE_PDF_OUTLINE`: that one is a publisher's bookmark tree
#: BRIDGED from (page, y) onto a line, and its arm reports what did not bridge.
#: NOT `RULE_HEADING` either, and that distinction is the one this rule exists
#: for -- an ATX line reaching the proposer says nothing about who wrote it, so
#: a `<sec><title>` and a heading a converter guessed out of a font size were
#: indistinguishable in the artifact and were judged by the same two steps.
#: Measured on R761: the orphan check removed 710 of 2 761 declared sections
#: (710 of 710 removed are followed immediately by another heading -- they are
#: containers) and Arm F folded 2 066 more, 2 089 -> 23 at shipped defaults.
RULE_XML_SECTION = "rule:xml-section"
#: The extractor ids whose reader WRITES the heading line from an element the
#: source declared, so the structure is transcribed rather than recovered. One
#: row, and it is deliberately not "every type whose headings look declared":
#: an office document's headings arrive through an external converter's
#: rendering decisions, and moving that row is a measurement over the whole
#: unit worksheet, which this rule did not make.
DECLARED_STRUCTURE_IDS = frozenset({"xml"})
RULE_NAMES = ( RULE_NAMES = (
RULE_HEADING, RULE_HEADING,
RULE_TABLE_BLOCK, RULE_TABLE_BLOCK,
@ -152,6 +172,7 @@ RULE_NAMES = (
RULE_SHEET_SECTION, RULE_SHEET_SECTION,
RULE_BOLD_TITLE, RULE_BOLD_TITLE,
RULE_PDF_OUTLINE, RULE_PDF_OUTLINE,
RULE_XML_SECTION,
) )
#: How many characters of context each side of a quote anchor carries. Enough #: How many characters of context each side of a quote anchor carries. Enough
@ -606,10 +627,12 @@ def _sheet_section_rows(lines: list[str]) -> dict[int, tuple[str, str]]:
#: Rules the orphan check is not asked about. D3 because a sheet row carries #: Rules the orphan check is not asked about. D3 because a sheet row carries
#: its content in its own cells, and the bookmark arm because the check judges #: its content in its own cells, and the two DECLARED-structure rules because
#: whether a GUESS was a heading -- a question a publisher's own tree has #: the check judges whether a GUESS was a heading -- a question a publisher's
#: already answered, and one that deletes every container section if asked. #: own tree has already answered, and one that deletes every container section
_ORPHAN_EXEMPT = (RULE_SHEET_SECTION, RULE_PDF_OUTLINE) #: if asked. Measured on one 701-page process code: 683 of 2 762 bookmark nodes
#: and 710 of 2 761 STS sections are containers.
_ORPHAN_EXEMPT = (RULE_SHEET_SECTION, RULE_PDF_OUTLINE, RULE_XML_SECTION)
def _split_outline_title(title: str) -> tuple[str | None, str]: def _split_outline_title(title: str) -> tuple[str | None, str]:
@ -648,6 +671,7 @@ def find_candidates(
contents_name: bool = False, contents_name: bool = False,
bold_title: bool = False, bold_title: bool = False,
outline_marks: Sequence[OutlineMark] | None = None, outline_marks: Sequence[OutlineMark] | None = None,
outline_rule: str = RULE_PDF_OUTLINE,
) -> list[Candidate]: ) -> list[Candidate]:
"""Every boundary the mechanical rules propose, in document order. """Every boundary the mechanical rules propose, in document order.
@ -719,8 +743,12 @@ def find_candidates(
before spans are closed, so the text they opened is carried by the mark before spans are closed, so the text they opened is carried by the mark
above rather than lost. above rather than lost.
`outline_marks` is the PDF bookmark arm, and it is the only input here that `outline_marks` is the DECLARED-STRUCTURE route, and it is the only input
REPLACES the rules rather than gating one of them. A non-empty list is the here that REPLACES the rules rather than gating one of them. Two readers
reach it -- `extract.pdf_outline` for a `/Outlines` tree and
`extract.xml_outline` for NISO-STS `<sec><title>` -- and `outline_rule`
names which, because an artifact that cannot tell a bridged bookmark from
an element the reader wrote itself cannot be audited. A non-empty list is the
publisher's own declaration of the document's structure, so nothing below publisher's own declaration of the document's structure, so nothing below
votes against it: the text heuristics, the two gates and Arm F's fold are votes against it: the text heuristics, the two gates and Arm F's fold are
all skipped, and the orphan check is not applied to its marks. An EMPTY all skipped, and the orphan check is not applied to its marks. An EMPTY
@ -782,7 +810,7 @@ def find_candidates(
title=title, title=title,
level=mark.level, level=mark.level,
number=number, number=number,
rule=RULE_PDF_OUTLINE, rule=outline_rule,
start=offsets[mark.line], start=offsets[mark.line],
end=end_of_text, end=end_of_text,
), ),
@ -1458,6 +1486,17 @@ def build_plan(
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated.""" """The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
taken: set[str] = set() taken: set[str] = set()
extractor_id = source.suffix.lower().lstrip(".") or "none" extractor_id = source.suffix.lower().lstrip(".") or "none"
# The declared-structure route, chosen by the ROW and never by the text.
# That is what keeps every other file type byte-identical: the same
# markdown arriving from a `.md` file carries a heading somebody guessed,
# and the reader that wrote a `<sec><title>` is the only witness that the
# source declared it. `--pdf-outline` stays a flag because a bookmark tree
# is a publisher's CLAIM about a document it also typeset; an STS element
# is the document.
outline_rule = RULE_PDF_OUTLINE
if extractor_id in DECLARED_STRUCTURE_IDS:
outline_marks = xml_outline(source.name, source_bytes)
outline_rule = RULE_XML_SECTION
entries: list[dict[str, Any]] = [] entries: list[dict[str, Any]] = []
candidates = find_candidates( candidates = find_candidates(
text, text,
@ -1473,6 +1512,7 @@ def build_plan(
contents_name=contents_name, contents_name=contents_name,
bold_title=bold_title, bold_title=bold_title,
outline_marks=outline_marks, outline_marks=outline_marks,
outline_rule=outline_rule,
) )
for candidate in subdivide(text, candidates, max_segment_chars): for candidate in subdivide(text, candidates, max_segment_chars):
entries.append( entries.append(