feat(propose): a section the source DECLARES takes the declared-structure route
`extract.xml_outline` reports the marks the XML reader wrote itself -- no bridge, no tolerance constant, no unresolved bucket, because the reader appended the line it is naming. `find_candidates` gains `outline_rule` so the route it already had for a PDF bookmark tree can carry a second reader's name, and `build_plan` chooses it by the ROW (`DECLARED_STRUCTURE_IDS`), never by the text: the same markdown from a `.md` file is still a guess and still keeps `rule:heading`. `rule:xml-section` is orphan-exempt for the reason the bookmark arm is -- the check asks whether a guess was a heading, and a container section is not a false positive. MEASURED on R761 at SHIPPED DEFAULTS, no flag: 23 concepts -> 2761, and 2761 of 2761 declared sections became a concept with the directory and the title the source states (0 unmatched, 0 concepts matching no declaration). `a)`-points as concepts 0 of 4954, table blocks 10 of 10. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
958e9bc998
commit
ee12f644ca
2 changed files with 103 additions and 11 deletions
|
|
@ -108,6 +108,16 @@ _PANDOC_FORMATS: dict[str, str] = {
|
||||||
# format from ONE publisher, and the file boundaries and `<h1>`s are a
|
# format from ONE publisher, and the file boundaries and `<h1>`s are a
|
||||||
# generator's cut of that document, not 828 documents anyone wrote.
|
# generator's cut of that document, not 828 documents anyone wrote.
|
||||||
#
|
#
|
||||||
|
# `.xml` JOINED THE TABLE 2026-09-11, as `measured`, and the class was read off
|
||||||
|
# the definitions above rather than inherited: the one file is a publisher's own
|
||||||
|
# NISO-STS delivery of R761, written for their purposes years before any lookup
|
||||||
|
# of ours, and its 2 761 titled `<sec>` are a fasit nobody here authored. The
|
||||||
|
# honesty limit that travels with it and does NOT move when the build reaches
|
||||||
|
# the reader's ceiling: the denominator is ONE file, ONE publisher, ONE schema.
|
||||||
|
# `.xml` as a file type is far wider than NISO-STS, and a document in any other
|
||||||
|
# schema keeps its text in document order and gets no structure at all -- which
|
||||||
|
# is measured on fixtures, not on a corpus.
|
||||||
|
#
|
||||||
# `.pdf` JOINED THE TABLE 2026-09-10, as `measured`, and it enters on the
|
# `.pdf` JOINED THE TABLE 2026-09-10, as `measured`, and it enters on the
|
||||||
# strongest evidence of any row here: eight real corpus PDFs with a fasit the
|
# strongest evidence of any row here: eight real corpus PDFs with a fasit the
|
||||||
# operator hand-counted document by document, plus a 701-page process code
|
# operator hand-counted document by document, plus a 701-page process code
|
||||||
|
|
@ -467,6 +477,14 @@ class _XmlTextExtractor:
|
||||||
self._lines: list[str] = []
|
self._lines: list[str] = []
|
||||||
self._current: list[str] = []
|
self._current: list[str] = []
|
||||||
self._prefix = ""
|
self._prefix = ""
|
||||||
|
# The declared structure, recorded WHERE it is written rather than
|
||||||
|
# recovered from the finished string. The PDF arm has to bridge from
|
||||||
|
# (page, `/XYZ` top) onto a line index and was wrong on 1 840 of 2 762
|
||||||
|
# nodes under the naive rule; here the reader appended the line, so the
|
||||||
|
# index is not a guess and carries no tolerance. Empty for a document
|
||||||
|
# that is not STS -- that is "this schema declares no section", and it
|
||||||
|
# must not collapse into "this document has no structure to state".
|
||||||
|
self.marks: list[OutlineMark] = []
|
||||||
|
|
||||||
def _break(self, prefix: str = "") -> None:
|
def _break(self, prefix: str = "") -> None:
|
||||||
"""Close the line being accumulated and open the next one.
|
"""Close the line being accumulated and open the next one.
|
||||||
|
|
@ -543,7 +561,11 @@ class _XmlTextExtractor:
|
||||||
level = min(depth, _ATX_MAX_LEVEL)
|
level = min(depth, _ATX_MAX_LEVEL)
|
||||||
parts = [self._text_of(label)] if label is not None else []
|
parts = [self._text_of(label)] if label is not None else []
|
||||||
parts.append(self._text_of(title))
|
parts.append(self._text_of(title))
|
||||||
self._emit("#" * level + " " + " ".join(part for part in parts if part))
|
heading = " ".join(part for part in parts if part)
|
||||||
|
self._emit("#" * level + " " + heading)
|
||||||
|
self.marks.append(
|
||||||
|
OutlineMark(line=len(self._lines) - 1, level=level, title=heading)
|
||||||
|
)
|
||||||
skip = {id(title)} | ({id(label)} if label is not None else set())
|
skip = {id(title)} | ({id(label)} if label is not None else set())
|
||||||
elif label is not None:
|
elif label is not None:
|
||||||
# NEVER a heading. The label goes in FRONT of the body line the
|
# NEVER a heading. The label goes in FRONT of the body line the
|
||||||
|
|
@ -573,7 +595,7 @@ class _XmlTextExtractor:
|
||||||
return "\n".join(self._lines)
|
return "\n".join(self._lines)
|
||||||
|
|
||||||
|
|
||||||
def _extract_xml(data: bytes) -> str:
|
def _xml_document(data: bytes) -> tuple[str, tuple[OutlineMark, ...]]:
|
||||||
"""`xml`: NISO-STS structure as markdown, any other schema as its text.
|
"""`xml`: NISO-STS structure as markdown, any other schema as its text.
|
||||||
|
|
||||||
A DTD IS REFUSED RATHER THAN PARSED, and that is a guarantee about this
|
A DTD IS REFUSED RATHER THAN PARSED, and that is a guarantee about this
|
||||||
|
|
@ -604,7 +626,37 @@ def _extract_xml(data: bytes) -> str:
|
||||||
f"the XML parser failed on this file: {exc}", code="extractor_xml_parse_error"
|
f"the XML parser failed on this file: {exc}", code="extractor_xml_parse_error"
|
||||||
) from exc
|
) from exc
|
||||||
sts = _local_name(root.tag) == _STS_ROOT or next(root.iter("sec"), None) is not None
|
sts = _local_name(root.tag) == _STS_ROOT or next(root.iter("sec"), None) is not None
|
||||||
return _XmlTextExtractor(sts=sts).text(root)
|
reader = _XmlTextExtractor(sts=sts)
|
||||||
|
return reader.text(root), tuple(reader.marks)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_xml(data: bytes) -> str:
|
||||||
|
return _xml_document(data)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def xml_outline(name: str, data: bytes) -> tuple[OutlineMark, ...]:
|
||||||
|
"""`xml`: the sections the document DECLARES, as marks on the extracted text.
|
||||||
|
|
||||||
|
The counterpart of `pdf_outline`, and the difference between them is the
|
||||||
|
whole point of the row. A bookmark states a page and a y position, so that
|
||||||
|
arm has to BRIDGE onto a line and reports what did not bridge; an STS
|
||||||
|
`<sec><title>` is written into the output by this reader, so the line index
|
||||||
|
is the one it appended at -- nothing is recovered, nothing is unresolved,
|
||||||
|
and there is no tolerance constant to choose.
|
||||||
|
|
||||||
|
RE-READS the bytes rather than returning both from one call, for the same
|
||||||
|
reason `pdf_outline` does: `extract_text` has one signature that every
|
||||||
|
caller and every registry entry is keyed to, and a second return value
|
||||||
|
would change it for eight rows to serve one. The parse is stdlib and the
|
||||||
|
document is read twice; measured on a 2.4 MB NISO-STS file, that is the
|
||||||
|
smaller cost by a wide margin.
|
||||||
|
|
||||||
|
Empty for every schema that is not STS. That is a statement about the
|
||||||
|
document -- it declares no section -- and `find_candidates` reads an empty
|
||||||
|
list as "leave every rule untouched", never as a route.
|
||||||
|
"""
|
||||||
|
del name # the registry decides which reader runs; kept for `pdf_outline`'s shape
|
||||||
|
return _xml_document(data)[1]
|
||||||
|
|
||||||
|
|
||||||
def _local_name(tag: str) -> str:
|
def _local_name(tag: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,7 @@ from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .errors import IngestError
|
from .errors import IngestError
|
||||||
from .extract import OutlineMark, extract_text, strip_converter_attribute
|
from .extract import OutlineMark, extract_text, strip_converter_attribute, xml_outline
|
||||||
from .extract import pdf_outline as extract_pdf_outline
|
from .extract import pdf_outline as extract_pdf_outline
|
||||||
from .materialize import reduce_to_id_grammar
|
from .materialize import reduce_to_id_grammar
|
||||||
from .segmentation import observed_extractor_version
|
from .segmentation import observed_extractor_version
|
||||||
|
|
@ -142,6 +142,26 @@ RULE_BOLD_TITLE = "rule:bold-title"
|
||||||
#: a structure index the file already carries. A reader who cannot tell the two
|
#: a structure index the file already carries. A reader who cannot tell the two
|
||||||
#: apart in an artifact cannot tell a recovered heading from a declared one.
|
#: apart in an artifact cannot tell a recovered heading from a declared one.
|
||||||
RULE_PDF_OUTLINE = "rule:pdf-outline"
|
RULE_PDF_OUTLINE = "rule:pdf-outline"
|
||||||
|
|
||||||
|
#: A section the SOURCE FORMAT declares as an element, read by a reader in this
|
||||||
|
#: package. NOT `RULE_PDF_OUTLINE`: that one is a publisher's bookmark tree
|
||||||
|
#: BRIDGED from (page, y) onto a line, and its arm reports what did not bridge.
|
||||||
|
#: NOT `RULE_HEADING` either, and that distinction is the one this rule exists
|
||||||
|
#: for -- an ATX line reaching the proposer says nothing about who wrote it, so
|
||||||
|
#: a `<sec><title>` and a heading a converter guessed out of a font size were
|
||||||
|
#: indistinguishable in the artifact and were judged by the same two steps.
|
||||||
|
#: Measured on R761: the orphan check removed 710 of 2 761 declared sections
|
||||||
|
#: (710 of 710 removed are followed immediately by another heading -- they are
|
||||||
|
#: containers) and Arm F folded 2 066 more, 2 089 -> 23 at shipped defaults.
|
||||||
|
RULE_XML_SECTION = "rule:xml-section"
|
||||||
|
|
||||||
|
#: The extractor ids whose reader WRITES the heading line from an element the
|
||||||
|
#: source declared, so the structure is transcribed rather than recovered. One
|
||||||
|
#: row, and it is deliberately not "every type whose headings look declared":
|
||||||
|
#: an office document's headings arrive through an external converter's
|
||||||
|
#: rendering decisions, and moving that row is a measurement over the whole
|
||||||
|
#: unit worksheet, which this rule did not make.
|
||||||
|
DECLARED_STRUCTURE_IDS = frozenset({"xml"})
|
||||||
RULE_NAMES = (
|
RULE_NAMES = (
|
||||||
RULE_HEADING,
|
RULE_HEADING,
|
||||||
RULE_TABLE_BLOCK,
|
RULE_TABLE_BLOCK,
|
||||||
|
|
@ -152,6 +172,7 @@ RULE_NAMES = (
|
||||||
RULE_SHEET_SECTION,
|
RULE_SHEET_SECTION,
|
||||||
RULE_BOLD_TITLE,
|
RULE_BOLD_TITLE,
|
||||||
RULE_PDF_OUTLINE,
|
RULE_PDF_OUTLINE,
|
||||||
|
RULE_XML_SECTION,
|
||||||
)
|
)
|
||||||
|
|
||||||
#: How many characters of context each side of a quote anchor carries. Enough
|
#: How many characters of context each side of a quote anchor carries. Enough
|
||||||
|
|
@ -606,10 +627,12 @@ def _sheet_section_rows(lines: list[str]) -> dict[int, tuple[str, str]]:
|
||||||
|
|
||||||
|
|
||||||
#: Rules the orphan check is not asked about. D3 because a sheet row carries
|
#: Rules the orphan check is not asked about. D3 because a sheet row carries
|
||||||
#: its content in its own cells, and the bookmark arm because the check judges
|
#: its content in its own cells, and the two DECLARED-structure rules because
|
||||||
#: whether a GUESS was a heading -- a question a publisher's own tree has
|
#: the check judges whether a GUESS was a heading -- a question a publisher's
|
||||||
#: already answered, and one that deletes every container section if asked.
|
#: own tree has already answered, and one that deletes every container section
|
||||||
_ORPHAN_EXEMPT = (RULE_SHEET_SECTION, RULE_PDF_OUTLINE)
|
#: if asked. Measured on one 701-page process code: 683 of 2 762 bookmark nodes
|
||||||
|
#: and 710 of 2 761 STS sections are containers.
|
||||||
|
_ORPHAN_EXEMPT = (RULE_SHEET_SECTION, RULE_PDF_OUTLINE, RULE_XML_SECTION)
|
||||||
|
|
||||||
|
|
||||||
def _split_outline_title(title: str) -> tuple[str | None, str]:
|
def _split_outline_title(title: str) -> tuple[str | None, str]:
|
||||||
|
|
@ -648,6 +671,7 @@ def find_candidates(
|
||||||
contents_name: bool = False,
|
contents_name: bool = False,
|
||||||
bold_title: bool = False,
|
bold_title: bool = False,
|
||||||
outline_marks: Sequence[OutlineMark] | None = None,
|
outline_marks: Sequence[OutlineMark] | None = None,
|
||||||
|
outline_rule: str = RULE_PDF_OUTLINE,
|
||||||
) -> list[Candidate]:
|
) -> list[Candidate]:
|
||||||
"""Every boundary the mechanical rules propose, in document order.
|
"""Every boundary the mechanical rules propose, in document order.
|
||||||
|
|
||||||
|
|
@ -719,8 +743,12 @@ def find_candidates(
|
||||||
before spans are closed, so the text they opened is carried by the mark
|
before spans are closed, so the text they opened is carried by the mark
|
||||||
above rather than lost.
|
above rather than lost.
|
||||||
|
|
||||||
`outline_marks` is the PDF bookmark arm, and it is the only input here that
|
`outline_marks` is the DECLARED-STRUCTURE route, and it is the only input
|
||||||
REPLACES the rules rather than gating one of them. A non-empty list is the
|
here that REPLACES the rules rather than gating one of them. Two readers
|
||||||
|
reach it -- `extract.pdf_outline` for a `/Outlines` tree and
|
||||||
|
`extract.xml_outline` for NISO-STS `<sec><title>` -- and `outline_rule`
|
||||||
|
names which, because an artifact that cannot tell a bridged bookmark from
|
||||||
|
an element the reader wrote itself cannot be audited. A non-empty list is the
|
||||||
publisher's own declaration of the document's structure, so nothing below
|
publisher's own declaration of the document's structure, so nothing below
|
||||||
votes against it: the text heuristics, the two gates and Arm F's fold are
|
votes against it: the text heuristics, the two gates and Arm F's fold are
|
||||||
all skipped, and the orphan check is not applied to its marks. An EMPTY
|
all skipped, and the orphan check is not applied to its marks. An EMPTY
|
||||||
|
|
@ -782,7 +810,7 @@ def find_candidates(
|
||||||
title=title,
|
title=title,
|
||||||
level=mark.level,
|
level=mark.level,
|
||||||
number=number,
|
number=number,
|
||||||
rule=RULE_PDF_OUTLINE,
|
rule=outline_rule,
|
||||||
start=offsets[mark.line],
|
start=offsets[mark.line],
|
||||||
end=end_of_text,
|
end=end_of_text,
|
||||||
),
|
),
|
||||||
|
|
@ -1458,6 +1486,17 @@ def build_plan(
|
||||||
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
|
"""The artifact. Every entry PROPOSED, the plan itself never adjudicated."""
|
||||||
taken: set[str] = set()
|
taken: set[str] = set()
|
||||||
extractor_id = source.suffix.lower().lstrip(".") or "none"
|
extractor_id = source.suffix.lower().lstrip(".") or "none"
|
||||||
|
# The declared-structure route, chosen by the ROW and never by the text.
|
||||||
|
# That is what keeps every other file type byte-identical: the same
|
||||||
|
# markdown arriving from a `.md` file carries a heading somebody guessed,
|
||||||
|
# and the reader that wrote a `<sec><title>` is the only witness that the
|
||||||
|
# source declared it. `--pdf-outline` stays a flag because a bookmark tree
|
||||||
|
# is a publisher's CLAIM about a document it also typeset; an STS element
|
||||||
|
# is the document.
|
||||||
|
outline_rule = RULE_PDF_OUTLINE
|
||||||
|
if extractor_id in DECLARED_STRUCTURE_IDS:
|
||||||
|
outline_marks = xml_outline(source.name, source_bytes)
|
||||||
|
outline_rule = RULE_XML_SECTION
|
||||||
entries: list[dict[str, Any]] = []
|
entries: list[dict[str, Any]] = []
|
||||||
candidates = find_candidates(
|
candidates = find_candidates(
|
||||||
text,
|
text,
|
||||||
|
|
@ -1473,6 +1512,7 @@ def build_plan(
|
||||||
contents_name=contents_name,
|
contents_name=contents_name,
|
||||||
bold_title=bold_title,
|
bold_title=bold_title,
|
||||||
outline_marks=outline_marks,
|
outline_marks=outline_marks,
|
||||||
|
outline_rule=outline_rule,
|
||||||
)
|
)
|
||||||
for candidate in subdivide(text, candidates, max_segment_chars):
|
for candidate in subdivide(text, candidates, max_segment_chars):
|
||||||
entries.append(
|
entries.append(
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue