feat(extract): one normalisation door removes U+00AD before the guard
Operator decision 2026-09-18, and the whole of it: okf removes the soft hyphen, the guard is untouched, and the number is BOOKED. `normalise_extracted(text) -> (text, removed)` in `extract`, applied once at the end of `extract_document`, so both entry points and both sides of a plan's `text_sha256` see one string. `_pdf_units` measures its page offsets through the same door, because that table is CHARACTER offsets rebuilt from the pages while the text it indexes comes back normalised -- two readings of one document, and a table built against the other names the wrong page with full confidence. Removing a character removes no newline, so every line-indexed rule downstream is unmoved by construction. `ExtractedDocument.soft_hyphens` carries the count out; `InboxResult.normalised` carries it per document; `account_run` reads it off the RUN rather than recounting the source, because a second count would be a second reader. It reaches the accounting JSON as `normalised_soft_hyphen` at both levels and `log.md` as a `**Normalisation**` bullet naming the count, the documents touched and that no other character is moved. EXPOSURE, measured and not assumed: 0 of the 78 readable documents of the K2 reference corpus carry U+00AD or any of the four real zero-width characters (the 8 unreadable ones raise before extraction), 0 in the pinned K2 bundle's concept bodies, and 0 across `tests/fixtures`, `examples`, `skills`, `docs`, `src`, README and CHANGELOG. The door cannot have moved a byte anyone here has measured. Suite 2179 passed, 1 skipped; ruff and `mypy --strict src/` clean. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
68cfa905a3
commit
eebaf534fa
3 changed files with 103 additions and 2 deletions
|
|
@ -732,6 +732,12 @@ class DocumentAccount:
|
|||
counts: dict[str, int]
|
||||
fates: dict[str, Fate]
|
||||
error: str | None = None
|
||||
#: How many U+00AD the normalisation door removed from this document's
|
||||
#: text before the persist gate saw it (operator decision 2026-09-18).
|
||||
#: Booked rather than silently applied: a door that changes a source's
|
||||
#: bytes and says nothing is the same class of absence this whole module
|
||||
#: exists to close.
|
||||
normalised_soft_hyphen: int = 0
|
||||
|
||||
@property
|
||||
def unaccounted(self) -> dict[str, int]:
|
||||
|
|
@ -758,6 +764,7 @@ class DocumentAccount:
|
|||
"source_file": self.source_file,
|
||||
"status": self.status,
|
||||
"code": self.code,
|
||||
"normalised_soft_hyphen": self.normalised_soft_hyphen,
|
||||
"inventory": dict(self.counts),
|
||||
"fates": {kind: self.fates[kind].to_json() for kind in self.counts},
|
||||
"unaccounted": self.unaccounted,
|
||||
|
|
@ -806,6 +813,11 @@ class Accounting:
|
|||
"""
|
||||
return sum(1 for d in self.documents if d.status == REJECTED)
|
||||
|
||||
@property
|
||||
def normalised_soft_hyphen(self) -> int:
|
||||
"""Soft hyphens the normalisation door removed across the whole run."""
|
||||
return sum(d.normalised_soft_hyphen for d in self.documents)
|
||||
|
||||
@property
|
||||
def images_found(self) -> int:
|
||||
return sum(d.counts.get("image", 0) for d in self.documents)
|
||||
|
|
@ -820,6 +832,7 @@ class Accounting:
|
|||
"unaccounted": self.unaccounted,
|
||||
"double_booked": self.double_booked,
|
||||
"refused": self.refused,
|
||||
"normalised_soft_hyphen": self.normalised_soft_hyphen,
|
||||
"documents": [d.to_json() for d in self.documents],
|
||||
"files": [f.to_json() for f in self.files],
|
||||
}
|
||||
|
|
@ -831,6 +844,12 @@ class Accounting:
|
|||
f"{self.unaccounted} unaccounted, {self.double_booked} double-booked; "
|
||||
f"{self.refused} of {len(self.documents)} document(s) refused whole."
|
||||
]
|
||||
touched = sum(1 for d in self.documents if d.normalised_soft_hyphen)
|
||||
lines.append(
|
||||
f"* **Normalisation**: {self.normalised_soft_hyphen} soft hyphen(s) (U+00AD) "
|
||||
f"removed from {touched} of {len(self.documents)} document(s) before the persist "
|
||||
"gate. No other character is touched."
|
||||
)
|
||||
for doc in self.documents:
|
||||
if doc.status == REJECTED:
|
||||
lines.append(
|
||||
|
|
@ -986,6 +1005,12 @@ def account_run(inbox: Path, walked: Sequence[Path], result: InboxResult) -> Acc
|
|||
documents.append(
|
||||
DocumentAccount(name, "", None, inv.counts(), {k: Fate() for k in inv.counts()})
|
||||
)
|
||||
# Read off the run, never recounted from the source: a second count would
|
||||
# be a second reader, and the number this publishes has to be the number
|
||||
# the normalisation door acted on.
|
||||
removed = {item.source_file: item.soft_hyphens for item in result.normalised}
|
||||
for document in documents:
|
||||
document.normalised_soft_hyphen = removed.get(document.source_file, 0)
|
||||
return Accounting(documents, files)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue