feat(accounting): okf build accounts for every source element
okf build --accounting PATH inventories every source document before extraction, in the gate's per-format vocabulary, and after the run gives each element one fate (carried / pointer / coded rejection), written as JSON and summarised in log.md. "carried" is checked against the written concept bodies, so a gate that drops a line is found (test). Exit 1 on anything unaccounted or double-booked. Opt-in: +744 s (+19 %) on the 43-document reference corpus, and that corpus fails the check on 24 real losses (22 images on text-less PDF pages, 2 docx Title paragraphs). Changed without the flag: - okf build exits 1 when it extracted documents and persisted none. Door B and corpus.measure are unchanged. One test relied on exit 0. - An image file carried through a persisted document is its own K1b column, no longer also extractor_unknown. The set is what the resolver actually carried (ExtractedDocument.files), never a byte match. tools/okf_accounting_gate.py (checks untouched) is green on all six rows, R761 110 of 110 under both gates. Report: docs/2026-09-17-innholdsregnskapet-bygget.md Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
0b00de4408
commit
864570b320
13 changed files with 1751 additions and 59 deletions
984
src/llm_ingestion_okf/accounting.py
Normal file
984
src/llm_ingestion_okf/accounting.py
Normal file
|
|
@ -0,0 +1,984 @@
|
|||
"""Content accounting for `okf build`: what the SOURCE held, and where each part went.
|
||||
|
||||
`okf build`'s conservation identity counts FILES. A file can be merged while
|
||||
text inside it is gone, and nothing about the identity can say so. This module
|
||||
adds the element level:
|
||||
|
||||
1. an INVENTORY per document, read from the source bytes by the format's own
|
||||
rules, independent of whether extraction or the persist gate later succeed
|
||||
-- so a refused document still says what it held;
|
||||
2. an ACCOUNT after the run, giving every inventoried element exactly one
|
||||
fate: `carried`, `pointer` or a coded rejection. What has none is
|
||||
UNACCOUNTED, what has two is DOUBLE-BOOKED, and a build asked for the
|
||||
account fails on either.
|
||||
|
||||
**The vocabulary is the gate's.** `tools/okf_accounting_gate.py` compares the
|
||||
inventory here against `tools/okf_witness.py`, an implementation that imports
|
||||
nothing from this package. The two implement ONE set of definitions (named per
|
||||
format below) twice; agreeing on the fixtures is evidence that the definitions
|
||||
are implementable as written, and disagreeing is a finding about one of them.
|
||||
|
||||
**`carried` is checked, not declared.** A persisted document's element is
|
||||
carried when every piece of its text is found in the concept bodies written
|
||||
for that document, compared on letters and digits alone (case-folded), so
|
||||
markdown escapes, table pipes and whitespace do not count as loss. A container
|
||||
(a table, a section, a slide) is carried when everything inside it is. An
|
||||
element with no text of its own is carried with its document. This finds text
|
||||
that the extractor, the segmentation, the gate or the writer lost; it cannot
|
||||
find text the extractor never saw AND no inventory counts -- that is what the
|
||||
per-format vocabulary bounds, and what the gate's exceptions name.
|
||||
|
||||
**Images** are booked from what the reader did with each placement, never
|
||||
from the bundle: carried (bytes in `assets/`), `pointer` (a remote reference,
|
||||
which extraction never fetches) or rejected with the reader's code. A format
|
||||
whose reader does not handle images at all books an image as `pointer` when
|
||||
its reference is present in the carried text (markdown), and leaves it
|
||||
unaccounted otherwise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import posixpath
|
||||
import re
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from .inbox import DocumentAssets, InboxResult, relative_source
|
||||
|
||||
ACCOUNTING_VERSION = 1
|
||||
|
||||
CARRIED = "carried"
|
||||
POINTER = "pointer"
|
||||
MERGED = "merged"
|
||||
REJECTED = "rejected"
|
||||
PERSISTED = "persisted"
|
||||
|
||||
#: The reader's code for an image it deliberately did not fetch.
|
||||
_REMOTE_CODE = "asset_remote"
|
||||
|
||||
_REMOTE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:|//)")
|
||||
_NOT_ALNUM = re.compile(r"[\W_]+")
|
||||
|
||||
|
||||
def _norm(text: str) -> str:
|
||||
return _NOT_ALNUM.sub("", text.casefold())
|
||||
|
||||
|
||||
# --- inventory ---------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Inventory:
|
||||
"""What one source document holds, in the gate's element vocabulary.
|
||||
|
||||
`chunks` is the text of the document in pieces; each element names the
|
||||
pieces it consists of. `refs` holds, per image element, what the source
|
||||
points at: an inbox-relative path, a remote reference, or None for an
|
||||
embedded picture.
|
||||
"""
|
||||
|
||||
source_file: str
|
||||
vocabulary: tuple[str, ...]
|
||||
elements: list[tuple[str, tuple[int, ...]]] = field(default_factory=list)
|
||||
chunks: list[str] = field(default_factory=list)
|
||||
refs: list[tuple[str, str | None]] = field(default_factory=list)
|
||||
|
||||
def chunk(self, text: str) -> int:
|
||||
self.chunks.append(text)
|
||||
return len(self.chunks) - 1
|
||||
|
||||
def add(self, kind: str, chunks: Iterable[int] = ()) -> None:
|
||||
if kind not in self.vocabulary:
|
||||
raise ValueError(f"{kind!r} is not in this format's vocabulary")
|
||||
self.elements.append((kind, tuple(chunks)))
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
result = dict.fromkeys(self.vocabulary, 0)
|
||||
for kind, _ in self.elements:
|
||||
result[kind] += 1
|
||||
return result
|
||||
|
||||
def pointed_files(self) -> set[str]:
|
||||
return {target for kind, target in self.refs if kind == "local" and target}
|
||||
|
||||
|
||||
def _resolve(inbox: Path, document: Path, ref: str, *, sts: bool) -> str | None:
|
||||
"""The inbox-relative file a local reference names, contained in the
|
||||
document's own directory, or None."""
|
||||
directory = document.parent
|
||||
tries = [ref] + ([f"graphics/{PurePosixPath(ref).name}"] if sts else [])
|
||||
for candidate in tries:
|
||||
normal = posixpath.normpath(candidate)
|
||||
if normal.startswith(("/", "..")) or normal == ".":
|
||||
continue
|
||||
target = directory / normal
|
||||
if target.is_file():
|
||||
return relative_source(target, inbox)
|
||||
return None
|
||||
|
||||
|
||||
def _add_image(inv: Inventory, inbox: Path, document: Path, ref: str | None, sts: bool) -> None:
|
||||
if ref is None:
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
return
|
||||
if not ref or _REMOTE.match(ref):
|
||||
inv.refs.append(("remote", ref or None))
|
||||
inv.add("image", (inv.chunk(ref),) if ref else ())
|
||||
return
|
||||
inv.refs.append(("local", _resolve(inbox, document, ref, sts=sts)))
|
||||
inv.add("image", (inv.chunk(ref),))
|
||||
|
||||
|
||||
# markdown -- fences as the proposer reads them, so the two cannot disagree
|
||||
_ATX_LINE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]|$)")
|
||||
_TABLE_DELIMITER = re.compile(r"^\s*\|?(?:\s*:?-{3,}:?\s*\|)*\s*:?-{3,}:?\s*\|?\s*$")
|
||||
_IMAGE_REF = re.compile(r"!\[[^\]]*\]\(\s*<?([^)\s>]+)>?[^)]*\)")
|
||||
|
||||
_MARKDOWN = ("heading", "paragraph", "table", "table_row", "image", "code_block")
|
||||
|
||||
|
||||
def _inventory_markdown(inv: Inventory, inbox: Path, path: Path, text: str) -> None:
|
||||
from .propose import fenced_lines
|
||||
|
||||
lines = text.split("\n")
|
||||
fenced = fenced_lines(lines)
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
line = lines[index]
|
||||
if index in fenced:
|
||||
start = index
|
||||
while index < len(lines) and index in fenced:
|
||||
index += 1
|
||||
inv.add("code_block", (inv.chunk("\n".join(lines[start:index])),))
|
||||
continue
|
||||
if not line.strip():
|
||||
index += 1
|
||||
continue
|
||||
if _ATX_LINE.match(line):
|
||||
inv.add("heading", (inv.chunk(line),))
|
||||
index += 1
|
||||
continue
|
||||
nxt = lines[index + 1] if index + 1 < len(lines) else ""
|
||||
if "|" in line and index + 1 not in fenced and _TABLE_DELIMITER.match(nxt):
|
||||
header = inv.chunk(line)
|
||||
index += 2
|
||||
rows: list[int] = []
|
||||
while index < len(lines) and index not in fenced and "|" in lines[index]:
|
||||
row = inv.chunk(lines[index])
|
||||
inv.add("table_row", (row,))
|
||||
rows.append(row)
|
||||
index += 1
|
||||
inv.add("table", (header, *rows))
|
||||
continue
|
||||
chunks: list[int] = []
|
||||
while (
|
||||
index < len(lines)
|
||||
and index not in fenced
|
||||
and lines[index].strip()
|
||||
and not _ATX_LINE.match(lines[index])
|
||||
):
|
||||
current = lines[index]
|
||||
for ref in _IMAGE_REF.findall(current):
|
||||
_add_image(inv, inbox, path, ref, sts=False)
|
||||
rest = _IMAGE_REF.sub("", current)
|
||||
if rest.strip():
|
||||
chunks.append(inv.chunk(rest))
|
||||
index += 1
|
||||
if chunks:
|
||||
inv.add("paragraph", chunks)
|
||||
|
||||
|
||||
_TEXT = ("paragraph", "line")
|
||||
|
||||
|
||||
def _inventory_text(inv: Inventory, text: str) -> None:
|
||||
block: list[int] = []
|
||||
for line in [*text.split("\n"), ""]:
|
||||
if line.strip():
|
||||
piece = inv.chunk(line)
|
||||
inv.add("line", (piece,))
|
||||
block.append(piece)
|
||||
elif block:
|
||||
inv.add("paragraph", block)
|
||||
block = []
|
||||
|
||||
|
||||
_CSV = ("header_cell", "row", "cell")
|
||||
|
||||
|
||||
def _inventory_csv(inv: Inventory, text: str) -> None:
|
||||
rows = [row for row in csv.reader(io.StringIO(text)) if row]
|
||||
for position, row in enumerate(rows):
|
||||
pieces = [inv.chunk(value) for value in row]
|
||||
if position == 0:
|
||||
for piece in pieces:
|
||||
inv.add("header_cell", (piece,))
|
||||
continue
|
||||
for piece in pieces:
|
||||
inv.add("cell", (piece,))
|
||||
inv.add("row", pieces)
|
||||
|
||||
|
||||
_JSON = ("key", "value")
|
||||
|
||||
|
||||
def _inventory_json(inv: Inventory, text: str) -> None:
|
||||
"""Fenced verbatim by the reader, so every key and value is carried exactly
|
||||
when the document's own text is: each names the whole text."""
|
||||
whole = (inv.chunk(text),)
|
||||
stack: list[Any] = [json.loads(text)]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
inv.add("key", whole)
|
||||
stack.append(value)
|
||||
elif isinstance(node, list):
|
||||
stack.extend(node)
|
||||
else:
|
||||
inv.add("value", whole)
|
||||
|
||||
|
||||
_HTML = ("cell", "heading", "image", "list_item", "paragraph", "table")
|
||||
_HTML_KIND = {
|
||||
"p": "paragraph",
|
||||
"li": "list_item",
|
||||
"table": "table",
|
||||
"td": "cell",
|
||||
"th": "cell",
|
||||
**{f"h{n}": "heading" for n in range(1, 7)},
|
||||
}
|
||||
_HTML_VOID = {"img", "br", "hr", "meta", "link", "input", "col", "area", "base", "wbr", "source"}
|
||||
|
||||
|
||||
class _HtmlInventory(HTMLParser):
|
||||
def __init__(self, inv: Inventory, inbox: Path, path: Path) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.inv, self.inbox, self.path = inv, inbox, path
|
||||
self.open: list[tuple[str, list[int]]] = []
|
||||
self.skip = 0
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag == "img":
|
||||
_add_image(self.inv, self.inbox, self.path, dict(attrs).get("src") or "", sts=False)
|
||||
return
|
||||
if tag in ("script", "style"):
|
||||
self.skip += 1
|
||||
if tag not in _HTML_VOID:
|
||||
self.open.append((tag, []))
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in ("script", "style") and self.skip:
|
||||
self.skip -= 1
|
||||
for position in range(len(self.open) - 1, -1, -1):
|
||||
if self.open[position][0] == tag:
|
||||
break
|
||||
else:
|
||||
return
|
||||
while len(self.open) > position:
|
||||
self._close(*self.open.pop())
|
||||
|
||||
def _close(self, tag: str, pieces: list[int]) -> None:
|
||||
kind = _HTML_KIND.get(tag)
|
||||
if kind is not None:
|
||||
self.inv.add(kind, pieces)
|
||||
if self.open:
|
||||
self.open[-1][1].extend(pieces)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self.skip or not data.strip():
|
||||
return
|
||||
if self.open:
|
||||
self.open[-1][1].append(self.inv.chunk(data))
|
||||
|
||||
def finish(self) -> None:
|
||||
self.close()
|
||||
while self.open:
|
||||
self._close(*self.open.pop())
|
||||
|
||||
|
||||
# xml / sts
|
||||
_STS = (
|
||||
"cell",
|
||||
"footnote",
|
||||
"image",
|
||||
"list_item",
|
||||
"paragraph",
|
||||
"section",
|
||||
"section_label",
|
||||
"table",
|
||||
"table_label",
|
||||
"title",
|
||||
)
|
||||
_STS_KIND = {
|
||||
"sec": "section",
|
||||
"p": "paragraph",
|
||||
"table-wrap": "table",
|
||||
"td": "cell",
|
||||
"th": "cell",
|
||||
"list-item": "list_item",
|
||||
"fn": "footnote",
|
||||
}
|
||||
|
||||
|
||||
def _bare(tag: str) -> str:
|
||||
return tag.rpartition("}")[2]
|
||||
|
||||
|
||||
def _inventory_xml(inv: Inventory, inbox: Path, path: Path, data: bytes) -> None:
|
||||
if b"<!DOCTYPE" in data:
|
||||
raise ValueError("a DOCTYPE is not read")
|
||||
root = ElementTree.fromstring(data)
|
||||
sts = _bare(root.tag) == "standard" or any(_bare(e.tag) == "sec" for e in root.iter())
|
||||
if not sts:
|
||||
inv.vocabulary = ("element",)
|
||||
inv.add("element", (inv.chunk("".join(root.itertext())),))
|
||||
return
|
||||
|
||||
def walk(node: ElementTree.Element, parent: str) -> list[int]:
|
||||
tag = _bare(node.tag)
|
||||
pieces: list[int] = []
|
||||
if node.text and node.text.strip():
|
||||
pieces.append(inv.chunk(node.text))
|
||||
if tag in ("graphic", "inline-graphic"):
|
||||
href = next((v for k, v in node.attrib.items() if _bare(k) == "href"), "")
|
||||
_add_image(inv, inbox, path, href, sts=True)
|
||||
for child in node:
|
||||
pieces.extend(walk(child, tag))
|
||||
if child.tail and child.tail.strip():
|
||||
pieces.append(inv.chunk(child.tail))
|
||||
kind = _STS_KIND.get(tag)
|
||||
if tag == "title" and parent == "sec":
|
||||
kind = "title"
|
||||
elif tag == "label" and parent == "sec":
|
||||
kind = "section_label"
|
||||
elif tag == "label" and parent == "table-wrap":
|
||||
kind = "table_label"
|
||||
if kind is not None:
|
||||
inv.add(kind, pieces)
|
||||
return pieces
|
||||
|
||||
walk(root, "")
|
||||
|
||||
|
||||
# office containers
|
||||
_WNS = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
||||
_ANS = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
|
||||
_PNS = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
|
||||
_SNS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
_XDRNS = "{http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing}"
|
||||
_TNS = "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}"
|
||||
_TBNS = "{urn:oasis:names:tc:opendocument:xmlns:table:1.0}"
|
||||
_DNS = "{urn:oasis:names:tc:opendocument:xmlns:drawing:1.0}"
|
||||
|
||||
_DOCX = ("cell", "footnote", "heading", "image", "paragraph", "table")
|
||||
_HEADING_STYLES = re.compile(r"(?i)^(?:heading|overskrift|title|tittel)\s*\d*$")
|
||||
|
||||
|
||||
def _joined(node: ElementTree.Element, text_tag: str) -> str:
|
||||
return "".join(t.text or "" for t in node.iter(text_tag))
|
||||
|
||||
|
||||
def _docx_lines(para: ElementTree.Element) -> list[str]:
|
||||
"""A paragraph's text, split where it breaks a line (`w:br`, `w:cr`).
|
||||
|
||||
A break is a line boundary in the source, and the converter keeps it one:
|
||||
inside a grid-table cell the two halves land on different rows, with other
|
||||
cells' text between them (measured on K2).
|
||||
"""
|
||||
lines = [""]
|
||||
for node in para.iter():
|
||||
if node.tag == f"{_WNS}t":
|
||||
lines[-1] += node.text or ""
|
||||
elif node.tag in (f"{_WNS}br", f"{_WNS}cr"):
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def _inventory_docx(inv: Inventory, data: bytes) -> None:
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
body = ElementTree.fromstring(archive.read("word/document.xml"))
|
||||
notes = (
|
||||
ElementTree.fromstring(archive.read("word/footnotes.xml"))
|
||||
if "word/footnotes.xml" in archive.namelist()
|
||||
else None
|
||||
)
|
||||
paragraph_pieces: dict[int, list[int]] = {}
|
||||
for para in body.iter(f"{_WNS}p"):
|
||||
pieces = [inv.chunk(line) for line in _docx_lines(para) if line.strip()]
|
||||
paragraph_pieces[id(para)] = pieces
|
||||
style = para.find(f"{_WNS}pPr/{_WNS}pStyle")
|
||||
if style is not None and _HEADING_STYLES.match(style.get(f"{_WNS}val", "")):
|
||||
inv.add("heading", pieces)
|
||||
elif pieces:
|
||||
inv.add("paragraph", pieces)
|
||||
for kind, tag in (("table", "tbl"), ("cell", "tc")):
|
||||
for node in body.iter(f"{_WNS}{tag}"):
|
||||
inv.add(
|
||||
kind,
|
||||
[c for p in node.iter(f"{_WNS}p") for c in paragraph_pieces.get(id(p), [])],
|
||||
)
|
||||
for _ in body.iter(f"{_ANS}blip"):
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
if notes is not None:
|
||||
for note in notes.iter(f"{_WNS}footnote"):
|
||||
if int(note.get(f"{_WNS}id", "0")) > 0:
|
||||
text = _joined(note, f"{_WNS}t")
|
||||
inv.add("footnote", (inv.chunk(text),) if text.strip() else ())
|
||||
|
||||
|
||||
_PPTX = ("cell", "image", "paragraph", "slide", "table", "title")
|
||||
_SLIDE = re.compile(r"ppt/slides/slide\d+\.xml")
|
||||
|
||||
|
||||
def _inventory_pptx(inv: Inventory, data: bytes) -> None:
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
slides = [
|
||||
ElementTree.fromstring(archive.read(n))
|
||||
for n in archive.namelist()
|
||||
if _SLIDE.fullmatch(n)
|
||||
]
|
||||
for slide in slides:
|
||||
pieces: list[int] = []
|
||||
for frame in slide.iter(f"{_ANS}tbl"):
|
||||
cells: list[int] = []
|
||||
for cell in frame.iter(f"{_ANS}tc"):
|
||||
text = _joined(cell, f"{_ANS}t")
|
||||
own = (inv.chunk(text),) if text.strip() else ()
|
||||
inv.add("cell", own)
|
||||
cells.extend(own)
|
||||
inv.add("table", cells)
|
||||
pieces.extend(cells)
|
||||
for shape in slide.iter(f"{_PNS}sp"):
|
||||
holder = shape.find(f"{_PNS}nvSpPr/{_PNS}nvPr/{_PNS}ph")
|
||||
texts = [
|
||||
inv.chunk(t)
|
||||
for t in (_joined(p, f"{_ANS}t") for p in shape.iter(f"{_ANS}p"))
|
||||
if t.strip()
|
||||
]
|
||||
if holder is not None and holder.get("type") in ("title", "ctrTitle") and texts:
|
||||
inv.add("title", texts)
|
||||
else:
|
||||
for piece in texts:
|
||||
inv.add("paragraph", (piece,))
|
||||
pieces.extend(texts)
|
||||
for _ in slide.iter(f"{_PNS}pic"):
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
inv.add("slide", pieces)
|
||||
|
||||
|
||||
_XLSX = ("cell", "image", "row", "sheet")
|
||||
_SHEET = re.compile(r"xl/worksheets/sheet\d+\.xml")
|
||||
_DRAWING = re.compile(r"xl/drawings/drawing\d+\.xml")
|
||||
|
||||
|
||||
def _inventory_xlsx(inv: Inventory, data: bytes) -> None:
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
names = archive.namelist()
|
||||
shared: list[str] = []
|
||||
if "xl/sharedStrings.xml" in names:
|
||||
table = ElementTree.fromstring(archive.read("xl/sharedStrings.xml"))
|
||||
shared = ["".join(t.text or "" for t in si.iter(f"{_SNS}t")) for si in table]
|
||||
sheets = [ElementTree.fromstring(archive.read(n)) for n in names if _SHEET.fullmatch(n)]
|
||||
drawings = [ElementTree.fromstring(archive.read(n)) for n in names if _DRAWING.fullmatch(n)]
|
||||
for sheet in sheets:
|
||||
sheet_pieces: list[int] = []
|
||||
for row in sheet.iter(f"{_SNS}row"):
|
||||
row_pieces: list[int] = []
|
||||
valued = False
|
||||
for cell in row.iter(f"{_SNS}c"):
|
||||
value = cell.find(f"{_SNS}v")
|
||||
inline = cell.find(f"{_SNS}is")
|
||||
if value is None and inline is None:
|
||||
continue
|
||||
valued = True
|
||||
if inline is not None:
|
||||
text = "".join(t.text or "" for t in inline.iter(f"{_SNS}t"))
|
||||
elif cell.get("t") == "s" and value is not None and value.text is not None:
|
||||
text = shared[int(value.text)]
|
||||
else:
|
||||
text = (value.text or "") if value is not None else ""
|
||||
own = (inv.chunk(text),) if text.strip() else ()
|
||||
inv.add("cell", own)
|
||||
row_pieces.extend(own)
|
||||
if valued:
|
||||
inv.add("row", row_pieces)
|
||||
sheet_pieces.extend(row_pieces)
|
||||
inv.add("sheet", sheet_pieces)
|
||||
for drawing in drawings:
|
||||
for _ in drawing.iter(f"{_XDRNS}pic"):
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
|
||||
|
||||
_ODT = ("cell", "heading", "image", "list_item", "paragraph", "table")
|
||||
|
||||
|
||||
def _inventory_odt(inv: Inventory, data: bytes) -> None:
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
content = ElementTree.fromstring(archive.read("content.xml"))
|
||||
pieces: dict[int, int] = {}
|
||||
in_cells: set[int] = set()
|
||||
for cell in content.iter(f"{_TBNS}table-cell"):
|
||||
in_cells.update(id(p) for p in cell.iter(f"{_TNS}p"))
|
||||
for tag in (f"{_TNS}p", f"{_TNS}h"):
|
||||
for node in content.iter(tag):
|
||||
text = "".join(node.itertext())
|
||||
if text.strip():
|
||||
pieces[id(node)] = inv.chunk(text)
|
||||
|
||||
def within(node: ElementTree.Element) -> list[int]:
|
||||
return [pieces[id(n)] for n in node.iter() if id(n) in pieces]
|
||||
|
||||
for node in content.iter(f"{_TNS}h"):
|
||||
inv.add("heading", within(node))
|
||||
for node in content.iter(f"{_TNS}p"):
|
||||
if id(node) not in in_cells and id(node) in pieces:
|
||||
inv.add("paragraph", (pieces[id(node)],))
|
||||
for kind, tag in (
|
||||
("table", f"{_TBNS}table"),
|
||||
("cell", f"{_TBNS}table-cell"),
|
||||
("list_item", f"{_TNS}list-item"),
|
||||
):
|
||||
for node in content.iter(tag):
|
||||
inv.add(kind, within(node))
|
||||
for _ in content.iter(f"{_DNS}image"):
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
|
||||
|
||||
_RTF = ("cell", "image", "paragraph", "table_row")
|
||||
_RTF_TOKEN = re.compile(
|
||||
r"\\([a-zA-Z]+)(-?\d+)? ?|\\'([0-9a-fA-F]{2})|\\([^a-zA-Z])|([{}])|([^\\{}\r\n]+)|[\r\n]"
|
||||
)
|
||||
_RTF_SKIP = {"fonttbl", "colortbl", "stylesheet", "info", "pict", "header", "footer", "*"}
|
||||
|
||||
|
||||
def _inventory_rtf(inv: Inventory, data: bytes) -> None:
|
||||
"""Paragraph: `\\par`. Table row: `\\row`. Cell: `\\cell`. Image: `\\pict`.
|
||||
|
||||
The text between two such words is the element's text: `\\'hh` is read as
|
||||
cp1252, `\\uN` as the code point N with its one fallback character
|
||||
skipped, and destination groups (font table, pictures, ...) carry none.
|
||||
"""
|
||||
raw = data.decode("latin-1")
|
||||
depth = 0
|
||||
skip_below: int | None = None
|
||||
skip_fallback = 0
|
||||
buffer: list[str] = []
|
||||
row: list[int] = []
|
||||
for match in _RTF_TOKEN.finditer(raw):
|
||||
word, number, hexa, symbol, brace, text = match.groups()
|
||||
if brace == "{":
|
||||
depth += 1
|
||||
continue
|
||||
if brace == "}":
|
||||
if skip_below is not None and depth <= skip_below:
|
||||
skip_below = None
|
||||
depth -= 1
|
||||
continue
|
||||
if word == "pict":
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
if skip_below is not None:
|
||||
continue
|
||||
if word in _RTF_SKIP or symbol == "*":
|
||||
skip_below = depth
|
||||
continue
|
||||
if word == "u" and number is not None:
|
||||
value = int(number)
|
||||
buffer.append(chr(value + 65536 if value < 0 else value))
|
||||
skip_fallback = 1
|
||||
continue
|
||||
if hexa is not None:
|
||||
if skip_fallback:
|
||||
skip_fallback -= 1
|
||||
else:
|
||||
buffer.append(bytes([int(hexa, 16)]).decode("cp1252", errors="replace"))
|
||||
continue
|
||||
if text is not None:
|
||||
if skip_fallback:
|
||||
text = text[1:]
|
||||
skip_fallback = 0
|
||||
buffer.append(text)
|
||||
continue
|
||||
if symbol is not None:
|
||||
buffer.append({"~": " ", "-": "", "_": "-"}.get(symbol, symbol))
|
||||
continue
|
||||
if word in ("par", "cell", "row"):
|
||||
piece = inv.chunk("".join(buffer))
|
||||
buffer = []
|
||||
if word == "par":
|
||||
inv.add("paragraph", (piece,))
|
||||
elif word == "cell":
|
||||
inv.add("cell", (piece,))
|
||||
row.append(piece)
|
||||
else:
|
||||
inv.add("table_row", row)
|
||||
row = []
|
||||
|
||||
|
||||
_PDF = ("image", "page")
|
||||
|
||||
|
||||
def _inventory_pdf(inv: Inventory, data: bytes) -> None:
|
||||
"""Page: one per page. Image: one per placement on a page, as pdfplumber
|
||||
lists them. A page's own text is not inventoried here: a PDF without a
|
||||
structure tree declares no headings or paragraphs, which is one of the
|
||||
gate's proposed exceptions, and its pages are checked through the images
|
||||
they carry and the document's persist outcome alone."""
|
||||
import pdfplumber
|
||||
|
||||
with pdfplumber.open(io.BytesIO(data)) as pdf:
|
||||
for page in pdf.pages:
|
||||
inv.add("page")
|
||||
for _ in page.images:
|
||||
inv.refs.append(("embedded", None))
|
||||
inv.add("image")
|
||||
page.close()
|
||||
|
||||
|
||||
Reader = Callable[[Inventory, Path, Path, bytes], None]
|
||||
|
||||
|
||||
def _text_reader(function: Callable[[Inventory, str], None]) -> Reader:
|
||||
def read(inv: Inventory, inbox: Path, path: Path, data: bytes) -> None:
|
||||
function(inv, data.decode("utf-8-sig"))
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def _bytes_reader(function: Callable[[Inventory, bytes], None]) -> Reader:
|
||||
def read(inv: Inventory, inbox: Path, path: Path, data: bytes) -> None:
|
||||
function(inv, data)
|
||||
|
||||
return read
|
||||
|
||||
|
||||
def _markdown(inv: Inventory, inbox: Path, path: Path, data: bytes) -> None:
|
||||
_inventory_markdown(inv, inbox, path, data.decode("utf-8-sig"))
|
||||
|
||||
|
||||
def _html(inv: Inventory, inbox: Path, path: Path, data: bytes) -> None:
|
||||
parser = _HtmlInventory(inv, inbox, path)
|
||||
parser.feed(data.decode("utf-8-sig"))
|
||||
parser.finish()
|
||||
|
||||
|
||||
_READERS: dict[str, tuple[tuple[str, ...], Reader]] = {
|
||||
".md": (_MARKDOWN, _markdown),
|
||||
".txt": (_TEXT, _text_reader(_inventory_text)),
|
||||
".csv": (_CSV, _text_reader(_inventory_csv)),
|
||||
".json": (_JSON, _text_reader(_inventory_json)),
|
||||
".html": (_HTML, _html),
|
||||
".htm": (_HTML, _html),
|
||||
".xml": (_STS, _inventory_xml),
|
||||
".docx": (_DOCX, _bytes_reader(_inventory_docx)),
|
||||
".pptx": (_PPTX, _bytes_reader(_inventory_pptx)),
|
||||
".xlsx": (_XLSX, _bytes_reader(_inventory_xlsx)),
|
||||
".odt": (_ODT, _bytes_reader(_inventory_odt)),
|
||||
".rtf": (_RTF, _bytes_reader(_inventory_rtf)),
|
||||
".pdf": (_PDF, _bytes_reader(_inventory_pdf)),
|
||||
}
|
||||
|
||||
|
||||
def inventory(inbox: Path, path: Path, data: bytes | None = None) -> Inventory:
|
||||
"""What the source file at `path` holds. A file of a type this module does
|
||||
not inventory has an empty vocabulary; one it cannot read raises."""
|
||||
reader = _READERS.get(path.suffix.lower())
|
||||
inv = Inventory(relative_source(path, inbox), reader[0] if reader else ())
|
||||
if reader is not None:
|
||||
reader[1](inv, inbox, path, path.read_bytes() if data is None else data)
|
||||
return inv
|
||||
|
||||
|
||||
# --- the account -------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fate:
|
||||
carried: int = 0
|
||||
pointer: int = 0
|
||||
rejected: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return self.carried + self.pointer + sum(self.rejected.values())
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
CARRIED: self.carried,
|
||||
POINTER: self.pointer,
|
||||
REJECTED: dict(sorted(self.rejected.items())),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DocumentAccount:
|
||||
source_file: str
|
||||
status: str
|
||||
code: str | None
|
||||
counts: dict[str, int]
|
||||
fates: dict[str, Fate]
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def unaccounted(self) -> dict[str, int]:
|
||||
return {
|
||||
kind: count - self.fates[kind].total
|
||||
for kind, count in self.counts.items()
|
||||
if count > self.fates[kind].total
|
||||
}
|
||||
|
||||
@property
|
||||
def double_booked(self) -> dict[str, int]:
|
||||
return {
|
||||
kind: self.fates[kind].total - count
|
||||
for kind, count in self.counts.items()
|
||||
if self.fates[kind].total > count
|
||||
}
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return sum(self.counts.values())
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
entry: dict[str, Any] = {
|
||||
"source_file": self.source_file,
|
||||
"status": self.status,
|
||||
"code": self.code,
|
||||
"inventory": dict(self.counts),
|
||||
"fates": {kind: self.fates[kind].to_json() for kind in self.counts},
|
||||
"unaccounted": self.unaccounted,
|
||||
"double_booked": self.double_booked,
|
||||
}
|
||||
if self.error is not None:
|
||||
entry["inventory_error"] = self.error
|
||||
return entry
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileAccount:
|
||||
source_file: str
|
||||
fate: str
|
||||
code: str | None
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {"source_file": self.source_file, "fate": self.fate, "code": self.code}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Accounting:
|
||||
documents: list[DocumentAccount]
|
||||
files: list[FileAccount]
|
||||
|
||||
@property
|
||||
def unaccounted(self) -> int:
|
||||
return sum(sum(d.unaccounted.values()) for d in self.documents) + sum(
|
||||
1 for f in self.files if not f.fate
|
||||
)
|
||||
|
||||
@property
|
||||
def double_booked(self) -> int:
|
||||
return sum(sum(d.double_booked.values()) for d in self.documents)
|
||||
|
||||
@property
|
||||
def images_found(self) -> int:
|
||||
return sum(d.counts.get("image", 0) for d in self.documents)
|
||||
|
||||
@property
|
||||
def elements(self) -> int:
|
||||
return sum(d.total for d in self.documents)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"accounting_version": ACCOUNTING_VERSION,
|
||||
"unaccounted": self.unaccounted,
|
||||
"double_booked": self.double_booked,
|
||||
"documents": [d.to_json() for d in self.documents],
|
||||
"files": [f.to_json() for f in self.files],
|
||||
}
|
||||
|
||||
def log_lines(self) -> list[str]:
|
||||
lines = [
|
||||
f"* **Accounting**: {len(self.documents)} document(s) and {len(self.files)} "
|
||||
f"other file(s); {self.elements} elements found in the sources; "
|
||||
f"{self.unaccounted} unaccounted, {self.double_booked} double-booked."
|
||||
]
|
||||
for doc in self.documents:
|
||||
if doc.status == REJECTED:
|
||||
lines.append(
|
||||
f"* {doc.source_file}: {doc.total} elements found in the source, 0 carried: "
|
||||
f"document rejected `{doc.code}`."
|
||||
)
|
||||
for kind, count in sorted(doc.unaccounted.items()):
|
||||
lines.append(f"* {doc.source_file}: {kind} {count} unaccounted.")
|
||||
for kind, count in sorted(doc.double_booked.items()):
|
||||
lines.append(f"* {doc.source_file}: {kind} {count} double-booked.")
|
||||
return lines
|
||||
|
||||
|
||||
def _bodies(paths: Sequence[Path]) -> str:
|
||||
parts = []
|
||||
for path in paths:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if text.startswith("---\n"):
|
||||
end = text.find("\n---\n", 4)
|
||||
text = text[end + 5 :] if end >= 0 else text
|
||||
parts.append(text)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
#: A converter attribute block such as `{.mark}` or `{#slide-1}`. Its letters
|
||||
#: stand between the words of text that WAS carried (`[Sted]{.mark}`), so it is
|
||||
#: removed before comparing -- measured on K2, where a highlighted placeholder
|
||||
#: otherwise read as lost.
|
||||
_CONVERTER_ATTRIBUTE = re.compile(r"\{[#.][^{}\n]*\}")
|
||||
|
||||
|
||||
class _Finder:
|
||||
"""Is a piece of text in the carried text? Searched from where the last
|
||||
piece was found first, because readers keep the source order."""
|
||||
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = _norm(_CONVERTER_ATTRIBUTE.sub("", text))
|
||||
self.cursor = 0
|
||||
|
||||
def __call__(self, piece: str) -> bool:
|
||||
needle = _norm(piece)
|
||||
if not needle:
|
||||
return True
|
||||
at = self.text.find(needle, self.cursor)
|
||||
if at < 0:
|
||||
at = self.text.find(needle)
|
||||
if at < 0:
|
||||
return False
|
||||
self.cursor = at + len(needle)
|
||||
return True
|
||||
|
||||
|
||||
def _book_images(
|
||||
inv: Inventory, fate: Fate, assets: DocumentAssets | None, found: Callable[[str], bool]
|
||||
) -> None:
|
||||
placements = [chunks for kind, chunks in inv.elements if kind == "image"]
|
||||
handled = assets is not None and (assets.carried or assets.rejected)
|
||||
if assets is not None and handled:
|
||||
fate.carried += assets.carried
|
||||
for rejection in assets.rejected:
|
||||
if rejection.code == _REMOTE_CODE:
|
||||
fate.pointer += 1
|
||||
else:
|
||||
fate.rejected[rejection.code] = fate.rejected.get(rejection.code, 0) + 1
|
||||
return
|
||||
# The reader did not handle images for this document: a reference kept in
|
||||
# the carried text is a pointer; anything else has no fate.
|
||||
for chunks in placements:
|
||||
if chunks and all(found(inv.chunks[c]) for c in chunks):
|
||||
fate.pointer += 1
|
||||
|
||||
|
||||
def _persisted_account(
|
||||
inv: Inventory, bodies: str, assets: DocumentAssets | None
|
||||
) -> DocumentAccount:
|
||||
finder = _Finder(bodies)
|
||||
seen: dict[int, bool] = {}
|
||||
|
||||
def found(index: int) -> bool:
|
||||
if index not in seen:
|
||||
seen[index] = finder(inv.chunks[index])
|
||||
return seen[index]
|
||||
|
||||
counts = inv.counts()
|
||||
fates = {kind: Fate() for kind in counts}
|
||||
for kind, chunks in inv.elements:
|
||||
if kind == "image":
|
||||
continue
|
||||
if all(found(c) for c in chunks):
|
||||
fates[kind].carried += 1
|
||||
if "image" in fates:
|
||||
_book_images(inv, fates["image"], assets, finder)
|
||||
return DocumentAccount(inv.source_file, PERSISTED, None, counts, fates)
|
||||
|
||||
|
||||
def _refused_account(inv: Inventory, code: str) -> DocumentAccount:
|
||||
counts = inv.counts()
|
||||
fates = {kind: Fate(rejected={code: n} if n else {}) for kind, n in counts.items()}
|
||||
return DocumentAccount(inv.source_file, REJECTED, code, counts, fates)
|
||||
|
||||
|
||||
def account_run(inbox: Path, walked: Sequence[Path], result: InboxResult) -> Accounting:
|
||||
"""Every walked file's account after one `process_inbox` run."""
|
||||
persisted = {item.source_file for item in result.persisted}
|
||||
concepts: dict[str, list[Path]] = {}
|
||||
for item in result.concepts:
|
||||
concepts.setdefault(item.source_file, []).append(item.path)
|
||||
refused = {item.source_file: item.disposition for item in result.quarantined + result.rejected}
|
||||
failed = {item.source_file: item.error.code for item in result.failed}
|
||||
assets = {item.source_file: item for item in result.document_assets}
|
||||
carried = set(result.carried_files)
|
||||
documents: list[DocumentAccount] = []
|
||||
files: list[FileAccount] = []
|
||||
for path in walked:
|
||||
name = relative_source(path, inbox)
|
||||
if path.suffix.lower() not in _READERS:
|
||||
if name in carried:
|
||||
files.append(FileAccount(name, CARRIED, None))
|
||||
elif name in persisted:
|
||||
files.append(FileAccount(name, MERGED, None))
|
||||
else:
|
||||
files.append(
|
||||
FileAccount(name, REJECTED if name in failed else "", failed.get(name))
|
||||
)
|
||||
continue
|
||||
try:
|
||||
inv = inventory(inbox, path)
|
||||
# Broad on purpose, and it diagnoses nothing: the row carries the
|
||||
# exception's own type and text, never a guessed cause.
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
code = refused.get(name) or failed.get(name) or "inventory_unreadable"
|
||||
documents.append(
|
||||
DocumentAccount(
|
||||
name,
|
||||
REJECTED if name not in persisted else PERSISTED,
|
||||
code,
|
||||
{},
|
||||
{},
|
||||
error=error,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if name in persisted:
|
||||
documents.append(
|
||||
_persisted_account(inv, _bodies(concepts.get(name, [])), assets.get(name))
|
||||
)
|
||||
elif name in refused:
|
||||
documents.append(_refused_account(inv, refused[name]))
|
||||
elif name in failed:
|
||||
documents.append(_refused_account(inv, failed[name]))
|
||||
else:
|
||||
documents.append(
|
||||
DocumentAccount(name, "", None, inv.counts(), {k: Fate() for k in inv.counts()})
|
||||
)
|
||||
return Accounting(documents, files)
|
||||
|
||||
|
||||
def summary(accounting: Accounting) -> Mapping[str, int]:
|
||||
return {
|
||||
"documents": len(accounting.documents),
|
||||
"files": len(accounting.files),
|
||||
"elements": accounting.elements,
|
||||
"unaccounted": accounting.unaccounted,
|
||||
"double_booked": accounting.double_booked,
|
||||
}
|
||||
|
|
@ -68,6 +68,7 @@ caller.
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
|
@ -500,6 +501,7 @@ def build(
|
|||
frontmatter: Mapping[str, str] | None = None,
|
||||
gate: str = DEFAULT_GATE,
|
||||
assets: bool = DEFAULT_ASSETS,
|
||||
account: bool = False,
|
||||
) -> CorpusReport:
|
||||
"""Folder in, bundle out. The whole command, minus argument parsing.
|
||||
|
||||
|
|
@ -541,6 +543,7 @@ def build(
|
|||
concept_frontmatter_values=concept_values,
|
||||
gate=gate,
|
||||
assets=assets,
|
||||
account=account,
|
||||
)
|
||||
_write_log(bundle, report, profile=STRUCTURED_V1)
|
||||
return report
|
||||
|
|
@ -608,6 +611,7 @@ def build(
|
|||
concept_frontmatter_values=concept_values,
|
||||
gate=gate,
|
||||
assets=assets,
|
||||
account=account,
|
||||
)
|
||||
_write_log(bundle, report, profile=SEGMENTED_OKF_V0_2)
|
||||
return report
|
||||
|
|
@ -934,6 +938,18 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
|||
"bundle of documents that had none"
|
||||
),
|
||||
)
|
||||
build_parser.add_argument(
|
||||
"--accounting",
|
||||
type=Path,
|
||||
default=None,
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"take an inventory of every source before extraction and give every "
|
||||
"element one fate after the run -- carried, pointer or a coded "
|
||||
"rejection -- written as JSON to PATH and summarised in log.md. The "
|
||||
"build fails (exit 1) when any element is unaccounted or booked twice"
|
||||
),
|
||||
)
|
||||
build_parser.add_argument(
|
||||
"--gate",
|
||||
choices=GATE_NAMES,
|
||||
|
|
@ -1122,6 +1138,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
shell_parent=args.shell_parent,
|
||||
gate=args.gate,
|
||||
assets=args.assets,
|
||||
account=args.accounting is not None,
|
||||
frontmatter=frontmatter_from_flags(args.frontmatter or ()),
|
||||
)
|
||||
except (IngestError, OSError, ValueError) as exc:
|
||||
|
|
@ -1132,15 +1149,42 @@ def main(argv: list[str] | None = None) -> int:
|
|||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(report.render(), encoding="utf-8", newline="")
|
||||
print(report.render())
|
||||
if report.unaccounted or report.merged + report.rejected != report.n:
|
||||
if report.conservation_failed:
|
||||
print(
|
||||
f"{CLI_ID}: K1b FAILED - merged ({report.merged}) + coded rejections "
|
||||
f"({report.rejected}) != N ({report.n}). Unaccounted: "
|
||||
f"{CLI_ID}: K1b FAILED - {report.identity()}. Unaccounted: "
|
||||
f"{', '.join(report.unaccounted) or '(none named)'}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
failed = False
|
||||
if report.accounting is not None and args.accounting is not None:
|
||||
args.accounting.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.accounting.write_text(
|
||||
json.dumps(report.accounting.to_json(), indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="",
|
||||
)
|
||||
if report.accounting.unaccounted or report.accounting.double_booked:
|
||||
print(
|
||||
f"{CLI_ID}: accounting FAILED - {report.accounting.unaccounted} element(s) "
|
||||
f"unaccounted, {report.accounting.double_booked} double-booked; see "
|
||||
f"{args.accounting} and log.md",
|
||||
file=sys.stderr,
|
||||
)
|
||||
failed = True
|
||||
# A run that read documents and kept none is not a success, whatever the
|
||||
# conservation identity says: every refusal is coded, and the bundle is
|
||||
# still empty. Door B's library function keeps "all rejected" as a normal
|
||||
# outcome -- for a hostile inbox it is one -- but this command is an
|
||||
# operator pointing at their own folder.
|
||||
if report.extracted and not report.persisted:
|
||||
print(
|
||||
f"{CLI_ID}: FAILED - 0 of {report.extracted} extracted document(s) persisted; "
|
||||
f"rejection codes: {', '.join(f'{c} {n}' for c, n in report.codes)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
failed = True
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from collections.abc import Callable, Mapping
|
|||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
from .accounting import Accounting, account_run
|
||||
from .assets import ASSETS_DIR
|
||||
from .errors import IngestError
|
||||
from .extract import extract_text
|
||||
|
|
@ -236,11 +237,42 @@ class CorpusReport:
|
|||
assets: bool = False
|
||||
assets_carried: int = 0
|
||||
assets_found: int = 0
|
||||
#: Walked files that are not documents and whose bytes a persisted
|
||||
#: document carried as an image. Their own column in the conservation
|
||||
#: identity: before this they were ALSO counted as `extractor_unknown`
|
||||
#: rejections, so one file had two fates.
|
||||
carried_files: int = 0
|
||||
#: The content accounting of this run, or None when it was not asked for.
|
||||
accounting: Accounting | None = None
|
||||
|
||||
@property
|
||||
def merged(self) -> int:
|
||||
return self.substantive + self.degenerate
|
||||
|
||||
@property
|
||||
def conservation_failed(self) -> bool:
|
||||
"""K1b: every walked file is merged, carried or a coded rejection."""
|
||||
return bool(self.unaccounted) or (
|
||||
self.merged + self.carried_files + self.rejected != self.n
|
||||
)
|
||||
|
||||
def identity(self) -> str:
|
||||
"""The conservation identity with its numbers, in words.
|
||||
|
||||
The carried column is written only when it is non-zero, so a run with
|
||||
no image files beside its documents keeps the line it always had.
|
||||
"""
|
||||
if not self.carried_files:
|
||||
return (
|
||||
f"merged + coded rejections = {self.merged} + {self.rejected} = "
|
||||
f"{self.merged + self.rejected}; N = {self.n}"
|
||||
)
|
||||
total = self.merged + self.carried_files + self.rejected
|
||||
return (
|
||||
"merged + files carried through a document + coded rejections = "
|
||||
f"{self.merged} + {self.carried_files} + {self.rejected} = {total}; N = {self.n}"
|
||||
)
|
||||
|
||||
def render(self) -> str:
|
||||
per_file = self.seconds_total / self.n if self.n else 0.0
|
||||
lines = [
|
||||
|
|
@ -266,7 +298,11 @@ class CorpusReport:
|
|||
f"- degenerate: {self.degenerate}/{self.n}",
|
||||
f"- rejected (coded): {self.rejected}/{self.n}",
|
||||
"",
|
||||
f"merged + coded rejections = {self.merged + self.rejected}; N = {self.n}",
|
||||
(
|
||||
f"merged + coded rejections = {self.merged + self.rejected}; N = {self.n}"
|
||||
if not self.carried_files
|
||||
else self.identity()
|
||||
),
|
||||
"",
|
||||
"## Converter",
|
||||
"",
|
||||
|
|
@ -320,15 +356,16 @@ class CorpusReport:
|
|||
f"merged = {self.merged} ({self.substantive} substantive, "
|
||||
f"{self.degenerate} degenerate), coded rejections = {self.rejected}.",
|
||||
f"* **Rejected**: {rejections}.",
|
||||
f"* **Conservation (K1b)**: merged + coded rejections = "
|
||||
f"{self.merged} + {self.rejected} = {self.merged + self.rejected}; "
|
||||
f"N = {self.n}. The run exits non-zero when these differ.",
|
||||
f"* **Conservation (K1b)**: {self.identity()}. "
|
||||
"The run exits non-zero when these differ.",
|
||||
f"* **Converter**: {self.converter_path}, version {self.converter_version}.",
|
||||
f"* **Gate**: {self.gate} "
|
||||
f"({_GATE_DESCRIPTIONS.get(self.gate, 'unrecognised gate name')}). "
|
||||
"Every persisted byte of this bundle passed it.",
|
||||
self._assets_line(),
|
||||
]
|
||||
if self.accounting is not None:
|
||||
lines.extend(self.accounting.log_lines())
|
||||
if self.unaccounted:
|
||||
lines.append("* **Unaccounted**: " + ", ".join(self.unaccounted) + " — K1b FAILED.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
|
@ -351,8 +388,11 @@ class CorpusReport:
|
|||
"document whose table is a picture reached the bundle as text with a "
|
||||
"gap in it. Absence of an image here is not evidence the sources had none."
|
||||
)
|
||||
# With an account, "found" is what the SOURCES declare, so a refused
|
||||
# document's pictures are not reported as never having existed.
|
||||
found = self.assets_found if self.accounting is None else self.accounting.images_found
|
||||
return (
|
||||
f"* **Images**: {self.assets_carried} carried of {self.assets_found} found, "
|
||||
f"* **Images**: {self.assets_carried} carried of {found} found, "
|
||||
f"written to `{ASSETS_DIR}/` and pointed at from the concepts they stand in. "
|
||||
"The image BYTES were not screened: the gate above reads text, and a "
|
||||
"picture is not text."
|
||||
|
|
@ -420,9 +460,13 @@ def measure(
|
|||
concept_frontmatter_values: Mapping[str, str] | None = None,
|
||||
gate: str = GATE_NONE,
|
||||
assets: bool = False,
|
||||
account: bool = False,
|
||||
) -> CorpusReport:
|
||||
"""Run the corpus through the door and count what happened.
|
||||
|
||||
`account` adds the element-level content accounting (`accounting.py`):
|
||||
an inventory of every source and one fate per element, on the report.
|
||||
|
||||
Keyword-only with defaults, so the flat call that produced the published
|
||||
K1/K2 numbers stays source-compatible and byte-identical.
|
||||
"""
|
||||
|
|
@ -451,11 +495,13 @@ def measure(
|
|||
|
||||
merged_names = tuple(item.source_file for item in result.persisted)
|
||||
blocked = result.quarantined + result.rejected
|
||||
coded_names = tuple(item.source_file for item in result.failed) + tuple(
|
||||
carried = set(result.carried_files)
|
||||
failed = tuple(item for item in result.failed if item.source_file not in carried)
|
||||
coded_names = tuple(item.source_file for item in failed) + tuple(
|
||||
item.source_file for item in blocked
|
||||
)
|
||||
counts: dict[str, int] = {}
|
||||
for failure in result.failed:
|
||||
for failure in failed:
|
||||
counts[failure.error.code] = counts.get(failure.error.code, 0) + 1
|
||||
for item in blocked:
|
||||
counts[item.disposition] = counts.get(item.disposition, 0) + 1
|
||||
|
|
@ -478,11 +524,15 @@ def measure(
|
|||
converter_path=path,
|
||||
converter_version=version,
|
||||
codes=tuple(sorted(counts.items())),
|
||||
unaccounted=unaccounted_names(dropped=dropped, merged=merged_names, coded=coded_names),
|
||||
unaccounted=unaccounted_names(
|
||||
dropped=dropped, merged=merged_names + tuple(sorted(carried)), coded=coded_names
|
||||
),
|
||||
gate=gate,
|
||||
assets=assets,
|
||||
assets_carried=len(result.assets),
|
||||
assets_found=len(result.assets) + len(result.assets_rejected),
|
||||
carried_files=len(carried),
|
||||
accounting=account_run(corpus, walked, result) if account else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -633,10 +683,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||
bundle.mkdir(parents=True, exist_ok=True)
|
||||
(bundle / LOG_NAME).write_text(report.render_log(), encoding="utf-8", newline="")
|
||||
print(report.render())
|
||||
if report.unaccounted or report.merged + report.rejected != report.n:
|
||||
if report.conservation_failed:
|
||||
print(
|
||||
f"{HARNESS_ID}: K1b FAILED - merged ({report.merged}) + coded rejections "
|
||||
f"({report.rejected}) != N ({report.n}). Unaccounted: "
|
||||
f"{HARNESS_ID}: K1b FAILED - {report.identity()}. Unaccounted: "
|
||||
f"{', '.join(report.unaccounted) or '(none named)'}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -386,6 +386,12 @@ class ExtractedDocument:
|
|||
text: str
|
||||
images: tuple[ExtractedImage, ...] = ()
|
||||
rejected: tuple[AssetRejection, ...] = ()
|
||||
#: The references, relative to the document's own directory, whose bytes
|
||||
#: the resolver returned and that were CARRIED as images. Recorded where
|
||||
#: the resolution happened rather than inferred from bytes afterwards: an
|
||||
#: unpointed file with the same bytes as a carried one was carried through
|
||||
#: nothing (R761 ships eight such duplicates).
|
||||
files: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class _AssetCollector:
|
||||
|
|
@ -402,6 +408,7 @@ class _AssetCollector:
|
|||
self._resolve = resolve
|
||||
self.images: list[ExtractedImage] = []
|
||||
self.rejected: list[AssetRejection] = []
|
||||
self.files: list[str] = []
|
||||
|
||||
def carry(self, data: bytes, *, name: str, label: str | None = None) -> str:
|
||||
"""Bytes the reader already holds, as the block that stands in the text."""
|
||||
|
|
@ -445,8 +452,10 @@ class _AssetCollector:
|
|||
label=label,
|
||||
href=source,
|
||||
)
|
||||
found = source
|
||||
data = self._resolve(source) if self._resolve is not None else None
|
||||
if data is None and sibling is not None and sibling != source and self._resolve is not None:
|
||||
found = sibling
|
||||
data = self._resolve(sibling)
|
||||
if data is None:
|
||||
return self.reject(
|
||||
|
|
@ -455,7 +464,11 @@ class _AssetCollector:
|
|||
reason="the file the document points at was not found beside it",
|
||||
label=label,
|
||||
)
|
||||
return self.carry(data, name=source, label=label)
|
||||
carried = len(self.images)
|
||||
block = self.carry(data, name=source, label=label)
|
||||
if len(self.images) > carried:
|
||||
self.files.append(found)
|
||||
return block
|
||||
|
||||
def _data_uri(self, match: re.Match[str], *, label: str | None) -> str:
|
||||
payload = match.group("payload")
|
||||
|
|
@ -2408,4 +2421,5 @@ def extract_document(
|
|||
text=renderer(text) if renderer is not None else text,
|
||||
images=tuple(collector.images) if collector is not None else (),
|
||||
rejected=tuple(collector.rejected) if collector is not None else (),
|
||||
files=tuple(collector.files) if collector is not None else (),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from __future__ import annotations
|
|||
|
||||
import hashlib
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
|
|
@ -633,6 +634,24 @@ class InboxResult:
|
|||
# exactly like a run over documents that had none.
|
||||
assets: tuple[str, ...] = ()
|
||||
assets_rejected: tuple[AssetRejection, ...] = ()
|
||||
# Inbox files whose bytes a PERSISTED document carried as an image, as
|
||||
# inbox-relative paths. Such a file has one fate -- carried -- and is not
|
||||
# also a coded rejection of the walk; the conservation identity counts it
|
||||
# in its own column.
|
||||
carried_files: tuple[str, ...] = ()
|
||||
# Per persisted document: how many image placements were carried, and the
|
||||
# ones that were found and not carried, with their codes. The content
|
||||
# accounting books a document's images from this, never from the bundle.
|
||||
document_assets: tuple[DocumentAssets, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentAssets:
|
||||
"""One persisted document's image outcome."""
|
||||
|
||||
source_file: str
|
||||
carried: int
|
||||
rejected: tuple[AssetRejection, ...]
|
||||
|
||||
|
||||
def relative_source(path: Path, inbox: Path) -> str:
|
||||
|
|
@ -1087,6 +1106,8 @@ def process_inbox(
|
|||
# retire it.
|
||||
carried_assets: dict[str, bytes] = {}
|
||||
refused_assets: list[AssetRejection] = []
|
||||
carried_files: set[str] = set()
|
||||
document_assets: list[DocumentAssets] = []
|
||||
|
||||
# Phase 1: name every file BEFORE any gate call or write, so an intra-run
|
||||
# collision is caught while both files can still be refused together. Under
|
||||
|
|
@ -1391,6 +1412,19 @@ def process_inbox(
|
|||
# orphan no pointer names and no retirement pass reaches.
|
||||
_write_assets(bundle, document.images, carried_assets)
|
||||
refused_assets.extend(document.rejected)
|
||||
directory = PurePosixPath(source_name(path)).parent
|
||||
carried_files.update(
|
||||
posixpath.normpath((directory / reference).as_posix())
|
||||
for reference in document.files
|
||||
)
|
||||
if outputs:
|
||||
document_assets.append(
|
||||
DocumentAssets(
|
||||
source_file=source_name(path),
|
||||
carried=len(document.images),
|
||||
rejected=document.rejected,
|
||||
)
|
||||
)
|
||||
for target_name, content, reasons in outputs:
|
||||
# `write_bytes` resolves a subpath through `safe_resolve` but never
|
||||
# creates one. Without this the very first hierarchical write fails.
|
||||
|
|
@ -1449,6 +1483,8 @@ def process_inbox(
|
|||
skipped=skipped,
|
||||
assets=tuple(sorted(carried_assets)),
|
||||
assets_rejected=tuple(refused_assets),
|
||||
carried_files=tuple(sorted(carried_files)),
|
||||
document_assets=tuple(document_assets),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -190,10 +190,9 @@ def create(
|
|||
bundle_id=identity,
|
||||
okf_version=PROJECT_OKF_VERSION,
|
||||
)
|
||||
if report.unaccounted or report.merged + report.rejected != report.n:
|
||||
if report.conservation_failed:
|
||||
raise IngestError(
|
||||
f"K1b FAILED - merged ({report.merged}) + coded rejections "
|
||||
f"({report.rejected}) != N ({report.n}). Unaccounted: "
|
||||
f"K1b FAILED - {report.identity()}. Unaccounted: "
|
||||
f"{', '.join(report.unaccounted) or '(none named)'}",
|
||||
code="conservation_failed",
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue