okf-wiki/okfwiki/readers.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

429 lines
19 KiB
Python

"""A reader per file type: every unit of text in a document with its exact place. No model call (CLAUDE.md rule 1).
Usage:
python3 -m okfwiki.readers locate FILE QUERIES
Places, as the file-type gate defines them (gates/filetypes.manifest.json): pdf {"page": N}, the physical page;
docx {"paragraph": N}, the w:p elements of the body in order, empty ones and those in tables included, those in a text
box or a compatibility fallback not; xlsx {"sheet": name, "cell": "B7"}; pptx {"slide": N, "notes": bool}, the slide
list's order, hidden slides included; md, txt and html {"line": N}, for html the raw line of a visible text node
(not in head, title, script, style, noscript, template or svg). Numbers are 1-based.
Hidden Office content (a hidden sheet, row or column, a hidden slide, hidden text, a tracked deletion) is kept apart
from what a reader of the document sees. `locate` marks it hidden; `render` leaves it out and counts it.
Office files are read from their package XML with the standard library. PDF needs pypdf at exactly PDF_PIN, because
the quote check needs the same raw text on every run (rule decisions R9 and R37); any other version is refused.
locate FILE and a JSON file {"file", "queries": [{"id", "quote"}]}: prints {"answers": [{"id", "found":
[{"locator", "hidden"}]}]}, every place where the quote occurs, runs of whitespace collapsed to one space.
Exit 0 = done, 2 = could not run.
"""
from __future__ import annotations
import argparse
import html.parser
import json
import logging
import posixpath
import re
import sys
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from xml.etree import ElementTree
PDF_PIN = "6.14.2"
OFFICE = (".docx", ".xlsx", ".pptx")
TYPES = (".pdf",) + OFFICE
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
P = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
A = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}"
MC_FALLBACK = "{http://schemas.openxmlformats.org/markup-compatibility/2006}Fallback"
SPACE = re.compile(r"\s+")
# A line of a rendering that Markdown would read as structure gets a backslash, so a paragraph is always a paragraph.
STRUCTURE = re.compile(r"^(#|```|~~~|[-*+] |\d+[.)] |>|\||<)")
# A sheet name heads its rows as their section, below the levels the skeleton proposes as concepts (title, 2 and 3):
# "T1" or "Note" is a container, not a concept (trial of 07.11).
SHEET_LEVEL = 4
# A run of spreadsheet rows becomes one block until it holds about this many characters; each further block starts
# with the run's first row again (most often its header), so a number far down a long table keeps its column names.
# On the 22 real workbooks of the mixed set, one run without a limit held 1,190 rows and 1.07 million characters.
ROW_BLOCK_CHARS = 4000
STYLE_HEADING = re.compile(r"^heading ([1-6])$")
class NoReader(Exception):
"""The reader this file type needs is missing or not the pinned version."""
class Unreadable(Exception):
"""The file is not a sound file of its type."""
@dataclass(frozen=True)
class Unit:
locator: dict[str, Any]
visible: str # what a reader of the document sees, whitespace collapsed
text: str # visible plus hidden content, whitespace collapsed
heading: int = 0 # the Markdown heading level the unit renders as, 0 for none
group: str = "" # units of one group render as one block (a run of spreadsheet rows)
line: str = "" # units of one line render as one line, each after its column letter (a spreadsheet row)
@dataclass(frozen=True)
class Rendering:
text: str
locators: list[dict[str, Any] | None] # the place of line N at index N-1, None for a blank line or a heading
left_out: int # units with hidden content, left out of the text
def collapse(text: str) -> str:
return SPACE.sub(" ", text).strip()
def xml(z: zipfile.ZipFile, part: str) -> ElementTree.Element:
try:
return ElementTree.fromstring(z.read(part))
except KeyError as e:
raise Unreadable(f"the package has no {part}") from e
except ElementTree.ParseError as e:
raise Unreadable(f"{part} is not XML: {e}") from e
def targets(z: zipfile.ZipFile, part: str) -> dict[str, str]:
"""Relationship id -> package part, for the relationships of part."""
folder, name = posixpath.split(part)
rels = posixpath.join(folder, "_rels", name + ".rels")
if rels not in z.namelist():
return {}
found: dict[str, str] = {}
for rel in xml(z, rels).iter(f"{PKG}Relationship"):
target = rel.get("Target", "")
if rel.get("TargetMode") == "External":
continue
found[rel.get("Id", "")] = (target[1:] if target.startswith("/")
else posixpath.normpath(posixpath.join(folder, target)))
return found
def walk(node: ElementTree.Element, skip: tuple[str, ...]) -> list[ElementTree.Element]:
"""node and its descendants in document order, without the subtrees of the tags in skip."""
out = [node]
for child in node:
if child.tag not in skip:
out += walk(child, skip)
return out
def docx_units(z: zipfile.ZipFile) -> list[Unit]:
levels: dict[str, int] = {}
if "word/styles.xml" in z.namelist():
for style in xml(z, "word/styles.xml").iter(f"{W}style"):
name = style.find(f"{W}name")
m = STYLE_HEADING.match((name.get(f"{W}val", "") if name is not None else "").lower())
if m or (name is not None and name.get(f"{W}val", "").lower() == "title"):
levels[style.get(f"{W}styleId", "")] = int(m.group(1)) if m else 1
body = xml(z, "word/document.xml").find(f"{W}body")
if body is None:
raise Unreadable("word/document.xml has no body")
box = (f"{W}txbxContent", MC_FALLBACK)
units: list[Unit] = []
for p in (e for e in walk(body, box) if e.tag == f"{W}p"):
visible: list[str] = []
full: list[str] = []
for run in (e for e in walk(p, box) if e.tag == f"{W}r"):
props = run.find(f"{W}rPr")
vanish = props.find(f"{W}vanish") if props is not None else None
hidden = vanish is not None and vanish.get(f"{W}val", "true") not in ("0", "false")
for e in run:
piece = (e.text or "" if e.tag in (f"{W}t", f"{W}delText") else
"\t" if e.tag == f"{W}tab" else " " if e.tag in (f"{W}br", f"{W}cr") else "")
full.append(piece)
if not hidden and e.tag != f"{W}delText":
visible.append(piece)
mark = p.find(f"{W}pPr/{W}pStyle")
level = levels.get(mark.get(f"{W}val", "") if mark is not None else "", 0)
units.append(Unit({"paragraph": len(units) + 1}, collapse("".join(visible)), collapse("".join(full)), level))
return units
def cell_text(cell: ElementTree.Element, shared: list[str]) -> str:
kind = cell.get("t", "n")
if kind == "inlineStr":
node = cell.find(f"{S}is")
return "" if node is None else string_item(node)
value = cell.find(f"{S}v")
raw = "" if value is None or value.text is None else value.text
if kind == "s":
try:
return shared[int(raw)]
except (ValueError, IndexError) as e:
raise Unreadable(f"cell {cell.get('r')} names shared string {raw!r}, which does not exist") from e
if kind == "b":
return {"1": "TRUE", "0": "FALSE"}.get(raw, raw)
return raw
def string_item(si: ElementTree.Element) -> str:
"""The text of a shared or inline string, rich runs joined, phonetic runs left out."""
return "".join(t.text or "" for t in walk(si, (f"{S}rPh",)) if t.tag == f"{S}t")
def letters(ref: str) -> str:
"""The column letters of a cell reference: "AB" of "AB12"."""
return re.match(r"[A-Za-z]*", ref).group(0) # type: ignore[union-attr]
def column(ref: str) -> int:
n = 0
for ch in letters(ref).upper():
n = n * 26 + ord(ch) - 64
return n
def xlsx_units(z: zipfile.ZipFile) -> list[Unit]:
shared = ([string_item(si) for si in xml(z, "xl/sharedStrings.xml").iter(f"{S}si")]
if "xl/sharedStrings.xml" in z.namelist() else [])
parts = targets(z, "xl/workbook.xml")
units: list[Unit] = []
for sheet in xml(z, "xl/workbook.xml").iter(f"{S}sheet"):
name = sheet.get("name", "")
part = parts.get(sheet.get(f"{R}id", ""))
if part is None:
raise Unreadable(f"sheet {name!r} has no worksheet part")
root = xml(z, part)
sheet_hidden = sheet.get("state", "visible") != "visible"
hidden_cols: set[int] = set()
for col in root.iter(f"{S}col"):
if col.get("hidden") in ("1", "true"):
hidden_cols.update(range(int(col.get("min", "0")), int(col.get("max", "0")) + 1))
first = True
block, last = 0, 0 # an empty row between two rows ends a block; a hidden row does not
for row in root.iter(f"{S}row"):
row_hidden = row.get("hidden") in ("1", "true")
number = int(row.get("r", "0") or "0")
shown = not row_hidden and any(collapse(cell_text(c, shared)) for c in row.iter(f"{S}c"))
if shown:
if number != last + 1:
block += 1
last = number
elif row_hidden and number == last + 1:
last = number
for cell in row.iter(f"{S}c"):
ref = cell.get("r", "")
text = collapse(cell_text(cell, shared))
if not text:
continue
hidden = sheet_hidden or row_hidden or column(ref) in hidden_cols
heading = name if first and not sheet_hidden else ""
if heading:
units.append(Unit({"sheet": name}, heading, heading, SHEET_LEVEL))
first = False
units.append(Unit({"sheet": name, "cell": ref}, "" if hidden else text, text, 0,
f"{name}\n{block}", f"{name}\n{row.get('r', '')}"))
return units
def slide_paragraphs(root: ElementTree.Element) -> list[tuple[str, bool]]:
"""(text, is the title) of every paragraph in a slide or notes part, in document order."""
out: list[tuple[str, bool]] = []
for shape in (e for e in walk(root, (MC_FALLBACK,)) if e.tag == f"{P}sp" or e.tag == f"{P}graphicFrame"):
ph = shape.find(f"{P}nvSpPr/{P}nvPr/{P}ph")
if ph is not None and ph.get("type") in ("sldNum", "sldImg", "dt", "ftr", "hdr"):
continue
title = ph is not None and ph.get("type") in ("title", "ctrTitle")
for para in shape.iter(f"{A}p"):
text = "".join((e.text or "") if e.tag == f"{A}t" else " " if e.tag == f"{A}br" else ""
for e in para.iter())
out.append((collapse(text), title))
return out
def pptx_units(z: zipfile.ZipFile) -> list[Unit]:
parts = targets(z, "ppt/presentation.xml")
units: list[Unit] = []
for n, sld in enumerate(xml(z, "ppt/presentation.xml").iter(f"{P}sldId"), 1):
part = parts.get(sld.get(f"{R}id", ""))
if part is None:
raise Unreadable(f"slide {n} has no part")
root = xml(z, part)
hidden = root.get("show") in ("0", "false")
for text, title in slide_paragraphs(root):
if text:
units.append(Unit({"slide": n, "notes": False}, "" if hidden else text, text, 2 if title else 0))
notes = [t for t in targets(z, part).values() if "/notesSlides/" in "/" + t]
for note in notes[:1]:
for text, _ in slide_paragraphs(xml(z, note)):
if text:
units.append(Unit({"slide": n, "notes": True}, "" if hidden else text, text))
return units
def pdf_units(path: Path) -> list[Unit]:
try:
import pypdf # type: ignore[import-not-found, unused-ignore]
except ImportError as e:
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN} (python3 -m pip install pypdf=={PDF_PIN})") from e
logging.getLogger("pypdf").setLevel(logging.ERROR) # repairs of a sloppy file are not the user's business
if pypdf.__version__ != PDF_PIN:
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN}, found {pypdf.__version__} "
f"(python3 -m pip install pypdf=={PDF_PIN})")
try:
doc = pypdf.PdfReader(str(path))
if doc.is_encrypted and not doc.decrypt(""):
raise Unreadable(f"{path.name} is encrypted")
texts = [page.extract_text() or "" for page in doc.pages]
except Unreadable:
raise
except Exception as e: # pypdf raises many kinds on a broken file
raise Unreadable(f"{path.name} is not a readable PDF: {e}") from e
return [Unit({"page": n}, collapse(t), collapse(t)) for n, t in enumerate(texts, 1)]
INVISIBLE = frozenset({"head", "title", "script", "style", "noscript", "template", "svg"})
class TextNodes(html.parser.HTMLParser):
"""Every visible text node of an HTML page, cut at its line breaks: (line of the raw source, text)."""
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.open: list[str] = []
self.found: list[tuple[int, str]] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
if tag in INVISIBLE:
self.open.append(tag)
def handle_endtag(self, tag: str) -> None:
if tag in self.open:
del self.open[len(self.open) - 1 - self.open[::-1].index(tag):]
def handle_data(self, data: str) -> None:
if not self.open:
line = self.getpos()[0]
self.found += [(line + i, piece) for i, piece in enumerate(data.split("\n"))]
def html_units(path: Path) -> list[Unit]:
parser = TextNodes()
try:
parser.feed(path.read_text(encoding="utf-8"))
parser.close()
except UnicodeDecodeError as e:
raise Unreadable(f"{path.name} is not UTF-8") from e
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in parser.found if t.strip()]
def text_units(path: Path) -> list[Unit]:
try:
lines = path.read_text(encoding="utf-8").split("\n")
except UnicodeDecodeError as e:
raise Unreadable(f"{path.name} is not UTF-8") from e
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in enumerate(lines, 1)]
def units(path: Path) -> list[Unit]:
ext = path.suffix.lower()
if ext == ".pdf":
return pdf_units(path)
if ext in (".html", ".htm"):
return html_units(path)
if ext not in OFFICE:
return text_units(path)
try:
with zipfile.ZipFile(path) as z:
return {".docx": docx_units, ".xlsx": xlsx_units, ".pptx": pptx_units}[ext](z)
except zipfile.BadZipFile as e:
raise Unreadable(f"{path.name} is not an Office package: {e}") from e
def render(path: Path) -> Rendering:
"""The text a reader of the document sees, as Markdown the block splitter reads: a heading line per heading unit,
one line per unit, a blank line between blocks (a unit, or a run of spreadsheet rows). A spreadsheet row is one
line, each cell after its column letter ("A: Quantity | C: 101000"), so a number keeps the column its header
names; its place lists the cells with the offset each starts at (common.place_at). Hidden units are left out."""
lines: list[str] = []
places: list[dict[str, Any] | None] = []
left_out = 0
group: str | None = None
line: str | None = None
block, first_row = 0, 0 # where the current block, and the current run of rows, start in lines
for u in units(path):
if u.visible != u.text:
left_out += 1
if not u.visible:
continue
if u.line and u.line == line:
lines[-1] += " | "
places[-1]["cells"].append([u.locator["cell"], len(lines[-1])]) # type: ignore[index]
lines[-1] += f"{letters(u.locator['cell'])}: {u.visible}"
continue
same_run = bool(u.group and u.group == group)
if lines and not same_run:
lines.append("")
places.append(None)
block = first_row = len(lines)
elif same_run and u.line and sum(len(x) for x in lines[block:]) >= ROW_BLOCK_CHARS:
head, at = lines[first_row], places[first_row] or {}
lines += ["", head]
places += [None, {"sheet": at.get("sheet"), "cells": [list(c) for c in at.get("cells", [])]}]
block = len(lines) - 1
group, line = u.group or None, u.line or None
if u.line:
lines.append(f"{letters(u.locator['cell'])}: {u.visible}")
places.append({"sheet": u.locator["sheet"], "cells": [[u.locator["cell"], 0]]})
elif u.heading:
lines.append("#" * u.heading + " " + u.visible)
places.append(None)
else:
lines.append(("\\" + u.visible) if STRUCTURE.match(u.visible) else u.visible)
places.append(u.locator)
return Rendering("\n".join(lines) + "\n" if lines else "", places, left_out)
def locate(path: Path, quotes: list[str]) -> list[list[tuple[dict[str, Any], bool]]]:
"""For each quote, every (place, hidden) where it occurs, in the order of the document."""
found = units(path)
out: list[list[tuple[dict[str, Any], bool]]] = []
for quote in quotes:
q = collapse(quote)
places: list[tuple[dict[str, Any], bool]] = []
for u in found:
if "cell" not in u.locator and "sheet" in u.locator:
continue # a sheet's own heading unit is not a place in the workbook
if q and q in u.visible:
places.append((u.locator, False))
elif q and q in u.text:
places.append((u.locator, True))
out.append(places)
return out
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="python3 -m okfwiki.readers", description=__doc__.split("\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
p = sub.add_parser("locate")
p.add_argument("file", type=Path)
p.add_argument("queries", type=Path)
args = parser.parse_args(argv)
try:
doc = json.loads(args.queries.read_text(encoding="utf-8"))
queries = [(q["id"], q["quote"]) for q in doc["queries"]]
answers = locate(args.file, [quote for _, quote in queries])
except (OSError, ValueError, KeyError, TypeError, NoReader, Unreadable) as e:
print(f"readers cannot run: {e}", file=sys.stderr)
return 2
print(json.dumps({"answers": [{"id": i, "found": [{"locator": loc, "hidden": h} for loc, h in places]}
for (i, _), places in zip(queries, answers)]}, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())