"""A reader per file type: every unit of text in a document with its exact place. No model call (CLAUDE.md rule 1). Usage: python3 -m okfwiki.readers locate FILE QUERIES Places, as the file-type gate defines them (gates/filetypes.manifest.json): pdf {"page": N}, the physical page; docx {"paragraph": N}, the w:p elements of the body in order, empty ones and those in tables included, those in a text box or a compatibility fallback not; xlsx {"sheet": name, "cell": "B7"}; pptx {"slide": N, "notes": bool}, the slide list's order, hidden slides included; md, txt and html {"line": N}, for html the raw line of a visible text node (not in head, title, script, style, noscript, template or svg). Numbers are 1-based. Hidden Office content (a hidden sheet, row or column, a hidden slide, hidden text, a tracked deletion) is kept apart from what a reader of the document sees. `locate` marks it hidden; `render` leaves it out and counts it. Office files are read from their package XML with the standard library. PDF needs pypdf at exactly PDF_PIN, because the quote check needs the same raw text on every run (rule decisions R9 and R37); any other version is refused. locate FILE and a JSON file {"file", "queries": [{"id", "quote"}]}: prints {"answers": [{"id", "found": [{"locator", "hidden"}]}]}, every place where the quote occurs, runs of whitespace collapsed to one space. Exit 0 = done, 2 = could not run. """ from __future__ import annotations import argparse import html.parser import json import logging import posixpath import re import sys import zipfile from dataclasses import dataclass from pathlib import Path from typing import Any from xml.etree import ElementTree PDF_PIN = "6.14.2" OFFICE = (".docx", ".xlsx", ".pptx") TYPES = (".pdf",) + OFFICE W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" P = "{http://schemas.openxmlformats.org/presentationml/2006/main}" A = "{http://schemas.openxmlformats.org/drawingml/2006/main}" R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}" PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}" MC_FALLBACK = "{http://schemas.openxmlformats.org/markup-compatibility/2006}Fallback" SPACE = re.compile(r"\s+") # A line of a rendering that Markdown would read as structure gets a backslash, so a paragraph is always a paragraph. STRUCTURE = re.compile(r"^(#|```|~~~|[-*+] |\d+[.)] |>|\||<)") # A sheet name heads its rows as their section, below the levels the skeleton proposes as concepts (title, 2 and 3): # "T1" or "Note" is a container, not a concept (trial of 07.11). SHEET_LEVEL = 4 # A run of spreadsheet rows becomes one block until it holds about this many characters; each further block starts # with the run's first row again (most often its header), so a number far down a long table keeps its column names. # On the 22 real workbooks of the mixed set, one run without a limit held 1,190 rows and 1.07 million characters. ROW_BLOCK_CHARS = 4000 STYLE_HEADING = re.compile(r"^heading ([1-6])$") class NoReader(Exception): """The reader this file type needs is missing or not the pinned version.""" class Unreadable(Exception): """The file is not a sound file of its type.""" @dataclass(frozen=True) class Unit: locator: dict[str, Any] visible: str # what a reader of the document sees, whitespace collapsed text: str # visible plus hidden content, whitespace collapsed heading: int = 0 # the Markdown heading level the unit renders as, 0 for none group: str = "" # units of one group render as one block (a run of spreadsheet rows) line: str = "" # units of one line render as one line, each after its column letter (a spreadsheet row) @dataclass(frozen=True) class Rendering: text: str locators: list[dict[str, Any] | None] # the place of line N at index N-1, None for a blank line or a heading left_out: int # units with hidden content, left out of the text def collapse(text: str) -> str: return SPACE.sub(" ", text).strip() def xml(z: zipfile.ZipFile, part: str) -> ElementTree.Element: try: return ElementTree.fromstring(z.read(part)) except KeyError as e: raise Unreadable(f"the package has no {part}") from e except ElementTree.ParseError as e: raise Unreadable(f"{part} is not XML: {e}") from e def targets(z: zipfile.ZipFile, part: str) -> dict[str, str]: """Relationship id -> package part, for the relationships of part.""" folder, name = posixpath.split(part) rels = posixpath.join(folder, "_rels", name + ".rels") if rels not in z.namelist(): return {} found: dict[str, str] = {} for rel in xml(z, rels).iter(f"{PKG}Relationship"): target = rel.get("Target", "") if rel.get("TargetMode") == "External": continue found[rel.get("Id", "")] = (target[1:] if target.startswith("/") else posixpath.normpath(posixpath.join(folder, target))) return found def walk(node: ElementTree.Element, skip: tuple[str, ...]) -> list[ElementTree.Element]: """node and its descendants in document order, without the subtrees of the tags in skip.""" out = [node] for child in node: if child.tag not in skip: out += walk(child, skip) return out def docx_units(z: zipfile.ZipFile) -> list[Unit]: levels: dict[str, int] = {} if "word/styles.xml" in z.namelist(): for style in xml(z, "word/styles.xml").iter(f"{W}style"): name = style.find(f"{W}name") m = STYLE_HEADING.match((name.get(f"{W}val", "") if name is not None else "").lower()) if m or (name is not None and name.get(f"{W}val", "").lower() == "title"): levels[style.get(f"{W}styleId", "")] = int(m.group(1)) if m else 1 body = xml(z, "word/document.xml").find(f"{W}body") if body is None: raise Unreadable("word/document.xml has no body") box = (f"{W}txbxContent", MC_FALLBACK) units: list[Unit] = [] for p in (e for e in walk(body, box) if e.tag == f"{W}p"): visible: list[str] = [] full: list[str] = [] for run in (e for e in walk(p, box) if e.tag == f"{W}r"): props = run.find(f"{W}rPr") vanish = props.find(f"{W}vanish") if props is not None else None hidden = vanish is not None and vanish.get(f"{W}val", "true") not in ("0", "false") for e in run: piece = (e.text or "" if e.tag in (f"{W}t", f"{W}delText") else "\t" if e.tag == f"{W}tab" else " " if e.tag in (f"{W}br", f"{W}cr") else "") full.append(piece) if not hidden and e.tag != f"{W}delText": visible.append(piece) mark = p.find(f"{W}pPr/{W}pStyle") level = levels.get(mark.get(f"{W}val", "") if mark is not None else "", 0) units.append(Unit({"paragraph": len(units) + 1}, collapse("".join(visible)), collapse("".join(full)), level)) return units def cell_text(cell: ElementTree.Element, shared: list[str]) -> str: kind = cell.get("t", "n") if kind == "inlineStr": node = cell.find(f"{S}is") return "" if node is None else string_item(node) value = cell.find(f"{S}v") raw = "" if value is None or value.text is None else value.text if kind == "s": try: return shared[int(raw)] except (ValueError, IndexError) as e: raise Unreadable(f"cell {cell.get('r')} names shared string {raw!r}, which does not exist") from e if kind == "b": return {"1": "TRUE", "0": "FALSE"}.get(raw, raw) return raw def string_item(si: ElementTree.Element) -> str: """The text of a shared or inline string, rich runs joined, phonetic runs left out.""" return "".join(t.text or "" for t in walk(si, (f"{S}rPh",)) if t.tag == f"{S}t") def letters(ref: str) -> str: """The column letters of a cell reference: "AB" of "AB12".""" return re.match(r"[A-Za-z]*", ref).group(0) # type: ignore[union-attr] def column(ref: str) -> int: n = 0 for ch in letters(ref).upper(): n = n * 26 + ord(ch) - 64 return n def xlsx_units(z: zipfile.ZipFile) -> list[Unit]: shared = ([string_item(si) for si in xml(z, "xl/sharedStrings.xml").iter(f"{S}si")] if "xl/sharedStrings.xml" in z.namelist() else []) parts = targets(z, "xl/workbook.xml") units: list[Unit] = [] for sheet in xml(z, "xl/workbook.xml").iter(f"{S}sheet"): name = sheet.get("name", "") part = parts.get(sheet.get(f"{R}id", "")) if part is None: raise Unreadable(f"sheet {name!r} has no worksheet part") root = xml(z, part) sheet_hidden = sheet.get("state", "visible") != "visible" hidden_cols: set[int] = set() for col in root.iter(f"{S}col"): if col.get("hidden") in ("1", "true"): hidden_cols.update(range(int(col.get("min", "0")), int(col.get("max", "0")) + 1)) first = True block, last = 0, 0 # an empty row between two rows ends a block; a hidden row does not for row in root.iter(f"{S}row"): row_hidden = row.get("hidden") in ("1", "true") number = int(row.get("r", "0") or "0") shown = not row_hidden and any(collapse(cell_text(c, shared)) for c in row.iter(f"{S}c")) if shown: if number != last + 1: block += 1 last = number elif row_hidden and number == last + 1: last = number for cell in row.iter(f"{S}c"): ref = cell.get("r", "") text = collapse(cell_text(cell, shared)) if not text: continue hidden = sheet_hidden or row_hidden or column(ref) in hidden_cols heading = name if first and not sheet_hidden else "" if heading: units.append(Unit({"sheet": name}, heading, heading, SHEET_LEVEL)) first = False units.append(Unit({"sheet": name, "cell": ref}, "" if hidden else text, text, 0, f"{name}\n{block}", f"{name}\n{row.get('r', '')}")) return units def slide_paragraphs(root: ElementTree.Element) -> list[tuple[str, bool]]: """(text, is the title) of every paragraph in a slide or notes part, in document order.""" out: list[tuple[str, bool]] = [] for shape in (e for e in walk(root, (MC_FALLBACK,)) if e.tag == f"{P}sp" or e.tag == f"{P}graphicFrame"): ph = shape.find(f"{P}nvSpPr/{P}nvPr/{P}ph") if ph is not None and ph.get("type") in ("sldNum", "sldImg", "dt", "ftr", "hdr"): continue title = ph is not None and ph.get("type") in ("title", "ctrTitle") for para in shape.iter(f"{A}p"): text = "".join((e.text or "") if e.tag == f"{A}t" else " " if e.tag == f"{A}br" else "" for e in para.iter()) out.append((collapse(text), title)) return out def pptx_units(z: zipfile.ZipFile) -> list[Unit]: parts = targets(z, "ppt/presentation.xml") units: list[Unit] = [] for n, sld in enumerate(xml(z, "ppt/presentation.xml").iter(f"{P}sldId"), 1): part = parts.get(sld.get(f"{R}id", "")) if part is None: raise Unreadable(f"slide {n} has no part") root = xml(z, part) hidden = root.get("show") in ("0", "false") for text, title in slide_paragraphs(root): if text: units.append(Unit({"slide": n, "notes": False}, "" if hidden else text, text, 2 if title else 0)) notes = [t for t in targets(z, part).values() if "/notesSlides/" in "/" + t] for note in notes[:1]: for text, _ in slide_paragraphs(xml(z, note)): if text: units.append(Unit({"slide": n, "notes": True}, "" if hidden else text, text)) return units def pdf_units(path: Path) -> list[Unit]: try: import pypdf # type: ignore[import-not-found, unused-ignore] except ImportError as e: raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN} (python3 -m pip install pypdf=={PDF_PIN})") from e logging.getLogger("pypdf").setLevel(logging.ERROR) # repairs of a sloppy file are not the user's business if pypdf.__version__ != PDF_PIN: raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN}, found {pypdf.__version__} " f"(python3 -m pip install pypdf=={PDF_PIN})") try: doc = pypdf.PdfReader(str(path)) if doc.is_encrypted and not doc.decrypt(""): raise Unreadable(f"{path.name} is encrypted") texts = [page.extract_text() or "" for page in doc.pages] except Unreadable: raise except Exception as e: # pypdf raises many kinds on a broken file raise Unreadable(f"{path.name} is not a readable PDF: {e}") from e return [Unit({"page": n}, collapse(t), collapse(t)) for n, t in enumerate(texts, 1)] INVISIBLE = frozenset({"head", "title", "script", "style", "noscript", "template", "svg"}) class TextNodes(html.parser.HTMLParser): """Every visible text node of an HTML page, cut at its line breaks: (line of the raw source, text).""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.open: list[str] = [] self.found: list[tuple[int, str]] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag in INVISIBLE: self.open.append(tag) def handle_endtag(self, tag: str) -> None: if tag in self.open: del self.open[len(self.open) - 1 - self.open[::-1].index(tag):] def handle_data(self, data: str) -> None: if not self.open: line = self.getpos()[0] self.found += [(line + i, piece) for i, piece in enumerate(data.split("\n"))] def html_units(path: Path) -> list[Unit]: parser = TextNodes() try: parser.feed(path.read_text(encoding="utf-8")) parser.close() except UnicodeDecodeError as e: raise Unreadable(f"{path.name} is not UTF-8") from e return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in parser.found if t.strip()] def text_units(path: Path) -> list[Unit]: try: lines = path.read_text(encoding="utf-8").split("\n") except UnicodeDecodeError as e: raise Unreadable(f"{path.name} is not UTF-8") from e return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in enumerate(lines, 1)] def units(path: Path) -> list[Unit]: ext = path.suffix.lower() if ext == ".pdf": return pdf_units(path) if ext in (".html", ".htm"): return html_units(path) if ext not in OFFICE: return text_units(path) try: with zipfile.ZipFile(path) as z: return {".docx": docx_units, ".xlsx": xlsx_units, ".pptx": pptx_units}[ext](z) except zipfile.BadZipFile as e: raise Unreadable(f"{path.name} is not an Office package: {e}") from e def render(path: Path) -> Rendering: """The text a reader of the document sees, as Markdown the block splitter reads: a heading line per heading unit, one line per unit, a blank line between blocks (a unit, or a run of spreadsheet rows). A spreadsheet row is one line, each cell after its column letter ("A: Quantity | C: 101000"), so a number keeps the column its header names; its place lists the cells with the offset each starts at (common.place_at). Hidden units are left out.""" lines: list[str] = [] places: list[dict[str, Any] | None] = [] left_out = 0 group: str | None = None line: str | None = None block, first_row = 0, 0 # where the current block, and the current run of rows, start in lines for u in units(path): if u.visible != u.text: left_out += 1 if not u.visible: continue if u.line and u.line == line: lines[-1] += " | " places[-1]["cells"].append([u.locator["cell"], len(lines[-1])]) # type: ignore[index] lines[-1] += f"{letters(u.locator['cell'])}: {u.visible}" continue same_run = bool(u.group and u.group == group) if lines and not same_run: lines.append("") places.append(None) block = first_row = len(lines) elif same_run and u.line and sum(len(x) for x in lines[block:]) >= ROW_BLOCK_CHARS: head, at = lines[first_row], places[first_row] or {} lines += ["", head] places += [None, {"sheet": at.get("sheet"), "cells": [list(c) for c in at.get("cells", [])]}] block = len(lines) - 1 group, line = u.group or None, u.line or None if u.line: lines.append(f"{letters(u.locator['cell'])}: {u.visible}") places.append({"sheet": u.locator["sheet"], "cells": [[u.locator["cell"], 0]]}) elif u.heading: lines.append("#" * u.heading + " " + u.visible) places.append(None) else: lines.append(("\\" + u.visible) if STRUCTURE.match(u.visible) else u.visible) places.append(u.locator) return Rendering("\n".join(lines) + "\n" if lines else "", places, left_out) def locate(path: Path, quotes: list[str]) -> list[list[tuple[dict[str, Any], bool]]]: """For each quote, every (place, hidden) where it occurs, in the order of the document.""" found = units(path) out: list[list[tuple[dict[str, Any], bool]]] = [] for quote in quotes: q = collapse(quote) places: list[tuple[dict[str, Any], bool]] = [] for u in found: if "cell" not in u.locator and "sheet" in u.locator: continue # a sheet's own heading unit is not a place in the workbook if q and q in u.visible: places.append((u.locator, False)) elif q and q in u.text: places.append((u.locator, True)) out.append(places) return out def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="python3 -m okfwiki.readers", description=__doc__.split("\n")[0]) sub = parser.add_subparsers(dest="command", required=True) p = sub.add_parser("locate") p.add_argument("file", type=Path) p.add_argument("queries", type=Path) args = parser.parse_args(argv) try: doc = json.loads(args.queries.read_text(encoding="utf-8")) queries = [(q["id"], q["quote"]) for q in doc["queries"]] answers = locate(args.file, [quote for _, quote in queries]) except (OSError, ValueError, KeyError, TypeError, NoReader, Unreadable) as e: print(f"readers cannot run: {e}", file=sys.stderr) return 2 print(json.dumps({"answers": [{"id": i, "found": [{"locator": loc, "hidden": h} for loc, h in places]} for (i, _), places in zip(queries, answers)]}, ensure_ascii=False)) return 0 if __name__ == "__main__": sys.exit(main())