First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
429 lines
19 KiB
Python
429 lines
19 KiB
Python
"""A reader per file type: every unit of text in a document with its exact place. No model call (CLAUDE.md rule 1).
|
|
|
|
Usage:
|
|
python3 -m okfwiki.readers locate FILE QUERIES
|
|
|
|
Places, as the file-type gate defines them (gates/filetypes.manifest.json): pdf {"page": N}, the physical page;
|
|
docx {"paragraph": N}, the w:p elements of the body in order, empty ones and those in tables included, those in a text
|
|
box or a compatibility fallback not; xlsx {"sheet": name, "cell": "B7"}; pptx {"slide": N, "notes": bool}, the slide
|
|
list's order, hidden slides included; md, txt and html {"line": N}, for html the raw line of a visible text node
|
|
(not in head, title, script, style, noscript, template or svg). Numbers are 1-based.
|
|
|
|
Hidden Office content (a hidden sheet, row or column, a hidden slide, hidden text, a tracked deletion) is kept apart
|
|
from what a reader of the document sees. `locate` marks it hidden; `render` leaves it out and counts it.
|
|
|
|
Office files are read from their package XML with the standard library. PDF needs pypdf at exactly PDF_PIN, because
|
|
the quote check needs the same raw text on every run (rule decisions R9 and R37); any other version is refused.
|
|
|
|
locate FILE and a JSON file {"file", "queries": [{"id", "quote"}]}: prints {"answers": [{"id", "found":
|
|
[{"locator", "hidden"}]}]}, every place where the quote occurs, runs of whitespace collapsed to one space.
|
|
Exit 0 = done, 2 = could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import html.parser
|
|
import json
|
|
import logging
|
|
import posixpath
|
|
import re
|
|
import sys
|
|
import zipfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from xml.etree import ElementTree
|
|
|
|
PDF_PIN = "6.14.2"
|
|
OFFICE = (".docx", ".xlsx", ".pptx")
|
|
TYPES = (".pdf",) + OFFICE
|
|
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
|
S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
|
P = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
|
|
A = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
|
|
R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
|
PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
|
MC_FALLBACK = "{http://schemas.openxmlformats.org/markup-compatibility/2006}Fallback"
|
|
SPACE = re.compile(r"\s+")
|
|
# A line of a rendering that Markdown would read as structure gets a backslash, so a paragraph is always a paragraph.
|
|
STRUCTURE = re.compile(r"^(#|```|~~~|[-*+] |\d+[.)] |>|\||<)")
|
|
# A sheet name heads its rows as their section, below the levels the skeleton proposes as concepts (title, 2 and 3):
|
|
# "T1" or "Note" is a container, not a concept (trial of 07.11).
|
|
SHEET_LEVEL = 4
|
|
# A run of spreadsheet rows becomes one block until it holds about this many characters; each further block starts
|
|
# with the run's first row again (most often its header), so a number far down a long table keeps its column names.
|
|
# On the 22 real workbooks of the mixed set, one run without a limit held 1,190 rows and 1.07 million characters.
|
|
ROW_BLOCK_CHARS = 4000
|
|
STYLE_HEADING = re.compile(r"^heading ([1-6])$")
|
|
|
|
|
|
class NoReader(Exception):
|
|
"""The reader this file type needs is missing or not the pinned version."""
|
|
|
|
|
|
class Unreadable(Exception):
|
|
"""The file is not a sound file of its type."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Unit:
|
|
locator: dict[str, Any]
|
|
visible: str # what a reader of the document sees, whitespace collapsed
|
|
text: str # visible plus hidden content, whitespace collapsed
|
|
heading: int = 0 # the Markdown heading level the unit renders as, 0 for none
|
|
group: str = "" # units of one group render as one block (a run of spreadsheet rows)
|
|
line: str = "" # units of one line render as one line, each after its column letter (a spreadsheet row)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Rendering:
|
|
text: str
|
|
locators: list[dict[str, Any] | None] # the place of line N at index N-1, None for a blank line or a heading
|
|
left_out: int # units with hidden content, left out of the text
|
|
|
|
|
|
def collapse(text: str) -> str:
|
|
return SPACE.sub(" ", text).strip()
|
|
|
|
|
|
def xml(z: zipfile.ZipFile, part: str) -> ElementTree.Element:
|
|
try:
|
|
return ElementTree.fromstring(z.read(part))
|
|
except KeyError as e:
|
|
raise Unreadable(f"the package has no {part}") from e
|
|
except ElementTree.ParseError as e:
|
|
raise Unreadable(f"{part} is not XML: {e}") from e
|
|
|
|
|
|
def targets(z: zipfile.ZipFile, part: str) -> dict[str, str]:
|
|
"""Relationship id -> package part, for the relationships of part."""
|
|
folder, name = posixpath.split(part)
|
|
rels = posixpath.join(folder, "_rels", name + ".rels")
|
|
if rels not in z.namelist():
|
|
return {}
|
|
found: dict[str, str] = {}
|
|
for rel in xml(z, rels).iter(f"{PKG}Relationship"):
|
|
target = rel.get("Target", "")
|
|
if rel.get("TargetMode") == "External":
|
|
continue
|
|
found[rel.get("Id", "")] = (target[1:] if target.startswith("/")
|
|
else posixpath.normpath(posixpath.join(folder, target)))
|
|
return found
|
|
|
|
|
|
def walk(node: ElementTree.Element, skip: tuple[str, ...]) -> list[ElementTree.Element]:
|
|
"""node and its descendants in document order, without the subtrees of the tags in skip."""
|
|
out = [node]
|
|
for child in node:
|
|
if child.tag not in skip:
|
|
out += walk(child, skip)
|
|
return out
|
|
|
|
|
|
def docx_units(z: zipfile.ZipFile) -> list[Unit]:
|
|
levels: dict[str, int] = {}
|
|
if "word/styles.xml" in z.namelist():
|
|
for style in xml(z, "word/styles.xml").iter(f"{W}style"):
|
|
name = style.find(f"{W}name")
|
|
m = STYLE_HEADING.match((name.get(f"{W}val", "") if name is not None else "").lower())
|
|
if m or (name is not None and name.get(f"{W}val", "").lower() == "title"):
|
|
levels[style.get(f"{W}styleId", "")] = int(m.group(1)) if m else 1
|
|
body = xml(z, "word/document.xml").find(f"{W}body")
|
|
if body is None:
|
|
raise Unreadable("word/document.xml has no body")
|
|
box = (f"{W}txbxContent", MC_FALLBACK)
|
|
units: list[Unit] = []
|
|
for p in (e for e in walk(body, box) if e.tag == f"{W}p"):
|
|
visible: list[str] = []
|
|
full: list[str] = []
|
|
for run in (e for e in walk(p, box) if e.tag == f"{W}r"):
|
|
props = run.find(f"{W}rPr")
|
|
vanish = props.find(f"{W}vanish") if props is not None else None
|
|
hidden = vanish is not None and vanish.get(f"{W}val", "true") not in ("0", "false")
|
|
for e in run:
|
|
piece = (e.text or "" if e.tag in (f"{W}t", f"{W}delText") else
|
|
"\t" if e.tag == f"{W}tab" else " " if e.tag in (f"{W}br", f"{W}cr") else "")
|
|
full.append(piece)
|
|
if not hidden and e.tag != f"{W}delText":
|
|
visible.append(piece)
|
|
mark = p.find(f"{W}pPr/{W}pStyle")
|
|
level = levels.get(mark.get(f"{W}val", "") if mark is not None else "", 0)
|
|
units.append(Unit({"paragraph": len(units) + 1}, collapse("".join(visible)), collapse("".join(full)), level))
|
|
return units
|
|
|
|
|
|
def cell_text(cell: ElementTree.Element, shared: list[str]) -> str:
|
|
kind = cell.get("t", "n")
|
|
if kind == "inlineStr":
|
|
node = cell.find(f"{S}is")
|
|
return "" if node is None else string_item(node)
|
|
value = cell.find(f"{S}v")
|
|
raw = "" if value is None or value.text is None else value.text
|
|
if kind == "s":
|
|
try:
|
|
return shared[int(raw)]
|
|
except (ValueError, IndexError) as e:
|
|
raise Unreadable(f"cell {cell.get('r')} names shared string {raw!r}, which does not exist") from e
|
|
if kind == "b":
|
|
return {"1": "TRUE", "0": "FALSE"}.get(raw, raw)
|
|
return raw
|
|
|
|
|
|
def string_item(si: ElementTree.Element) -> str:
|
|
"""The text of a shared or inline string, rich runs joined, phonetic runs left out."""
|
|
return "".join(t.text or "" for t in walk(si, (f"{S}rPh",)) if t.tag == f"{S}t")
|
|
|
|
|
|
def letters(ref: str) -> str:
|
|
"""The column letters of a cell reference: "AB" of "AB12"."""
|
|
return re.match(r"[A-Za-z]*", ref).group(0) # type: ignore[union-attr]
|
|
|
|
|
|
def column(ref: str) -> int:
|
|
n = 0
|
|
for ch in letters(ref).upper():
|
|
n = n * 26 + ord(ch) - 64
|
|
return n
|
|
|
|
|
|
def xlsx_units(z: zipfile.ZipFile) -> list[Unit]:
|
|
shared = ([string_item(si) for si in xml(z, "xl/sharedStrings.xml").iter(f"{S}si")]
|
|
if "xl/sharedStrings.xml" in z.namelist() else [])
|
|
parts = targets(z, "xl/workbook.xml")
|
|
units: list[Unit] = []
|
|
for sheet in xml(z, "xl/workbook.xml").iter(f"{S}sheet"):
|
|
name = sheet.get("name", "")
|
|
part = parts.get(sheet.get(f"{R}id", ""))
|
|
if part is None:
|
|
raise Unreadable(f"sheet {name!r} has no worksheet part")
|
|
root = xml(z, part)
|
|
sheet_hidden = sheet.get("state", "visible") != "visible"
|
|
hidden_cols: set[int] = set()
|
|
for col in root.iter(f"{S}col"):
|
|
if col.get("hidden") in ("1", "true"):
|
|
hidden_cols.update(range(int(col.get("min", "0")), int(col.get("max", "0")) + 1))
|
|
first = True
|
|
block, last = 0, 0 # an empty row between two rows ends a block; a hidden row does not
|
|
for row in root.iter(f"{S}row"):
|
|
row_hidden = row.get("hidden") in ("1", "true")
|
|
number = int(row.get("r", "0") or "0")
|
|
shown = not row_hidden and any(collapse(cell_text(c, shared)) for c in row.iter(f"{S}c"))
|
|
if shown:
|
|
if number != last + 1:
|
|
block += 1
|
|
last = number
|
|
elif row_hidden and number == last + 1:
|
|
last = number
|
|
for cell in row.iter(f"{S}c"):
|
|
ref = cell.get("r", "")
|
|
text = collapse(cell_text(cell, shared))
|
|
if not text:
|
|
continue
|
|
hidden = sheet_hidden or row_hidden or column(ref) in hidden_cols
|
|
heading = name if first and not sheet_hidden else ""
|
|
if heading:
|
|
units.append(Unit({"sheet": name}, heading, heading, SHEET_LEVEL))
|
|
first = False
|
|
units.append(Unit({"sheet": name, "cell": ref}, "" if hidden else text, text, 0,
|
|
f"{name}\n{block}", f"{name}\n{row.get('r', '')}"))
|
|
return units
|
|
|
|
|
|
def slide_paragraphs(root: ElementTree.Element) -> list[tuple[str, bool]]:
|
|
"""(text, is the title) of every paragraph in a slide or notes part, in document order."""
|
|
out: list[tuple[str, bool]] = []
|
|
for shape in (e for e in walk(root, (MC_FALLBACK,)) if e.tag == f"{P}sp" or e.tag == f"{P}graphicFrame"):
|
|
ph = shape.find(f"{P}nvSpPr/{P}nvPr/{P}ph")
|
|
if ph is not None and ph.get("type") in ("sldNum", "sldImg", "dt", "ftr", "hdr"):
|
|
continue
|
|
title = ph is not None and ph.get("type") in ("title", "ctrTitle")
|
|
for para in shape.iter(f"{A}p"):
|
|
text = "".join((e.text or "") if e.tag == f"{A}t" else " " if e.tag == f"{A}br" else ""
|
|
for e in para.iter())
|
|
out.append((collapse(text), title))
|
|
return out
|
|
|
|
|
|
def pptx_units(z: zipfile.ZipFile) -> list[Unit]:
|
|
parts = targets(z, "ppt/presentation.xml")
|
|
units: list[Unit] = []
|
|
for n, sld in enumerate(xml(z, "ppt/presentation.xml").iter(f"{P}sldId"), 1):
|
|
part = parts.get(sld.get(f"{R}id", ""))
|
|
if part is None:
|
|
raise Unreadable(f"slide {n} has no part")
|
|
root = xml(z, part)
|
|
hidden = root.get("show") in ("0", "false")
|
|
for text, title in slide_paragraphs(root):
|
|
if text:
|
|
units.append(Unit({"slide": n, "notes": False}, "" if hidden else text, text, 2 if title else 0))
|
|
notes = [t for t in targets(z, part).values() if "/notesSlides/" in "/" + t]
|
|
for note in notes[:1]:
|
|
for text, _ in slide_paragraphs(xml(z, note)):
|
|
if text:
|
|
units.append(Unit({"slide": n, "notes": True}, "" if hidden else text, text))
|
|
return units
|
|
|
|
|
|
def pdf_units(path: Path) -> list[Unit]:
|
|
try:
|
|
import pypdf # type: ignore[import-not-found, unused-ignore]
|
|
except ImportError as e:
|
|
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN} (python3 -m pip install pypdf=={PDF_PIN})") from e
|
|
logging.getLogger("pypdf").setLevel(logging.ERROR) # repairs of a sloppy file are not the user's business
|
|
if pypdf.__version__ != PDF_PIN:
|
|
raise NoReader(f"{path.name}: reading PDF needs pypdf=={PDF_PIN}, found {pypdf.__version__} "
|
|
f"(python3 -m pip install pypdf=={PDF_PIN})")
|
|
try:
|
|
doc = pypdf.PdfReader(str(path))
|
|
if doc.is_encrypted and not doc.decrypt(""):
|
|
raise Unreadable(f"{path.name} is encrypted")
|
|
texts = [page.extract_text() or "" for page in doc.pages]
|
|
except Unreadable:
|
|
raise
|
|
except Exception as e: # pypdf raises many kinds on a broken file
|
|
raise Unreadable(f"{path.name} is not a readable PDF: {e}") from e
|
|
return [Unit({"page": n}, collapse(t), collapse(t)) for n, t in enumerate(texts, 1)]
|
|
|
|
|
|
INVISIBLE = frozenset({"head", "title", "script", "style", "noscript", "template", "svg"})
|
|
|
|
|
|
class TextNodes(html.parser.HTMLParser):
|
|
"""Every visible text node of an HTML page, cut at its line breaks: (line of the raw source, text)."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.open: list[str] = []
|
|
self.found: list[tuple[int, str]] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
if tag in INVISIBLE:
|
|
self.open.append(tag)
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in self.open:
|
|
del self.open[len(self.open) - 1 - self.open[::-1].index(tag):]
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if not self.open:
|
|
line = self.getpos()[0]
|
|
self.found += [(line + i, piece) for i, piece in enumerate(data.split("\n"))]
|
|
|
|
|
|
def html_units(path: Path) -> list[Unit]:
|
|
parser = TextNodes()
|
|
try:
|
|
parser.feed(path.read_text(encoding="utf-8"))
|
|
parser.close()
|
|
except UnicodeDecodeError as e:
|
|
raise Unreadable(f"{path.name} is not UTF-8") from e
|
|
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in parser.found if t.strip()]
|
|
|
|
|
|
def text_units(path: Path) -> list[Unit]:
|
|
try:
|
|
lines = path.read_text(encoding="utf-8").split("\n")
|
|
except UnicodeDecodeError as e:
|
|
raise Unreadable(f"{path.name} is not UTF-8") from e
|
|
return [Unit({"line": n}, collapse(t), collapse(t)) for n, t in enumerate(lines, 1)]
|
|
|
|
|
|
def units(path: Path) -> list[Unit]:
|
|
ext = path.suffix.lower()
|
|
if ext == ".pdf":
|
|
return pdf_units(path)
|
|
if ext in (".html", ".htm"):
|
|
return html_units(path)
|
|
if ext not in OFFICE:
|
|
return text_units(path)
|
|
try:
|
|
with zipfile.ZipFile(path) as z:
|
|
return {".docx": docx_units, ".xlsx": xlsx_units, ".pptx": pptx_units}[ext](z)
|
|
except zipfile.BadZipFile as e:
|
|
raise Unreadable(f"{path.name} is not an Office package: {e}") from e
|
|
|
|
|
|
def render(path: Path) -> Rendering:
|
|
"""The text a reader of the document sees, as Markdown the block splitter reads: a heading line per heading unit,
|
|
one line per unit, a blank line between blocks (a unit, or a run of spreadsheet rows). A spreadsheet row is one
|
|
line, each cell after its column letter ("A: Quantity | C: 101000"), so a number keeps the column its header
|
|
names; its place lists the cells with the offset each starts at (common.place_at). Hidden units are left out."""
|
|
lines: list[str] = []
|
|
places: list[dict[str, Any] | None] = []
|
|
left_out = 0
|
|
group: str | None = None
|
|
line: str | None = None
|
|
block, first_row = 0, 0 # where the current block, and the current run of rows, start in lines
|
|
for u in units(path):
|
|
if u.visible != u.text:
|
|
left_out += 1
|
|
if not u.visible:
|
|
continue
|
|
if u.line and u.line == line:
|
|
lines[-1] += " | "
|
|
places[-1]["cells"].append([u.locator["cell"], len(lines[-1])]) # type: ignore[index]
|
|
lines[-1] += f"{letters(u.locator['cell'])}: {u.visible}"
|
|
continue
|
|
same_run = bool(u.group and u.group == group)
|
|
if lines and not same_run:
|
|
lines.append("")
|
|
places.append(None)
|
|
block = first_row = len(lines)
|
|
elif same_run and u.line and sum(len(x) for x in lines[block:]) >= ROW_BLOCK_CHARS:
|
|
head, at = lines[first_row], places[first_row] or {}
|
|
lines += ["", head]
|
|
places += [None, {"sheet": at.get("sheet"), "cells": [list(c) for c in at.get("cells", [])]}]
|
|
block = len(lines) - 1
|
|
group, line = u.group or None, u.line or None
|
|
if u.line:
|
|
lines.append(f"{letters(u.locator['cell'])}: {u.visible}")
|
|
places.append({"sheet": u.locator["sheet"], "cells": [[u.locator["cell"], 0]]})
|
|
elif u.heading:
|
|
lines.append("#" * u.heading + " " + u.visible)
|
|
places.append(None)
|
|
else:
|
|
lines.append(("\\" + u.visible) if STRUCTURE.match(u.visible) else u.visible)
|
|
places.append(u.locator)
|
|
return Rendering("\n".join(lines) + "\n" if lines else "", places, left_out)
|
|
|
|
|
|
def locate(path: Path, quotes: list[str]) -> list[list[tuple[dict[str, Any], bool]]]:
|
|
"""For each quote, every (place, hidden) where it occurs, in the order of the document."""
|
|
found = units(path)
|
|
out: list[list[tuple[dict[str, Any], bool]]] = []
|
|
for quote in quotes:
|
|
q = collapse(quote)
|
|
places: list[tuple[dict[str, Any], bool]] = []
|
|
for u in found:
|
|
if "cell" not in u.locator and "sheet" in u.locator:
|
|
continue # a sheet's own heading unit is not a place in the workbook
|
|
if q and q in u.visible:
|
|
places.append((u.locator, False))
|
|
elif q and q in u.text:
|
|
places.append((u.locator, True))
|
|
out.append(places)
|
|
return out
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(prog="python3 -m okfwiki.readers", description=__doc__.split("\n")[0])
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
p = sub.add_parser("locate")
|
|
p.add_argument("file", type=Path)
|
|
p.add_argument("queries", type=Path)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
doc = json.loads(args.queries.read_text(encoding="utf-8"))
|
|
queries = [(q["id"], q["quote"]) for q in doc["queries"]]
|
|
answers = locate(args.file, [quote for _, quote in queries])
|
|
except (OSError, ValueError, KeyError, TypeError, NoReader, Unreadable) as e:
|
|
print(f"readers cannot run: {e}", file=sys.stderr)
|
|
return 2
|
|
print(json.dumps({"answers": [{"id": i, "found": [{"locator": loc, "hidden": h} for loc, h in places]}
|
|
for (i, _), places in zip(queries, answers)]}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|