"""Motor step 3 (A1): a reader per file type gives every unit of text its exact place.
pdf {"page": N}, docx {"paragraph": N}, xlsx {"sheet", "cell"}, pptx {"slide", "notes"}, md and txt {"line": N}: the
locators of the file-type gate (gates/filetypes.manifest.json). Hidden Office content is marked, never flattened.
The Office files here are built by hand from their package XML, so no real document is in the repository.
Run: python3 -m unittest discover -s tests -p 'test_readers.py'
"""
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from okfwiki import readers # noqa: E402
W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'
WP = ('xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape" '
'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"')
REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
PKG = "http://schemas.openxmlformats.org/package/2006/relationships"
S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"'
P = ('xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" '
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '
f'xmlns:r="{REL}"')
def package(path: Path, parts: dict[str, str]) -> Path:
with zipfile.ZipFile(path, "w") as z:
for name, text in parts.items():
z.writestr(name, text)
return path
def rels(*targets: tuple[str, str]) -> str:
body = "".join(f'' for i, t in targets)
return f'{body}'
def docx(path: Path) -> Path:
body = (
'Årlig plan'
'' # an empty paragraph counts
'Kompost vendes hver uke.'
' Skjult merknad.'
'Celletekst i tabell'
f'Før boksen.'
'Tekst i boks'
'Reserve'
''
'Slettet setning her.'
'Ny setning her.'
'# Ikke en overskrift'
)
styles = (f''
'')
return package(path, {"word/document.xml": f'{body}',
"word/styles.xml": styles})
def xlsx(path: Path) -> Path:
workbook = (f''
''
'')
shared = (f'BikuberSvermer i juni'
'fonetiskSkjult rad')
sheet1 = (f''
'012'
'Skjult kolonne
'
'2
'
'1A1Formeltekst
'
'7
'
'')
sheet2 = (f'Gjemt ark
'
'')
return package(path, {"xl/workbook.xml": workbook,
"xl/_rels/workbook.xml.rels": rels(("rId1", "worksheets/a.xml"),
("rId2", "/xl/worksheets/b.xml")),
"xl/sharedStrings.xml": shared, "xl/worksheets/a.xml": sheet1,
"xl/worksheets/b.xml": sheet2})
def slide(title: str, text: str, show: bool = True) -> str:
hide = "" if show else ' show="0"'
return (f''
f'{title}'
''
f'{text}andre linje'
'')
def pptx(path: Path) -> Path:
presentation = (f''
'')
notes = (f''
'Husk røykpusteren.'
'')
return package(path, {
"ppt/presentation.xml": presentation,
"ppt/_rels/presentation.xml.rels": rels(("rId9", "slides/slide2.xml"), ("rId2", "slides/slide1.xml"),
("rId3", "slides/slide3.xml")),
"ppt/slides/slide2.xml": slide("Våren", "Første lysbilde"),
"ppt/slides/slide1.xml": slide("Sommeren", "Andre lysbilde"),
"ppt/slides/_rels/slide1.xml.rels": rels(("rId1", "../notesSlides/notesSlide1.xml")),
"ppt/notesSlides/notesSlide1.xml": notes,
"ppt/slides/slide3.xml": slide("Gjemt", "Skjult lysbilde", show=False),
})
def pdf(path: Path, pages: list[str]) -> Path:
"""A minimal PDF with one line of Helvetica text per page."""
objects = ["<< /Type /Catalog /Pages 2 0 R >>", ""]
kids = []
font = 3
objects.append("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
for text in pages:
stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("latin-1")
objects.append(f"<< /Length {len(stream)} >>\nstream\n{stream.decode('latin-1')}\nendstream")
content = len(objects)
objects.append(f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {content} 0 R "
f"/Resources << /Font << /F1 {font} 0 R >> >> >>")
kids.append(f"{len(objects)} 0 R")
objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>"
out = b"%PDF-1.4\n"
offsets = []
for i, obj in enumerate(objects, 1):
offsets.append(len(out))
out += f"{i} 0 obj\n{obj}\nendobj\n".encode("latin-1")
xref = len(out)
out += f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode()
out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets)
out += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()
path.write_bytes(out)
return path
def places(path: Path, quote: str) -> list[tuple[dict[str, object], bool]]:
return readers.locate(path, [quote])[0]
class DocxTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp())
self.file = docx(self.tmp / "plan.docx")
def test_paragraphs_count_empty_and_table_paragraphs_but_not_text_boxes_or_fallbacks(self) -> None:
self.assertEqual(places(self.file, "Årlig plan"), [({"paragraph": 1}, False)])
self.assertEqual(places(self.file, "Kompost vendes hver uke."), [({"paragraph": 3}, False)])
self.assertEqual(places(self.file, "Celletekst i tabell"), [({"paragraph": 4}, False)])
self.assertEqual(places(self.file, "Før boksen."), [({"paragraph": 5}, False)])
self.assertEqual(places(self.file, "Ny setning her."), [({"paragraph": 6}, False)])
self.assertEqual(places(self.file, "Reserve"), [])
def test_hidden_text_and_tracked_deletions_are_marked_hidden(self) -> None:
self.assertEqual(places(self.file, "Skjult merknad."), [({"paragraph": 3}, True)])
self.assertEqual(places(self.file, "Slettet setning her."), [({"paragraph": 6}, True)])
def test_the_rendering_has_headings_from_styles_one_paragraph_a_line_and_nothing_hidden(self) -> None:
r = readers.render(self.file)
lines = r.text.split("\n")
self.assertEqual(lines[0], "# Årlig plan")
self.assertIn("Kompost vendes hver uke.", lines)
self.assertNotIn("Skjult", r.text)
self.assertNotIn("Slettet", r.text)
self.assertIn("\\# Ikke en overskrift", lines) # a paragraph never becomes Markdown structure
n = lines.index("Kompost vendes hver uke.")
self.assertEqual(r.locators[n], {"paragraph": 3})
self.assertIsNone(r.locators[n + 1]) # the blank line between paragraphs has no place
self.assertEqual(r.left_out, 2)
class XlsxTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp())
self.file = xlsx(self.tmp / "kuber.xlsx")
def test_cells_have_sheet_and_cell_and_rich_text_without_phonetics(self) -> None:
self.assertEqual(places(self.file, "Bikuber"), [({"sheet": "Plan", "cell": "A1"}, False)])
self.assertEqual(places(self.file, "Svermer i juni"), [({"sheet": "Plan", "cell": "B3"}, False)])
self.assertEqual(places(self.file, "Formeltekst"), [({"sheet": "Plan", "cell": "D3"}, False)])
self.assertEqual(places(self.file, "fonetisk"), [])
def test_hidden_sheets_rows_and_columns_are_marked_hidden(self) -> None:
self.assertEqual(places(self.file, "Gjemt ark"), [({"sheet": "Gjemt", "cell": "A1"}, True)])
self.assertEqual(places(self.file, "Skjult rad"), [({"sheet": "Plan", "cell": "A2"}, True)])
self.assertEqual(places(self.file, "Skjult kolonne"), [({"sheet": "Plan", "cell": "C1"}, True)])
def test_the_rendering_has_a_line_per_row_and_a_block_per_run_of_rows(self) -> None:
r = readers.render(self.file)
# a sheet name is the section of its rows, never a concept the skeleton proposes (title and levels 2-3);
# a row is one line, each cell after its column letter, so a number keeps the column its header names;
# rows with only hidden rows between them are one block, an empty row ends it
self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\nB: Svermer i juni | D: Formeltekst\n\nA: 7\n")
self.assertEqual(r.locators, [None, None, {"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]},
{"sheet": "Plan", "cells": [["B3", 0], ["D3", 20]]}, None,
{"sheet": "Plan", "cells": [["A5", 0]]}])
self.assertEqual(r.left_out, 3)
def test_a_long_run_of_rows_is_split_and_each_further_block_repeats_its_first_row(self) -> None:
with mock.patch.object(readers, "ROW_BLOCK_CHARS", 10):
r = readers.render(self.file)
self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\n\nA: Bikuber | B: 12\n"
"B: Svermer i juni | D: Formeltekst\n\nA: 7\n")
self.assertEqual(r.locators[2], r.locators[4])
class PptxTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp())
self.file = pptx(self.tmp / "kurs.pptx")
def test_slides_follow_the_slide_list_and_notes_are_flagged(self) -> None:
self.assertEqual(places(self.file, "Første lysbilde andre linje"), [({"slide": 1, "notes": False}, False)])
self.assertEqual(places(self.file, "Sommeren"), [({"slide": 2, "notes": False}, False)])
self.assertEqual(places(self.file, "Husk røykpusteren."), [({"slide": 2, "notes": True}, False)])
def test_a_hidden_slide_is_marked_hidden(self) -> None:
self.assertEqual(places(self.file, "Skjult lysbilde"), [({"slide": 3, "notes": False}, True)])
def test_the_rendering_has_the_title_as_heading_and_leaves_hidden_slides_out(self) -> None:
r = readers.render(self.file)
self.assertEqual(r.text, "## Våren\n\nFørste lysbilde andre linje\n\n## Sommeren\n\nAndre lysbilde andre linje"
"\n\nHusk røykpusteren.\n")
self.assertEqual(r.locators[8], {"slide": 2, "notes": True})
self.assertEqual(r.left_out, 2)
class PdfTest(unittest.TestCase):
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp())
self.file = pdf(self.tmp / "rapport.pdf", ["Side en om bier", "Side to om honning"])
def test_text_is_found_on_its_physical_page(self) -> None:
self.assertEqual(places(self.file, "Side to om honning"), [({"page": 2}, False)])
self.assertEqual(readers.render(self.file).locators, [{"page": 1}, None, {"page": 2}])
def test_a_pdf_library_other_than_the_pinned_one_is_refused(self) -> None:
with mock.patch.object(readers, "PDF_PIN", "0.0.1"):
with self.assertRaises(readers.NoReader) as caught:
readers.render(self.file)
self.assertIn("pypdf==0.0.1", str(caught.exception))
class TextTest(unittest.TestCase):
def test_md_and_txt_are_found_on_their_line_with_whitespace_collapsed(self) -> None:
tmp = Path(tempfile.mkdtemp())
(tmp / "a.txt").write_text("første\nto ord\nto ord\n", encoding="utf-8")
self.assertEqual(places(tmp / "a.txt", "to ord"), [({"line": 2}, False), ({"line": 3}, False)])
def test_html_text_is_found_on_its_raw_line_outside_the_head_and_scripts(self) -> None:
tmp = Path(tempfile.mkdtemp())
(tmp / "a.html").write_text("
Om bier\nOm bier & honning\n"
"og voks
\n", encoding="utf-8")
self.assertEqual(places(tmp / "a.html", "Om bier & honning"), [({"line": 2}, False)])
self.assertEqual(places(tmp / "a.html", "og voks"), [({"line": 3}, False)])
self.assertEqual(places(tmp / "a.html", "Om bier"), [({"line": 2}, False)])
def test_a_broken_package_is_unreadable(self) -> None:
tmp = Path(tempfile.mkdtemp())
(tmp / "x.docx").write_bytes(b"not a zip")
with self.assertRaises(readers.Unreadable):
readers.render(tmp / "x.docx")
class LocateCliTest(unittest.TestCase):
def test_the_gate_contract(self) -> None:
tmp = Path(tempfile.mkdtemp())
file = docx(tmp / "plan.docx")
queries = tmp / "q.json"
queries.write_text(json.dumps({"file": "plan.docx", "queries": [{"id": "p1", "quote": "Skjult merknad."},
{"id": "p2", "quote": "finnes ikke"}]}),
encoding="utf-8")
done = subprocess.run([sys.executable, "-m", "okfwiki.readers", "locate", str(file), str(queries)],
cwd=ROOT, capture_output=True, text=True, timeout=60)
self.assertEqual(done.returncode, 0, done.stderr)
self.assertEqual(json.loads(done.stdout), {"answers": [
{"id": "p1", "found": [{"locator": {"paragraph": 3}, "hidden": True}]},
{"id": "p2", "found": []}]})
if __name__ == "__main__":
unittest.main()