"""Motor step 3 (A1): a reader per file type gives every unit of text its exact place. pdf {"page": N}, docx {"paragraph": N}, xlsx {"sheet", "cell"}, pptx {"slide", "notes"}, md and txt {"line": N}: the locators of the file-type gate (gates/filetypes.manifest.json). Hidden Office content is marked, never flattened. The Office files here are built by hand from their package XML, so no real document is in the repository. Run: python3 -m unittest discover -s tests -p 'test_readers.py' """ from __future__ import annotations import json import subprocess import sys import tempfile import unittest import zipfile from pathlib import Path from unittest import mock ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from okfwiki import readers # noqa: E402 W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' WP = ('xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape" ' 'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"') REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" PKG = "http://schemas.openxmlformats.org/package/2006/relationships" S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"' P = ('xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" ' 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" ' f'xmlns:r="{REL}"') def package(path: Path, parts: dict[str, str]) -> Path: with zipfile.ZipFile(path, "w") as z: for name, text in parts.items(): z.writestr(name, text) return path def rels(*targets: tuple[str, str]) -> str: body = "".join(f'' for i, t in targets) return f'{body}' def docx(path: Path) -> Path: body = ( 'Årlig plan' '' # an empty paragraph counts 'Kompost vendes hver uke.' ' Skjult merknad.' 'Celletekst i tabell' f'Før boksen.' 'Tekst i boks' 'Reserve' '' 'Slettet setning her.' 'Ny setning her.' '# Ikke en overskrift' ) styles = (f'' '') return package(path, {"word/document.xml": f'{body}', "word/styles.xml": styles}) def xlsx(path: Path) -> Path: workbook = (f'' '' '') shared = (f'BikuberSvermer i juni' 'fonetiskSkjult rad') sheet1 = (f'' '012' 'Skjult kolonne' '' '1A1Formeltekst' '7' '') sheet2 = (f'Gjemt ark' '') return package(path, {"xl/workbook.xml": workbook, "xl/_rels/workbook.xml.rels": rels(("rId1", "worksheets/a.xml"), ("rId2", "/xl/worksheets/b.xml")), "xl/sharedStrings.xml": shared, "xl/worksheets/a.xml": sheet1, "xl/worksheets/b.xml": sheet2}) def slide(title: str, text: str, show: bool = True) -> str: hide = "" if show else ' show="0"' return (f'' f'{title}' '' f'{text}andre linje' '') def pptx(path: Path) -> Path: presentation = (f'' '') notes = (f'' 'Husk røykpusteren.' '') return package(path, { "ppt/presentation.xml": presentation, "ppt/_rels/presentation.xml.rels": rels(("rId9", "slides/slide2.xml"), ("rId2", "slides/slide1.xml"), ("rId3", "slides/slide3.xml")), "ppt/slides/slide2.xml": slide("Våren", "Første lysbilde"), "ppt/slides/slide1.xml": slide("Sommeren", "Andre lysbilde"), "ppt/slides/_rels/slide1.xml.rels": rels(("rId1", "../notesSlides/notesSlide1.xml")), "ppt/notesSlides/notesSlide1.xml": notes, "ppt/slides/slide3.xml": slide("Gjemt", "Skjult lysbilde", show=False), }) def pdf(path: Path, pages: list[str]) -> Path: """A minimal PDF with one line of Helvetica text per page.""" objects = ["<< /Type /Catalog /Pages 2 0 R >>", ""] kids = [] font = 3 objects.append("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") for text in pages: stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("latin-1") objects.append(f"<< /Length {len(stream)} >>\nstream\n{stream.decode('latin-1')}\nendstream") content = len(objects) objects.append(f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {content} 0 R " f"/Resources << /Font << /F1 {font} 0 R >> >> >>") kids.append(f"{len(objects)} 0 R") objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>" out = b"%PDF-1.4\n" offsets = [] for i, obj in enumerate(objects, 1): offsets.append(len(out)) out += f"{i} 0 obj\n{obj}\nendobj\n".encode("latin-1") xref = len(out) out += f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets) out += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode() path.write_bytes(out) return path def places(path: Path, quote: str) -> list[tuple[dict[str, object], bool]]: return readers.locate(path, [quote])[0] class DocxTest(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp()) self.file = docx(self.tmp / "plan.docx") def test_paragraphs_count_empty_and_table_paragraphs_but_not_text_boxes_or_fallbacks(self) -> None: self.assertEqual(places(self.file, "Årlig plan"), [({"paragraph": 1}, False)]) self.assertEqual(places(self.file, "Kompost vendes hver uke."), [({"paragraph": 3}, False)]) self.assertEqual(places(self.file, "Celletekst i tabell"), [({"paragraph": 4}, False)]) self.assertEqual(places(self.file, "Før boksen."), [({"paragraph": 5}, False)]) self.assertEqual(places(self.file, "Ny setning her."), [({"paragraph": 6}, False)]) self.assertEqual(places(self.file, "Reserve"), []) def test_hidden_text_and_tracked_deletions_are_marked_hidden(self) -> None: self.assertEqual(places(self.file, "Skjult merknad."), [({"paragraph": 3}, True)]) self.assertEqual(places(self.file, "Slettet setning her."), [({"paragraph": 6}, True)]) def test_the_rendering_has_headings_from_styles_one_paragraph_a_line_and_nothing_hidden(self) -> None: r = readers.render(self.file) lines = r.text.split("\n") self.assertEqual(lines[0], "# Årlig plan") self.assertIn("Kompost vendes hver uke.", lines) self.assertNotIn("Skjult", r.text) self.assertNotIn("Slettet", r.text) self.assertIn("\\# Ikke en overskrift", lines) # a paragraph never becomes Markdown structure n = lines.index("Kompost vendes hver uke.") self.assertEqual(r.locators[n], {"paragraph": 3}) self.assertIsNone(r.locators[n + 1]) # the blank line between paragraphs has no place self.assertEqual(r.left_out, 2) class XlsxTest(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp()) self.file = xlsx(self.tmp / "kuber.xlsx") def test_cells_have_sheet_and_cell_and_rich_text_without_phonetics(self) -> None: self.assertEqual(places(self.file, "Bikuber"), [({"sheet": "Plan", "cell": "A1"}, False)]) self.assertEqual(places(self.file, "Svermer i juni"), [({"sheet": "Plan", "cell": "B3"}, False)]) self.assertEqual(places(self.file, "Formeltekst"), [({"sheet": "Plan", "cell": "D3"}, False)]) self.assertEqual(places(self.file, "fonetisk"), []) def test_hidden_sheets_rows_and_columns_are_marked_hidden(self) -> None: self.assertEqual(places(self.file, "Gjemt ark"), [({"sheet": "Gjemt", "cell": "A1"}, True)]) self.assertEqual(places(self.file, "Skjult rad"), [({"sheet": "Plan", "cell": "A2"}, True)]) self.assertEqual(places(self.file, "Skjult kolonne"), [({"sheet": "Plan", "cell": "C1"}, True)]) def test_the_rendering_has_a_line_per_row_and_a_block_per_run_of_rows(self) -> None: r = readers.render(self.file) # a sheet name is the section of its rows, never a concept the skeleton proposes (title and levels 2-3); # a row is one line, each cell after its column letter, so a number keeps the column its header names; # rows with only hidden rows between them are one block, an empty row ends it self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\nB: Svermer i juni | D: Formeltekst\n\nA: 7\n") self.assertEqual(r.locators, [None, None, {"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]}, {"sheet": "Plan", "cells": [["B3", 0], ["D3", 20]]}, None, {"sheet": "Plan", "cells": [["A5", 0]]}]) self.assertEqual(r.left_out, 3) def test_a_long_run_of_rows_is_split_and_each_further_block_repeats_its_first_row(self) -> None: with mock.patch.object(readers, "ROW_BLOCK_CHARS", 10): r = readers.render(self.file) self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\n\nA: Bikuber | B: 12\n" "B: Svermer i juni | D: Formeltekst\n\nA: 7\n") self.assertEqual(r.locators[2], r.locators[4]) class PptxTest(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp()) self.file = pptx(self.tmp / "kurs.pptx") def test_slides_follow_the_slide_list_and_notes_are_flagged(self) -> None: self.assertEqual(places(self.file, "Første lysbilde andre linje"), [({"slide": 1, "notes": False}, False)]) self.assertEqual(places(self.file, "Sommeren"), [({"slide": 2, "notes": False}, False)]) self.assertEqual(places(self.file, "Husk røykpusteren."), [({"slide": 2, "notes": True}, False)]) def test_a_hidden_slide_is_marked_hidden(self) -> None: self.assertEqual(places(self.file, "Skjult lysbilde"), [({"slide": 3, "notes": False}, True)]) def test_the_rendering_has_the_title_as_heading_and_leaves_hidden_slides_out(self) -> None: r = readers.render(self.file) self.assertEqual(r.text, "## Våren\n\nFørste lysbilde andre linje\n\n## Sommeren\n\nAndre lysbilde andre linje" "\n\nHusk røykpusteren.\n") self.assertEqual(r.locators[8], {"slide": 2, "notes": True}) self.assertEqual(r.left_out, 2) class PdfTest(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp()) self.file = pdf(self.tmp / "rapport.pdf", ["Side en om bier", "Side to om honning"]) def test_text_is_found_on_its_physical_page(self) -> None: self.assertEqual(places(self.file, "Side to om honning"), [({"page": 2}, False)]) self.assertEqual(readers.render(self.file).locators, [{"page": 1}, None, {"page": 2}]) def test_a_pdf_library_other_than_the_pinned_one_is_refused(self) -> None: with mock.patch.object(readers, "PDF_PIN", "0.0.1"): with self.assertRaises(readers.NoReader) as caught: readers.render(self.file) self.assertIn("pypdf==0.0.1", str(caught.exception)) class TextTest(unittest.TestCase): def test_md_and_txt_are_found_on_their_line_with_whitespace_collapsed(self) -> None: tmp = Path(tempfile.mkdtemp()) (tmp / "a.txt").write_text("første\nto ord\nto ord\n", encoding="utf-8") self.assertEqual(places(tmp / "a.txt", "to ord"), [({"line": 2}, False), ({"line": 3}, False)]) def test_html_text_is_found_on_its_raw_line_outside_the_head_and_scripts(self) -> None: tmp = Path(tempfile.mkdtemp()) (tmp / "a.html").write_text("Om bier\n

Om bier & honning\n" "og voks

\n", encoding="utf-8") self.assertEqual(places(tmp / "a.html", "Om bier & honning"), [({"line": 2}, False)]) self.assertEqual(places(tmp / "a.html", "og voks"), [({"line": 3}, False)]) self.assertEqual(places(tmp / "a.html", "Om bier"), [({"line": 2}, False)]) def test_a_broken_package_is_unreadable(self) -> None: tmp = Path(tempfile.mkdtemp()) (tmp / "x.docx").write_bytes(b"not a zip") with self.assertRaises(readers.Unreadable): readers.render(tmp / "x.docx") class LocateCliTest(unittest.TestCase): def test_the_gate_contract(self) -> None: tmp = Path(tempfile.mkdtemp()) file = docx(tmp / "plan.docx") queries = tmp / "q.json" queries.write_text(json.dumps({"file": "plan.docx", "queries": [{"id": "p1", "quote": "Skjult merknad."}, {"id": "p2", "quote": "finnes ikke"}]}), encoding="utf-8") done = subprocess.run([sys.executable, "-m", "okfwiki.readers", "locate", str(file), str(queries)], cwd=ROOT, capture_output=True, text=True, timeout=60) self.assertEqual(done.returncode, 0, done.stderr) self.assertEqual(json.loads(done.stdout), {"answers": [ {"id": "p1", "found": [{"locator": {"paragraph": 3}, "hidden": True}]}, {"id": "p2", "found": []}]}) if __name__ == "__main__": unittest.main()