First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
290 lines
16 KiB
Python
290 lines
16 KiB
Python
"""Motor step 3 (A1): a reader per file type gives every unit of text its exact place.
|
|
|
|
pdf {"page": N}, docx {"paragraph": N}, xlsx {"sheet", "cell"}, pptx {"slide", "notes"}, md and txt {"line": N}: the
|
|
locators of the file-type gate (gates/filetypes.manifest.json). Hidden Office content is marked, never flattened.
|
|
The Office files here are built by hand from their package XML, so no real document is in the repository.
|
|
Run: python3 -m unittest discover -s tests -p 'test_readers.py'
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import zipfile
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from okfwiki import readers # noqa: E402
|
|
|
|
W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'
|
|
WP = ('xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape" '
|
|
'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"')
|
|
REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
|
PKG = "http://schemas.openxmlformats.org/package/2006/relationships"
|
|
S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"'
|
|
P = ('xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" '
|
|
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '
|
|
f'xmlns:r="{REL}"')
|
|
|
|
|
|
def package(path: Path, parts: dict[str, str]) -> Path:
|
|
with zipfile.ZipFile(path, "w") as z:
|
|
for name, text in parts.items():
|
|
z.writestr(name, text)
|
|
return path
|
|
|
|
|
|
def rels(*targets: tuple[str, str]) -> str:
|
|
body = "".join(f'<Relationship Id="{i}" Type="{REL}/x" Target="{t}"/>' for i, t in targets)
|
|
return f'<Relationships xmlns="{PKG}">{body}</Relationships>'
|
|
|
|
|
|
def docx(path: Path) -> Path:
|
|
body = (
|
|
'<w:p><w:pPr><w:pStyle w:val="Overskrift1"/></w:pPr><w:r><w:t>Årlig plan</w:t></w:r></w:p>'
|
|
'<w:p/>' # an empty paragraph counts
|
|
'<w:p><w:r><w:t xml:space="preserve">Kompost vendes </w:t></w:r><w:r><w:t>hver uke.</w:t></w:r>'
|
|
'<w:r><w:rPr><w:vanish/></w:rPr><w:t> Skjult merknad.</w:t></w:r></w:p>'
|
|
'<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Celletekst i tabell</w:t></w:r></w:p></w:tc></w:tr></w:tbl>'
|
|
f'<w:p><w:r><w:t>Før boksen.</w:t></w:r><w:r><mc:AlternateContent {WP}><mc:Choice Requires="wps">'
|
|
'<w:drawing><wps:txbx><w:txbxContent><w:p><w:r><w:t>Tekst i boks</w:t></w:r></w:p></w:txbxContent>'
|
|
'</wps:txbx></w:drawing></mc:Choice><mc:Fallback><w:pict><w:p><w:r><w:t>Reserve</w:t></w:r></w:p>'
|
|
'</w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p>'
|
|
'<w:p><w:del><w:r><w:delText>Slettet setning her.</w:delText></w:r></w:del>'
|
|
'<w:ins><w:r><w:t>Ny setning her.</w:t></w:r></w:ins></w:p>'
|
|
'<w:p><w:r><w:t># Ikke en overskrift</w:t></w:r></w:p>'
|
|
)
|
|
styles = (f'<w:styles {W}><w:style w:type="paragraph" w:styleId="Overskrift1">'
|
|
'<w:name w:val="heading 1"/></w:style></w:styles>')
|
|
return package(path, {"word/document.xml": f'<w:document {W}><w:body>{body}</w:body></w:document>',
|
|
"word/styles.xml": styles})
|
|
|
|
|
|
def xlsx(path: Path) -> Path:
|
|
workbook = (f'<workbook {S} xmlns:r="{REL}"><sheets>'
|
|
'<sheet name="Plan" sheetId="1" r:id="rId1"/>'
|
|
'<sheet name="Gjemt" sheetId="2" state="hidden" r:id="rId2"/></sheets></workbook>')
|
|
shared = (f'<sst {S}><si><t>Bikuber</t></si><si><r><t>Svermer i </t></r><r><t>juni</t></r>'
|
|
'<rPh><t>fonetisk</t></rPh></si><si><t>Skjult rad</t></si></sst>')
|
|
sheet1 = (f'<worksheet {S}><cols><col min="3" max="3" hidden="1"/></cols><sheetData>'
|
|
'<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1"><v>12</v></c><c r="C1" t="inlineStr">'
|
|
'<is><t>Skjult kolonne</t></is></c></row>'
|
|
'<row r="2" hidden="1"><c r="A2" t="s"><v>2</v></c></row>'
|
|
'<row r="3"><c r="B3" t="s"><v>1</v></c><c r="D3" t="str"><f>A1</f><v>Formeltekst</v></c></row>'
|
|
'<row r="5"><c r="A5"><v>7</v></c></row>'
|
|
'</sheetData></worksheet>')
|
|
sheet2 = (f'<worksheet {S}><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>Gjemt ark</t></is></c></row>'
|
|
'</sheetData></worksheet>')
|
|
return package(path, {"xl/workbook.xml": workbook,
|
|
"xl/_rels/workbook.xml.rels": rels(("rId1", "worksheets/a.xml"),
|
|
("rId2", "/xl/worksheets/b.xml")),
|
|
"xl/sharedStrings.xml": shared, "xl/worksheets/a.xml": sheet1,
|
|
"xl/worksheets/b.xml": sheet2})
|
|
|
|
|
|
def slide(title: str, text: str, show: bool = True) -> str:
|
|
hide = "" if show else ' show="0"'
|
|
return (f'<p:sld {P}{hide}><p:cSld><p:spTree>'
|
|
f'<p:sp><p:nvSpPr><p:nvPr><p:ph type="title"/></p:nvPr></p:nvSpPr><p:txBody><a:p><a:r><a:t>{title}'
|
|
'</a:t></a:r></a:p></p:txBody></p:sp>'
|
|
f'<p:sp><p:txBody><a:p><a:r><a:t>{text}</a:t></a:r><a:br/><a:r><a:t>andre linje</a:t></a:r></a:p>'
|
|
'</p:txBody></p:sp></p:spTree></p:cSld></p:sld>')
|
|
|
|
|
|
def pptx(path: Path) -> Path:
|
|
presentation = (f'<p:presentation {P}><p:sldIdLst><p:sldId id="256" r:id="rId9"/>'
|
|
'<p:sldId id="257" r:id="rId2"/><p:sldId id="258" r:id="rId3"/></p:sldIdLst></p:presentation>')
|
|
notes = (f'<p:notes {P}><p:cSld><p:spTree><p:sp><p:nvSpPr><p:nvPr><p:ph type="body"/></p:nvPr></p:nvSpPr>'
|
|
'<p:txBody><a:p><a:r><a:t>Husk røykpusteren.</a:t></a:r></a:p></p:txBody></p:sp>'
|
|
'</p:spTree></p:cSld></p:notes>')
|
|
return package(path, {
|
|
"ppt/presentation.xml": presentation,
|
|
"ppt/_rels/presentation.xml.rels": rels(("rId9", "slides/slide2.xml"), ("rId2", "slides/slide1.xml"),
|
|
("rId3", "slides/slide3.xml")),
|
|
"ppt/slides/slide2.xml": slide("Våren", "Første lysbilde"),
|
|
"ppt/slides/slide1.xml": slide("Sommeren", "Andre lysbilde"),
|
|
"ppt/slides/_rels/slide1.xml.rels": rels(("rId1", "../notesSlides/notesSlide1.xml")),
|
|
"ppt/notesSlides/notesSlide1.xml": notes,
|
|
"ppt/slides/slide3.xml": slide("Gjemt", "Skjult lysbilde", show=False),
|
|
})
|
|
|
|
|
|
def pdf(path: Path, pages: list[str]) -> Path:
|
|
"""A minimal PDF with one line of Helvetica text per page."""
|
|
objects = ["<< /Type /Catalog /Pages 2 0 R >>", ""]
|
|
kids = []
|
|
font = 3
|
|
objects.append("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
|
|
for text in pages:
|
|
stream = f"BT /F1 12 Tf 72 720 Td ({text}) Tj ET".encode("latin-1")
|
|
objects.append(f"<< /Length {len(stream)} >>\nstream\n{stream.decode('latin-1')}\nendstream")
|
|
content = len(objects)
|
|
objects.append(f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {content} 0 R "
|
|
f"/Resources << /Font << /F1 {font} 0 R >> >> >>")
|
|
kids.append(f"{len(objects)} 0 R")
|
|
objects[1] = f"<< /Type /Pages /Kids [{' '.join(kids)}] /Count {len(kids)} >>"
|
|
out = b"%PDF-1.4\n"
|
|
offsets = []
|
|
for i, obj in enumerate(objects, 1):
|
|
offsets.append(len(out))
|
|
out += f"{i} 0 obj\n{obj}\nendobj\n".encode("latin-1")
|
|
xref = len(out)
|
|
out += f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode()
|
|
out += b"".join(f"{o:010d} 00000 n \n".encode() for o in offsets)
|
|
out += f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n".encode()
|
|
path.write_bytes(out)
|
|
return path
|
|
|
|
|
|
def places(path: Path, quote: str) -> list[tuple[dict[str, object], bool]]:
|
|
return readers.locate(path, [quote])[0]
|
|
|
|
|
|
class DocxTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.file = docx(self.tmp / "plan.docx")
|
|
|
|
def test_paragraphs_count_empty_and_table_paragraphs_but_not_text_boxes_or_fallbacks(self) -> None:
|
|
self.assertEqual(places(self.file, "Årlig plan"), [({"paragraph": 1}, False)])
|
|
self.assertEqual(places(self.file, "Kompost vendes hver uke."), [({"paragraph": 3}, False)])
|
|
self.assertEqual(places(self.file, "Celletekst i tabell"), [({"paragraph": 4}, False)])
|
|
self.assertEqual(places(self.file, "Før boksen."), [({"paragraph": 5}, False)])
|
|
self.assertEqual(places(self.file, "Ny setning her."), [({"paragraph": 6}, False)])
|
|
self.assertEqual(places(self.file, "Reserve"), [])
|
|
|
|
def test_hidden_text_and_tracked_deletions_are_marked_hidden(self) -> None:
|
|
self.assertEqual(places(self.file, "Skjult merknad."), [({"paragraph": 3}, True)])
|
|
self.assertEqual(places(self.file, "Slettet setning her."), [({"paragraph": 6}, True)])
|
|
|
|
def test_the_rendering_has_headings_from_styles_one_paragraph_a_line_and_nothing_hidden(self) -> None:
|
|
r = readers.render(self.file)
|
|
lines = r.text.split("\n")
|
|
self.assertEqual(lines[0], "# Årlig plan")
|
|
self.assertIn("Kompost vendes hver uke.", lines)
|
|
self.assertNotIn("Skjult", r.text)
|
|
self.assertNotIn("Slettet", r.text)
|
|
self.assertIn("\\# Ikke en overskrift", lines) # a paragraph never becomes Markdown structure
|
|
n = lines.index("Kompost vendes hver uke.")
|
|
self.assertEqual(r.locators[n], {"paragraph": 3})
|
|
self.assertIsNone(r.locators[n + 1]) # the blank line between paragraphs has no place
|
|
self.assertEqual(r.left_out, 2)
|
|
|
|
|
|
class XlsxTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.file = xlsx(self.tmp / "kuber.xlsx")
|
|
|
|
def test_cells_have_sheet_and_cell_and_rich_text_without_phonetics(self) -> None:
|
|
self.assertEqual(places(self.file, "Bikuber"), [({"sheet": "Plan", "cell": "A1"}, False)])
|
|
self.assertEqual(places(self.file, "Svermer i juni"), [({"sheet": "Plan", "cell": "B3"}, False)])
|
|
self.assertEqual(places(self.file, "Formeltekst"), [({"sheet": "Plan", "cell": "D3"}, False)])
|
|
self.assertEqual(places(self.file, "fonetisk"), [])
|
|
|
|
def test_hidden_sheets_rows_and_columns_are_marked_hidden(self) -> None:
|
|
self.assertEqual(places(self.file, "Gjemt ark"), [({"sheet": "Gjemt", "cell": "A1"}, True)])
|
|
self.assertEqual(places(self.file, "Skjult rad"), [({"sheet": "Plan", "cell": "A2"}, True)])
|
|
self.assertEqual(places(self.file, "Skjult kolonne"), [({"sheet": "Plan", "cell": "C1"}, True)])
|
|
|
|
def test_the_rendering_has_a_line_per_row_and_a_block_per_run_of_rows(self) -> None:
|
|
r = readers.render(self.file)
|
|
# a sheet name is the section of its rows, never a concept the skeleton proposes (title and levels 2-3);
|
|
# a row is one line, each cell after its column letter, so a number keeps the column its header names;
|
|
# rows with only hidden rows between them are one block, an empty row ends it
|
|
self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\nB: Svermer i juni | D: Formeltekst\n\nA: 7\n")
|
|
self.assertEqual(r.locators, [None, None, {"sheet": "Plan", "cells": [["A1", 0], ["B1", 13]]},
|
|
{"sheet": "Plan", "cells": [["B3", 0], ["D3", 20]]}, None,
|
|
{"sheet": "Plan", "cells": [["A5", 0]]}])
|
|
self.assertEqual(r.left_out, 3)
|
|
|
|
|
|
def test_a_long_run_of_rows_is_split_and_each_further_block_repeats_its_first_row(self) -> None:
|
|
with mock.patch.object(readers, "ROW_BLOCK_CHARS", 10):
|
|
r = readers.render(self.file)
|
|
self.assertEqual(r.text, "#### Plan\n\nA: Bikuber | B: 12\n\nA: Bikuber | B: 12\n"
|
|
"B: Svermer i juni | D: Formeltekst\n\nA: 7\n")
|
|
self.assertEqual(r.locators[2], r.locators[4])
|
|
|
|
class PptxTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.file = pptx(self.tmp / "kurs.pptx")
|
|
|
|
def test_slides_follow_the_slide_list_and_notes_are_flagged(self) -> None:
|
|
self.assertEqual(places(self.file, "Første lysbilde andre linje"), [({"slide": 1, "notes": False}, False)])
|
|
self.assertEqual(places(self.file, "Sommeren"), [({"slide": 2, "notes": False}, False)])
|
|
self.assertEqual(places(self.file, "Husk røykpusteren."), [({"slide": 2, "notes": True}, False)])
|
|
|
|
def test_a_hidden_slide_is_marked_hidden(self) -> None:
|
|
self.assertEqual(places(self.file, "Skjult lysbilde"), [({"slide": 3, "notes": False}, True)])
|
|
|
|
def test_the_rendering_has_the_title_as_heading_and_leaves_hidden_slides_out(self) -> None:
|
|
r = readers.render(self.file)
|
|
self.assertEqual(r.text, "## Våren\n\nFørste lysbilde andre linje\n\n## Sommeren\n\nAndre lysbilde andre linje"
|
|
"\n\nHusk røykpusteren.\n")
|
|
self.assertEqual(r.locators[8], {"slide": 2, "notes": True})
|
|
self.assertEqual(r.left_out, 2)
|
|
|
|
|
|
class PdfTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = Path(tempfile.mkdtemp())
|
|
self.file = pdf(self.tmp / "rapport.pdf", ["Side en om bier", "Side to om honning"])
|
|
|
|
def test_text_is_found_on_its_physical_page(self) -> None:
|
|
self.assertEqual(places(self.file, "Side to om honning"), [({"page": 2}, False)])
|
|
self.assertEqual(readers.render(self.file).locators, [{"page": 1}, None, {"page": 2}])
|
|
|
|
def test_a_pdf_library_other_than_the_pinned_one_is_refused(self) -> None:
|
|
with mock.patch.object(readers, "PDF_PIN", "0.0.1"):
|
|
with self.assertRaises(readers.NoReader) as caught:
|
|
readers.render(self.file)
|
|
self.assertIn("pypdf==0.0.1", str(caught.exception))
|
|
|
|
|
|
class TextTest(unittest.TestCase):
|
|
def test_md_and_txt_are_found_on_their_line_with_whitespace_collapsed(self) -> None:
|
|
tmp = Path(tempfile.mkdtemp())
|
|
(tmp / "a.txt").write_text("første\nto ord\nto ord\n", encoding="utf-8")
|
|
self.assertEqual(places(tmp / "a.txt", "to ord"), [({"line": 2}, False), ({"line": 3}, False)])
|
|
|
|
def test_html_text_is_found_on_its_raw_line_outside_the_head_and_scripts(self) -> None:
|
|
tmp = Path(tempfile.mkdtemp())
|
|
(tmp / "a.html").write_text("<html><head><meta><title>Om bier</title></head>\n<body><p>Om bier & honning\n"
|
|
"og voks</p><script>var x = 'Om bier';</script></body></html>\n", encoding="utf-8")
|
|
self.assertEqual(places(tmp / "a.html", "Om bier & honning"), [({"line": 2}, False)])
|
|
self.assertEqual(places(tmp / "a.html", "og voks"), [({"line": 3}, False)])
|
|
self.assertEqual(places(tmp / "a.html", "Om bier"), [({"line": 2}, False)])
|
|
|
|
def test_a_broken_package_is_unreadable(self) -> None:
|
|
tmp = Path(tempfile.mkdtemp())
|
|
(tmp / "x.docx").write_bytes(b"not a zip")
|
|
with self.assertRaises(readers.Unreadable):
|
|
readers.render(tmp / "x.docx")
|
|
|
|
|
|
class LocateCliTest(unittest.TestCase):
|
|
def test_the_gate_contract(self) -> None:
|
|
tmp = Path(tempfile.mkdtemp())
|
|
file = docx(tmp / "plan.docx")
|
|
queries = tmp / "q.json"
|
|
queries.write_text(json.dumps({"file": "plan.docx", "queries": [{"id": "p1", "quote": "Skjult merknad."},
|
|
{"id": "p2", "quote": "finnes ikke"}]}),
|
|
encoding="utf-8")
|
|
done = subprocess.run([sys.executable, "-m", "okfwiki.readers", "locate", str(file), str(queries)],
|
|
cwd=ROOT, capture_output=True, text=True, timeout=60)
|
|
self.assertEqual(done.returncode, 0, done.stderr)
|
|
self.assertEqual(json.loads(done.stdout), {"answers": [
|
|
{"id": "p1", "found": [{"locator": {"paragraph": 3}, "hidden": True}]},
|
|
{"id": "p2", "found": []}]})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|