"""File-type gate for the reader (E4): does a reader find each frozen quote at its one true place, per file type? No model call happens here. The gate is fail-closed: a reader that is missing, crashes, times out or answers in the wrong shape has located nothing. Reader contract (the L1b reader, or any stub standing in for it): - argv is the command after `--`, with `{file}` replaced by a fresh copy of one source file (its name and extension kept), `{queries}` by a JSON file {"file": "", "queries": [{"id": "...", "quote": "..."}]}, and `{sources}` by the --sources root. The reader never sees the expected locator. - stdout is one JSON object {"answers": [{"id": "...", "found": [{"locator": {...}, "hidden": false}]}]}: every place in the file where the quote occurs, compared with runs of whitespace collapsed to one space. - exit 0. Anything else is an error for every pair on that file. A pair is *located* when its answer names exactly one place, that place is the expected locator, and the `hidden` flag equals the expected one (hidden Office content must be marked, not flattened into ordinary text). Every quote was chosen to occur exactly once in its file, so a second place is a wrong answer. Locators (see the manifest for why each was chosen): pdf {"page": N} · docx {"paragraph": N} · xlsx {"sheet": name, "cell": "B7"} · pptx {"slide": N, "notes": bool} · md, txt, html {"line": N}. Numbers are 1-based integers (a boolean is not a number). Cell references ignore case; `notes` defaults to false. Before anything runs, the pairs file and every source file a pair names are checked against the manifest's sha256; a mismatch, a missing file or a file the manifest does not list stops the gate (exit 2). Usage: python3 gates/filetype_gate.py --manifest gates/filetypes.manifest.json --pairs gates-local/filetypes/pairs.json \\ --sources --cases gates-local/filetypes/files [--timeout 120] \\ -- {file} {queries} Exit: 0 GREEN, 1 RED, 2 the gate could not run. """ from __future__ import annotations import argparse import hashlib import json import shutil import subprocess import sys import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any PAIRS_SCHEMA = "okf-wiki/filetype-pairs/1" MANIFEST_SCHEMA = "okf-wiki/filetype-manifest/1" TYPES = ("pdf", "docx", "xlsx", "pptx", "md", "txt", "html") ROOTS = ("sources", "cases") GROUPS = ("filetype", "hidden", "hidden-control") LOCATOR_KEYS: dict[str, frozenset[str]] = { "pdf": frozenset({"page"}), "docx": frozenset({"paragraph"}), "xlsx": frozenset({"sheet", "cell"}), "pptx": frozenset({"slide", "notes"}), "md": frozenset({"line"}), "txt": frozenset({"line"}), "html": frozenset({"line"}), } class GateError(Exception): """The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments).""" @dataclass(frozen=True) class Pair: id: str type: str root: str path: str group: str hidden_class: str | None quote: str locator: dict[str, Any] hidden: bool @property def source(self) -> tuple[str, str]: return (self.root, self.path) @dataclass(frozen=True) class Verdict: located: bool reason: str = "" def sha256_file(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def _is_int(v: object) -> bool: return isinstance(v, int) and not isinstance(v, bool) def canonical_locator(ftype: str, loc: object) -> dict[str, Any] | None: """The locator in comparable form, or None when it does not have the type's shape.""" if not isinstance(loc, dict): return None out = dict(loc) if ftype == "pptx": out.setdefault("notes", False) if set(out) != LOCATOR_KEYS[ftype]: return None if ftype == "xlsx": if not isinstance(out["sheet"], str) or not isinstance(out["cell"], str): return None out["cell"] = out["cell"].upper() return out for key in LOCATOR_KEYS[ftype] - {"notes"}: if not _is_int(out[key]): return None if ftype == "pptx" and not isinstance(out["notes"], bool): return None return out def same_locator(ftype: str, expected: object, got: object) -> bool: a, b = canonical_locator(ftype, expected), canonical_locator(ftype, got) return a is not None and a == b def load_json(path: Path, what: str) -> Any: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise GateError(f"cannot read {what} {path}: {exc}") from exc def load_pairs(path: Path) -> tuple[str, list[Pair]]: data = load_json(path, "pairs file") if not isinstance(data, dict) or data.get("schema") != PAIRS_SCHEMA: raise GateError(f"pairs file schema must be {PAIRS_SCHEMA!r}") pairs: list[Pair] = [] for raw in data.get("pairs", []): try: p = Pair(id=str(raw["id"]), type=str(raw["type"]), root=str(raw["root"]), path=str(raw["path"]), group=str(raw["group"]), hidden_class=raw.get("hidden_class"), quote=str(raw["quote"]), locator=raw["locator"], hidden=raw["hidden"]) except (KeyError, TypeError) as exc: raise GateError(f"pair {raw!r} is malformed: {exc}") from exc if p.type not in TYPES or p.root not in ROOTS or p.group not in GROUPS or not isinstance(p.hidden, bool): raise GateError(f"pair {p.id}: bad type, root, group or hidden flag") if canonical_locator(p.type, p.locator) is None: raise GateError(f"pair {p.id}: locator {p.locator!r} does not have the {p.type} shape") if p.group != "filetype" and not p.hidden_class: raise GateError(f"pair {p.id}: a hidden case needs a hidden_class") if not p.quote.strip(): raise GateError(f"pair {p.id}: empty quote") pairs.append(p) ids = [p.id for p in pairs] if not pairs or len(ids) != len(set(ids)): raise GateError("pairs file has no pairs or repeats an id") return str(data.get("set", "")), pairs def verify_fixtures(manifest_path: Path, pairs_path: Path, pairs: list[Pair], roots: dict[str, Path]) -> None: manifest = load_json(manifest_path, "manifest") if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: raise GateError(f"manifest schema must be {MANIFEST_SCHEMA!r}") problems: list[str] = [] pairs_sha = sha256_file(pairs_path) known_sets = {str(s.get("sha256")) for s in manifest.get("sets", {}).values()} if pairs_sha not in known_sets: problems.append(f"pairs file {pairs_path} matches no set in the manifest (sha256 {pairs_sha[:12]}...)") listed = {(str(s["root"]), str(s["path"])): str(s["sha256"]) for s in manifest.get("sources", [])} for root, rel in sorted({p.source for p in pairs}): if (root, rel) not in listed: problems.append(f"{root}:{rel} is not in the manifest") continue f = roots[root] / rel if not f.is_file(): problems.append(f"source missing: {f}") elif sha256_file(f) != listed[(root, rel)]: problems.append(f"source sha256 mismatch: {f}") if problems: raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems)) def run_reader(template: list[str], pairs: list[Pair], source: Path, sources_root: Path, timeout: float) -> tuple[dict[str, list[Any]] | None, str]: """Answers per pair id for one file, or (None, reason) when the run is an error.""" with tempfile.TemporaryDirectory(prefix="filetype-gate-") as tmp: copy = Path(tmp) / "file" / source.name copy.parent.mkdir() shutil.copyfile(source, copy) queries = Path(tmp) / "queries.json" queries.write_text(json.dumps({"file": source.name, "queries": [{"id": p.id, "quote": p.quote} for p in pairs]}), encoding="utf-8") argv = [a.replace("{file}", str(copy)).replace("{queries}", str(queries)).replace("{sources}", str(sources_root)) for a in template] try: proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False) except FileNotFoundError: return None, f"reader not found: {argv[0]}" except PermissionError: return None, f"reader not executable: {argv[0]}" except subprocess.TimeoutExpired: return None, f"reader timed out after {timeout:g}s" if proc.returncode != 0: tail = proc.stderr.strip().splitlines()[-1:] or [""] return None, f"reader exited {proc.returncode} {tail[0]}".rstrip() try: payload = json.loads(proc.stdout) except json.JSONDecodeError: return None, "reader output is not valid JSON" answers = payload.get("answers") if isinstance(payload, dict) else None if not isinstance(answers, list) or not all(isinstance(a, dict) and isinstance(a.get("id"), str) and isinstance(a.get("found"), list) for a in answers): return None, 'reader output lacks an "answers" list of {"id", "found": [...]} objects' return {str(a["id"]): list(a["found"]) for a in answers}, "" def judge(pair: Pair, found: list[Any] | None) -> Verdict: if found is None: return Verdict(False, "no answer") if not found: return Verdict(False, "not found") if len(found) > 1: return Verdict(False, f"{len(found)} places claimed") place = found[0] if not isinstance(place, dict) or not same_locator(pair.type, pair.locator, place.get("locator")): got = place.get("locator") if isinstance(place, dict) else place return Verdict(False, f"wrong place {json.dumps(got)}") if place.get("hidden") is not pair.hidden: return Verdict(False, f"hidden flag wrong (expected {str(pair.hidden).lower()})") return Verdict(True) def report(set_name: str, pairs: list[Pair], verdicts: dict[str, Verdict], errors: dict[tuple[str, str], str], files: int, command: list[str]) -> bool: print(f"filetype gate: {len(pairs)} pairs in {files} files (set {set_name}); reader: {' '.join(command)}") for (root, rel), reason in sorted(errors.items()): print(f" ERROR {root}:{rel}: {reason}") for p in pairs: v = verdicts[p.id] if not v.located: print(f" {p.id:<8} {p.type:<5} {json.dumps(p.locator):<40} NOT LOCATED ({v.reason})") def line(label: str, group: list[Pair]) -> str: return f"{label}: {sum(verdicts[p.id].located for p in group)} of {len(group)}" ordinary = [p for p in pairs if p.group == "filetype"] present = [t for t in TYPES if any(p.type == t for p in ordinary)] print("per file type: " + " · ".join(line(t, [p for p in ordinary if p.type == t]) for t in present)) full = [t for t in present if all(verdicts[p.id].located for p in ordinary if p.type == t)] print(f"file types with every pair located: {len(full)} of {len(present)}") hidden = [p for p in pairs if p.group == "hidden"] if hidden: classes = sorted({str(p.hidden_class) for p in hidden}) print("hidden, per class: " + " · ".join(line(c, [p for p in hidden if p.hidden_class == c]) for c in classes)) controls = [p for p in pairs if p.group == "hidden-control"] if controls: print(line("visible controls in the hidden cases", controls)) located = sum(v.located for v in verdicts.values()) print(f"located {located} of {len(pairs)} · reader errors {len(errors)} of {files} files") green = located == len(pairs) and not errors print(f"verdict: {'GREEN' if green else 'RED'}") return green def main(argv: list[str] | None = None) -> int: args_in = sys.argv[1:] if argv is None else argv if "--" in args_in: split = args_in.index("--") own, command = args_in[:split], args_in[split + 1:] else: own, command = args_in, [] parser = argparse.ArgumentParser(description="Run the reader's file-type gate (fail-closed, no model calls).") parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--pairs", type=Path, required=True, help="the working set or the held-out set") parser.add_argument("--sources", type=Path, required=True, help="root for pairs with root 'sources'") parser.add_argument("--cases", type=Path, required=True, help="root for pairs with root 'cases' (gates-local)") parser.add_argument("--timeout", type=float, default=120.0, help="seconds per reader run (one run per file)") args = parser.parse_args(own) if not command: print("usage error: give the reader command after --, e.g. -- python3 reader.py {file} {queries}") return 2 roots = {"sources": args.sources, "cases": args.cases} try: set_name, pairs = load_pairs(args.pairs) verify_fixtures(args.manifest, args.pairs, pairs, roots) except GateError as exc: print(f"GATE CANNOT RUN: {exc}") return 2 by_file: dict[tuple[str, str], list[Pair]] = {} for p in pairs: by_file.setdefault(p.source, []).append(p) verdicts: dict[str, Verdict] = {} errors: dict[tuple[str, str], str] = {} for (root, rel), group in sorted(by_file.items()): answers, reason = run_reader(command, group, roots[root] / rel, args.sources, args.timeout) if answers is None: errors[(root, rel)] = reason verdicts.update({p.id: Verdict(False, "reader error") for p in group}) continue verdicts.update({p.id: judge(p, answers.get(p.id)) for p in group}) return 0 if report(set_name, pairs, verdicts, errors, len(by_file), command) else 1 if __name__ == "__main__": sys.exit(main())