okf-wiki/gates/filetype_gate.py
Kjell Tore Guttormsen 0fd86a23a3
okf-wiki 1.1.1
First public release, published as one snapshot. What changed between
versions is in CHANGELOG.md.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-14 21:03:20 +01:00

302 lines
14 KiB
Python

"""File-type gate for the reader (E4): does a reader find each frozen quote at its one true place, per file type?
No model call happens here. The gate is fail-closed: a reader that is missing, crashes, times out or answers in
the wrong shape has located nothing.
Reader contract (the L1b reader, or any stub standing in for it):
- argv is the command after `--`, with `{file}` replaced by a fresh copy of one source file (its name and
extension kept), `{queries}` by a JSON file {"file": "<name>", "queries": [{"id": "...", "quote": "..."}]},
and `{sources}` by the --sources root. The reader never sees the expected locator.
- stdout is one JSON object {"answers": [{"id": "...", "found": [{"locator": {...}, "hidden": false}]}]}:
every place in the file where the quote occurs, compared with runs of whitespace collapsed to one space.
- exit 0. Anything else is an error for every pair on that file.
A pair is *located* when its answer names exactly one place, that place is the expected locator, and the
`hidden` flag equals the expected one (hidden Office content must be marked, not flattened into ordinary text).
Every quote was chosen to occur exactly once in its file, so a second place is a wrong answer.
Locators (see the manifest for why each was chosen): pdf {"page": N} · docx {"paragraph": N} ·
xlsx {"sheet": name, "cell": "B7"} · pptx {"slide": N, "notes": bool} · md, txt, html {"line": N}.
Numbers are 1-based integers (a boolean is not a number). Cell references ignore case; `notes` defaults to false.
Before anything runs, the pairs file and every source file a pair names are checked against the manifest's
sha256; a mismatch, a missing file or a file the manifest does not list stops the gate (exit 2).
Usage:
python3 gates/filetype_gate.py --manifest gates/filetypes.manifest.json --pairs gates-local/filetypes/pairs.json \\
--sources <root of blandet/ and wiki/> --cases gates-local/filetypes/files [--timeout 120] \\
-- <reader> {file} {queries}
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import shutil
import subprocess
import sys
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
PAIRS_SCHEMA = "okf-wiki/filetype-pairs/1"
MANIFEST_SCHEMA = "okf-wiki/filetype-manifest/1"
TYPES = ("pdf", "docx", "xlsx", "pptx", "md", "txt", "html")
ROOTS = ("sources", "cases")
GROUPS = ("filetype", "hidden", "hidden-control")
LOCATOR_KEYS: dict[str, frozenset[str]] = {
"pdf": frozenset({"page"}),
"docx": frozenset({"paragraph"}),
"xlsx": frozenset({"sheet", "cell"}),
"pptx": frozenset({"slide", "notes"}),
"md": frozenset({"line"}),
"txt": frozenset({"line"}),
"html": frozenset({"line"}),
}
class GateError(Exception):
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments)."""
@dataclass(frozen=True)
class Pair:
id: str
type: str
root: str
path: str
group: str
hidden_class: str | None
quote: str
locator: dict[str, Any]
hidden: bool
@property
def source(self) -> tuple[str, str]:
return (self.root, self.path)
@dataclass(frozen=True)
class Verdict:
located: bool
reason: str = ""
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _is_int(v: object) -> bool:
return isinstance(v, int) and not isinstance(v, bool)
def canonical_locator(ftype: str, loc: object) -> dict[str, Any] | None:
"""The locator in comparable form, or None when it does not have the type's shape."""
if not isinstance(loc, dict):
return None
out = dict(loc)
if ftype == "pptx":
out.setdefault("notes", False)
if set(out) != LOCATOR_KEYS[ftype]:
return None
if ftype == "xlsx":
if not isinstance(out["sheet"], str) or not isinstance(out["cell"], str):
return None
out["cell"] = out["cell"].upper()
return out
for key in LOCATOR_KEYS[ftype] - {"notes"}:
if not _is_int(out[key]):
return None
if ftype == "pptx" and not isinstance(out["notes"], bool):
return None
return out
def same_locator(ftype: str, expected: object, got: object) -> bool:
a, b = canonical_locator(ftype, expected), canonical_locator(ftype, got)
return a is not None and a == b
def load_json(path: Path, what: str) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise GateError(f"cannot read {what} {path}: {exc}") from exc
def load_pairs(path: Path) -> tuple[str, list[Pair]]:
data = load_json(path, "pairs file")
if not isinstance(data, dict) or data.get("schema") != PAIRS_SCHEMA:
raise GateError(f"pairs file schema must be {PAIRS_SCHEMA!r}")
pairs: list[Pair] = []
for raw in data.get("pairs", []):
try:
p = Pair(id=str(raw["id"]), type=str(raw["type"]), root=str(raw["root"]), path=str(raw["path"]),
group=str(raw["group"]), hidden_class=raw.get("hidden_class"), quote=str(raw["quote"]),
locator=raw["locator"], hidden=raw["hidden"])
except (KeyError, TypeError) as exc:
raise GateError(f"pair {raw!r} is malformed: {exc}") from exc
if p.type not in TYPES or p.root not in ROOTS or p.group not in GROUPS or not isinstance(p.hidden, bool):
raise GateError(f"pair {p.id}: bad type, root, group or hidden flag")
if canonical_locator(p.type, p.locator) is None:
raise GateError(f"pair {p.id}: locator {p.locator!r} does not have the {p.type} shape")
if p.group != "filetype" and not p.hidden_class:
raise GateError(f"pair {p.id}: a hidden case needs a hidden_class")
if not p.quote.strip():
raise GateError(f"pair {p.id}: empty quote")
pairs.append(p)
ids = [p.id for p in pairs]
if not pairs or len(ids) != len(set(ids)):
raise GateError("pairs file has no pairs or repeats an id")
return str(data.get("set", "")), pairs
def verify_fixtures(manifest_path: Path, pairs_path: Path, pairs: list[Pair], roots: dict[str, Path]) -> None:
manifest = load_json(manifest_path, "manifest")
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
raise GateError(f"manifest schema must be {MANIFEST_SCHEMA!r}")
problems: list[str] = []
pairs_sha = sha256_file(pairs_path)
known_sets = {str(s.get("sha256")) for s in manifest.get("sets", {}).values()}
if pairs_sha not in known_sets:
problems.append(f"pairs file {pairs_path} matches no set in the manifest (sha256 {pairs_sha[:12]}...)")
listed = {(str(s["root"]), str(s["path"])): str(s["sha256"]) for s in manifest.get("sources", [])}
for root, rel in sorted({p.source for p in pairs}):
if (root, rel) not in listed:
problems.append(f"{root}:{rel} is not in the manifest")
continue
f = roots[root] / rel
if not f.is_file():
problems.append(f"source missing: {f}")
elif sha256_file(f) != listed[(root, rel)]:
problems.append(f"source sha256 mismatch: {f}")
if problems:
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
def run_reader(template: list[str], pairs: list[Pair], source: Path, sources_root: Path,
timeout: float) -> tuple[dict[str, list[Any]] | None, str]:
"""Answers per pair id for one file, or (None, reason) when the run is an error."""
with tempfile.TemporaryDirectory(prefix="filetype-gate-") as tmp:
copy = Path(tmp) / "file" / source.name
copy.parent.mkdir()
shutil.copyfile(source, copy)
queries = Path(tmp) / "queries.json"
queries.write_text(json.dumps({"file": source.name,
"queries": [{"id": p.id, "quote": p.quote} for p in pairs]}), encoding="utf-8")
argv = [a.replace("{file}", str(copy)).replace("{queries}", str(queries)).replace("{sources}", str(sources_root))
for a in template]
try:
proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
except FileNotFoundError:
return None, f"reader not found: {argv[0]}"
except PermissionError:
return None, f"reader not executable: {argv[0]}"
except subprocess.TimeoutExpired:
return None, f"reader timed out after {timeout:g}s"
if proc.returncode != 0:
tail = proc.stderr.strip().splitlines()[-1:] or [""]
return None, f"reader exited {proc.returncode} {tail[0]}".rstrip()
try:
payload = json.loads(proc.stdout)
except json.JSONDecodeError:
return None, "reader output is not valid JSON"
answers = payload.get("answers") if isinstance(payload, dict) else None
if not isinstance(answers, list) or not all(isinstance(a, dict) and isinstance(a.get("id"), str)
and isinstance(a.get("found"), list) for a in answers):
return None, 'reader output lacks an "answers" list of {"id", "found": [...]} objects'
return {str(a["id"]): list(a["found"]) for a in answers}, ""
def judge(pair: Pair, found: list[Any] | None) -> Verdict:
if found is None:
return Verdict(False, "no answer")
if not found:
return Verdict(False, "not found")
if len(found) > 1:
return Verdict(False, f"{len(found)} places claimed")
place = found[0]
if not isinstance(place, dict) or not same_locator(pair.type, pair.locator, place.get("locator")):
got = place.get("locator") if isinstance(place, dict) else place
return Verdict(False, f"wrong place {json.dumps(got)}")
if place.get("hidden") is not pair.hidden:
return Verdict(False, f"hidden flag wrong (expected {str(pair.hidden).lower()})")
return Verdict(True)
def report(set_name: str, pairs: list[Pair], verdicts: dict[str, Verdict], errors: dict[tuple[str, str], str],
files: int, command: list[str]) -> bool:
print(f"filetype gate: {len(pairs)} pairs in {files} files (set {set_name}); reader: {' '.join(command)}")
for (root, rel), reason in sorted(errors.items()):
print(f" ERROR {root}:{rel}: {reason}")
for p in pairs:
v = verdicts[p.id]
if not v.located:
print(f" {p.id:<8} {p.type:<5} {json.dumps(p.locator):<40} NOT LOCATED ({v.reason})")
def line(label: str, group: list[Pair]) -> str:
return f"{label}: {sum(verdicts[p.id].located for p in group)} of {len(group)}"
ordinary = [p for p in pairs if p.group == "filetype"]
present = [t for t in TYPES if any(p.type == t for p in ordinary)]
print("per file type: " + " · ".join(line(t, [p for p in ordinary if p.type == t]) for t in present))
full = [t for t in present if all(verdicts[p.id].located for p in ordinary if p.type == t)]
print(f"file types with every pair located: {len(full)} of {len(present)}")
hidden = [p for p in pairs if p.group == "hidden"]
if hidden:
classes = sorted({str(p.hidden_class) for p in hidden})
print("hidden, per class: " + " · ".join(line(c, [p for p in hidden if p.hidden_class == c]) for c in classes))
controls = [p for p in pairs if p.group == "hidden-control"]
if controls:
print(line("visible controls in the hidden cases", controls))
located = sum(v.located for v in verdicts.values())
print(f"located {located} of {len(pairs)} · reader errors {len(errors)} of {files} files")
green = located == len(pairs) and not errors
print(f"verdict: {'GREEN' if green else 'RED'}")
return green
def main(argv: list[str] | None = None) -> int:
args_in = sys.argv[1:] if argv is None else argv
if "--" in args_in:
split = args_in.index("--")
own, command = args_in[:split], args_in[split + 1:]
else:
own, command = args_in, []
parser = argparse.ArgumentParser(description="Run the reader's file-type gate (fail-closed, no model calls).")
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--pairs", type=Path, required=True, help="the working set or the held-out set")
parser.add_argument("--sources", type=Path, required=True, help="root for pairs with root 'sources'")
parser.add_argument("--cases", type=Path, required=True, help="root for pairs with root 'cases' (gates-local)")
parser.add_argument("--timeout", type=float, default=120.0, help="seconds per reader run (one run per file)")
args = parser.parse_args(own)
if not command:
print("usage error: give the reader command after --, e.g. -- python3 reader.py {file} {queries}")
return 2
roots = {"sources": args.sources, "cases": args.cases}
try:
set_name, pairs = load_pairs(args.pairs)
verify_fixtures(args.manifest, args.pairs, pairs, roots)
except GateError as exc:
print(f"GATE CANNOT RUN: {exc}")
return 2
by_file: dict[tuple[str, str], list[Pair]] = {}
for p in pairs:
by_file.setdefault(p.source, []).append(p)
verdicts: dict[str, Verdict] = {}
errors: dict[tuple[str, str], str] = {}
for (root, rel), group in sorted(by_file.items()):
answers, reason = run_reader(command, group, roots[root] / rel, args.sources, args.timeout)
if answers is None:
errors[(root, rel)] = reason
verdicts.update({p.id: Verdict(False, "reader error") for p in group})
continue
verdicts.update({p.id: judge(p, answers.get(p.id)) for p in group})
return 0 if report(set_name, pairs, verdicts, errors, len(by_file), command) else 1
if __name__ == "__main__":
sys.exit(main())