First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
302 lines
14 KiB
Python
302 lines
14 KiB
Python
"""File-type gate for the reader (E4): does a reader find each frozen quote at its one true place, per file type?
|
|
|
|
No model call happens here. The gate is fail-closed: a reader that is missing, crashes, times out or answers in
|
|
the wrong shape has located nothing.
|
|
|
|
Reader contract (the L1b reader, or any stub standing in for it):
|
|
- argv is the command after `--`, with `{file}` replaced by a fresh copy of one source file (its name and
|
|
extension kept), `{queries}` by a JSON file {"file": "<name>", "queries": [{"id": "...", "quote": "..."}]},
|
|
and `{sources}` by the --sources root. The reader never sees the expected locator.
|
|
- stdout is one JSON object {"answers": [{"id": "...", "found": [{"locator": {...}, "hidden": false}]}]}:
|
|
every place in the file where the quote occurs, compared with runs of whitespace collapsed to one space.
|
|
- exit 0. Anything else is an error for every pair on that file.
|
|
|
|
A pair is *located* when its answer names exactly one place, that place is the expected locator, and the
|
|
`hidden` flag equals the expected one (hidden Office content must be marked, not flattened into ordinary text).
|
|
Every quote was chosen to occur exactly once in its file, so a second place is a wrong answer.
|
|
|
|
Locators (see the manifest for why each was chosen): pdf {"page": N} · docx {"paragraph": N} ·
|
|
xlsx {"sheet": name, "cell": "B7"} · pptx {"slide": N, "notes": bool} · md, txt, html {"line": N}.
|
|
Numbers are 1-based integers (a boolean is not a number). Cell references ignore case; `notes` defaults to false.
|
|
|
|
Before anything runs, the pairs file and every source file a pair names are checked against the manifest's
|
|
sha256; a mismatch, a missing file or a file the manifest does not list stops the gate (exit 2).
|
|
|
|
Usage:
|
|
python3 gates/filetype_gate.py --manifest gates/filetypes.manifest.json --pairs gates-local/filetypes/pairs.json \\
|
|
--sources <root of blandet/ and wiki/> --cases gates-local/filetypes/files [--timeout 120] \\
|
|
-- <reader> {file} {queries}
|
|
Exit: 0 GREEN, 1 RED, 2 the gate could not run.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
PAIRS_SCHEMA = "okf-wiki/filetype-pairs/1"
|
|
MANIFEST_SCHEMA = "okf-wiki/filetype-manifest/1"
|
|
TYPES = ("pdf", "docx", "xlsx", "pptx", "md", "txt", "html")
|
|
ROOTS = ("sources", "cases")
|
|
GROUPS = ("filetype", "hidden", "hidden-control")
|
|
LOCATOR_KEYS: dict[str, frozenset[str]] = {
|
|
"pdf": frozenset({"page"}),
|
|
"docx": frozenset({"paragraph"}),
|
|
"xlsx": frozenset({"sheet", "cell"}),
|
|
"pptx": frozenset({"slide", "notes"}),
|
|
"md": frozenset({"line"}),
|
|
"txt": frozenset({"line"}),
|
|
"html": frozenset({"line"}),
|
|
}
|
|
|
|
|
|
class GateError(Exception):
|
|
"""The gate cannot produce a measurement (bad manifest, drifted fixtures, bad arguments)."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Pair:
|
|
id: str
|
|
type: str
|
|
root: str
|
|
path: str
|
|
group: str
|
|
hidden_class: str | None
|
|
quote: str
|
|
locator: dict[str, Any]
|
|
hidden: bool
|
|
|
|
@property
|
|
def source(self) -> tuple[str, str]:
|
|
return (self.root, self.path)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Verdict:
|
|
located: bool
|
|
reason: str = ""
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _is_int(v: object) -> bool:
|
|
return isinstance(v, int) and not isinstance(v, bool)
|
|
|
|
|
|
def canonical_locator(ftype: str, loc: object) -> dict[str, Any] | None:
|
|
"""The locator in comparable form, or None when it does not have the type's shape."""
|
|
if not isinstance(loc, dict):
|
|
return None
|
|
out = dict(loc)
|
|
if ftype == "pptx":
|
|
out.setdefault("notes", False)
|
|
if set(out) != LOCATOR_KEYS[ftype]:
|
|
return None
|
|
if ftype == "xlsx":
|
|
if not isinstance(out["sheet"], str) or not isinstance(out["cell"], str):
|
|
return None
|
|
out["cell"] = out["cell"].upper()
|
|
return out
|
|
for key in LOCATOR_KEYS[ftype] - {"notes"}:
|
|
if not _is_int(out[key]):
|
|
return None
|
|
if ftype == "pptx" and not isinstance(out["notes"], bool):
|
|
return None
|
|
return out
|
|
|
|
|
|
def same_locator(ftype: str, expected: object, got: object) -> bool:
|
|
a, b = canonical_locator(ftype, expected), canonical_locator(ftype, got)
|
|
return a is not None and a == b
|
|
|
|
|
|
def load_json(path: Path, what: str) -> Any:
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise GateError(f"cannot read {what} {path}: {exc}") from exc
|
|
|
|
|
|
def load_pairs(path: Path) -> tuple[str, list[Pair]]:
|
|
data = load_json(path, "pairs file")
|
|
if not isinstance(data, dict) or data.get("schema") != PAIRS_SCHEMA:
|
|
raise GateError(f"pairs file schema must be {PAIRS_SCHEMA!r}")
|
|
pairs: list[Pair] = []
|
|
for raw in data.get("pairs", []):
|
|
try:
|
|
p = Pair(id=str(raw["id"]), type=str(raw["type"]), root=str(raw["root"]), path=str(raw["path"]),
|
|
group=str(raw["group"]), hidden_class=raw.get("hidden_class"), quote=str(raw["quote"]),
|
|
locator=raw["locator"], hidden=raw["hidden"])
|
|
except (KeyError, TypeError) as exc:
|
|
raise GateError(f"pair {raw!r} is malformed: {exc}") from exc
|
|
if p.type not in TYPES or p.root not in ROOTS or p.group not in GROUPS or not isinstance(p.hidden, bool):
|
|
raise GateError(f"pair {p.id}: bad type, root, group or hidden flag")
|
|
if canonical_locator(p.type, p.locator) is None:
|
|
raise GateError(f"pair {p.id}: locator {p.locator!r} does not have the {p.type} shape")
|
|
if p.group != "filetype" and not p.hidden_class:
|
|
raise GateError(f"pair {p.id}: a hidden case needs a hidden_class")
|
|
if not p.quote.strip():
|
|
raise GateError(f"pair {p.id}: empty quote")
|
|
pairs.append(p)
|
|
ids = [p.id for p in pairs]
|
|
if not pairs or len(ids) != len(set(ids)):
|
|
raise GateError("pairs file has no pairs or repeats an id")
|
|
return str(data.get("set", "")), pairs
|
|
|
|
|
|
def verify_fixtures(manifest_path: Path, pairs_path: Path, pairs: list[Pair], roots: dict[str, Path]) -> None:
|
|
manifest = load_json(manifest_path, "manifest")
|
|
if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA:
|
|
raise GateError(f"manifest schema must be {MANIFEST_SCHEMA!r}")
|
|
problems: list[str] = []
|
|
pairs_sha = sha256_file(pairs_path)
|
|
known_sets = {str(s.get("sha256")) for s in manifest.get("sets", {}).values()}
|
|
if pairs_sha not in known_sets:
|
|
problems.append(f"pairs file {pairs_path} matches no set in the manifest (sha256 {pairs_sha[:12]}...)")
|
|
listed = {(str(s["root"]), str(s["path"])): str(s["sha256"]) for s in manifest.get("sources", [])}
|
|
for root, rel in sorted({p.source for p in pairs}):
|
|
if (root, rel) not in listed:
|
|
problems.append(f"{root}:{rel} is not in the manifest")
|
|
continue
|
|
f = roots[root] / rel
|
|
if not f.is_file():
|
|
problems.append(f"source missing: {f}")
|
|
elif sha256_file(f) != listed[(root, rel)]:
|
|
problems.append(f"source sha256 mismatch: {f}")
|
|
if problems:
|
|
raise GateError("fixtures do not match the manifest:\n " + "\n ".join(problems))
|
|
|
|
|
|
def run_reader(template: list[str], pairs: list[Pair], source: Path, sources_root: Path,
|
|
timeout: float) -> tuple[dict[str, list[Any]] | None, str]:
|
|
"""Answers per pair id for one file, or (None, reason) when the run is an error."""
|
|
with tempfile.TemporaryDirectory(prefix="filetype-gate-") as tmp:
|
|
copy = Path(tmp) / "file" / source.name
|
|
copy.parent.mkdir()
|
|
shutil.copyfile(source, copy)
|
|
queries = Path(tmp) / "queries.json"
|
|
queries.write_text(json.dumps({"file": source.name,
|
|
"queries": [{"id": p.id, "quote": p.quote} for p in pairs]}), encoding="utf-8")
|
|
argv = [a.replace("{file}", str(copy)).replace("{queries}", str(queries)).replace("{sources}", str(sources_root))
|
|
for a in template]
|
|
try:
|
|
proc = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
|
|
except FileNotFoundError:
|
|
return None, f"reader not found: {argv[0]}"
|
|
except PermissionError:
|
|
return None, f"reader not executable: {argv[0]}"
|
|
except subprocess.TimeoutExpired:
|
|
return None, f"reader timed out after {timeout:g}s"
|
|
if proc.returncode != 0:
|
|
tail = proc.stderr.strip().splitlines()[-1:] or [""]
|
|
return None, f"reader exited {proc.returncode} {tail[0]}".rstrip()
|
|
try:
|
|
payload = json.loads(proc.stdout)
|
|
except json.JSONDecodeError:
|
|
return None, "reader output is not valid JSON"
|
|
answers = payload.get("answers") if isinstance(payload, dict) else None
|
|
if not isinstance(answers, list) or not all(isinstance(a, dict) and isinstance(a.get("id"), str)
|
|
and isinstance(a.get("found"), list) for a in answers):
|
|
return None, 'reader output lacks an "answers" list of {"id", "found": [...]} objects'
|
|
return {str(a["id"]): list(a["found"]) for a in answers}, ""
|
|
|
|
|
|
def judge(pair: Pair, found: list[Any] | None) -> Verdict:
|
|
if found is None:
|
|
return Verdict(False, "no answer")
|
|
if not found:
|
|
return Verdict(False, "not found")
|
|
if len(found) > 1:
|
|
return Verdict(False, f"{len(found)} places claimed")
|
|
place = found[0]
|
|
if not isinstance(place, dict) or not same_locator(pair.type, pair.locator, place.get("locator")):
|
|
got = place.get("locator") if isinstance(place, dict) else place
|
|
return Verdict(False, f"wrong place {json.dumps(got)}")
|
|
if place.get("hidden") is not pair.hidden:
|
|
return Verdict(False, f"hidden flag wrong (expected {str(pair.hidden).lower()})")
|
|
return Verdict(True)
|
|
|
|
|
|
def report(set_name: str, pairs: list[Pair], verdicts: dict[str, Verdict], errors: dict[tuple[str, str], str],
|
|
files: int, command: list[str]) -> bool:
|
|
print(f"filetype gate: {len(pairs)} pairs in {files} files (set {set_name}); reader: {' '.join(command)}")
|
|
for (root, rel), reason in sorted(errors.items()):
|
|
print(f" ERROR {root}:{rel}: {reason}")
|
|
for p in pairs:
|
|
v = verdicts[p.id]
|
|
if not v.located:
|
|
print(f" {p.id:<8} {p.type:<5} {json.dumps(p.locator):<40} NOT LOCATED ({v.reason})")
|
|
|
|
def line(label: str, group: list[Pair]) -> str:
|
|
return f"{label}: {sum(verdicts[p.id].located for p in group)} of {len(group)}"
|
|
|
|
ordinary = [p for p in pairs if p.group == "filetype"]
|
|
present = [t for t in TYPES if any(p.type == t for p in ordinary)]
|
|
print("per file type: " + " · ".join(line(t, [p for p in ordinary if p.type == t]) for t in present))
|
|
full = [t for t in present if all(verdicts[p.id].located for p in ordinary if p.type == t)]
|
|
print(f"file types with every pair located: {len(full)} of {len(present)}")
|
|
hidden = [p for p in pairs if p.group == "hidden"]
|
|
if hidden:
|
|
classes = sorted({str(p.hidden_class) for p in hidden})
|
|
print("hidden, per class: " + " · ".join(line(c, [p for p in hidden if p.hidden_class == c]) for c in classes))
|
|
controls = [p for p in pairs if p.group == "hidden-control"]
|
|
if controls:
|
|
print(line("visible controls in the hidden cases", controls))
|
|
located = sum(v.located for v in verdicts.values())
|
|
print(f"located {located} of {len(pairs)} · reader errors {len(errors)} of {files} files")
|
|
green = located == len(pairs) and not errors
|
|
print(f"verdict: {'GREEN' if green else 'RED'}")
|
|
return green
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args_in = sys.argv[1:] if argv is None else argv
|
|
if "--" in args_in:
|
|
split = args_in.index("--")
|
|
own, command = args_in[:split], args_in[split + 1:]
|
|
else:
|
|
own, command = args_in, []
|
|
parser = argparse.ArgumentParser(description="Run the reader's file-type gate (fail-closed, no model calls).")
|
|
parser.add_argument("--manifest", type=Path, required=True)
|
|
parser.add_argument("--pairs", type=Path, required=True, help="the working set or the held-out set")
|
|
parser.add_argument("--sources", type=Path, required=True, help="root for pairs with root 'sources'")
|
|
parser.add_argument("--cases", type=Path, required=True, help="root for pairs with root 'cases' (gates-local)")
|
|
parser.add_argument("--timeout", type=float, default=120.0, help="seconds per reader run (one run per file)")
|
|
args = parser.parse_args(own)
|
|
if not command:
|
|
print("usage error: give the reader command after --, e.g. -- python3 reader.py {file} {queries}")
|
|
return 2
|
|
roots = {"sources": args.sources, "cases": args.cases}
|
|
try:
|
|
set_name, pairs = load_pairs(args.pairs)
|
|
verify_fixtures(args.manifest, args.pairs, pairs, roots)
|
|
except GateError as exc:
|
|
print(f"GATE CANNOT RUN: {exc}")
|
|
return 2
|
|
by_file: dict[tuple[str, str], list[Pair]] = {}
|
|
for p in pairs:
|
|
by_file.setdefault(p.source, []).append(p)
|
|
verdicts: dict[str, Verdict] = {}
|
|
errors: dict[tuple[str, str], str] = {}
|
|
for (root, rel), group in sorted(by_file.items()):
|
|
answers, reason = run_reader(command, group, roots[root] / rel, args.sources, args.timeout)
|
|
if answers is None:
|
|
errors[(root, rel)] = reason
|
|
verdicts.update({p.id: Verdict(False, "reader error") for p in group})
|
|
continue
|
|
verdicts.update({p.id: judge(p, answers.get(p.id)) for p in group})
|
|
return 0 if report(set_name, pairs, verdicts, errors, len(by_file), command) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|