Kjell Tore Guttormsen
751441c17d
The content accounting stays OPT-IN until the losses it reports on the reference corpus are fixed; a default-on door would fail builds that pass today. Of the three exceptions the gate proposed, only the PDF one (heading, paragraph, table) is approved. Approving it moves no number, because no witness counts a heading in a PDF: what it changes is that the gap is a stated limit of the instrument rather than an open question about the build. An image in a workbook, or in md/txt/csv/json/odt/rtf, stays unaccounted and therefore stays red. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
683 lines
26 KiB
Python
683 lines
26 KiB
Python
"""The content-accounting gate for `okf build` (capability loop, step 3).
|
|
|
|
One command, one exit code. For every supported file type it asks whether the
|
|
bundle accounts for what the SOURCE holds: of M elements the source carries,
|
|
how many does the build account for as carried, as a pointer or as a coded
|
|
rejection -- and how many does it account for NOT AT ALL (u) or TWICE (d).
|
|
|
|
Written RED, before any capability. `okf build`'s conservation identity,
|
|
`merged + coded rejections == N`, counts FILES: a file can be "merged" while
|
|
content inside it is gone, and a file can be "rejected" while its bytes ride
|
|
into the bundle through a document that points at it. Neither is visible to
|
|
the identity, and both are measured here.
|
|
|
|
THE FASIT NEVER COMES FROM THE READER IT JUDGES. Element counts come from
|
|
`tools/okf_witness.py`, which imports no `llm_ingestion_okf` module (a test
|
|
proves it on the live import graph) and is committed as data in
|
|
`tests/fixtures/accounting/*inventory.json`. This module imports the package
|
|
only to RUN the build it judges.
|
|
|
|
THE DOOR THE CAPABILITY MUST OPEN (the contract this gate reads). `okf build`
|
|
accepts `--accounting PATH` and writes one JSON object there:
|
|
|
|
{"accounting_version": 1,
|
|
"documents": [
|
|
{"source_file": "<inbox-relative path>",
|
|
"status": "persisted" | "rejected", "code": "<rejection code>" | null,
|
|
"inventory": {"<element>": <count>, ...},
|
|
"fates": {"<element>": {"carried": n, "pointer": n,
|
|
"rejected": {"<code>": n}}}}],
|
|
"files": [
|
|
{"source_file": "<inbox-relative path>",
|
|
"fate": "carried" | "merged" | "rejected", "code": "<code>" | null}]}
|
|
|
|
`inventory` is taken BEFORE extraction and before the persist gate, in the
|
|
witness's element vocabulary (per file type, defined in `okf_witness.py`), so
|
|
a document the gate refuses still has one. `files` covers every inbox file
|
|
that is not a document the build reads; `fate` is exactly one value, so a file
|
|
whose bytes were carried through a document is `carried` and never also
|
|
`rejected`. Until the flag exists, rows 2 and 3 say so and stay red.
|
|
|
|
A rejected document is reported in `log.md` as ONE line, and row 4 reads it:
|
|
|
|
<source_file>: <M> elements found in the source, 0 carried: document rejected `<code>`
|
|
|
|
with M the document's inventory total. The `Images` bullet's "found" count is
|
|
the SOURCE's (every image the documents declare), never what a reader got to.
|
|
|
|
EXCEPTIONS to 100 % are listed in the output and are NOT APPROVED: none of
|
|
them lowers a denominator until the operator approves it by name.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import warnings
|
|
import zipfile
|
|
from collections.abc import Iterable, Mapping
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
TOOLS = Path(__file__).resolve().parent
|
|
REPO = TOOLS.parent
|
|
if str(TOOLS) not in sys.path:
|
|
sys.path.insert(0, str(TOOLS))
|
|
|
|
import okf_witness as witness # noqa: E402
|
|
|
|
FIXTURES = REPO / "tests" / "fixtures" / "accounting"
|
|
CORPUS = FIXTURES / "corpus"
|
|
REJECTED = FIXTURES / "rejected"
|
|
INVENTORY = FIXTURES / "inventory.json"
|
|
REJECTED_INVENTORY = FIXTURES / "rejected-inventory.json"
|
|
STS_FIXTURE = CORPUS / "prosess-84-sts.xml"
|
|
STS_TWIN = FIXTURES / "witness" / "prosess-84-sts.twin.json"
|
|
PDF_FIXTURE = CORPUS / "prosess-84-tabell.pdf"
|
|
README = REPO / "README.md"
|
|
|
|
R761_DEFAULT = Path.home() / "repos" / "vegnormal-okf" / "data" / "raw" / "860019"
|
|
R761_ZIP = "14ce59dc-2150-480b-b661-6ea605fe3b24.zip"
|
|
R761_JSON = "R761-2025-860019.json"
|
|
R761_PDF = "R761-prosesskoden-2025.pdf"
|
|
|
|
ACCOUNTING_FLAG = "--accounting"
|
|
ACCOUNTING_VERSION = 1
|
|
BUNDLE_ID = "accounting-gate"
|
|
OKF_VERSION = "0.2"
|
|
CONSUME_QUESTION = "Hvilken toleranseklasse gjelder for konstruksjoner av betong?"
|
|
|
|
GREEN = "GREEN"
|
|
RED = "RED"
|
|
SKIPPED = "SKIPPED"
|
|
DIAGNOSTIC = "DIAGNOSTIC"
|
|
|
|
#: Exceptions the operator has APPROVED, by (suffix, element), with the date.
|
|
#: Approving one does not move a number here: the witness counts no such
|
|
#: element in the first place, which is the whole reason the exception exists.
|
|
#: What approval changes is that the gap is now a stated limit of this
|
|
#: instrument rather than an open question about the build.
|
|
APPROVED_EXCEPTIONS: frozenset[tuple[str, str]] = frozenset(
|
|
{(".pdf", "heading"), (".pdf", "paragraph"), (".pdf", "table")}
|
|
)
|
|
|
|
#: When, and by whom.
|
|
APPROVED_ON = "2026-09-17, operator"
|
|
|
|
#: Exceptions this gate PROPOSES. Listed in every run; none of them is applied.
|
|
PROPOSED_EXCEPTIONS: tuple[dict[str, str], ...] = (
|
|
{
|
|
"suffix": ".xlsx",
|
|
"element": "image",
|
|
"reason": "the converter writes one pipe table per sheet and a pointer "
|
|
"block inside it would break source_rows (README, 0.10.0)",
|
|
"carried_instead": "nothing; the image is absent from the bundle",
|
|
},
|
|
{
|
|
"suffix": ".md .txt .csv .json .odt .rtf",
|
|
"element": "image",
|
|
"reason": "no reader for these types carries image bytes (README, 0.10.0)",
|
|
"carried_instead": "the reference text as written, if the format has one",
|
|
},
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Row:
|
|
number: int
|
|
name: str
|
|
k: int
|
|
m: int
|
|
status: str
|
|
reason: str
|
|
details: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def fails(self) -> bool:
|
|
return self.status == RED and self.number <= 6
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"row": self.number,
|
|
"name": self.name,
|
|
"k": self.k,
|
|
"m": self.m,
|
|
"status": self.status,
|
|
"reason": self.reason,
|
|
"details": self.details,
|
|
}
|
|
|
|
|
|
def _row(number: int, name: str, k: int, m: int, reason: str, details: list[str]) -> Row:
|
|
return Row(number, name, k, m, GREEN if m > 0 and k == m else RED, reason, details)
|
|
|
|
|
|
# --- inputs ------------------------------------------------------------------
|
|
|
|
|
|
def readme_types(readme: Path = README) -> list[str]:
|
|
"""T: the rows of README's supported-file-types table."""
|
|
text = readme.read_text(encoding="utf-8")
|
|
section = text.split("## Supported file types", 1)[1].split("\n## ", 1)[0]
|
|
return sorted(set(re.findall(r"^\| `(\.[a-z0-9]+)` \|", section, re.MULTILINE)))
|
|
|
|
|
|
def load_inventory(path: Path) -> dict[str, Any]:
|
|
data: dict[str, Any] = json.loads(path.read_text(encoding="utf-8"))
|
|
return data
|
|
|
|
|
|
def door_available() -> bool:
|
|
"""Does `okf build` accept the accounting flag?"""
|
|
from llm_ingestion_okf import cli
|
|
|
|
argv = ["build", "in", "--bundle", "out", "--bundle-id", "x", "--okf-version", "0.2"]
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
try:
|
|
cli.parse_args([*argv, ACCOUNTING_FLAG, "accounting.json"])
|
|
except SystemExit:
|
|
return False
|
|
return True
|
|
|
|
|
|
@dataclass
|
|
class Build:
|
|
"""What one `okf build` run left behind, read back from the artifacts."""
|
|
|
|
exit_code: int
|
|
log: str
|
|
accounting: dict[str, Any] | None
|
|
source_files: set[str]
|
|
asset_prefixes: set[str]
|
|
|
|
|
|
def _frontmatter_source_file(text: str) -> str | None:
|
|
if not text.startswith("---\n"):
|
|
return None
|
|
head = text[4:].split("\n---\n", 1)[0]
|
|
match = re.search(r"^source_file:\s*(.+?)\s*$", head, re.MULTILINE)
|
|
if match is None:
|
|
return None
|
|
value = match.group(1)
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
value = value[1:-1]
|
|
return value
|
|
|
|
|
|
def read_bundle(bundle: Path, exit_code: int, accounting_path: Path | None) -> Build:
|
|
sources: set[str] = set()
|
|
for path in bundle.rglob("*.md"):
|
|
if "assets" in path.relative_to(bundle).parts:
|
|
continue
|
|
found = _frontmatter_source_file(path.read_text(encoding="utf-8"))
|
|
if found:
|
|
sources.add(found)
|
|
assets = bundle / "assets"
|
|
prefixes = {p.name[:12] for p in assets.iterdir()} if assets.is_dir() else set()
|
|
log_path = bundle / "log.md"
|
|
accounting = None
|
|
if accounting_path is not None and accounting_path.is_file():
|
|
accounting = json.loads(accounting_path.read_text(encoding="utf-8"))
|
|
return Build(
|
|
exit_code=exit_code,
|
|
log=log_path.read_text(encoding="utf-8") if log_path.is_file() else "",
|
|
accounting=accounting,
|
|
source_files=sources,
|
|
asset_prefixes=prefixes,
|
|
)
|
|
|
|
|
|
def run_build(corpus: Path, workdir: Path, *, door: bool, gate: str | None = None) -> Build:
|
|
"""Run the real `okf build` in-process and read back what it wrote."""
|
|
from llm_ingestion_okf import cli
|
|
|
|
bundle = workdir / "bundle"
|
|
accounting_path = workdir / "accounting.json" if door else None
|
|
argv = [
|
|
"build",
|
|
str(corpus),
|
|
"--bundle",
|
|
str(bundle),
|
|
"--bundle-id",
|
|
BUNDLE_ID,
|
|
"--okf-version",
|
|
OKF_VERSION,
|
|
]
|
|
if gate is not None:
|
|
argv += ["--gate", gate]
|
|
if accounting_path is not None:
|
|
argv += [ACCOUNTING_FLAG, str(accounting_path)]
|
|
sink = io.StringIO()
|
|
with (
|
|
contextlib.redirect_stdout(sink),
|
|
contextlib.redirect_stderr(sink),
|
|
warnings.catch_warnings(),
|
|
):
|
|
warnings.simplefilter("ignore")
|
|
try:
|
|
code = cli.main(argv)
|
|
except SystemExit as exc:
|
|
code = exc.code if isinstance(exc.code, int) else 2
|
|
return read_bundle(bundle, code, accounting_path)
|
|
|
|
|
|
def _sha12(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()[:12]
|
|
|
|
|
|
# --- accounting --------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Unit:
|
|
"""One inventoried thing: a document, or an inbox file that is not one."""
|
|
|
|
name: str
|
|
kind: str
|
|
unaccounted: int
|
|
double: int
|
|
notes: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def clean(self) -> bool:
|
|
return self.unaccounted == 0 and self.double == 0
|
|
|
|
|
|
def _conservation_held(build: Build) -> bool:
|
|
return build.exit_code == 0 and "K1b FAILED" not in build.log
|
|
|
|
|
|
def account(inventory: Mapping[str, Any], build: Build, corpus: Path) -> list[Unit]:
|
|
"""Give every inventoried element and file its fate, or say it has none."""
|
|
declared_docs = {}
|
|
declared_files = {}
|
|
if build.accounting is not None:
|
|
declared_docs = {d["source_file"]: d for d in build.accounting.get("documents", [])}
|
|
declared_files = {f["source_file"]: f for f in build.accounting.get("files", [])}
|
|
units: list[Unit] = []
|
|
for name, entry in sorted(inventory["documents"].items()):
|
|
elements: dict[str, int] = entry["elements"]
|
|
declared = declared_docs.get(name)
|
|
if declared is None:
|
|
units.append(Unit(name, "document", sum(elements.values()), 0, ["no declared fates"]))
|
|
continue
|
|
unaccounted = double = 0
|
|
notes: list[str] = []
|
|
fates: dict[str, Any] = declared.get("fates", {})
|
|
for element in sorted(set(elements) | set(fates)):
|
|
fate = fates.get(element, {})
|
|
booked = (
|
|
int(fate.get("carried", 0))
|
|
+ int(fate.get("pointer", 0))
|
|
+ sum(int(v) for v in fate.get("rejected", {}).values())
|
|
)
|
|
have = elements.get(element, 0)
|
|
if booked < have:
|
|
unaccounted += have - booked
|
|
notes.append(f"{element}: {booked} booked of {have}")
|
|
elif booked > have:
|
|
double += booked - have
|
|
notes.append(f"{element}: {booked} booked, source has {have}")
|
|
units.append(Unit(name, "document", unaccounted, double, notes))
|
|
for name, entry in sorted(inventory["files"].items()):
|
|
pointed_by = entry["pointed_at_by"]
|
|
# Bytes in assets/ prove a carry only for a file a document points at:
|
|
# an unpointed file with the same bytes (R761 ships 8 such duplicates)
|
|
# was not carried through anything.
|
|
carried = bool(pointed_by) and _sha12(corpus / name) in build.asset_prefixes
|
|
merged = name in build.source_files
|
|
declared = declared_files.get(name)
|
|
notes = []
|
|
false_claim = False
|
|
if declared is not None:
|
|
rejected = declared.get("fate") == "rejected"
|
|
false_claim = declared.get("fate") == "carried" and not carried
|
|
if false_claim:
|
|
notes.append("declared carried, bytes absent from assets/")
|
|
else:
|
|
# K1b: every walked file is merged or a coded rejection, so a file
|
|
# that is not merged was booked as a rejection.
|
|
rejected = not merged and _conservation_held(build)
|
|
fates = sum((carried, merged, rejected))
|
|
if carried and rejected:
|
|
notes.append(f"carried via {', '.join(pointed_by) or 'a document'} AND rejected")
|
|
unaccounted = 1 if fates == 0 or false_claim else 0
|
|
units.append(Unit(name, "file", unaccounted, max(0, fates - 1), notes))
|
|
return units
|
|
|
|
|
|
# --- rows --------------------------------------------------------------------
|
|
|
|
|
|
def row1(table: Iterable[str], inventory: Mapping[str, Any], fresh: Mapping[str, Any]) -> Row:
|
|
table = sorted(table)
|
|
covered: set[str] = set()
|
|
stale: list[str] = []
|
|
for name, entry in inventory["documents"].items():
|
|
if fresh["documents"].get(name) == entry:
|
|
covered.add(entry["suffix"])
|
|
else:
|
|
stale.append(name)
|
|
missing = [t for t in table if t not in covered]
|
|
k = len(table) - len(missing)
|
|
reason = "every README type has a fixture with a reproducible witness count"
|
|
if missing:
|
|
reason = f"no fixture fasit for {', '.join(missing)}"
|
|
details = [f"committed fasit differs from a fresh witness count: {n}" for n in stale]
|
|
return _row(1, "file types with a fasit fixture", k, len(table), reason, details)
|
|
|
|
|
|
def row2(table: Iterable[str], inventory: Mapping[str, Any], build: Build, door: bool) -> Row:
|
|
table = sorted(table)
|
|
name = "source inventory before build"
|
|
if not door:
|
|
return _row(
|
|
2, name, 0, len(table), f"`okf build` has no `{ACCOUNTING_FLAG}` door; no inventory", []
|
|
)
|
|
declared = {d["source_file"]: d for d in (build.accounting or {}).get("documents", [])}
|
|
good: list[str] = []
|
|
details: list[str] = []
|
|
for suffix in table:
|
|
docs = [n for n, e in inventory["documents"].items() if e["suffix"] == suffix]
|
|
ok = bool(docs)
|
|
for doc in docs:
|
|
got = declared.get(doc, {}).get("inventory")
|
|
want = inventory["documents"][doc]["elements"]
|
|
if got != want:
|
|
ok = False
|
|
details.append(f"{doc}: declared {got}, witness {want}")
|
|
if ok:
|
|
good.append(suffix)
|
|
missing = [t for t in table if t not in good]
|
|
reason = (
|
|
"every type's inventory equals the witness"
|
|
if not missing
|
|
else (f"inventory absent or wrong for {', '.join(missing)}")
|
|
)
|
|
return _row(2, name, len(good), len(table), reason, details)
|
|
|
|
|
|
def row3(units: list[Unit], door: bool) -> Row:
|
|
clean = sum(1 for u in units if u.clean)
|
|
u_total = sum(u.unaccounted for u in units)
|
|
d_total = sum(u.double for u in units)
|
|
reason = f"u = {u_total} unaccounted, d = {d_total} double-booked"
|
|
if not door:
|
|
reason += f"; no `{ACCOUNTING_FLAG}` door, so no element has a declared fate"
|
|
details = [
|
|
f"{u.kind} {u.name}: u={u.unaccounted} d={u.double}"
|
|
+ (f" ({'; '.join(u.notes)})" if u.notes else "")
|
|
for u in units
|
|
if not u.clean
|
|
]
|
|
return _row(3, "accounting after build (u = 0 and d = 0)", clean, len(units), reason, details)
|
|
|
|
|
|
def row4(inventory: Mapping[str, Any], build: Build) -> Row:
|
|
name = "a rejected document is reported honestly"
|
|
rejected = [n for n in sorted(inventory["documents"]) if n not in build.source_files]
|
|
if not rejected:
|
|
return _row(4, name, 0, 0, "the fixture was not rejected; the row cannot judge", [])
|
|
declared_images = sum(len(e["images"]) for e in inventory["documents"].values())
|
|
found = re.search(r"\*\*Images\*\*: (\d+) carried of (\d+) found", build.log)
|
|
good = 0
|
|
details: list[str] = []
|
|
for doc in rejected:
|
|
total = sum(inventory["documents"][doc]["elements"].values())
|
|
line = re.compile(
|
|
rf"{re.escape(doc)}: {total} elements found in the source, 0 carried: "
|
|
r"document rejected `[a-z_]+`"
|
|
)
|
|
problems = []
|
|
if not line.search(build.log):
|
|
problems.append(f"no line '{doc}: {total} elements found in the source, 0 carried'")
|
|
if found is None:
|
|
problems.append("no Images bullet")
|
|
elif int(found.group(2)) != declared_images:
|
|
problems.append(
|
|
f"log says {found.group(1)} carried of {found.group(2)} found; "
|
|
f"the source declares {declared_images}"
|
|
)
|
|
if problems:
|
|
details.extend(f"{doc}: {p}" for p in problems)
|
|
else:
|
|
good += 1
|
|
reason = (
|
|
"every rejected document names what it held"
|
|
if good == len(rejected)
|
|
else (f"{len(rejected) - good} rejected document(s) reported as if they held less")
|
|
)
|
|
return _row(4, name, good, len(rejected), reason, details)
|
|
|
|
|
|
def compare(left: Mapping[str, int] | None, right: Mapping[str, int] | None) -> list[str]:
|
|
"""Disagreements between two witnesses, both numbers kept."""
|
|
if left is None or right is None:
|
|
return ["a witness is unavailable"]
|
|
return [
|
|
f"{e}: {left[e]} vs {right[e]}"
|
|
for e in sorted(set(left) & set(right))
|
|
if left[e] != right[e]
|
|
]
|
|
|
|
|
|
def row5(pairs: list[tuple[str, list[str]]], notes: list[str]) -> Row:
|
|
good = sum(1 for _, problems in pairs if not problems)
|
|
details = [f"{label}: {'; '.join(p)}" for label, p in pairs if p] + notes
|
|
reason = (
|
|
"both witnesses agree on every element"
|
|
if good == len(pairs)
|
|
else (f"{len(pairs) - good} pair(s) disagree")
|
|
)
|
|
return _row(5, "two witnesses agree", good, len(pairs), reason, details)
|
|
|
|
|
|
def witness_pairs(r761: Path | None) -> tuple[list[tuple[str, list[str]]], list[str]]:
|
|
sts_xml, _, _ = witness.count_sts_xml(STS_FIXTURE.read_bytes())
|
|
pairs = [
|
|
(
|
|
"sts fixture (xml | json)",
|
|
compare(sts_xml, witness.count_sts_json(STS_TWIN.read_bytes())),
|
|
),
|
|
(
|
|
"pdf fixture (pdfplumber | poppler)",
|
|
compare(witness.pdf_objects(PDF_FIXTURE), witness.pdf_poppler(PDF_FIXTURE)),
|
|
),
|
|
]
|
|
notes: list[str] = []
|
|
if r761 is None or not r761.is_dir():
|
|
notes.append("R761 pairs not measured: source missing")
|
|
return pairs, notes
|
|
with zipfile.ZipFile(r761 / R761_ZIP) as archive:
|
|
xml_name = next(n for n in archive.namelist() if n.endswith(".xml"))
|
|
r761_xml, _, _ = witness.count_sts_xml(archive.read(xml_name))
|
|
pairs.append(
|
|
(
|
|
"R761 sts (xml | json)",
|
|
compare(r761_xml, witness.count_sts_json((r761 / R761_JSON).read_bytes())),
|
|
)
|
|
)
|
|
pdf = r761 / R761_PDF
|
|
pairs.append(
|
|
(
|
|
"R761 pdf (pdfplumber | poppler)",
|
|
compare(witness.pdf_objects(pdf), witness.pdf_poppler(pdf)),
|
|
)
|
|
)
|
|
return pairs, notes
|
|
|
|
|
|
#: The two builds row 6 runs: the default gate is what a user gets; `none`
|
|
#: persists the document, which is the only way its pictures are carried and
|
|
#: the double booking of the files beside it becomes visible.
|
|
R761_GATES: tuple[str | None, ...] = (None, "none")
|
|
|
|
|
|
def row6(r761: Path | None, ci: bool) -> Row:
|
|
"""R761 through two builds; a unit is clean only if it is clean in both."""
|
|
name = "real corpus: R761 Prosesskoden:2025"
|
|
if r761 is None or not r761.is_dir():
|
|
status = SKIPPED if ci else RED
|
|
return Row(6, name, 0, 0, status, f"not measured, source missing: {r761}")
|
|
door = door_available()
|
|
runs: list[tuple[str, Build, list[Unit]]] = []
|
|
with zipfile.ZipFile(r761 / R761_ZIP) as archive, tempfile.TemporaryDirectory() as tmp:
|
|
inbox = Path(tmp) / "inbox"
|
|
archive.extractall(inbox)
|
|
inventory = witness.witness_inbox(inbox)
|
|
json_counts = witness.count_sts_json((r761 / R761_JSON).read_bytes())
|
|
for index, gate_name in enumerate(R761_GATES):
|
|
build = run_build(inbox, Path(tmp) / f"work{index}", door=door, gate=gate_name)
|
|
runs.append((gate_name or "default", build, account(inventory, build, inbox)))
|
|
files = inventory["files"]
|
|
pointed = sum(1 for f in files.values() if f["pointed_at_by"])
|
|
documents = inventory["documents"]
|
|
details = [
|
|
f"witness (json): {json.dumps(json_counts, sort_keys=True)}",
|
|
f"zip: {len(documents)} document(s), {len(files)} other files "
|
|
f"({pointed} pointed at, {len(files) - pointed} not)",
|
|
]
|
|
reasons = []
|
|
for label, build, units in runs:
|
|
persisted = sum(1 for d in documents if d in build.source_files)
|
|
u_total = sum(u.unaccounted for u in units)
|
|
d_total = sum(u.double for u in units)
|
|
reasons.append(f"gate {label}: u = {u_total}, d = {d_total}")
|
|
details.append(
|
|
f"gate {label}: exit {build.exit_code}, {persisted} of {len(documents)} "
|
|
f"document(s) persisted, {len(build.asset_prefixes)} asset file(s)"
|
|
)
|
|
for unit in units:
|
|
if unit.kind == "document":
|
|
details.append(
|
|
f" {unit.name}: u={unit.unaccounted} d={unit.double} ({'; '.join(unit.notes)})"
|
|
)
|
|
doubled = [u for u in units if u.kind == "file" and u.double]
|
|
if doubled:
|
|
details.append(
|
|
f" {len(doubled)} file(s) carried through the document AND rejected, "
|
|
f"e.g. {doubled[0].name}"
|
|
)
|
|
clean = sum(1 for parts in zip(*(units for _, _, units in runs)) if all(u.clean for u in parts))
|
|
total = len(runs[0][2])
|
|
return _row(6, name, clean, total, f"{'; '.join(reasons)} over {total} units", details)
|
|
|
|
|
|
def row7(workdir: Path) -> Row:
|
|
"""Diagnostic only: a question through `okf consume` on the fixture bundle."""
|
|
from llm_ingestion_okf import cli
|
|
|
|
bundle = workdir / "bundle"
|
|
out = workdir / "payload.json"
|
|
sink = io.StringIO()
|
|
with contextlib.redirect_stdout(sink), contextlib.redirect_stderr(sink):
|
|
try:
|
|
code = cli.main(
|
|
["consume", str(bundle), "--question", CONSUME_QUESTION, "--out", str(out)]
|
|
)
|
|
except SystemExit as exc:
|
|
code = exc.code if isinstance(exc.code, int) else 2
|
|
excerpts = 0
|
|
if out.is_file():
|
|
excerpts = len(json.loads(out.read_text(encoding="utf-8")).get("excerpts", []))
|
|
return Row(
|
|
7,
|
|
"question set via okf consume (diagnostic)",
|
|
excerpts,
|
|
0,
|
|
DIAGNOSTIC,
|
|
f"exit {code}, {excerpts} excerpt(s) for {CONSUME_QUESTION!r}",
|
|
)
|
|
|
|
|
|
# --- the run -----------------------------------------------------------------
|
|
|
|
|
|
def evaluate(*, r761: Path | None, ci: bool, consume: bool) -> list[Row]:
|
|
table = readme_types()
|
|
inventory = load_inventory(INVENTORY)
|
|
rejected_inventory = load_inventory(REJECTED_INVENTORY)
|
|
fresh = witness.witness_inbox(CORPUS)
|
|
door = door_available()
|
|
rows = [row1(table, inventory, fresh)]
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
work = Path(tmp) / "corpus"
|
|
build = run_build(CORPUS, work, door=door)
|
|
rows.append(row2(table, inventory, build, door))
|
|
rows.append(row3(account(inventory, build, CORPUS), door))
|
|
rejected_build = run_build(REJECTED, Path(tmp) / "rejected", door=door)
|
|
rows.append(row4(rejected_inventory, rejected_build))
|
|
rows.append(row5(*witness_pairs(r761)))
|
|
rows.append(row6(r761, ci))
|
|
if consume:
|
|
rows.append(row7(work))
|
|
return rows
|
|
|
|
|
|
def render(rows: list[Row]) -> str:
|
|
lines = ["row | k of M | status | reason"]
|
|
for row in rows:
|
|
count = "n/a" if row.status == DIAGNOSTIC else f"{row.k} of {row.m}"
|
|
lines.append(f"{row.number} {row.name} | {count} | {row.status} | {row.reason}")
|
|
lines.extend(f" - {d}" for d in row.details)
|
|
lines += ["", "exceptions (PROPOSED, NOT APPROVED -- none lowers a denominator):"]
|
|
for item in PROPOSED_EXCEPTIONS:
|
|
lines.append(
|
|
f" - {item['suffix']} {item['element']}: {item['reason']}; "
|
|
f"carried instead: {item['carried_instead']}"
|
|
)
|
|
lines.append(f"exceptions approved ({APPROVED_ON}), and none moves a denominator:")
|
|
for suffix, element in sorted(APPROVED_EXCEPTIONS) or [("(none)", "")]:
|
|
lines.append(f" - {suffix} {element}")
|
|
failing = [str(r.number) for r in rows if r.fails]
|
|
lines.append("")
|
|
lines.append(
|
|
f"GATE {'RED' if failing else 'GREEN'}"
|
|
+ (f": rows {', '.join(failing)}" if failing else "")
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
|
parser.add_argument("--json", action="store_true", help="emit the rows as JSON")
|
|
parser.add_argument(
|
|
"--r761",
|
|
type=Path,
|
|
default=R761_DEFAULT,
|
|
help="directory holding the R761 zip, JSON and PDF (read only)",
|
|
)
|
|
parser.add_argument(
|
|
"--consume", action="store_true", help="also run row 7 (diagnostic, never fails)"
|
|
)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
rows = evaluate(r761=args.r761, ci=bool(os.environ.get("CI")), consume=args.consume)
|
|
except (OSError, ValueError, KeyError) as exc:
|
|
print(f"okf-accounting-gate: did not run: {exc}", file=sys.stderr)
|
|
return 2
|
|
if args.json:
|
|
payload = {
|
|
"rows": [r.to_json() for r in rows],
|
|
"exceptions": {
|
|
"approved": sorted(APPROVED_EXCEPTIONS),
|
|
"proposed": PROPOSED_EXCEPTIONS,
|
|
},
|
|
"gate": RED if any(r.fails for r in rows) else GREEN,
|
|
}
|
|
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
else:
|
|
print(render(rows), end="")
|
|
return 1 if any(r.fails for r in rows) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|