open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(accounting): the gate opens the bundle itself (BLOCKER B-1)

Browse source 
An independent review of `0b00de4` found the judge was a calculator over a
report the judged writes: `account()` compared BOOKED NUMBERS with the
witness's counts and never opened a concept file. Reproduced here first --
a report that changes not one byte of the bundle and books every element as
carried gave `GATE GREEN`, exit 0, and so did booking every element as
rejected.

The witness now gives every element THE PIECES OF TEXT IT IS MADE OF, and
the gate looks for each of them in the concept bodies the run wrote. Pieces
and not one joined string: a reader writes a heading's marker and a
picture's pointer block between the parts of a container, so a section is
never one contiguous run even when every word of it is there.

Also in the judge, each with a test driving it from both sides:

- a negative booking, a document declared persisted that no concept names,
  a document declared rejected that the bundle holds, a rejection code
  outside a closed list, and an `accounting_version` the gate does not read
  are each REFUSED rather than summed;
- a document the build PERSISTED whose report carries nothing from it is
  never clean ("everything rejected" was);
- an asset proves a carry only when its BYTES hash to the source's and it
  stands under the name the layout gives it. The check was a name check, so
  a zero-byte file called `<sha12>-x.png` read as a carry (m-1).

NOT ONE ELEMENT COUNT MOVED: the 13 fixture documents' counts are identical
before and after, so this commit changes what the gate CHECKS and nothing
about what the witness counts. `texts` is additive in the committed fasit.

The rtf text scanner reads `\uN` escapes and skips `{\fonttbl}`-class
groups, or a fixture's font table reads as the first paragraph of its prose;
xlsx cell text is resolved through `sharedStrings.xml`, where a
spreadsheet's words actually live; a PDF page carries its own text lines,
which no row could see before.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-18 01:40:48 +02:00
commit 656cbe5d02
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 2177 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

481
tools/okf_accounting_gate.py
View file

@ -64,7 +64,7 @@ import warnings
import zipfile
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from pathlib import Path
from pathlib import Path, PurePosixPath
from typing import Any
TOOLS = Path(__file__).resolve().parent
@ -190,13 +190,20 @@ def door_available() -> bool:
@dataclass
class Build:
"""What one `okf build` run left behind, read back from the artifacts."""
"""What one `okf build` run left behind, read back from the artifacts.
`bundle_text` is every concept BODY the run wrote, joined. It is what
makes this gate a judge rather than a calculator: a booking that says an
element was carried is checked against these bytes, not against the
number beside it.
"""
exit_code: int
log: str
accounting: dict[str, Any] | None
source_files: set[str]
asset_prefixes: set[str]
assets: dict[str, str]
bundle_text: str = ""
def _frontmatter_source_file(text: str) -> str | None:
@ -212,16 +219,31 @@ def _frontmatter_source_file(text: str) -> str | None:
return value
def _body(text: str) -> str:
"""A concept file without its frontmatter block."""
if not text.startswith("---\n"):
return text
end = text.find("\n---\n", 4)
return text[end + 5 :] if end >= 0 else text
def read_bundle(bundle: Path, exit_code: int, accounting_path: Path | None) -> Build:
sources: set[str] = set()
for path in bundle.rglob("*.md"):
bodies: list[str] = []
for path in sorted(bundle.rglob("*.md"), key=lambda p: p.as_posix()):
if "assets" in path.relative_to(bundle).parts:
continue
found = _frontmatter_source_file(path.read_text(encoding="utf-8"))
text = path.read_text(encoding="utf-8")
found = _frontmatter_source_file(text)
if found:
sources.add(found)
assets = bundle / "assets"
prefixes = {p.name[:12] for p in assets.iterdir()} if assets.is_dir() else set()
bodies.append(_body(text))
assets_dir = bundle / "assets"
assets = (
{p.name: _sha256(p) for p in sorted(assets_dir.iterdir()) if p.is_file()}
if assets_dir.is_dir()
else {}
)
log_path = bundle / "log.md"
accounting = None
if accounting_path is not None and accounting_path.is_file():
@ -231,7 +253,8 @@ def read_bundle(bundle: Path, exit_code: int, accounting_path: Path | None) -> B
log=log_path.read_text(encoding="utf-8") if log_path.is_file() else "",
accounting=accounting,
source_files=sources,
asset_prefixes=prefixes,
assets=assets,
bundle_text="\n".join(bodies),
)
@ -269,8 +292,111 @@ def run_build(corpus: Path, workdir: Path, *, door: bool, gate: str | None = Non
return read_bundle(bundle, code, accounting_path)
def _sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def _sha256(path: Path) -> str:
return _sha256_bytes(path.read_bytes())
def _sha12(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()[:12]
return _sha256(path)[:12]
def asset_holds(build: Build, source: Path) -> bool:
"""Did the run carry THESE bytes, under the name the layout gives them?
Both halves are load-bearing. The name alone was the check until an
independent review wrote a zero-byte file called `<sha12>-x.png` and the
gate read it as a carry (m-1); the bytes alone would credit a file the
layout would have named something else.
"""
digest = _sha256(source)
return build.assets.get(f"{digest[:12]}-{source.name}") == digest
# --- the judge's own reading of the bundle ------------------------------------
#: Every rejection code this build can emit, read off the package's own source
#: 2026-09-18, plus the guard's two dispositions. A code outside this list is
#: RED: a report gets to say WHY it dropped something, not to invent the
#: vocabulary it says it in.
REJECTION_CODES: frozenset[str] = frozenset(
{
"asset_collision",
"asset_pdf_unsupported",
"asset_remote",
"asset_samples_invalid",
"asset_too_large",
"asset_type_unknown",
"asset_unresolved",
"extractor_binary_missing",
"extractor_binary_version",
"extractor_convert_error",
"extractor_decode_error",
"extractor_empty_conversion",
"extractor_empty_csv",
"extractor_empty_pdf",
"extractor_extra_missing",
"extractor_id",
"extractor_ocr_group_missing",
"extractor_pdf_error",
"extractor_unknown",
"extractor_version",
"extractor_xml_doctype",
"extractor_xml_parse_error",
"fail_secure",
"inbox_gate",
"inbox_slug_collision",
"inbox_slug_empty",
"inbox_slug_too_long",
"inbox_source_file_invalid",
"inbox_source_file_unaddressable",
"inbox_source_title_unaddressable",
"inbox_title_invalid",
"inventory_error",
"inventory_unreadable",
"quarantine_review",
}
)
_NOT_ALNUM = re.compile(r"[\W_]+")
#: A converter attribute block (`{.mark}`, `{#slide-1}`). Its letters stand
#: between words that WERE carried, so it is removed from the bundle text
#: before comparing -- the same allowance the build makes, and no looser.
_CONVERTER_ATTRIBUTE = re.compile(r"\{[#.][^{}\n]*\}")
def _norm(text: str) -> str:
return _NOT_ALNUM.sub("", text.casefold())
class Finder:
"""Is this piece of the SOURCE in the text the bundle holds?
Searched forward from the last hit first, because a reader keeps the
source's order; a miss there falls back to the whole text, so an element
that moved is still found. The gate owns this code: borrowing the build's
own finder would make the judge agree with the judged by construction.
"""
def __init__(self, text: str) -> None:
self.text = _norm(_CONVERTER_ATTRIBUTE.sub("", text))
self.cursor = 0
def __call__(self, piece: str) -> bool:
needle = _norm(piece)
if not needle:
return False
at = self.text.find(needle, self.cursor)
if at < 0:
at = self.text.find(needle)
if at < 0:
return False
self.cursor = at + len(needle)
return True
# --- accounting --------------------------------------------------------------
@ -278,79 +404,261 @@ def _sha12(path: Path) -> str:
@dataclass
class Unit:
"""One inventoried thing: a document, or an inbox file that is not one."""
"""One inventoried thing: a document, or an inbox file that is not one.
`unaccounted` and `double` are about the NUMBERS; `unverified` and
`invalid` are about the bundle and the declaration themselves. A booking
the gate could not verify is never clean, and `verified`/`unverifiable`
carry the denominator behind that word.
"""
name: str
kind: str
unaccounted: int
double: int
unverified: int = 0
invalid: int = 0
verified: int = 0
unverifiable: int = 0
notes: list[str] = field(default_factory=list)
@property
def clean(self) -> bool:
return self.unaccounted == 0 and self.double == 0
return not (self.unaccounted or self.double or self.unverified or self.invalid)
def _conservation_held(build: Build) -> bool:
return build.exit_code == 0 and "K1b FAILED" not in build.log
def _image_proof(
images: Iterable[Mapping[str, Any]], build: Build, corpus: Path, finder: Finder
) -> tuple[int, int, int]:
"""(bytes proved in `assets/`, references found as text, images the gate
cannot check).
An image element carries no text of its own, so the only proof it was
carried is the asset -- and the only proof a POINTER survived is the
reference standing in the bundle. An image embedded in a binary container
has no source file to hash, so the gate says it cannot check it rather
than passing it.
"""
proved = pointed = blind = 0
for image in images:
kind = image.get("kind")
target = image.get("target")
ref = str(image.get("ref") or "")
names = [n for n in (ref, PurePosixPath(ref).name if ref else "") if n]
if target:
names.append(PurePosixPath(target).name)
if asset_holds(build, corpus / target):
proved += 1
elif kind == witness.EMBEDDED:
blind += 1
if any(finder(name) for name in names):
pointed += 1
return proved, pointed, blind
def _document_unit(
name: str,
entry: Mapping[str, Any],
declared: Mapping[str, Any] | None,
build: Build,
corpus: Path,
version_ok: bool,
) -> Unit:
"""One document's account, checked against the bundle the run wrote."""
elements: dict[str, int] = dict(entry["elements"])
texts: Mapping[str, list[str]] = entry.get("texts", {})
images: list[Mapping[str, Any]] = list(entry.get("images", []))
total = sum(elements.values())
if declared is None:
return Unit(name, "document", total, 0, notes=["no declared fates"])
if not version_ok:
return Unit(
name,
"document",
total,
0,
invalid=1,
notes=[
f"accounting_version is not {ACCOUNTING_VERSION}: the declaration is unreadable"
],
)
fates: Mapping[str, Any] = declared.get("fates", {})
persisted = name in build.source_files
unaccounted = double = unverified = invalid = verified = unverifiable = 0
notes: list[str] = []
status = declared.get("status")
code = declared.get("code")
if status == "persisted" and not persisted:
invalid += 1
notes.append("declared persisted; no concept in the bundle names this document")
if status == "rejected" and persisted:
invalid += 1
notes.append("declared rejected; the bundle holds a concept from this document")
if code is not None and code not in REJECTION_CODES:
invalid += 1
notes.append(f"document code `{code}` is not one this gate knows")
finder = Finder(build.bundle_text)
booked_carried = 0
for element in sorted(set(elements) | set(fates)):
fate: Mapping[str, Any] = fates.get(element, {})
carried = int(fate.get("carried", 0))
pointer = int(fate.get("pointer", 0))
rejected = {str(k): int(v) for k, v in (fate.get("rejected") or {}).items()}
have = elements.get(element, 0)
booked_carried += max(carried, 0) + max(pointer, 0)
if min([carried, pointer, *rejected.values()], default=0) < 0:
invalid += 1
notes.append(f"{element}: a negative booking {json.dumps(fate, sort_keys=True)}")
unknown = sorted(c for c in rejected if c not in REJECTION_CODES)
if unknown:
invalid += 1
notes.append(
f"{element}: rejection code(s) {', '.join(unknown)} outside the closed list"
)
booked = carried + pointer + sum(rejected.values())
if booked < have:
unaccounted += have - booked
notes.append(f"{element}: {booked} booked of {have}")
elif booked > have:
double += booked - have
notes.append(f"{element}: {booked} booked, source has {have}")
want = max(carried, 0) + max(pointer, 0)
if want == 0:
continue
if not persisted:
invalid += 1
notes.append(
f"{element}: {want} booked carried, but the bundle holds nothing from here"
)
if element == "image":
proved, pointed, blind = _image_proof(images, build, corpus, finder)
reach = min(carried, proved) + min(pointer, pointed)
verified += reach
short = want - reach
if short > 0:
take = min(short, blind)
unverifiable += take
if short > take:
unverified += short - take
notes.append(
f"image: {want} booked, {proved} asset(s) with the source's bytes and "
f"{pointed} reference(s) found in the bundle"
)
continue
# An element is made of PIECES, and it is carried only if EVERY one of
# them is in the bundle: a reader writes its own markers between the
# parts of a container, so a section is never one contiguous run.
elements_pieces = [
[piece for piece in element_pieces if _norm(piece)]
for element_pieces in texts.get(element, [])
]
sayable = [pieces for pieces in elements_pieces if pieces]
found = sum(1 for pieces in sayable if all(finder(piece) for piece in pieces))
verified += min(found, want)
if want > found:
short = want - found
take = min(short, len(elements_pieces) - len(sayable))
unverifiable += take
if short > take:
unverified += short - take
notes.append(
f"{element}: {want} booked carried, {found} of {len(sayable)} "
"found in the bundle"
)
if persisted and total > 0 and booked_carried == 0:
invalid += 1
notes.append("the build persisted this document and the report carries nothing from it")
return Unit(
name,
"document",
unaccounted,
double,
unverified=unverified,
invalid=invalid,
verified=verified,
unverifiable=unverifiable,
notes=notes,
)
def _file_unit(
name: str,
entry: Mapping[str, Any],
declared: Mapping[str, Any] | None,
build: Build,
corpus: Path,
version_ok: bool,
) -> Unit:
"""One inbox file that is not a document the build reads."""
pointed_by = entry["pointed_at_by"]
# Bytes in assets/ prove a carry only for a file a document points at:
# an unpointed file with the same bytes (R761 ships 8 such duplicates)
# was not carried through anything.
carried = bool(pointed_by) and asset_holds(build, corpus / name)
merged = name in build.source_files
notes: list[str] = []
invalid = 0
if declared is not None and not version_ok:
return Unit(
name,
"file",
1,
0,
invalid=1,
notes=[
f"accounting_version is not {ACCOUNTING_VERSION}: the declaration is unreadable"
],
)
false_claim = False
if declared is not None:
fate = declared.get("fate")
code = declared.get("code")
rejected = fate == "rejected"
false_claim = fate == "carried" and not carried
if false_claim:
notes.append("declared carried; no asset holds this file's bytes under its own name")
if code is not None and code not in REJECTION_CODES:
invalid += 1
notes.append(f"file code `{code}` is not one this gate knows")
else:
# K1b: every walked file is merged or a coded rejection, so a file
# that is not merged was booked as a rejection.
rejected = not merged and _conservation_held(build)
fates = sum((carried, merged, rejected))
if carried and rejected:
notes.append(f"carried via {', '.join(pointed_by) or 'a document'} AND rejected")
unaccounted = 1 if fates == 0 or false_claim else 0
return Unit(
name,
"file",
unaccounted,
max(0, fates - 1),
invalid=invalid,
verified=1 if carried else 0,
notes=notes,
)
def account(inventory: Mapping[str, Any], build: Build, corpus: Path) -> list[Unit]:
"""Give every inventoried element and file its fate, or say it has none."""
declared_docs = {}
declared_files = {}
"""Give every inventoried element and file its fate, and CHECK it."""
declared_docs: dict[str, Any] = {}
declared_files: dict[str, Any] = {}
version_ok = True
if build.accounting is not None:
declared_docs = {d["source_file"]: d for d in build.accounting.get("documents", [])}
declared_files = {f["source_file"]: f for f in build.accounting.get("files", [])}
version_ok = build.accounting.get("accounting_version") == ACCOUNTING_VERSION
units: list[Unit] = []
for name, entry in sorted(inventory["documents"].items()):
elements: dict[str, int] = entry["elements"]
declared = declared_docs.get(name)
if declared is None:
units.append(Unit(name, "document", sum(elements.values()), 0, ["no declared fates"]))
continue
unaccounted = double = 0
notes: list[str] = []
fates: dict[str, Any] = declared.get("fates", {})
for element in sorted(set(elements) | set(fates)):
fate = fates.get(element, {})
booked = (
int(fate.get("carried", 0))
+ int(fate.get("pointer", 0))
+ sum(int(v) for v in fate.get("rejected", {}).values())
)
have = elements.get(element, 0)
if booked < have:
unaccounted += have - booked
notes.append(f"{element}: {booked} booked of {have}")
elif booked > have:
double += booked - have
notes.append(f"{element}: {booked} booked, source has {have}")
units.append(Unit(name, "document", unaccounted, double, notes))
units.append(
_document_unit(name, entry, declared_docs.get(name), build, corpus, version_ok)
)
for name, entry in sorted(inventory["files"].items()):
pointed_by = entry["pointed_at_by"]
# Bytes in assets/ prove a carry only for a file a document points at:
# an unpointed file with the same bytes (R761 ships 8 such duplicates)
# was not carried through anything.
carried = bool(pointed_by) and _sha12(corpus / name) in build.asset_prefixes
merged = name in build.source_files
declared = declared_files.get(name)
notes = []
false_claim = False
if declared is not None:
rejected = declared.get("fate") == "rejected"
false_claim = declared.get("fate") == "carried" and not carried
if false_claim:
notes.append("declared carried, bytes absent from assets/")
else:
# K1b: every walked file is merged or a coded rejection, so a file
# that is not merged was booked as a rejection.
rejected = not merged and _conservation_held(build)
fates = sum((carried, merged, rejected))
if carried and rejected:
notes.append(f"carried via {', '.join(pointed_by) or 'a document'} AND rejected")
unaccounted = 1 if fates == 0 or false_claim else 0
units.append(Unit(name, "file", unaccounted, max(0, fates - 1), notes))
units.append(_file_unit(name, entry, declared_files.get(name), build, corpus, version_ok))
return units
@ -405,20 +713,45 @@ def row2(table: Iterable[str], inventory: Mapping[str, Any], build: Build, door:
return _row(2, name, len(good), len(table), reason, details)
def _tally(units: Iterable[Unit]) -> str:
"""What the gate FOUND, with the denominator beside it."""
units = list(units)
verified = sum(u.verified for u in units)
unverified = sum(u.unverified for u in units)
blind = sum(u.unverifiable for u in units)
return (
f"{verified} carried element(s) found in the bundle, {unverified} claimed and not found, "
f"{blind} carrying no text of their own (the gate cannot check those)"
)
def row3(units: list[Unit], door: bool) -> Row:
clean = sum(1 for u in units if u.clean)
u_total = sum(u.unaccounted for u in units)
d_total = sum(u.double for u in units)
reason = f"u = {u_total} unaccounted, d = {d_total} double-booked"
unverified = sum(u.unverified for u in units)
invalid = sum(u.invalid for u in units)
reason = (
f"u = {u_total} unaccounted, d = {d_total} double-booked, "
f"{unverified} booked carried and not in the bundle, {invalid} declaration(s) the gate refuses"
)
if not door:
reason += f"; no `{ACCOUNTING_FLAG}` door, so no element has a declared fate"
details = [
f"{u.kind} {u.name}: u={u.unaccounted} d={u.double}"
details = [_tally(units)] + [
f"{u.kind} {u.name}: u={u.unaccounted} d={u.double} "
f"unverified={u.unverified} invalid={u.invalid}"
+ (f" ({'; '.join(u.notes)})" if u.notes else "")
for u in units
if not u.clean
]
return _row(3, "accounting after build (u = 0 and d = 0)", clean, len(units), reason, details)
return _row(
3,
"accounting after build (u = 0, d = 0, and every carried element found)",
clean,
len(units),
reason,
details,
)
def row4(inventory: Mapping[str, Any], build: Build) -> Row:
@ -480,16 +813,23 @@ def row5(pairs: list[tuple[str, list[str]]], notes: list[str]) -> Row:
return _row(5, "two witnesses agree", good, len(pairs), reason, details)
def _counts(count: witness.Count | None) -> dict[str, int] | None:
return None if count is None else dict(count.counts)
def witness_pairs(r761: Path | None) -> tuple[list[tuple[str, list[str]]], list[str]]:
sts_xml, _, _ = witness.count_sts_xml(STS_FIXTURE.read_bytes())
pairs = [
(
"sts fixture (xml | json)",
compare(sts_xml, witness.count_sts_json(STS_TWIN.read_bytes())),
compare(_counts(sts_xml), _counts(witness.count_sts_json(STS_TWIN.read_bytes()))),
),
(
"pdf fixture (pdfplumber | poppler)",
compare(witness.pdf_objects(PDF_FIXTURE), witness.pdf_poppler(PDF_FIXTURE)),
compare(
_counts(witness.pdf_objects(PDF_FIXTURE)),
_counts(witness.pdf_poppler(PDF_FIXTURE)),
),
),
]
notes: list[str] = []
@ -502,14 +842,17 @@ def witness_pairs(r761: Path | None) -> tuple[list[tuple[str, list[str]]], list[
pairs.append(
(
"R761 sts (xml | json)",
compare(r761_xml, witness.count_sts_json((r761 / R761_JSON).read_bytes())),
compare(
_counts(r761_xml),
_counts(witness.count_sts_json((r761 / R761_JSON).read_bytes())),
),
)
)
pdf = r761 / R761_PDF
pairs.append(
(
"R761 pdf (pdfplumber | poppler)",
compare(witness.pdf_objects(pdf), witness.pdf_poppler(pdf)),
compare(_counts(witness.pdf_objects(pdf)), _counts(witness.pdf_poppler(pdf))),
)
)
return pairs, notes
@ -533,7 +876,7 @@ def row6(r761: Path | None, ci: bool) -> Row:
inbox = Path(tmp) / "inbox"
archive.extractall(inbox)
inventory = witness.witness_inbox(inbox)
json_counts = witness.count_sts_json((r761 / R761_JSON).read_bytes())
json_counts = dict(witness.count_sts_json((r761 / R761_JSON).read_bytes()).counts)
for index, gate_name in enumerate(R761_GATES):
build = run_build(inbox, Path(tmp) / f"work{index}", door=door, gate=gate_name)
runs.append((gate_name or "default", build, account(inventory, build, inbox)))