feat(accounting): okf build accounts for every source element

okf build --accounting PATH inventories every source document before
extraction, in the gate's per-format vocabulary, and after the run gives
each element one fate (carried / pointer / coded rejection), written as
JSON and summarised in log.md. "carried" is checked against the written
concept bodies, so a gate that drops a line is found (test). Exit 1 on
anything unaccounted or double-booked. Opt-in: +744 s (+19 %) on the
43-document reference corpus, and that corpus fails the check on 24 real
losses (22 images on text-less PDF pages, 2 docx Title paragraphs).

Changed without the flag:
- okf build exits 1 when it extracted documents and persisted none.
  Door B and corpus.measure are unchanged. One test relied on exit 0.
- An image file carried through a persisted document is its own K1b
  column, no longer also extractor_unknown. The set is what the resolver
  actually carried (ExtractedDocument.files), never a byte match.

tools/okf_accounting_gate.py (checks untouched) is green on all six rows,
R761 110 of 110 under both gates.

Report: docs/2026-09-17-innholdsregnskapet-bygget.md

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-17 18:35:20 +02:00
commit 864570b320
13 changed files with 1751 additions and 59 deletions

View file

@ -21,6 +21,7 @@ from __future__ import annotations
import hashlib
import os
import posixpath
import re
import unicodedata
from collections.abc import Callable, Mapping, Sequence
@ -633,6 +634,24 @@ class InboxResult:
# exactly like a run over documents that had none.
assets: tuple[str, ...] = ()
assets_rejected: tuple[AssetRejection, ...] = ()
# Inbox files whose bytes a PERSISTED document carried as an image, as
# inbox-relative paths. Such a file has one fate -- carried -- and is not
# also a coded rejection of the walk; the conservation identity counts it
# in its own column.
carried_files: tuple[str, ...] = ()
# Per persisted document: how many image placements were carried, and the
# ones that were found and not carried, with their codes. The content
# accounting books a document's images from this, never from the bundle.
document_assets: tuple[DocumentAssets, ...] = ()
@dataclass(frozen=True)
class DocumentAssets:
"""One persisted document's image outcome."""
source_file: str
carried: int
rejected: tuple[AssetRejection, ...]
def relative_source(path: Path, inbox: Path) -> str:
@ -1087,6 +1106,8 @@ def process_inbox(
# retire it.
carried_assets: dict[str, bytes] = {}
refused_assets: list[AssetRejection] = []
carried_files: set[str] = set()
document_assets: list[DocumentAssets] = []
# Phase 1: name every file BEFORE any gate call or write, so an intra-run
# collision is caught while both files can still be refused together. Under
@ -1391,6 +1412,19 @@ def process_inbox(
# orphan no pointer names and no retirement pass reaches.
_write_assets(bundle, document.images, carried_assets)
refused_assets.extend(document.rejected)
directory = PurePosixPath(source_name(path)).parent
carried_files.update(
posixpath.normpath((directory / reference).as_posix())
for reference in document.files
)
if outputs:
document_assets.append(
DocumentAssets(
source_file=source_name(path),
carried=len(document.images),
rejected=document.rejected,
)
)
for target_name, content, reasons in outputs:
# `write_bytes` resolves a subpath through `safe_resolve` but never
# creates one. Without this the very first hierarchical write fails.
@ -1449,6 +1483,8 @@ def process_inbox(
skipped=skipped,
assets=tuple(sorted(carried_assets)),
assets_rejected=tuple(refused_assets),
carried_files=tuple(sorted(carried_files)),
document_assets=tuple(document_assets),
)