feat(accounting): okf build accounts for every source element
okf build --accounting PATH inventories every source document before extraction, in the gate's per-format vocabulary, and after the run gives each element one fate (carried / pointer / coded rejection), written as JSON and summarised in log.md. "carried" is checked against the written concept bodies, so a gate that drops a line is found (test). Exit 1 on anything unaccounted or double-booked. Opt-in: +744 s (+19 %) on the 43-document reference corpus, and that corpus fails the check on 24 real losses (22 images on text-less PDF pages, 2 docx Title paragraphs). Changed without the flag: - okf build exits 1 when it extracted documents and persisted none. Door B and corpus.measure are unchanged. One test relied on exit 0. - An image file carried through a persisted document is its own K1b column, no longer also extractor_unknown. The set is what the resolver actually carried (ExtractedDocument.files), never a byte match. tools/okf_accounting_gate.py (checks untouched) is green on all six rows, R761 110 of 110 under both gates. Report: docs/2026-09-17-innholdsregnskapet-bygget.md Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
0b00de4408
commit
864570b320
13 changed files with 1751 additions and 59 deletions
|
|
@ -21,6 +21,7 @@ from __future__ import annotations
|
|||
|
||||
import hashlib
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
|
|
@ -633,6 +634,24 @@ class InboxResult:
|
|||
# exactly like a run over documents that had none.
|
||||
assets: tuple[str, ...] = ()
|
||||
assets_rejected: tuple[AssetRejection, ...] = ()
|
||||
# Inbox files whose bytes a PERSISTED document carried as an image, as
|
||||
# inbox-relative paths. Such a file has one fate -- carried -- and is not
|
||||
# also a coded rejection of the walk; the conservation identity counts it
|
||||
# in its own column.
|
||||
carried_files: tuple[str, ...] = ()
|
||||
# Per persisted document: how many image placements were carried, and the
|
||||
# ones that were found and not carried, with their codes. The content
|
||||
# accounting books a document's images from this, never from the bundle.
|
||||
document_assets: tuple[DocumentAssets, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DocumentAssets:
|
||||
"""One persisted document's image outcome."""
|
||||
|
||||
source_file: str
|
||||
carried: int
|
||||
rejected: tuple[AssetRejection, ...]
|
||||
|
||||
|
||||
def relative_source(path: Path, inbox: Path) -> str:
|
||||
|
|
@ -1087,6 +1106,8 @@ def process_inbox(
|
|||
# retire it.
|
||||
carried_assets: dict[str, bytes] = {}
|
||||
refused_assets: list[AssetRejection] = []
|
||||
carried_files: set[str] = set()
|
||||
document_assets: list[DocumentAssets] = []
|
||||
|
||||
# Phase 1: name every file BEFORE any gate call or write, so an intra-run
|
||||
# collision is caught while both files can still be refused together. Under
|
||||
|
|
@ -1391,6 +1412,19 @@ def process_inbox(
|
|||
# orphan no pointer names and no retirement pass reaches.
|
||||
_write_assets(bundle, document.images, carried_assets)
|
||||
refused_assets.extend(document.rejected)
|
||||
directory = PurePosixPath(source_name(path)).parent
|
||||
carried_files.update(
|
||||
posixpath.normpath((directory / reference).as_posix())
|
||||
for reference in document.files
|
||||
)
|
||||
if outputs:
|
||||
document_assets.append(
|
||||
DocumentAssets(
|
||||
source_file=source_name(path),
|
||||
carried=len(document.images),
|
||||
rejected=document.rejected,
|
||||
)
|
||||
)
|
||||
for target_name, content, reasons in outputs:
|
||||
# `write_bytes` resolves a subpath through `safe_resolve` but never
|
||||
# creates one. Without this the very first hierarchical write fails.
|
||||
|
|
@ -1449,6 +1483,8 @@ def process_inbox(
|
|||
skipped=skipped,
|
||||
assets=tuple(sorted(carried_assets)),
|
||||
assets_rejected=tuple(refused_assets),
|
||||
carried_files=tuple(sorted(carried_files)),
|
||||
document_assets=tuple(document_assets),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue