feat(accounting): okf build accounts for every source element

okf build --accounting PATH inventories every source document before
extraction, in the gate's per-format vocabulary, and after the run gives
each element one fate (carried / pointer / coded rejection), written as
JSON and summarised in log.md. "carried" is checked against the written
concept bodies, so a gate that drops a line is found (test). Exit 1 on
anything unaccounted or double-booked. Opt-in: +744 s (+19 %) on the
43-document reference corpus, and that corpus fails the check on 24 real
losses (22 images on text-less PDF pages, 2 docx Title paragraphs).

Changed without the flag:
- okf build exits 1 when it extracted documents and persisted none.
  Door B and corpus.measure are unchanged. One test relied on exit 0.
- An image file carried through a persisted document is its own K1b
  column, no longer also extractor_unknown. The set is what the resolver
  actually carried (ExtractedDocument.files), never a byte match.

tools/okf_accounting_gate.py (checks untouched) is green on all six rows,
R761 110 of 110 under both gates.

Report: docs/2026-09-17-innholdsregnskapet-bygget.md

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-17 18:35:20 +02:00
commit 864570b320
13 changed files with 1751 additions and 59 deletions

View file

@ -68,6 +68,7 @@ caller.
from __future__ import annotations
import argparse
import json
import sys
import tempfile
from collections.abc import Mapping, Sequence
@ -500,6 +501,7 @@ def build(
frontmatter: Mapping[str, str] | None = None,
gate: str = DEFAULT_GATE,
assets: bool = DEFAULT_ASSETS,
account: bool = False,
) -> CorpusReport:
"""Folder in, bundle out. The whole command, minus argument parsing.
@ -541,6 +543,7 @@ def build(
concept_frontmatter_values=concept_values,
gate=gate,
assets=assets,
account=account,
)
_write_log(bundle, report, profile=STRUCTURED_V1)
return report
@ -608,6 +611,7 @@ def build(
concept_frontmatter_values=concept_values,
gate=gate,
assets=assets,
account=account,
)
_write_log(bundle, report, profile=SEGMENTED_OKF_V0_2)
return report
@ -934,6 +938,18 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
"bundle of documents that had none"
),
)
build_parser.add_argument(
"--accounting",
type=Path,
default=None,
metavar="PATH",
help=(
"take an inventory of every source before extraction and give every "
"element one fate after the run -- carried, pointer or a coded "
"rejection -- written as JSON to PATH and summarised in log.md. The "
"build fails (exit 1) when any element is unaccounted or booked twice"
),
)
build_parser.add_argument(
"--gate",
choices=GATE_NAMES,
@ -1122,6 +1138,7 @@ def main(argv: list[str] | None = None) -> int:
shell_parent=args.shell_parent,
gate=args.gate,
assets=args.assets,
account=args.accounting is not None,
frontmatter=frontmatter_from_flags(args.frontmatter or ()),
)
except (IngestError, OSError, ValueError) as exc:
@ -1132,15 +1149,42 @@ def main(argv: list[str] | None = None) -> int:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(report.render(), encoding="utf-8", newline="")
print(report.render())
if report.unaccounted or report.merged + report.rejected != report.n:
if report.conservation_failed:
print(
f"{CLI_ID}: K1b FAILED - merged ({report.merged}) + coded rejections "
f"({report.rejected}) != N ({report.n}). Unaccounted: "
f"{CLI_ID}: K1b FAILED - {report.identity()}. Unaccounted: "
f"{', '.join(report.unaccounted) or '(none named)'}",
file=sys.stderr,
)
return 1
return 0
failed = False
if report.accounting is not None and args.accounting is not None:
args.accounting.parent.mkdir(parents=True, exist_ok=True)
args.accounting.write_text(
json.dumps(report.accounting.to_json(), indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
newline="",
)
if report.accounting.unaccounted or report.accounting.double_booked:
print(
f"{CLI_ID}: accounting FAILED - {report.accounting.unaccounted} element(s) "
f"unaccounted, {report.accounting.double_booked} double-booked; see "
f"{args.accounting} and log.md",
file=sys.stderr,
)
failed = True
# A run that read documents and kept none is not a success, whatever the
# conservation identity says: every refusal is coded, and the bundle is
# still empty. Door B's library function keeps "all rejected" as a normal
# outcome -- for a hostile inbox it is one -- but this command is an
# operator pointing at their own folder.
if report.extracted and not report.persisted:
print(
f"{CLI_ID}: FAILED - 0 of {report.extracted} extracted document(s) persisted; "
f"rejection codes: {', '.join(f'{c} {n}' for c, n in report.codes)}",
file=sys.stderr,
)
failed = True
return 1 if failed else 0
if __name__ == "__main__":