feat(accounting): okf build accounts for every source element
okf build --accounting PATH inventories every source document before extraction, in the gate's per-format vocabulary, and after the run gives each element one fate (carried / pointer / coded rejection), written as JSON and summarised in log.md. "carried" is checked against the written concept bodies, so a gate that drops a line is found (test). Exit 1 on anything unaccounted or double-booked. Opt-in: +744 s (+19 %) on the 43-document reference corpus, and that corpus fails the check on 24 real losses (22 images on text-less PDF pages, 2 docx Title paragraphs). Changed without the flag: - okf build exits 1 when it extracted documents and persisted none. Door B and corpus.measure are unchanged. One test relied on exit 0. - An image file carried through a persisted document is its own K1b column, no longer also extractor_unknown. The set is what the resolver actually carried (ExtractedDocument.files), never a byte match. tools/okf_accounting_gate.py (checks untouched) is green on all six rows, R761 110 of 110 under both gates. Report: docs/2026-09-17-innholdsregnskapet-bygget.md Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
0b00de4408
commit
864570b320
13 changed files with 1751 additions and 59 deletions
|
|
@ -68,6 +68,7 @@ caller.
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
|
@ -500,6 +501,7 @@ def build(
|
|||
frontmatter: Mapping[str, str] | None = None,
|
||||
gate: str = DEFAULT_GATE,
|
||||
assets: bool = DEFAULT_ASSETS,
|
||||
account: bool = False,
|
||||
) -> CorpusReport:
|
||||
"""Folder in, bundle out. The whole command, minus argument parsing.
|
||||
|
||||
|
|
@ -541,6 +543,7 @@ def build(
|
|||
concept_frontmatter_values=concept_values,
|
||||
gate=gate,
|
||||
assets=assets,
|
||||
account=account,
|
||||
)
|
||||
_write_log(bundle, report, profile=STRUCTURED_V1)
|
||||
return report
|
||||
|
|
@ -608,6 +611,7 @@ def build(
|
|||
concept_frontmatter_values=concept_values,
|
||||
gate=gate,
|
||||
assets=assets,
|
||||
account=account,
|
||||
)
|
||||
_write_log(bundle, report, profile=SEGMENTED_OKF_V0_2)
|
||||
return report
|
||||
|
|
@ -934,6 +938,18 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
|
|||
"bundle of documents that had none"
|
||||
),
|
||||
)
|
||||
build_parser.add_argument(
|
||||
"--accounting",
|
||||
type=Path,
|
||||
default=None,
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"take an inventory of every source before extraction and give every "
|
||||
"element one fate after the run -- carried, pointer or a coded "
|
||||
"rejection -- written as JSON to PATH and summarised in log.md. The "
|
||||
"build fails (exit 1) when any element is unaccounted or booked twice"
|
||||
),
|
||||
)
|
||||
build_parser.add_argument(
|
||||
"--gate",
|
||||
choices=GATE_NAMES,
|
||||
|
|
@ -1122,6 +1138,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
shell_parent=args.shell_parent,
|
||||
gate=args.gate,
|
||||
assets=args.assets,
|
||||
account=args.accounting is not None,
|
||||
frontmatter=frontmatter_from_flags(args.frontmatter or ()),
|
||||
)
|
||||
except (IngestError, OSError, ValueError) as exc:
|
||||
|
|
@ -1132,15 +1149,42 @@ def main(argv: list[str] | None = None) -> int:
|
|||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(report.render(), encoding="utf-8", newline="")
|
||||
print(report.render())
|
||||
if report.unaccounted or report.merged + report.rejected != report.n:
|
||||
if report.conservation_failed:
|
||||
print(
|
||||
f"{CLI_ID}: K1b FAILED - merged ({report.merged}) + coded rejections "
|
||||
f"({report.rejected}) != N ({report.n}). Unaccounted: "
|
||||
f"{CLI_ID}: K1b FAILED - {report.identity()}. Unaccounted: "
|
||||
f"{', '.join(report.unaccounted) or '(none named)'}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
failed = False
|
||||
if report.accounting is not None and args.accounting is not None:
|
||||
args.accounting.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.accounting.write_text(
|
||||
json.dumps(report.accounting.to_json(), indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
newline="",
|
||||
)
|
||||
if report.accounting.unaccounted or report.accounting.double_booked:
|
||||
print(
|
||||
f"{CLI_ID}: accounting FAILED - {report.accounting.unaccounted} element(s) "
|
||||
f"unaccounted, {report.accounting.double_booked} double-booked; see "
|
||||
f"{args.accounting} and log.md",
|
||||
file=sys.stderr,
|
||||
)
|
||||
failed = True
|
||||
# A run that read documents and kept none is not a success, whatever the
|
||||
# conservation identity says: every refusal is coded, and the bundle is
|
||||
# still empty. Door B's library function keeps "all rejected" as a normal
|
||||
# outcome -- for a hostile inbox it is one -- but this command is an
|
||||
# operator pointing at their own folder.
|
||||
if report.extracted and not report.persisted:
|
||||
print(
|
||||
f"{CLI_ID}: FAILED - 0 of {report.extracted} extracted document(s) persisted; "
|
||||
f"rejection codes: {', '.join(f'{c} {n}' for c, n in report.codes)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
failed = True
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue