test(accounting): the gate opens the bundle itself (BLOCKER B-1)

An independent review of `0b00de4` found the judge was a calculator over a
report the judged writes: `account()` compared BOOKED NUMBERS with the
witness's counts and never opened a concept file. Reproduced here first --
a report that changes not one byte of the bundle and books every element as
carried gave `GATE GREEN`, exit 0, and so did booking every element as
rejected.

The witness now gives every element THE PIECES OF TEXT IT IS MADE OF, and
the gate looks for each of them in the concept bodies the run wrote. Pieces
and not one joined string: a reader writes a heading's marker and a
picture's pointer block between the parts of a container, so a section is
never one contiguous run even when every word of it is there.

Also in the judge, each with a test driving it from both sides:

- a negative booking, a document declared persisted that no concept names,
  a document declared rejected that the bundle holds, a rejection code
  outside a closed list, and an `accounting_version` the gate does not read
  are each REFUSED rather than summed;
- a document the build PERSISTED whose report carries nothing from it is
  never clean ("everything rejected" was);
- an asset proves a carry only when its BYTES hash to the source's and it
  stands under the name the layout gives it. The check was a name check, so
  a zero-byte file called `<sha12>-x.png` read as a carry (m-1).

NOT ONE ELEMENT COUNT MOVED: the 13 fixture documents' counts are identical
before and after, so this commit changes what the gate CHECKS and nothing
about what the witness counts. `texts` is additive in the committed fasit.

The rtf text scanner reads `\uN` escapes and skips `{\fonttbl}`-class
groups, or a fixture's font table reads as the first paragraph of its prose;
xlsx cell text is resolved through `sharedStrings.xml`, where a
spreadsheet's words actually live; a PDF page carries its own text lines,
which no row could see before.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-18 01:40:48 +02:00
commit 656cbe5d02
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
5 changed files with 2177 additions and 254 deletions

File diff suppressed because it is too large Load diff

View file

@ -17,6 +17,24 @@
}
],
"suffix": ".html",
"texts": {
"cell": [],
"heading": [
[
"Skjult"
]
],
"image": [
[]
],
"list_item": [],
"paragraph": [
[
"Denne teksten bærer et usynlig tegn."
]
],
"table": []
},
"witness": "html.parser"
}
},

View file

@ -22,6 +22,7 @@ from __future__ import annotations
import json
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any
@ -135,9 +136,10 @@ def test_the_witness_matches_a_hand_count(name: str) -> None:
def test_a_fenced_heading_is_not_a_heading_to_the_witness() -> None:
elements, _ = witness.count_markdown("# Real\n\n```bash\n# not one\n```\n")
assert elements["heading"] == 1
assert elements["code_block"] == 1
count, _ = witness.count_markdown("# Real\n\n```bash\n# not one\n```\n")
assert count.counts["heading"] == 1
assert count.counts["code_block"] == 1
assert count.texts["heading"] == [["# Real"]]
def test_the_sts_image_reference_resolves_through_the_graphics_directory() -> None:
@ -163,29 +165,52 @@ def test_the_witness_refuses_a_doctype() -> None:
# --- 2. every row can go both ways -------------------------------------------
#: The two headings of `a.md` as a bundle would carry them. The gate verifies
#: a booked `carried` against THIS, never against the declaration.
_BUNDLE_TEXT = (
"# Foerste overskrift\n\nProsa.\n\n"
"![](/assets/0123456789ab-x.png)\n_Source: x.png_\n\n# Andre overskrift\n"
)
def _inventory() -> dict[str, Any]:
return {
"documents": {
"a.md": {"suffix": ".md", "elements": {"heading": 2, "image": 1}, "images": []},
"a.md": {
"suffix": ".md",
"elements": {"heading": 2, "image": 1},
"texts": {
"heading": ["Foerste overskrift", "Andre overskrift"],
"image": [""],
},
"images": [{"kind": "local", "ref": "graphics/x.png", "target": "graphics/x.png"}],
},
},
"files": {"graphics/x.png": {"pointed_at_by": ["a.md"]}},
}
def _assets(*paths: Path) -> dict[str, str]:
"""The `assets/` directory a build wrote for these source files."""
return {f"{gate._sha12(p)}-{p.name}": gate._sha256(p) for p in paths}
def _build(
*,
accounting: dict[str, Any] | None = None,
sources: set[str] | None = None,
assets: set[str] | None = None,
assets: dict[str, str] | None = None,
log: str = "",
exit_code: int = 0,
bundle_text: str = _BUNDLE_TEXT,
) -> gate.Build:
return gate.Build(
exit_code=exit_code,
log=log,
accounting=accounting,
source_files={"a.md"} if sources is None else sources,
asset_prefixes=assets or set(),
assets=assets or {},
bundle_text=bundle_text,
)
@ -280,14 +305,14 @@ def test_row3_is_red_when_no_fate_is_declared(tmp_path: Path) -> None:
def test_a_file_carried_through_a_document_and_rejected_is_double_booked(tmp_path: Path) -> None:
corpus = _corpus(tmp_path)
carried = {gate._sha12(corpus / "graphics" / "x.png")}
carried = _assets(corpus / "graphics" / "x.png")
units = gate.account(_inventory(), _build(accounting=_declared(), assets=carried), corpus)
assert (units[1].unaccounted, units[1].double) == (0, 1)
def test_a_file_carried_through_a_document_and_declared_carried_is_clean(tmp_path: Path) -> None:
corpus = _corpus(tmp_path)
carried = {gate._sha12(corpus / "graphics" / "x.png")}
carried = _assets(corpus / "graphics" / "x.png")
build = _build(accounting=_declared(fate="carried"), assets=carried)
assert gate.account(_inventory(), build, corpus)[1].clean
@ -305,7 +330,7 @@ def test_an_unpointed_file_sharing_bytes_with_a_carried_one_is_not_carried(
(corpus / "graphics" / "twin.png").write_bytes(b"png bytes")
inventory = _inventory()
inventory["files"]["graphics/twin.png"] = {"pointed_at_by": []}
carried = {gate._sha12(corpus / "graphics" / "x.png")}
carried = _assets(corpus / "graphics" / "x.png")
units = gate.account(inventory, _build(assets=carried), corpus)
assert [(u.name, u.double) for u in units[1:]] == [
("graphics/twin.png", 0),
@ -315,11 +340,153 @@ def test_an_unpointed_file_sharing_bytes_with_a_carried_one_is_not_carried(
def test_without_the_door_double_booking_is_derived_from_conservation(tmp_path: Path) -> None:
corpus = _corpus(tmp_path)
carried = {gate._sha12(corpus / "graphics" / "x.png")}
carried = _assets(corpus / "graphics" / "x.png")
assert gate.account(_inventory(), _build(assets=carried), corpus)[1].double == 1
assert gate.account(_inventory(), _build(), corpus)[1].clean
# --- B-1: the judge opens the bundle itself ----------------------------------
#
# Written RED 2026-09-18 against the hardening order. At 864570b the gate
# compared BOOKED NUMBERS with the witness's counts and never opened a concept
# file, so a report that booked every element of every document as carried was
# `GATE GREEN` over a bundle holding nothing (independent review, B-1).
def _all_carried(headings: int = 2, images: int = 1) -> dict[str, Any]:
"""A report that books everything as carried, the cheat's shape."""
return {
"accounting_version": 1,
"documents": [
{
"source_file": "a.md",
"status": "persisted",
"code": None,
"inventory": {"heading": 2, "image": 1},
"fates": {
"heading": {"carried": headings},
"image": {"carried": images},
},
}
],
"files": [
{"source_file": "graphics/x.png", "fate": "rejected", "code": "extractor_unknown"}
],
}
def test_carried_text_the_bundle_does_not_hold_is_unverified(tmp_path: Path) -> None:
build = _build(accounting=_all_carried(images=0), bundle_text="")
unit = gate.account(_inventory(), build, _corpus(tmp_path))[0]
assert unit.unverified == 2
assert not unit.clean
def test_carried_text_the_bundle_holds_verifies(tmp_path: Path) -> None:
corpus = _corpus(tmp_path)
build = _build(accounting=_all_carried(images=0), assets=_assets(corpus / "graphics" / "x.png"))
unit = gate.account(_inventory(), build, corpus)[0]
assert (unit.unverified, unit.verified) == (0, 2)
def test_one_heading_carried_of_two_in_the_bundle_is_unverified(tmp_path: Path) -> None:
build = _build(accounting=_all_carried(images=0), bundle_text="# Foerste overskrift\n")
unit = gate.account(_inventory(), build, _corpus(tmp_path))[0]
assert unit.unverified == 1
def test_an_image_booked_carried_without_its_bytes_is_unverified(tmp_path: Path) -> None:
"""The image element has no text of its own, so the only proof it was
carried is the asset. Without it the booking is not verifiable, and an
unverifiable booking is never clean."""
unit = gate.account(_inventory(), _build(accounting=_all_carried()), _corpus(tmp_path))[0]
assert unit.unverified >= 1
assert not unit.clean
def test_a_negative_booking_is_never_clean(tmp_path: Path) -> None:
declared = _all_carried()
declared["documents"][0]["fates"]["heading"] = {"carried": 25, "rejected": {"x": -15}}
unit = gate.account(_inventory(), _build(accounting=declared), _corpus(tmp_path))[0]
assert unit.invalid >= 1
assert not unit.clean
def test_a_document_declared_persisted_that_is_not_in_the_bundle_is_never_clean(
tmp_path: Path,
) -> None:
build = _build(accounting=_all_carried(), sources=set())
unit = gate.account(_inventory(), build, _corpus(tmp_path))[0]
assert unit.invalid >= 1
def test_everything_rejected_is_never_clean_for_a_document_the_build_persisted(
tmp_path: Path,
) -> None:
declared = _all_carried()
declared["documents"][0]["fates"] = {
"heading": {"rejected": {"fail_secure": 2}},
"image": {"rejected": {"fail_secure": 1}},
}
unit = gate.account(_inventory(), _build(accounting=declared), _corpus(tmp_path))[0]
assert unit.invalid >= 1
assert "persisted" in "; ".join(unit.notes)
def test_everything_rejected_is_clean_for_a_document_the_build_refused(tmp_path: Path) -> None:
declared = _all_carried()
declared["documents"][0]["status"] = "rejected"
declared["documents"][0]["code"] = "fail_secure"
declared["documents"][0]["fates"] = {
"heading": {"rejected": {"fail_secure": 2}},
"image": {"rejected": {"fail_secure": 1}},
}
unit = gate.account(
_inventory(), _build(accounting=declared, sources=set()), _corpus(tmp_path)
)[0]
assert unit.clean
def test_a_rejection_code_outside_the_closed_list_is_never_clean(tmp_path: Path) -> None:
declared = _all_carried()
declared["documents"][0]["status"] = "rejected"
declared["documents"][0]["code"] = "because_i_said_so"
declared["documents"][0]["fates"] = {
"heading": {"rejected": {"because_i_said_so": 2}},
"image": {"rejected": {"because_i_said_so": 1}},
}
unit = gate.account(
_inventory(), _build(accounting=declared, sources=set()), _corpus(tmp_path)
)[0]
assert unit.invalid >= 1
def test_an_accounting_version_the_gate_does_not_read_is_never_clean(tmp_path: Path) -> None:
declared = _all_carried()
declared["accounting_version"] = 2
units = gate.account(_inventory(), _build(accounting=declared), _corpus(tmp_path))
assert not any(u.clean for u in units)
def test_an_asset_with_the_right_name_and_the_wrong_bytes_is_not_carried(tmp_path: Path) -> None:
"""m-1: the check was a NAME check, so a zero-byte file called
`<sha12>-x.png` proved a carry."""
corpus = _corpus(tmp_path)
source = corpus / "graphics" / "x.png"
lying = {f"{gate._sha12(source)}-x.png": gate._sha256_bytes(b"")}
build = _build(accounting=_declared(fate="carried"), assets=lying)
assert gate.account(_inventory(), build, corpus)[1].unaccounted == 1
def test_the_cheat_that_books_everything_carried_makes_row3_red(tmp_path: Path) -> None:
"""The review's `MODE=carried`: a report that changes not one byte of the
bundle and books every element as carried."""
units = gate.account(
_inventory(), _build(accounting=_all_carried(), bundle_text=""), _corpus(tmp_path)
)
assert gate.row3(units, door=True).status == gate.RED
_HONEST_LOG = (
"* **Images**: 0 carried of 1 found, written to `assets/`.\n"
"* a.md: 3 elements found in the source, 0 carried: document rejected `fail_secure`\n"
@ -417,6 +584,46 @@ def real_rows() -> list[gate.Row]:
return gate.evaluate(r761=None, ci=True, consume=False)
def _cheating_report(inventory: dict[str, Any], mode: str) -> dict[str, Any]:
"""The review's `cheat.py`, as data: a report that changes not one byte of
the bundle and books every element as carried (or as rejected)."""
documents = []
for name, entry in inventory["documents"].items():
if mode == "carried":
fates = {kind: {"carried": n} for kind, n in entry["elements"].items()}
else:
fates = {
kind: {"rejected": {"extractor_unknown": n}} if n else {"rejected": {}}
for kind, n in entry["elements"].items()
}
documents.append(
{
"source_file": name,
"status": "persisted",
"code": None,
"inventory": dict(entry["elements"]),
"fates": fates,
}
)
files = [{"source_file": name, "fate": "carried", "code": None} for name in inventory["files"]]
return {"accounting_version": 1, "documents": documents, "files": files}
@pytest.mark.parametrize("mode", ["carried", "empty"])
def test_a_report_the_build_did_not_write_cannot_make_row3_green(mode: str) -> None:
"""B-1, end to end on the real fixture bundle. Until 2026-09-18 both modes
gave `GATE GREEN`, exit 0: the gate compared the report's numbers with the
witness's and never opened a concept file."""
pytest.importorskip("pdfplumber")
pytest.importorskip("pypandoc")
inventory = gate.load_inventory(gate.INVENTORY)
with tempfile.TemporaryDirectory() as tmp:
build = gate.run_build(gate.CORPUS, Path(tmp), door=True)
build.accounting = _cheating_report(inventory, mode)
units = gate.account(inventory, build, gate.CORPUS)
assert gate.row3(units, door=True).status == gate.RED
def test_the_door_exists() -> None:
assert gate.door_available()
@ -433,4 +640,6 @@ def test_the_real_gate_is_green_on_every_fixture_row(real_rows: list[gate.Row])
(5, gate.GREEN),
(6, gate.SKIPPED),
]
assert real_rows[2].reason.startswith("u = 0 unaccounted, d = 0 double-booked")
assert real_rows[2].reason.startswith(
"u = 0 unaccounted, d = 0 double-booked, 0 booked carried and not in the bundle"
)