test(fixtures): replace sector-specific example material with generic, fictitious examples — green
Every fixture, test document, tool example and document now uses an invented kitchen-and-baking handbook series, written in this repository. The package's behaviour is unchanged; src/ changes are comments and help text only. - Generated fixtures are regenerated from their generators. Their structural counts are identical before and after: elements, images, rows, cells, headings, bookmarks and the witness inventory's per-document totals. The image-inbox and accounting documents are renamed kapittel-84-*. - tools/okf_accounting_gate.py: the two options that named one real corpus each are replaced by a generic, repeatable --corpus PATH with no default. Row 5 compares the PDF pair alone. Gate verdict unchanged: RED rows 2, 3, 6. - tools/okf_witness.py: the STS JSON reader for one publisher's delivery is removed, along with its three twins and five tests. The mutation harness loses W09. - docs/: 13 dated reports that documented runs on a retired reference corpus are removed, and 40 are neutralized. Dead links are removed, and no new dangling path is introduced. - The synthetic MCP-gate corpus and the residual probe words are neutral. Valgt: keep the `okf quality --fasit` bar value (the measured fraction, one corpus) and rewrite only its provenance, because the verdict stays unchanged and the number names nothing. Term check with the local list: 0 of 411 tracked files, 0 file names, 0 of 27 binary fixtures. Suite after git add: 2457 passed, 1 skipped. The base tree had 2460 passed and 2 skipped; five tests went with the JSON reader and four were added by the term check. ruff, ruff format and mypy --strict src/ are clean. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1e7345a401
commit
9d1f4b14ed
174 changed files with 1889 additions and 6512 deletions
|
|
@ -45,7 +45,7 @@ as a PNG -- and `asset_holds` proves such a carry by it, because the same
|
|||
pair stated in the bundle's prose can be written by a document. A door built
|
||||
from a contract without it writes a ledger this gate reads as "nothing was
|
||||
converted", and every converted image comes out claimed-and-not-found: 19 of
|
||||
50 on R761. `normalised_soft_hyphen`, `unaccounted` and `double_booked` had
|
||||
50 on a real reference delivery. `normalised_soft_hyphen`, `unaccounted` and `double_booked` had
|
||||
been written for a round longer than they were published.
|
||||
|
||||
`inventory` is taken BEFORE extraction and before the persist gate, in the
|
||||
|
|
@ -97,23 +97,18 @@ CORPUS = FIXTURES / "corpus"
|
|||
REJECTED = FIXTURES / "rejected"
|
||||
INVENTORY = FIXTURES / "inventory.json"
|
||||
REJECTED_INVENTORY = FIXTURES / "rejected-inventory.json"
|
||||
STS_FIXTURE = CORPUS / "prosess-84-sts.xml"
|
||||
STS_TWIN = FIXTURES / "witness" / "prosess-84-sts.twin.json"
|
||||
PDF_FIXTURE = CORPUS / "prosess-84-tabell.pdf"
|
||||
STS_FIXTURE = CORPUS / "kapittel-84-sts.xml"
|
||||
PDF_FIXTURE = CORPUS / "kapittel-84-tabell.pdf"
|
||||
README = REPO / "README.md"
|
||||
|
||||
#: No DEFAULT real corpus since 2026-09-21: the two this row was built on are
|
||||
#: material tied to the operator's employer, and that test track is retired.
|
||||
#: Row 6 measures a real corpus only when one is named on the command line.
|
||||
R761_ZIP = "14ce59dc-2150-480b-b661-6ea605fe3b24.zip"
|
||||
R761_JSON = "R761-2025-860019.json"
|
||||
R761_PDF = "R761-prosesskoden-2025.pdf"
|
||||
#: Row 6 has NO default corpus: it measures a real corpus only when one is
|
||||
#: named on the command line (`--corpus`, repeatable), and says so otherwise.
|
||||
|
||||
ACCOUNTING_FLAG = "--accounting"
|
||||
ACCOUNTING_VERSION = 1
|
||||
BUNDLE_ID = "accounting-gate"
|
||||
OKF_VERSION = "0.2"
|
||||
CONSUME_QUESTION = "Hvilken toleranseklasse gjelder for konstruksjoner av betong?"
|
||||
CONSUME_QUESTION = "Hvilken hevetidsklasse gjelder for brød og boller?"
|
||||
|
||||
GREEN = "GREEN"
|
||||
RED = "RED"
|
||||
|
|
@ -488,15 +483,15 @@ def asset_holds(build: Build, source: Path) -> bool:
|
|||
build lowercases the source's basename, folds its separator runs, cuts it
|
||||
to a maximum and takes the suffix from the BYTES rather than from the
|
||||
name. Re-implementing that here would make the judge agree with the judged
|
||||
by construction -- and it would be wrong: measured 2026-09-18 on R761,
|
||||
whose own hrefs carry spaces, capitals and parentheses, a judge checking
|
||||
by construction -- and it would be wrong: measured 2026-09-18 on a real
|
||||
reference delivery, whose own hrefs carry spaces, capitals and parentheses, a judge checking
|
||||
the full name reported 50 of 50 carried images as missing.
|
||||
|
||||
A SECOND ROUTE, for an image the build CONVERTS. Since the viewable-asset
|
||||
round a source in a format no model can be shown reaches the bundle as a
|
||||
PNG, so its own bytes are not in `assets/` and never will be -- measured,
|
||||
the day that landed R761 went from 0 to 19 claimed-and-not-found, which is
|
||||
exactly its RLE8 BMP count. The pair of digests comes from the RUN's own
|
||||
the day that landed a real reference delivery went from 0 to 19
|
||||
claimed-and-not-found, which is exactly its RLE8 BMP count. The pair of digests comes from the RUN's own
|
||||
accounting (`_conversions`), never from the bundle's prose, and this then
|
||||
HASHES THE ASSET ITSELF: the claim is accepted only when a file in
|
||||
`assets/` really holds the bytes the run says it wrote. A bundle claiming
|
||||
|
|
@ -506,7 +501,7 @@ def asset_holds(build: Build, source: Path) -> bool:
|
|||
AND NEITHER ROUTE WORKS WITHOUT THE RUN'S LEDGER. The second one reads the
|
||||
pair out of the `--accounting` file, so on a bundle built WITHOUT that flag
|
||||
a converted image cannot be proved at all and this returns False -- the
|
||||
reading R761 gave 19 times the day conversion landed. The gate itself
|
||||
reading that delivery gave 19 times the day conversion landed. The gate itself
|
||||
always passes the flag (`door_available()`), so the package build is
|
||||
untouched; any other caller on a doorless bundle gets the weak reading, and
|
||||
since 2026-09-19 the report says so in as many words rather than printing
|
||||
|
|
@ -844,7 +839,7 @@ def _file_unit(
|
|||
"""One inbox file that is not a document the build reads."""
|
||||
pointed_by = entry["pointed_at_by"]
|
||||
# Bytes in assets/ prove a carry only for a file a document points at:
|
||||
# an unpointed file with the same bytes (R761 ships 8 such duplicates)
|
||||
# an unpointed file with the same bytes (a real delivery shipped 8 such)
|
||||
# was not carried through anything.
|
||||
carried = bool(pointed_by) and asset_holds(build, corpus / name)
|
||||
merged = name in build.source_files
|
||||
|
|
@ -968,7 +963,8 @@ def _tally(units: Iterable[Unit], *, ledger: bool = True) -> str:
|
|||
`--accounting` file. Without one there is no record of what the run
|
||||
CONVERTED, so `asset_holds` falls back to its first route and a picture
|
||||
that reached the bundle under a new digest is counted claimed-and-not-found
|
||||
-- exactly the reading the conversion route removed (R761: 19 of 50). That
|
||||
-- exactly the reading the conversion route removed (19 of 50 on a real
|
||||
delivery). That
|
||||
fallback is honest and it used to be silent, which is a different thing:
|
||||
the number was printed and its cause was not, so a reader of a doorless
|
||||
bundle's report could only conclude the bundle had lost the pictures.
|
||||
|
|
@ -1085,13 +1081,9 @@ def _counts(count: witness.Count | None) -> dict[str, int] | None:
|
|||
return None if count is None else dict(count.counts)
|
||||
|
||||
|
||||
def witness_pairs(r761: Path | None) -> tuple[list[tuple[str, list[str]]], list[str]]:
|
||||
sts_xml, _, _ = witness.count_sts_xml(STS_FIXTURE.read_bytes())
|
||||
def witness_pairs() -> tuple[list[tuple[str, list[str]]], list[str]]:
|
||||
"""The pairs of independent witnesses over the same fixture bytes."""
|
||||
pairs = [
|
||||
(
|
||||
"sts fixture (xml | json)",
|
||||
compare(_counts(sts_xml), _counts(witness.count_sts_json(STS_TWIN.read_bytes()))),
|
||||
),
|
||||
(
|
||||
"pdf fixture (pdfplumber | poppler)",
|
||||
compare(
|
||||
|
|
@ -1100,36 +1092,13 @@ def witness_pairs(r761: Path | None) -> tuple[list[tuple[str, list[str]]], list[
|
|||
),
|
||||
),
|
||||
]
|
||||
notes: list[str] = []
|
||||
if r761 is None or not r761.is_dir():
|
||||
notes.append("R761 pairs not measured: source missing")
|
||||
return pairs, notes
|
||||
with zipfile.ZipFile(r761 / R761_ZIP) as archive:
|
||||
xml_name = next(n for n in archive.namelist() if n.endswith(".xml"))
|
||||
r761_xml, _, _ = witness.count_sts_xml(archive.read(xml_name))
|
||||
pairs.append(
|
||||
(
|
||||
"R761 sts (xml | json)",
|
||||
compare(
|
||||
_counts(r761_xml),
|
||||
_counts(witness.count_sts_json((r761 / R761_JSON).read_bytes())),
|
||||
),
|
||||
)
|
||||
)
|
||||
pdf = r761 / R761_PDF
|
||||
pairs.append(
|
||||
(
|
||||
"R761 pdf (pdfplumber | poppler)",
|
||||
compare(_counts(witness.pdf_objects(pdf)), _counts(witness.pdf_poppler(pdf))),
|
||||
)
|
||||
)
|
||||
return pairs, notes
|
||||
return pairs, []
|
||||
|
||||
|
||||
#: The two builds row 6 runs per corpus: the default gate is what a user gets;
|
||||
#: `none` persists the document, which is the only way its pictures are
|
||||
#: carried and the double booking of the files beside it becomes visible.
|
||||
R761_GATES: tuple[str | None, ...] = (None, "none")
|
||||
RUN_GATES: tuple[str | None, ...] = (None, "none")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -1137,7 +1106,7 @@ class RealCorpus:
|
|||
"""A corpus of real documents, read only, outside this repository."""
|
||||
|
||||
label: str
|
||||
kind: str # "zip" or "file"
|
||||
kind: str # "zip", "dir" or "file"
|
||||
path: Path
|
||||
|
||||
@property
|
||||
|
|
@ -1145,21 +1114,26 @@ class RealCorpus:
|
|||
return self.path.exists()
|
||||
|
||||
|
||||
def real_corpora(r761: Path | None, n200: Path | None) -> list[RealCorpus]:
|
||||
"""R761 is the gate's original corpus; N200 was added 2026-09-18 because
|
||||
R761 holds NONE of the STS classes the role map was missing -- 0 `fig`, 0
|
||||
formulas, 0 references -- so the only real corpus could not have found the
|
||||
hole. N200 carries 194 citations, 49 figures and 135 footnotes."""
|
||||
def real_corpora(paths: Sequence[Path]) -> list[RealCorpus]:
|
||||
"""One corpus per `--corpus` path: a zip is extracted, a directory is
|
||||
copied whole, and any other file becomes a one-document inbox."""
|
||||
corpora = []
|
||||
if r761 is not None:
|
||||
corpora.append(RealCorpus("R761 Prosesskoden:2025", "zip", r761 / R761_ZIP))
|
||||
if n200 is not None:
|
||||
corpora.append(RealCorpus("N200 Vegbygging:2024", "file", n200))
|
||||
for path in paths:
|
||||
if path.suffix.lower() == ".zip":
|
||||
kind = "zip"
|
||||
elif path.is_dir():
|
||||
kind = "dir"
|
||||
else:
|
||||
kind = "file"
|
||||
corpora.append(RealCorpus(path.name, kind, path))
|
||||
return corpora
|
||||
|
||||
|
||||
def _corpus_inbox(corpus: RealCorpus, root: Path) -> Path:
|
||||
inbox = root / "inbox"
|
||||
if corpus.kind == "dir":
|
||||
shutil.copytree(corpus.path, inbox)
|
||||
return inbox
|
||||
inbox.mkdir(parents=True)
|
||||
if corpus.kind == "zip":
|
||||
with zipfile.ZipFile(corpus.path) as archive:
|
||||
|
|
@ -1183,17 +1157,16 @@ def clean_in_every_run(runs: Sequence[Sequence[Unit]]) -> int:
|
|||
def measures_no_class(units: Sequence[Unit]) -> str | None:
|
||||
"""Did this corpus fail to measure ANY element class at all? (H6)
|
||||
|
||||
Reproduced on N200 Vegbygging:2024, 2026-09-19: `okf build` proposes 0
|
||||
Reproduced on a real JSON delivery, 2026-09-19: `okf build` proposes 0
|
||||
plans on it, prints `FAILED - no segmentation plans` and exits 2 BEFORE
|
||||
the accounting door -- no accounting file is written. Every element then
|
||||
lands as `u` with no declared fate, and 16 549 unaccounted reads like a
|
||||
finding about the build when it is a finding about the run.
|
||||
|
||||
What it would take is a capability and not a threshold: `.json` is read as
|
||||
generic JSON, and the publisher's STS delivery in that form would have to
|
||||
reach the same markdown grammar `_extract_xml` writes for the XML one. No
|
||||
such reader exists in the package -- `standardContent` occurs 0 times in
|
||||
`src/` and 4 times in this gate's witness (measured 2026-09-19).
|
||||
generic JSON, and an STS delivery in that form would have to reach the
|
||||
same markdown grammar `_extract_xml` writes for the XML one. No such
|
||||
reader exists in the package, and none in the witness either.
|
||||
"""
|
||||
documents = [u for u in units if u.kind == "document"]
|
||||
if not documents:
|
||||
|
|
@ -1214,7 +1187,7 @@ def refused_whole(documents: Mapping[str, Any], build: Build) -> str | None:
|
|||
|
||||
Booking every element of a refused document as a coded rejection gives
|
||||
u = 0 and d = 0, so the numbers are clean and the bundle is empty. Row 6
|
||||
read GREEN on R761 with 110 of 110 elements rejected and `okf build`
|
||||
read GREEN on a real delivery with 110 of 110 elements rejected and `okf build`
|
||||
exiting 1 unseen. The build order asked for an honest red there, so the
|
||||
row asks this question on its own.
|
||||
"""
|
||||
|
|
@ -1234,11 +1207,11 @@ def refused_whole(documents: Mapping[str, Any], build: Build) -> str | None:
|
|||
)
|
||||
|
||||
|
||||
def row6(r761: Path | None, n200: Path | None, ci: bool) -> Row:
|
||||
def row6(paths: Sequence[Path], ci: bool) -> Row:
|
||||
"""Every real corpus through two builds; a unit is clean only in both."""
|
||||
name = "real corpora"
|
||||
corpora = [c for c in real_corpora(r761, n200) if c.available]
|
||||
missing = [c for c in real_corpora(r761, n200) if not c.available]
|
||||
corpora = [c for c in real_corpora(paths) if c.available]
|
||||
missing = [c for c in real_corpora(paths) if not c.available]
|
||||
if not corpora:
|
||||
# SKIPPED is only free when there is nothing to measure. A source that
|
||||
# EXISTS and was not measured is a row that did not run, and a row
|
||||
|
|
@ -1256,7 +1229,7 @@ def row6(r761: Path | None, n200: Path | None, ci: bool) -> Row:
|
|||
inbox = _corpus_inbox(corpus, Path(tmp))
|
||||
inventory = witness.witness_inbox(inbox)
|
||||
runs: list[tuple[str, Build, list[Unit]]] = []
|
||||
for index, gate_name in enumerate(R761_GATES):
|
||||
for index, gate_name in enumerate(RUN_GATES):
|
||||
build = run_build(inbox, Path(tmp) / f"work{index}", door=door, gate=gate_name)
|
||||
runs.append((gate_name or "default", build, account(inventory, build, inbox)))
|
||||
documents = inventory["documents"]
|
||||
|
|
@ -1342,7 +1315,7 @@ def row7(workdir: Path) -> Row:
|
|||
# --- the run -----------------------------------------------------------------
|
||||
|
||||
|
||||
def evaluate(*, r761: Path | None, n200: Path | None, ci: bool, consume: bool) -> list[Row]:
|
||||
def evaluate(*, corpora: Sequence[Path] = (), ci: bool, consume: bool) -> list[Row]:
|
||||
table = readme_types()
|
||||
inventory = load_inventory(INVENTORY)
|
||||
rejected_inventory = load_inventory(REJECTED_INVENTORY)
|
||||
|
|
@ -1356,8 +1329,8 @@ def evaluate(*, r761: Path | None, n200: Path | None, ci: bool, consume: bool) -
|
|||
rows.append(row3(account(inventory, build, CORPUS), door))
|
||||
rejected_build = run_build(REJECTED, Path(tmp) / "rejected", door=door)
|
||||
rows.append(row4(rejected_inventory, rejected_build))
|
||||
rows.append(row5(*witness_pairs(r761)))
|
||||
rows.append(row6(r761, n200, ci))
|
||||
rows.append(row5(*witness_pairs()))
|
||||
rows.append(row6(corpora, ci))
|
||||
if consume:
|
||||
rows.append(row7(work))
|
||||
return rows
|
||||
|
|
@ -1407,16 +1380,12 @@ def main(argv: list[str] | None = None) -> int:
|
|||
parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
||||
parser.add_argument("--json", action="store_true", help="emit the rows as JSON")
|
||||
parser.add_argument(
|
||||
"--r761",
|
||||
"--corpus",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="directory holding the R761 zip, JSON and PDF (read only; no default)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--n200",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="the N200 JSON delivery, the second real corpus (read only; no default)",
|
||||
action="append",
|
||||
default=[],
|
||||
help="a real corpus for row 6: a directory, a zip or one file "
|
||||
"(repeatable, read only; no default)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--consume", action="store_true", help="also run row 7 (diagnostic, never fails)"
|
||||
|
|
@ -1424,8 +1393,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||
args = parser.parse_args(argv)
|
||||
try:
|
||||
rows = evaluate(
|
||||
r761=args.r761,
|
||||
n200=args.n200,
|
||||
corpora=args.corpus,
|
||||
ci=bool(os.environ.get("CI")),
|
||||
consume=args.consume,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ if __name__ == "__main__":
|
|||
# `importlib.util.spec_from_file_location` already holds the object
|
||||
# `module_from_spec` made, and `exec_module` leaves that object with this
|
||||
# file's own globals -- 3 public names against the packaged module's 76.
|
||||
# Reported by `vegnormal-okf` after v0.7.0 and reproduced in
|
||||
# Reported by a downstream consumer repository after v0.7.0 and reproduced in
|
||||
# `tests/test_okf_consume_shim.py` under two counting methods.
|
||||
#
|
||||
# THE DUNDER FILTER IS LOAD-BEARING. An unfiltered `vars(_impl)` overwrites
|
||||
|
|
|
|||
|
|
@ -57,10 +57,8 @@ class Mutant:
|
|||
file: str
|
||||
old: str
|
||||
new: str
|
||||
#: How many times `old` must occur. A role map written twice on purpose
|
||||
#: (`_sts_role_xml`, `_sts_role_json`) has two, and the mutant edits the
|
||||
#: first -- the XML witness -- so the JSON one stays honest and row 5 has
|
||||
#: a chance to see the disagreement.
|
||||
#: How many times `old` must occur; `first_only` edits the first of them
|
||||
#: when a line is written more than once on purpose.
|
||||
occurrences: int = 1
|
||||
first_only: bool = False
|
||||
#: The test file this mutant is judged by. Defaults to the gate's own
|
||||
|
|
@ -251,8 +249,6 @@ MUTANTS: tuple[Mutant, ...] = (
|
|||
WITNESS,
|
||||
' if tag == "fn":\n return "footnote"',
|
||||
' if tag == "fn":\n return None',
|
||||
occurrences=2,
|
||||
first_only=True,
|
||||
),
|
||||
Mutant(
|
||||
"W04 xlsx: images never counted",
|
||||
|
|
@ -277,8 +273,6 @@ MUTANTS: tuple[Mutant, ...] = (
|
|||
WITNESS,
|
||||
' if tag in ("td", "th"):\n return "cell"',
|
||||
' if tag in ("td", "th"):\n return None',
|
||||
occurrences=2,
|
||||
first_only=True,
|
||||
),
|
||||
Mutant(
|
||||
"W08 docx: a header or footer is never counted",
|
||||
|
|
@ -286,12 +280,6 @@ MUTANTS: tuple[Mutant, ...] = (
|
|||
' count.add("header_footer", *lines)',
|
||||
" pass",
|
||||
),
|
||||
Mutant(
|
||||
"W09 sts json: the twin reuses the XML map",
|
||||
WITNESS,
|
||||
" role = _sts_role_json(tag, parent, grandparent)",
|
||||
" role = _sts_role_xml(tag, parent, grandparent)",
|
||||
),
|
||||
# PM's own mutant, written outside this file 2026-09-18 and SURVIVING 155
|
||||
# green tests (H2). B-1 gave the judge six refusals; this was the one
|
||||
# nothing held from either side.
|
||||
|
|
|
|||
|
|
@ -390,12 +390,12 @@ def write_bundle(root: Path, bundle_id: str, concepts: Sequence[tuple[str, str,
|
|||
def corpus(scratch: Path) -> dict[str, Path]:
|
||||
"""Two bundles that share no vocabulary, so a cross-bundle answer cannot
|
||||
be produced by one of them alone."""
|
||||
bridges = write_bundle(
|
||||
scratch / "bundles" / "bridge-notes",
|
||||
"bridge-notes",
|
||||
gardens = write_bundle(
|
||||
scratch / "bundles" / "garden-notes",
|
||||
"garden-notes",
|
||||
[
|
||||
("spennvidde", "Spennvidde for gangbru", "En gangbru med spennvidde 24 meter."),
|
||||
("rekkverk", "Rekkverk paa gangbru", "Rekkverket skal vaere 1,2 meter hoeyt."),
|
||||
("bedbredde", "Bedbredde for drivhus", "Et drivhus med bedbredde 24 meter."),
|
||||
("gelender", "Gelender paa terrasse", "Gelenderet skal vaere 1,2 meter hoeyt."),
|
||||
],
|
||||
)
|
||||
kitchens = write_bundle(
|
||||
|
|
@ -406,7 +406,7 @@ def corpus(scratch: Path) -> dict[str, Path]:
|
|||
("ovn", "Ovnstemperatur", "Ovnen forvarmes til 250 grader."),
|
||||
],
|
||||
)
|
||||
return {"bridge-notes": bridges, "kitchen-notes": kitchens}
|
||||
return {"garden-notes": gardens, "kitchen-notes": kitchens}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
|
@ -522,7 +522,7 @@ def _probe_arguments(tool: str, bundle_id: str, concept_id: str, *, named: bool)
|
|||
if tool == "okf_describe":
|
||||
return bundle
|
||||
if tool == "okf_ask":
|
||||
return {**bundle, "question": "spennvidde for gangbru"}
|
||||
return {**bundle, "question": "bedbredde for drivhus"}
|
||||
return {**bundle, "concept_id": concept_id}
|
||||
|
||||
|
||||
|
|
@ -559,21 +559,21 @@ def row_one(bundles: Mapping[str, Path], reachable: tuple[bool, str]) -> Row:
|
|||
k = 0
|
||||
for variant, required in REQUIRED_TOOLS.items():
|
||||
named = variant == "one-to-many"
|
||||
target = bundles["bridge-notes"] if not named else bundles["bridge-notes"].parent
|
||||
target = bundles["garden-notes"] if not named else bundles["garden-notes"].parent
|
||||
try:
|
||||
with server(variant_argv(variant, target)) as client:
|
||||
client.handshake()
|
||||
listed = set(client.tool_names())
|
||||
concept = ""
|
||||
with suppress(Exception):
|
||||
concept = _first_concept_id(client, "bridge-notes", named=named)
|
||||
concept = _first_concept_id(client, "garden-notes", named=named)
|
||||
for tool in required:
|
||||
if tool not in listed:
|
||||
details.append(f"{variant}: {tool} not in tools/list")
|
||||
continue
|
||||
try:
|
||||
answer = client.call(
|
||||
tool, _probe_arguments(tool, "bridge-notes", concept, named=named)
|
||||
tool, _probe_arguments(tool, "garden-notes", concept, named=named)
|
||||
)
|
||||
except (RpcError, ServerGone) as error:
|
||||
details.append(f"{variant}: {tool} did not answer: {error}")
|
||||
|
|
@ -667,8 +667,8 @@ def read_anchor_set(questions: Path, freeze: Path, *, want_version: int) -> Anch
|
|||
# surface must be able to hand back verbatim. Without it, a row reporting
|
||||
# `0 of M` could be reporting that the lookup was never able to find anything.
|
||||
SYNTHETIC_ANCHORS: tuple[tuple[str, str, str], ...] = (
|
||||
("bridge-notes", "spennvidde", "En gangbru med spennvidde 24 meter."),
|
||||
("bridge-notes", "rekkverk", "Rekkverket skal vaere 1,2 meter hoeyt."),
|
||||
("garden-notes", "bedbredde", "Et drivhus med bedbredde 24 meter."),
|
||||
("garden-notes", "gelender", "Gelenderet skal vaere 1,2 meter hoeyt."),
|
||||
("kitchen-notes", "surdeig", "Surdeigen hever i 14 timer ved 22 grader."),
|
||||
("kitchen-notes", "ovn", "Ovnen forvarmes til 250 grader."),
|
||||
)
|
||||
|
|
@ -854,8 +854,8 @@ def row_two(
|
|||
def _served_name(target: Path, served: set[str]) -> str | None:
|
||||
"""The id the server calls this directory, read from the bundle itself.
|
||||
|
||||
Never guessed from the directory name: the frozen set names `n100-2023`
|
||||
and the bundle declares `vegnormal-n100-2023`, so a rule matching names
|
||||
Never guessed from the directory name: the frozen set names `q100-2023`
|
||||
and the bundle declares `kokebok-q100-2023`, so a rule matching names
|
||||
would have to invent an alias rule of its own.
|
||||
"""
|
||||
from llm_ingestion_okf import consume as okf_consume
|
||||
|
|
@ -924,7 +924,7 @@ def _drill_skill_one_to_one(bundle: Path, scratch: Path) -> DrillResult:
|
|||
return DrillResult("skill-one-to-one", loud=False, recreate=(), note=str(error))
|
||||
out = scratch / "skill-1-1"
|
||||
try:
|
||||
skill.generate(bundle, out=out, question="spennvidde for gangbru", force=True)
|
||||
skill.generate(bundle, out=out, question="bedbredde for drivhus", force=True)
|
||||
except Exception as error:
|
||||
return DrillResult(
|
||||
"skill-one-to-one", loud=False, recreate=(), note=f"not generated: {error}"
|
||||
|
|
@ -968,14 +968,14 @@ def _drill_skill_one_to_many(bundles: Mapping[str, Path], scratch: Path) -> Dril
|
|||
recreate=(),
|
||||
note="`skill.render_generic` does not exist: the candidate has not been built",
|
||||
)
|
||||
first, second = (bundles["bridge-notes"], bundles["kitchen-notes"])
|
||||
first, second = (bundles["garden-notes"], bundles["kitchen-notes"])
|
||||
text_first = render_generic()
|
||||
text_second = render_generic()
|
||||
identical = text_first == text_second
|
||||
leaked = [
|
||||
token
|
||||
for token in (
|
||||
"bridge-notes",
|
||||
"garden-notes",
|
||||
"kitchen-notes",
|
||||
okf_consume.bundle_ref(first),
|
||||
okf_consume.bundle_ref(second),
|
||||
|
|
@ -1007,11 +1007,11 @@ def row_three(scratch: Path) -> Row:
|
|||
stage.mkdir(parents=True, exist_ok=True)
|
||||
bundles = corpus(stage)
|
||||
if artefact == "mcp-one-to-one":
|
||||
results.append(_drill_mcp("one-to-one", bundles["bridge-notes"], "bridge-notes"))
|
||||
results.append(_drill_mcp("one-to-one", bundles["garden-notes"], "garden-notes"))
|
||||
elif artefact == "mcp-one-to-many":
|
||||
results.append(_drill_mcp("one-to-many", bundles["bridge-notes"], "bridge-notes"))
|
||||
results.append(_drill_mcp("one-to-many", bundles["garden-notes"], "garden-notes"))
|
||||
elif artefact == "skill-one-to-one":
|
||||
results.append(_drill_skill_one_to_one(bundles["bridge-notes"], stage))
|
||||
results.append(_drill_skill_one_to_one(bundles["garden-notes"], stage))
|
||||
else:
|
||||
results.append(_drill_skill_one_to_many(bundles, stage))
|
||||
for result in results:
|
||||
|
|
@ -1037,7 +1037,7 @@ def row_four(scratch: Path) -> Row:
|
|||
stage = scratch / "discovery"
|
||||
stage.mkdir(parents=True, exist_ok=True)
|
||||
bundles = corpus(stage)
|
||||
root = bundles["bridge-notes"].parent
|
||||
root = bundles["garden-notes"].parent
|
||||
newcomers = {
|
||||
"ferry-notes": ("avgang", "Avgangstider", "Fergen gaar hver time fra klokken 06."),
|
||||
"lichen-notes": ("vekst", "Vekstrate", "Lav vokser under en millimeter i aaret."),
|
||||
|
|
@ -1092,18 +1092,18 @@ def row_five(scratch: Path) -> Row:
|
|||
stage = scratch / "cross"
|
||||
stage.mkdir(parents=True, exist_ok=True)
|
||||
bundles = corpus(stage)
|
||||
root = bundles["bridge-notes"].parent
|
||||
root = bundles["garden-notes"].parent
|
||||
details: list[str] = []
|
||||
k = 0
|
||||
try:
|
||||
with server(variant_argv("one-to-many", root)) as client:
|
||||
client.handshake()
|
||||
calls = 0
|
||||
answer = client.call("okf_ask", {"question": "spennvidde og surdeig"})
|
||||
answer = client.call("okf_ask", {"question": "bedbredde og surdeig"})
|
||||
calls += 1
|
||||
named = _bundle_ids(answer)
|
||||
sources = _source_ids(answer)
|
||||
if {"bridge-notes", "kitchen-notes"} <= named:
|
||||
if {"garden-notes", "kitchen-notes"} <= named:
|
||||
k += 1
|
||||
else:
|
||||
details.append(f"one answer named {sorted(named) or 'no bundle'}")
|
||||
|
|
@ -1135,7 +1135,7 @@ def row_six(scratch: Path) -> Row:
|
|||
stage = scratch / "hostile"
|
||||
stage.mkdir(parents=True, exist_ok=True)
|
||||
bundles = corpus(stage)
|
||||
root = bundles["bridge-notes"].parent
|
||||
root = bundles["garden-notes"].parent
|
||||
|
||||
outside = stage / "outside"
|
||||
outside.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -1147,12 +1147,12 @@ def row_six(scratch: Path) -> Row:
|
|||
write_bundle(root / "broken-notes", "broken-notes", [("x", "X", "y")])
|
||||
(root / "broken-notes" / "index.md").write_bytes(b"\xff\xfe not a manifest at all\x00")
|
||||
|
||||
huge = root / "bridge-notes" / "svulmende.md"
|
||||
huge = root / "garden-notes" / "svulmende.md"
|
||||
huge.write_text(
|
||||
CONCEPT.format(
|
||||
title="Svulmende",
|
||||
source="svulmende.txt",
|
||||
bundle_id="bridge-notes",
|
||||
bundle_id="garden-notes",
|
||||
segment="s9",
|
||||
body="A" * OVERSIZED_BYTES,
|
||||
),
|
||||
|
|
@ -1160,7 +1160,7 @@ def row_six(scratch: Path) -> Row:
|
|||
)
|
||||
# Named in the index, or it is not a concept and the ceiling never runs:
|
||||
# the bundle's own index is what makes a file reachable at all.
|
||||
index = root / "bridge-notes" / "index.md"
|
||||
index = root / "garden-notes" / "index.md"
|
||||
index.write_text(
|
||||
index.read_text(encoding="utf-8").rstrip("\n")
|
||||
+ "\n- [Svulmende](svulmende.md) \u2014 adjudication: proposed\n",
|
||||
|
|
@ -1174,11 +1174,11 @@ def row_six(scratch: Path) -> Row:
|
|||
(
|
||||
"traversal-in-concept-id",
|
||||
"okf_fetch",
|
||||
{BUNDLE_KEY: "bridge-notes", "concept_id": "../../outside/secret-notes/hemmelig"},
|
||||
{BUNDLE_KEY: "garden-notes", "concept_id": "../../outside/secret-notes/hemmelig"},
|
||||
),
|
||||
("symlink-out-of-root", "okf_describe", {BUNDLE_KEY: "linked-notes"}),
|
||||
("broken-manifest", "okf_describe", {BUNDLE_KEY: "broken-notes"}),
|
||||
("oversized-concept", "okf_fetch", {BUNDLE_KEY: "bridge-notes", "concept_id": "svulmende"}),
|
||||
("oversized-concept", "okf_fetch", {BUNDLE_KEY: "garden-notes", "concept_id": "svulmende"}),
|
||||
("unknown-bundle-id", "okf_describe", {BUNDLE_KEY: "does-not-exist"}),
|
||||
]
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -16,9 +16,8 @@ real set lives outside this repository. Row 7 is red where a mutant survives,
|
|||
and each survivor is printed with what it moved.
|
||||
|
||||
THE PUBLIC ROWS RUN ON INVENTED MATERIAL ONLY (operator decision 2026-09-21).
|
||||
The test track built on material tied to the operator's employer -- the K2
|
||||
corpus and the two road-standard sets -- is retired: not re-measured, not
|
||||
frozen. Its row (9) and its sets' adapters are gone, and row 8 reads one
|
||||
The earlier test track -- the K2 corpus and two further reference sets -- is
|
||||
retired: not re-measured, not frozen. Its row (9) and its sets' adapters are gone, and row 8 reads one
|
||||
local set.
|
||||
|
||||
THE FASIT IS AN INPUT, NEVER A CONSTANT HERE -- `tools/okf_consume_measure.py`
|
||||
|
|
@ -2262,8 +2261,8 @@ def read_real_set(name: str, path: Path, expected_sha256: str) -> QuestionSet:
|
|||
|
||||
The adapter below is the whole of this gate's knowledge of it, and none of
|
||||
the question text ever reaches a tracked file here. Two more adapters, for
|
||||
sets built on material tied to the operator's employer, were removed
|
||||
2026-09-21 with the test track they belonged to.
|
||||
the sets of a retired test track, were removed 2026-09-21 with the track
|
||||
they belonged to.
|
||||
"""
|
||||
raw = path.read_bytes()
|
||||
measured = hashlib.sha256(raw).hexdigest()
|
||||
|
|
@ -2343,8 +2342,8 @@ def read_hold_out_set(path: Path, expected_sha256: str) -> QuestionSet:
|
|||
#: The sets row 8 is the measurement of, by name: a run that hands over some
|
||||
#: of them has measured some of them, and the row says so. Left to
|
||||
#: `len(real)` the row came back `6 of 6 GREEN` on one set of three (PM's J2,
|
||||
#: 2026-09-19). One set since 2026-09-21, when the two built on material tied
|
||||
#: to the operator's employer were retired; the rule stays for the day a
|
||||
#: 2026-09-19). One set since 2026-09-21, when the two of a retired test track
|
||||
#: were removed; the rule stays for the day a
|
||||
#: second set joins.
|
||||
REQUIRED_REAL_SETS: tuple[str, ...] = ("wiki-20",)
|
||||
|
||||
|
|
@ -2638,7 +2637,7 @@ def render(rows: Sequence[Row]) -> str:
|
|||
|
||||
|
||||
def _bundle_map(value: str) -> dict[str, Path]:
|
||||
"""`/a/bundle`, or `N100:2023=/a,N200:2024=/b` for a set spanning bundles."""
|
||||
"""`/a/bundle`, or `Q100:2023=/a,Q200:2024=/b` for a set spanning bundles."""
|
||||
if "=" not in value:
|
||||
return {"": Path(value).expanduser()}
|
||||
pairs = {}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ meant to judge is the reader agreeing with itself.
|
|||
Each counter reads the container the way the format defines it:
|
||||
|
||||
- XML (NISO-STS): `xml.etree.ElementTree` straight on the bytes.
|
||||
- STS JSON twin: the publisher's own node tree (`standardContent`, nodes with
|
||||
`e`/`t`/`x`), walked with the same element roles as the XML.
|
||||
- docx / pptx / xlsx / odt: the zip members' own XML.
|
||||
- PDF: pdfplumber OBJECTS (pages, image placements) and, as a second witness,
|
||||
poppler (`pdfinfo`, `pdfimages -list`). pdfplumber is also what the reader
|
||||
|
|
@ -44,7 +42,7 @@ import re
|
|||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
|
@ -447,15 +445,9 @@ STS_ROLES = (
|
|||
def _sts_role_xml(tag: str, parent: str | None, grandparent: str | None) -> str | None:
|
||||
"""An STS XML element's accounting role.
|
||||
|
||||
Written for the XML delivery ALONE. Until 2026-09-18 one function served
|
||||
both deliveries, so row 5 -- "two witnesses agree" -- could not see a hole
|
||||
in it: a role missing here was missing there, and the two agreed on a
|
||||
number neither of them should have produced (independent review, M-2).
|
||||
|
||||
In this delivery a section's label is `sec/label` and a table's label is
|
||||
`table-wrap/label`. Measured on R761 2026-09-18: `sec/label` 7 714,
|
||||
`sec/title/label` **0** -- the nested placement is a fact about the JSON
|
||||
delivery and does not belong in this map.
|
||||
A section's label is `sec/label` and a table's label is `table-wrap/label`.
|
||||
Measured on a real NISO-STS delivery 2026-09-18: `sec/label` 7 714,
|
||||
`sec/title/label` **0**, so a label nested in a title is not in this map.
|
||||
"""
|
||||
if tag == "sec":
|
||||
return "section"
|
||||
|
|
@ -490,70 +482,6 @@ def _sts_role_xml(tag: str, parent: str | None, grandparent: str | None) -> str
|
|||
return None
|
||||
|
||||
|
||||
def _sts_role_json(tag: str, parent: str | None, grandparent: str | None) -> str | None:
|
||||
"""The same roles, read from the publisher's JSON node tree.
|
||||
|
||||
Written apart from the XML map, because the publisher's two deliveries of
|
||||
ONE document place the same text differently (measured on R761
|
||||
Prosesskoden:2025, 2026-09-17):
|
||||
|
||||
- a section's label: XML `sec/label` on 7 714 sections; JSON `sec/label`
|
||||
on 4 954 and `sec/title/label` on the 2 760 that carry a title.
|
||||
- a table's label: XML `table-wrap/label` (10); JSON
|
||||
`table-wrap/table/caption` (10).
|
||||
|
||||
Counted by tag alone, the two witnesses disagree by 2 760 and by 10 on
|
||||
text both of them carry. The role, not the tag, is the unit.
|
||||
|
||||
The tag arrives through :func:`_local`, as the XML map's does: this
|
||||
delivery writes a namespace as a PREFIX on the tag string, so `mml:math`
|
||||
reached `tag == "math"` on nothing at all (H3, 74 of them in N200). The
|
||||
known-negative for stripping it is N101's 568 `tbx:` tags, none of whose
|
||||
local names is a role here.
|
||||
|
||||
`figcaption` is this delivery's own, measured and not assumed: NISO-STS
|
||||
writes a figure's caption as `fig/caption`, and the publisher's JSON puts
|
||||
it under the `graphic` instead -- 430 of them over the eight deliveries on
|
||||
this machine, 0 reached by the `caption` clause above. It is deliberately
|
||||
NOT in the XML map, which has never been measured carrying one.
|
||||
"""
|
||||
if tag == "sec":
|
||||
return "section"
|
||||
if tag == "title" and parent == "sec":
|
||||
return "title"
|
||||
if tag == "label" and parent == "sec":
|
||||
return "section_label"
|
||||
if tag == "label" and parent == "title" and grandparent == "sec":
|
||||
return "section_label"
|
||||
if tag == "label" and parent == "table-wrap":
|
||||
return "table_label"
|
||||
if tag == "caption" and parent == "table" and grandparent == "table-wrap":
|
||||
return "table_label"
|
||||
if tag == "caption" and parent == "fig":
|
||||
return "figure_caption"
|
||||
if tag == "figcaption":
|
||||
return "figure_caption"
|
||||
if tag == "fig":
|
||||
return "figure"
|
||||
if tag == "mixed-citation":
|
||||
return "citation"
|
||||
if tag == "math":
|
||||
return "math"
|
||||
if tag == "p":
|
||||
return "paragraph"
|
||||
if tag == "table-wrap":
|
||||
return "table"
|
||||
if tag in ("td", "th"):
|
||||
return "cell"
|
||||
if tag == "list-item":
|
||||
return "list_item"
|
||||
if tag in ("graphic", "inline-graphic"):
|
||||
return "image"
|
||||
if tag == "fn":
|
||||
return "footnote"
|
||||
return None
|
||||
|
||||
|
||||
def count_sts_xml(data: bytes) -> tuple[Count, list[str], bool]:
|
||||
"""Element roles of an STS document; `element` alone for other XML.
|
||||
|
||||
|
|
@ -594,47 +522,6 @@ def count_sts_xml(data: bytes) -> tuple[Count, list[str], bool]:
|
|||
return count, refs, True
|
||||
|
||||
|
||||
def is_sts_json(data: bytes) -> bool:
|
||||
"""Is this the publisher's JSON delivery of an STS document?
|
||||
|
||||
A standard shipped as JSON holds sections, titles, citations and tables.
|
||||
Counted as generic JSON it holds keys and leaves: the container, not the
|
||||
content -- and then no row can see that a citation left the bundle."""
|
||||
if b'"standardContent"' not in data:
|
||||
return False
|
||||
try:
|
||||
document = json.loads(data)
|
||||
except ValueError:
|
||||
return False
|
||||
content = document.get("standardContent") if isinstance(document, dict) else None
|
||||
return isinstance(content, dict) and isinstance(content.get("c"), list)
|
||||
|
||||
|
||||
def count_sts_json(data: bytes) -> Count:
|
||||
"""The same roles, read from the publisher's JSON node tree."""
|
||||
document = json.loads(data)
|
||||
count = Count(STS_ROLES)
|
||||
|
||||
def walk(node: Mapping[str, Any], parent: str | None, grandparent: str | None) -> list[str]:
|
||||
pieces: list[str] = []
|
||||
if node.get("t") and str(node["t"]).strip():
|
||||
pieces.append(_normal(str(node["t"])))
|
||||
body = node.get("x")
|
||||
if not isinstance(body, dict):
|
||||
return pieces
|
||||
tag = _local(str(body.get("tag")))
|
||||
role = _sts_role_json(tag, parent, grandparent)
|
||||
for child in body.get("c") or []:
|
||||
pieces.extend(walk(child, tag, parent))
|
||||
if role is not None:
|
||||
count.add(role, *pieces)
|
||||
return pieces
|
||||
|
||||
for child in document["standardContent"]["c"]:
|
||||
walk(child, None, None)
|
||||
return count
|
||||
|
||||
|
||||
# --- office zips -------------------------------------------------------------
|
||||
|
||||
_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
||||
|
|
@ -1258,10 +1145,7 @@ def witness_file(inbox: Path, path: Path) -> Inventory:
|
|||
elif suffix == ".csv":
|
||||
count, name = count_csv(data.decode("utf-8-sig")), "csv"
|
||||
elif suffix == ".json":
|
||||
if is_sts_json(data):
|
||||
count, name = count_sts_json(data), "sts json node tree"
|
||||
else:
|
||||
count, name = count_json(data.decode("utf-8-sig")), "json"
|
||||
count, name = count_json(data.decode("utf-8-sig")), "json"
|
||||
elif suffix in (".html", ".htm"):
|
||||
parser = _HtmlCounter()
|
||||
parser.feed(data.decode("utf-8-sig"))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue