test(gates): retire the K2 track; re-measure the retrieval gate's premises for BM25
Operator decision 2026-09-21: the test track built on material tied to the operator's employer (K2) is retired -- not re-measured, not frozen. Public tests and gates run on invented material. Retrieval gate: - The four FUSION_PREMISE xfails are gone and pass through their INPUTS: the synthetic MISS, LOOKUP and QUOTA bundles were re-measured for BM25 (the miss fasit no longer shares the rare word `maa`; lookup and quota decoys carry the question's words so each partition and the quota decide their own fixture). SPECS_SHA256 moved with them. Rows 2 and 3 green again. - Row 7's mutants M04, M06, M07, M08, M10 now patch `bm25`, the code the default runs. Three survive with 0 ranks moved (passage body, title weight, bm25.RRF_K), each with its mechanism printed. M07 was not forced: every synthetic body carries its title as a heading. - Row 9 (K2) removed; row 8 requires `wiki-20` alone, the `r761` and `vegnormal` adapters are gone. Chose the broad reading of "K2" because the operator decision defines it as employer-tied material and the order's grep includes `vegnormal`. Also removed: tests/test_default_bundle_pin.py, the K2 arms of test_okf_consume, the four real-arm tests of test_quality, the R761 soft hyphen test, the N101/N200 delivery tests and okf_accounting_gate's default real corpus (and H5's guard, which only existed for those defaults). Two fixtures carrying road-standard identifiers are rewritten with invented ones. Gate after: 1 10/10, 2 7/7, 3 5/5, 4 6/6, 5 0/1, 6 10/10, 7 11/14, 8 NOT RUN -> GATE RED: rows 5, 7, 8. Suite 2423 passed, 1 skipped, 0 xfailed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
977040f575
commit
3d149f955a
12 changed files with 288 additions and 1096 deletions
|
|
@ -378,10 +378,10 @@ def test_a_prefixed_tag_that_names_no_role_stays_uncounted() -> None:
|
|||
assert witness._sts_role_json(witness._local(tag), "sec", "body") is None
|
||||
|
||||
|
||||
#: THE ONE PLACE THIS NUMBER LIVES. The count itself is measured over the
|
||||
#: delivery by `test_n101s_own_prefixed_tags_are_counted_here_and_name_no_role`
|
||||
#: -- but that test is `skipif`-gated on a file only this machine has, so on a
|
||||
#: fresh clone the four published sentences were unguarded again, which is how
|
||||
#: THE ONE PLACE THIS NUMBER LIVES. The count itself was measured over the
|
||||
#: delivery by a `skipif`-gated test that read a file only this machine has
|
||||
#: (removed 2026-09-21 with the retired test track), so on a fresh clone the
|
||||
#: four published sentences were unguarded again, which is how
|
||||
#: 574 survived in four docstrings until PM counted it. The guard below needs
|
||||
#: no delivery: it reads the published sentences and holds them to each other
|
||||
#: and to this constant.
|
||||
|
|
@ -408,14 +408,14 @@ _TBX_PUBLISHERS = (
|
|||
def test_the_published_tbx_count_is_one_number_and_needs_no_delivery() -> None:
|
||||
"""The published strings are held to each other, on any machine.
|
||||
|
||||
This is the half the measurement could not cover. `_tags_of` counts the
|
||||
real delivery and is right to; it also cannot run where the delivery is
|
||||
absent, and an assertion that skips guards nothing. Editing `CLAUDE.md` to
|
||||
This is the half the measurement could not cover. The delivery count
|
||||
cannot run where the delivery is absent, and an assertion that skips
|
||||
guards nothing. Editing `CLAUDE.md` to
|
||||
600 tomorrow is red here, on a fresh clone, with no corpus.
|
||||
|
||||
It proves nothing about the WORLD -- five files agreeing is agreement, not
|
||||
a count -- which is why the delivery test keeps its own measurement and
|
||||
this one only holds the sentences to the constant it asserts.
|
||||
a count -- and this test only holds the sentences to the constant it
|
||||
asserts.
|
||||
"""
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
for name in _TBX_PUBLISHERS:
|
||||
|
|
@ -440,58 +440,6 @@ def test_the_published_tbx_count_is_one_number_and_needs_no_delivery() -> None:
|
|||
assert _PUBLISHED_TBX.findall(wrong) == ["574"]
|
||||
|
||||
|
||||
N101_DELIVERY = gate.N200_DEFAULT.parent / "N101-2025-860031.json"
|
||||
|
||||
|
||||
def _tags_of(payload: bytes) -> dict[str, int]:
|
||||
"""Every `tag` string in a delivery, counted by a walk written HERE.
|
||||
|
||||
The witness's own reader is what the known-negative below judges, so
|
||||
counting through it would make the two agree by construction.
|
||||
"""
|
||||
names: dict[str, int] = {}
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
tag = node.get("tag")
|
||||
if isinstance(tag, str):
|
||||
names[tag] = names.get(tag, 0) + 1
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
walk(value)
|
||||
|
||||
walk(json.loads(payload.decode("utf-8")))
|
||||
return names
|
||||
|
||||
|
||||
@pytest.mark.skipif(not N101_DELIVERY.is_file(), reason="N101 is not on this machine")
|
||||
def test_n101s_own_prefixed_tags_are_counted_here_and_name_no_role() -> None:
|
||||
"""The published number for that known-negative was a measurement nothing
|
||||
could falsify: it lived in four docstrings and in no assertion, and it was
|
||||
wrong. The count is made HERE, over the delivery itself, so the sentence
|
||||
four files publish is red when it stops being true."""
|
||||
names = _tags_of(N101_DELIVERY.read_bytes())
|
||||
assert sum(names.values()) > 0, "the walk found no tag at all"
|
||||
tbx = {tag: n for tag, n in names.items() if tag.startswith("tbx:")}
|
||||
assert sum(tbx.values()) == N101_TBX_TAGS
|
||||
for tag in sorted(tbx):
|
||||
assert witness._sts_role_json(witness._local(tag), "sec", "body") is None, tag
|
||||
|
||||
|
||||
@pytest.mark.skipif(not gate.N200_DEFAULT.is_file(), reason="N200 is not on this machine")
|
||||
def test_the_json_role_map_counts_n200s_own_formulas_and_figure_captions() -> None:
|
||||
"""The same defect on the delivery it was found in, with PM's numbers.
|
||||
Skipped where the corpus is absent, and then this file's own fixture is
|
||||
the only thing holding the rule -- which is why both exist."""
|
||||
counts = witness.count_sts_json(gate.N200_DEFAULT.read_bytes()).counts
|
||||
assert counts["math"] == 74
|
||||
assert counts["figure_caption"] == 49
|
||||
assert counts["citation"] == 194
|
||||
assert counts["figure"] == 49
|
||||
|
||||
|
||||
def test_the_two_sts_role_maps_are_written_twice_and_not_shared() -> None:
|
||||
"""M-2: both STS witnesses went through ONE `_sts_role`, so row 5 could
|
||||
never see a hole in it. Two maps, each written for its own delivery."""
|
||||
|
|
@ -1547,35 +1495,11 @@ def test_a_unit_clean_in_only_one_of_the_two_builds_is_not_clean() -> None:
|
|||
assert gate.clean_in_every_run([[clean], [dirty]]) == 0
|
||||
|
||||
|
||||
def test_a_row_skipped_while_the_default_source_exists_exits_one(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
"""H5: the guard asked whether the corpora named by the ARGUMENTS are
|
||||
available -- and row 6 is SKIPPED precisely when none of them is, so the
|
||||
branch could never fire and no test covered it. The question it meant to
|
||||
ask is about the machine: a corpus that is HERE and was pointed away from
|
||||
is a row that did not run, and `CI=1` then printed a qualified GREEN and
|
||||
exited 0.
|
||||
|
||||
Measured against its own known-negative below, so a guard that fires on
|
||||
everything would not pass either."""
|
||||
present = tmp_path / "corpus.json"
|
||||
present.write_text("{}", encoding="utf-8")
|
||||
skipped = gate.Row(6, "real corpora", 0, 0, gate.SKIPPED, "not measured, source missing: x")
|
||||
monkeypatch.setattr(gate, "N200_DEFAULT", present)
|
||||
monkeypatch.setattr(gate, "evaluate", lambda **kwargs: [skipped])
|
||||
code = gate.main(["--r761", str(tmp_path / "absent"), "--n200", str(tmp_path / "absent.json")])
|
||||
assert code == 1
|
||||
assert "row 6 was skipped while its source exists" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_a_row_skipped_with_no_source_on_the_machine_exits_zero(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""The known-negative: nothing to measure is not a row that did not run."""
|
||||
skipped = gate.Row(6, "real corpora", 0, 0, gate.SKIPPED, "not measured, source missing: x")
|
||||
monkeypatch.setattr(gate, "R761_DEFAULT", tmp_path / "absent")
|
||||
monkeypatch.setattr(gate, "N200_DEFAULT", tmp_path / "absent.json")
|
||||
monkeypatch.setattr(gate, "evaluate", lambda **kwargs: [skipped])
|
||||
assert gate.main([]) == 0
|
||||
|
||||
|
|
|
|||
|
|
@ -35,14 +35,14 @@ FIXTURES = Path(__file__).parent / "fixtures"
|
|||
# The form the producer actually ships, copied from a bundle: a query string
|
||||
# carries `?`, `=` and `&`, and no flow form of it passes both PyYAML and the
|
||||
# guard -- which is why the producer moved to the block form at all.
|
||||
VEGNORMAL = """\
|
||||
PRODUCER_FORM = """\
|
||||
---
|
||||
type: Krav
|
||||
title: Krav 10.2-2 Beredskap
|
||||
source_file: normal.xml
|
||||
source_file: haandbok.xml
|
||||
sources:
|
||||
- resource: https://viewers.test.invalid/api/nisosts/860019?languageCode=nb&v=2
|
||||
title: N500:2024
|
||||
- resource: https://viewers.test.invalid/api/documents/4711?languageCode=nb&v=2
|
||||
title: H500:2024
|
||||
---
|
||||
|
||||
## Krav
|
||||
|
|
@ -70,12 +70,12 @@ Body.
|
|||
# the document's namespace.
|
||||
NESTED_TITLE = """\
|
||||
---
|
||||
title: N100.2 Kryss og avkjoersler
|
||||
title: H100.2 Hytter og uthus
|
||||
generated: true
|
||||
source_file: vegnormal.md
|
||||
source_file: haandbok.md
|
||||
sources:
|
||||
- resource: https://example.test/bruprosjektering.pdf
|
||||
title: N200.7 Bruprosjektering
|
||||
- resource: https://example.test/broeyting.pdf
|
||||
title: H200.7 Broeyting
|
||||
---
|
||||
|
||||
Body.
|
||||
|
|
@ -118,7 +118,7 @@ QUOTED_LEAVES = """\
|
|||
title: Krav 1
|
||||
sources:
|
||||
- resource: "a, b.pdf"
|
||||
title: 'N100'
|
||||
title: 'H100'
|
||||
---
|
||||
|
||||
Body.
|
||||
|
|
@ -161,12 +161,12 @@ def _read_sources(text: str, tmp_path: Path) -> tuple[tuple[object, ...], bool]:
|
|||
|
||||
|
||||
def test_control_read_sources_reads_the_block_form(tmp_path: Path) -> None:
|
||||
entries, present = _read_sources(VEGNORMAL, tmp_path)
|
||||
entries, present = _read_sources(PRODUCER_FORM, tmp_path)
|
||||
assert present
|
||||
assert [dict(entry) for entry in entries] == [
|
||||
{
|
||||
"resource": "https://viewers.test.invalid/api/nisosts/860019?languageCode=nb&v=2",
|
||||
"title": "N500:2024",
|
||||
"resource": "https://viewers.test.invalid/api/documents/4711?languageCode=nb&v=2",
|
||||
"title": "H500:2024",
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -174,12 +174,12 @@ def test_control_read_sources_reads_the_block_form(tmp_path: Path) -> None:
|
|||
def test_control_both_reference_readers_read_the_block_form() -> None:
|
||||
expected = [
|
||||
{
|
||||
"resource": "https://viewers.test.invalid/api/nisosts/860019?languageCode=nb&v=2",
|
||||
"title": "N500:2024",
|
||||
"resource": "https://viewers.test.invalid/api/documents/4711?languageCode=nb&v=2",
|
||||
"title": "H500:2024",
|
||||
}
|
||||
]
|
||||
assert _pyyaml_sources(VEGNORMAL) == expected
|
||||
assert _guard_sources(VEGNORMAL) == expected
|
||||
assert _pyyaml_sources(PRODUCER_FORM) == expected
|
||||
assert _guard_sources(PRODUCER_FORM) == expected
|
||||
|
||||
|
||||
# --- the defect, once per copy of the grammar -----------------------------
|
||||
|
|
@ -187,10 +187,10 @@ def test_control_both_reference_readers_read_the_block_form() -> None:
|
|||
|
||||
@pytest.mark.parametrize("reader", ["materialize", "structure", "profiles"])
|
||||
def test_block_sources_reaches_every_flat_reader(reader: str, tmp_path: Path) -> None:
|
||||
value = _flat_readings(VEGNORMAL, tmp_path)[reader]["sources"]
|
||||
value = _flat_readings(PRODUCER_FORM, tmp_path)[reader]["sources"]
|
||||
assert value != ""
|
||||
assert _parse_flow_mappings(value) == _pyyaml_sources(VEGNORMAL)
|
||||
assert _parse_flow_mappings(value) == _guard_sources(VEGNORMAL)
|
||||
assert _parse_flow_mappings(value) == _pyyaml_sources(PRODUCER_FORM)
|
||||
assert _parse_flow_mappings(value) == _guard_sources(PRODUCER_FORM)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reader", ["materialize", "structure", "profiles"])
|
||||
|
|
@ -213,13 +213,13 @@ def test_a_nested_title_still_does_not_substitute(reader: str, tmp_path: Path) -
|
|||
"""The most important control here: reading the block is not a licence to
|
||||
let a nested key into the document's namespace."""
|
||||
flat = _flat_readings(NESTED_TITLE, tmp_path)[reader]
|
||||
assert flat["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert flat["title"] == "H100.2 Hytter og uthus"
|
||||
assert "resource" not in flat
|
||||
assert set(flat) == {"title", "generated", "source_file", "sources"}
|
||||
assert _parse_flow_mappings(flat["sources"]) == [
|
||||
{
|
||||
"resource": "https://example.test/bruprosjektering.pdf",
|
||||
"title": "N200.7 Bruprosjektering",
|
||||
"resource": "https://example.test/broeyting.pdf",
|
||||
"title": "H200.7 Broeyting",
|
||||
}
|
||||
]
|
||||
|
||||
|
|
@ -255,7 +255,7 @@ def test_quoted_leaves_follow_the_k3_22_rule(reader: str, tmp_path: Path) -> Non
|
|||
value = _flat_readings(QUOTED_LEAVES, tmp_path)[reader]["sources"]
|
||||
entries, _present = _read_sources(QUOTED_LEAVES, tmp_path)
|
||||
assert _parse_flow_mappings(value) == [dict(entry) for entry in entries]
|
||||
assert _parse_flow_mappings(value) == [{"resource": "a, b.pdf", "title": "'N100'"}]
|
||||
assert _parse_flow_mappings(value) == [{"resource": "a, b.pdf", "title": "'H100'"}]
|
||||
|
||||
|
||||
# --- the shipped fixtures, all of them, not a sample ----------------------
|
||||
|
|
|
|||
|
|
@ -1,272 +0,0 @@
|
|||
"""The bundle the DEFAULT build produces, pinned where a regression goes red.
|
||||
|
||||
READ WITH THE FUSION RANKING since v1.1: these ranks were measured on the
|
||||
three-signal fusion, which `okf consume` no longer uses by default. The
|
||||
default BM25 reading of these bytes is a separate measurement and is not
|
||||
pinned here.
|
||||
|
||||
`tests/test_okf_consume.py` pinned hit@8 against the Arm B bundle alone -- the
|
||||
configuration `okf build` stopped emitting on 2026-09-08. A published number
|
||||
measured on a bundle nobody produces is a number that cannot regress, so the
|
||||
guarantee it looks like was never held by anything.
|
||||
|
||||
This file pins the CURRENT default: `--outline-run 3 --table-grid --unit-fold
|
||||
--drop-wrapped-outline --outline-gate --first-span-from-zero
|
||||
--sheet-section-rows --keep-table-heading --close-span-gaps`, plus the reading
|
||||
side's `tie_shared_rank`. Round 6 moved the first five on 2026-09-09, round 7
|
||||
moved four more on 2026-09-10 and round 8 moved the last on 2026-09-11, each
|
||||
after measuring hit@8 on exactly the bundle its own default produces.
|
||||
|
||||
The gold set is LOCAL-ONLY and stays that way: no question and no
|
||||
`gold_document` is reproduced here, and a row is named by its INDEX, the way
|
||||
`docs/2026-09-07-okf-konsumskill-maaling.md` already names them.
|
||||
|
||||
The bundle itself is a build artefact, not a fixture: it is 832 files of a
|
||||
consumer's corpus and this repository is public. Absent, these tests SKIP with
|
||||
the command that rebuilds it -- "not measured", never zero.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
||||
|
||||
import okf_consume # noqa: E402
|
||||
import okf_consume_measure # noqa: E402
|
||||
|
||||
#: Built by:
|
||||
#: okf build <corpus>/K2/trinn1 \
|
||||
#: --bundle ~/corpora/okf-telling-20260829/K2-bundle-default-20260911 \
|
||||
#: --bundle-id k2-trinn1-20260903 --okf-version 0.2
|
||||
#: with no arm flag at all -- the package default, which is the point.
|
||||
#:
|
||||
#: Rebuilt 2026-09-09 for `--contents-name` (round 9). Digest, from inside the
|
||||
#: bundle:
|
||||
#: find . -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256
|
||||
#: -> 21af4a1aa98315cf514c4cbc6b4a9b77ce63960224d6d7b31b34d55cc67fb2ad
|
||||
#: (The previous default, `K2-bundle-default-20260911`, was
|
||||
#: 8c93e5e3222577a2b3352ca83af980e403d3a571c3a467b83c3d8170b1df2b69 at 436
|
||||
#: concepts and stays on disk.)
|
||||
#: Two independent builds of it differ in NOTHING (`diff -rq`), including
|
||||
#: `log.md`, which carries the corpus path and never the bundle's own.
|
||||
#:
|
||||
#: CONCEPT IDS MOVED IN THIS REBUILD, and not only because the count did.
|
||||
#: Round 9 strips pandoc's `{#sheet-N}` / `{#slide-N}` anchor where a title is
|
||||
#: formed, and a concept's filename is reduced FROM its title, so TWO ids on
|
||||
#: this bundle are renamed:
|
||||
#: del-ii-bilag-7-prisskjema/prissammenstilling-sheet-1 -> .../prissammenstilling
|
||||
#: del-ii-bilag-0-dokumentliste-del-ii/ark1-sheet-1 -> .../ark1
|
||||
#: The first is an id `portfolio-optimiser` has cited in writing. The rename
|
||||
#: was authorised by the operator on 2026-09-09 after the exposure was counted:
|
||||
#: 2 of 810 concepts on the previous default and 2 of 1108 on Arm B.
|
||||
DEFAULT_BUNDLE = Path.home() / "corpora" / "okf-telling-20260829" / "K2-bundle-default-20260912"
|
||||
GOLD_SET = PROJECT_ROOT / ".claude/projects/2026-09-07-okf-consume-prepass/hit-at-k-questions.json"
|
||||
|
||||
requires_default_bundle = pytest.mark.skipif(
|
||||
not DEFAULT_BUNDLE.is_dir() or not GOLD_SET.is_file(),
|
||||
reason=(
|
||||
f"the default-configuration K2 bundle is not present at {DEFAULT_BUNDLE}. "
|
||||
"NOT MEASURED, not zero: rebuild it with `okf build <corpus>/K2/trinn1 "
|
||||
"--bundle <that path> --bundle-id k2-trinn1-20260903 --okf-version 0.2`"
|
||||
),
|
||||
)
|
||||
|
||||
#: Measured 2026-09-09 on the bundle above. The count moved 425 -> 436 with
|
||||
#: `--sheet-section-rows --keep-table-heading`; `--first-span-from-zero` and
|
||||
#: `--close-span-gaps` each moved it by NOTHING, which is the point of both --
|
||||
#: they add no boundary, they only move a span's start or its end. Round 8's
|
||||
#: rule closed 43 631 characters (2.51 % of the corpus) that were in no
|
||||
#: segment, and the count was byte-for-byte the same 436.
|
||||
#:
|
||||
#: 436 -> 453 with round 9's `--contents-name`, which does add concepts: a run
|
||||
#: of data rows is no longer read as a contents listing and discarded, so the
|
||||
#: candidates it was taking with it survive. Corpus-wide, 429 -> 447 candidates
|
||||
#: over 32 -> 33 documents with a plan, and characters in no segment stay 0.
|
||||
EXPECTED_CONCEPTS = 453
|
||||
EXPECTED_HITS = 6
|
||||
#: Rank per question INDEX. The identity is the index; the question stays in
|
||||
#: the local-only gold set.
|
||||
#:
|
||||
#: ROW 6 MOVED `None` -> 5 ON 2026-09-10, on these same bytes and with no
|
||||
#: bundle changing: `consume.DEFAULT_SOURCE_QUOTA` became 2. It had missed on
|
||||
#: every bundle and every configuration measured until then, and the test below
|
||||
#: names the quota as its cause by turning it off. What the gain is NOT: hit@8
|
||||
#: scores whether the gold DOCUMENT appears among the delivered excerpts, and a
|
||||
#: document quota directly raises how many distinct documents a payload holds,
|
||||
#: so this metric is not neutral with respect to this rule. The five rows that
|
||||
#: were already rank 1 are, and they did not move.
|
||||
EXPECTED_RANKS = (1, 1, 1, 1, 1, 5)
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_default_bundle_holds_its_concept_count() -> None:
|
||||
assert len(list(okf_consume.enumerate_concepts(DEFAULT_BUNDLE))) == EXPECTED_CONCEPTS
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_hit_at_eight_holds_rank_one_on_every_row_it_held() -> None:
|
||||
"""The acceptance criterion round 6's default move had to clear.
|
||||
|
||||
Not the hit COUNT alone: the count survived a configuration that lost a
|
||||
row from rank 1 to rank 2, which is exactly how the previous round's
|
||||
regression hid. The rank per row is the pin.
|
||||
|
||||
On THIS bundle that is not a hypothetical -- see the test below, which
|
||||
reproduces the fall on these exact bytes by turning the reading-side
|
||||
default off.
|
||||
"""
|
||||
questions = json.loads(GOLD_SET.read_text(encoding="utf-8"))["questions"]
|
||||
assert len(questions) == len(EXPECTED_RANKS), "the gold set changed shape"
|
||||
ranks = []
|
||||
for entry in questions:
|
||||
payload = okf_consume.build_payload(
|
||||
DEFAULT_BUNDLE, question=entry["question"], ranking="fusion"
|
||||
)
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
ranks.append(okf_consume_measure.hit_rank(excerpts, entry["gold_document"]))
|
||||
assert tuple(ranks) == EXPECTED_RANKS, f"hit@8 ranks moved: {ranks}"
|
||||
assert sum(rank is not None for rank in ranks) == EXPECTED_HITS
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_bundle_declares_the_identity_the_reader_needs() -> None:
|
||||
"""Whatever else moves, the bundle stays one the reading direction opens."""
|
||||
assert okf_consume.root_bundle_id_of(DEFAULT_BUNDLE) == "k2-trinn1-20260903"
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_reading_default_is_what_holds_row_one_on_these_bytes() -> None:
|
||||
"""The known-negative, on the shipped bundle rather than a fixture.
|
||||
|
||||
Round 7 moved `--sheet-section-rows --keep-table-heading` into the build
|
||||
default, which splits row 1's gold document from 1 concept into 12. Round 6
|
||||
measured that exact split costing row 1 its rank, and held the two rules
|
||||
back for it. What removed the cost is `consume.DEFAULT_TIE_SHARED_RANK`,
|
||||
and this test is the proof that it is still what removes it: turn it off
|
||||
on these bytes and the fall comes back.
|
||||
|
||||
Without this, `EXPECTED_RANKS` above would be a green assertion with no
|
||||
stated cause, and a later change to the fusion could take the cause away
|
||||
while the pin stayed green on some other accident.
|
||||
"""
|
||||
questions = json.loads(GOLD_SET.read_text(encoding="utf-8"))["questions"]
|
||||
ranks = []
|
||||
for entry in questions:
|
||||
payload = okf_consume.build_payload(
|
||||
DEFAULT_BUNDLE, question=entry["question"], ranking="fusion", tie_shared_rank=False
|
||||
)
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
ranks.append(okf_consume_measure.hit_rank(excerpts, entry["gold_document"]))
|
||||
assert ranks[0] == 2, "the known-negative stopped being negative"
|
||||
# Rows 2-5 are untouched by the tie-break. Row 6 is NOT asserted equal to
|
||||
# the pin: it is the row `DEFAULT_SOURCE_QUOTA` recovered, and the two
|
||||
# rules reach it independently, so pinning it here would assert a
|
||||
# coincidence rather than a cause.
|
||||
assert tuple(ranks[1:5]) == EXPECTED_RANKS[1:5]
|
||||
assert ranks[5] is not None
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_stem_rule_holds_every_rank_on_the_shipped_bytes() -> None:
|
||||
"""Round 10's reading-side default, pinned the way `--tie-shared-rank` is.
|
||||
|
||||
The rule requires a shared prefix to be a word the bundle uses. It moves a
|
||||
payload with NO bundle changing, which is the same class of change as the
|
||||
tie-break above, so it gets the same treatment: the ranks are pinned WITH
|
||||
it, and the test below pins what it costs when it is off.
|
||||
|
||||
Measured on these bytes, control run first: `under` occurs 79 times by
|
||||
equality and matches 172 concepts by prefix; `bilateral` occurs 0 times and
|
||||
matched 400 of 453 before this rule and 0 after; `standhaftig` 0 and
|
||||
219 -> 56. `undersjoisk` goes 172 -> 162 and stops there, because `under`
|
||||
IS a word here -- a genuine Norwegian morpheme rather than a residual.
|
||||
"""
|
||||
questions = json.loads(GOLD_SET.read_text(encoding="utf-8"))["questions"]
|
||||
ranks = []
|
||||
for entry in questions:
|
||||
payload = okf_consume.build_payload(
|
||||
DEFAULT_BUNDLE, question=entry["question"], ranking="fusion"
|
||||
)
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
ranks.append(okf_consume_measure.hit_rank(excerpts, entry["gold_document"]))
|
||||
assert tuple(ranks) == EXPECTED_RANKS
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_document_quota_is_what_reaches_row_six_on_these_bytes() -> None:
|
||||
"""The known-negative for round 11's default, on the shipped bundle.
|
||||
|
||||
Row 6 had missed on every bundle and every configuration measured before
|
||||
2026-09-10. `DEFAULT_SOURCE_QUOTA = 2` reaches it at rank 5, and this test
|
||||
is what keeps `EXPECTED_RANKS` from being green for an unstated reason:
|
||||
turn the quota off on these exact bytes and the miss comes back.
|
||||
|
||||
The five standing rank-1 rows are asserted UNMOVED in the same run, because
|
||||
"the new rule cost nothing" is the half of the claim a hit count cannot
|
||||
see.
|
||||
"""
|
||||
questions = json.loads(GOLD_SET.read_text(encoding="utf-8"))["questions"]
|
||||
ranks = []
|
||||
for entry in questions:
|
||||
payload = okf_consume.build_payload(
|
||||
DEFAULT_BUNDLE, question=entry["question"], ranking="fusion", source_quota=None
|
||||
)
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
ranks.append(okf_consume_measure.hit_rank(excerpts, entry["gold_document"]))
|
||||
assert ranks[5] is None, "the known-negative stopped being negative"
|
||||
assert tuple(ranks[:5]) == EXPECTED_RANKS[:5]
|
||||
|
||||
|
||||
@requires_default_bundle
|
||||
def test_the_stem_rule_is_what_keeps_bilateral_out_of_four_hundred_concepts() -> None:
|
||||
"""The known-negative for the stem rule, on the shipped bundle.
|
||||
|
||||
Without it the pin above would be green with no stated cause. The number
|
||||
that names the cause is the one the rule was built for: a word occurring
|
||||
ZERO times reaching 400 of 453 concepts.
|
||||
|
||||
The control is measured FIRST and in the same loop, because round 9's first
|
||||
attempt at this measurement returned 0 for the probe AND for the control --
|
||||
`enumerate_concepts` yields `str`, not `Path` -- and a zero that is really a
|
||||
broken query is indistinguishable from a zero that is a result.
|
||||
"""
|
||||
root_id = okf_consume.root_bundle_id_of(DEFAULT_BUNDLE)
|
||||
concepts = [
|
||||
okf_consume.read_concept(
|
||||
DEFAULT_BUNDLE / f"{concept_id}.md",
|
||||
bundle_root=DEFAULT_BUNDLE,
|
||||
root_bundle_id=root_id,
|
||||
)
|
||||
for concept_id in okf_consume.enumerate_concepts(DEFAULT_BUNDLE)
|
||||
]
|
||||
documents = [okf_consume.normalise(text) for text in okf_consume.searchable_text(concepts)]
|
||||
stems = frozenset(token for document in documents for token in document)
|
||||
|
||||
def reach(word: str, *, stemmed: bool) -> int:
|
||||
token = okf_consume.normalise(word)[0]
|
||||
return sum(
|
||||
1
|
||||
for document in documents
|
||||
for _ in (0,)
|
||||
if any(
|
||||
okf_consume.tokens_match(token, other, stems=stems if stemmed else None)
|
||||
for other in document
|
||||
)
|
||||
)
|
||||
|
||||
assert sum(1 for d in documents if okf_consume.normalise("under")[0] in d) == 79, (
|
||||
"the CONTROL first: `under` really does occur, so the query can find"
|
||||
)
|
||||
assert reach("bilateral", stemmed=False) == 400, "the defect, on these bytes"
|
||||
assert reach("bilateral", stemmed=True) == 0, "and what the rule does to it"
|
||||
assert reach("under", stemmed=True) == 172, "a real word keeps its compounds"
|
||||
|
|
@ -37,15 +37,15 @@ from llm_ingestion_okf.structure import derive_document_structure
|
|||
# rather than only asserting that they could.
|
||||
NESTED = """\
|
||||
---
|
||||
title: N100.2 Kryss og avkjoersler
|
||||
title: H100.2 Hytter og uthus
|
||||
generated: true
|
||||
source_file: vegnormal.md
|
||||
source_file: haandbok.md
|
||||
sources:
|
||||
- resource: https://example.test/bruprosjektering.pdf
|
||||
title: N200.7 Bruprosjektering
|
||||
- resource: https://example.test/broeyting.pdf
|
||||
title: H200.7 Broeyting
|
||||
---
|
||||
|
||||
# Kryss og avkjoersler
|
||||
# Hytter og uthus
|
||||
|
||||
Body text.
|
||||
"""
|
||||
|
|
@ -53,12 +53,12 @@ Body text.
|
|||
# The same document with the nested block removed. Nothing else differs.
|
||||
FLAT = """\
|
||||
---
|
||||
title: N100.2 Kryss og avkjoersler
|
||||
title: H100.2 Hytter og uthus
|
||||
generated: true
|
||||
source_file: vegnormal.md
|
||||
source_file: haandbok.md
|
||||
---
|
||||
|
||||
# Kryss og avkjoersler
|
||||
# Hytter og uthus
|
||||
|
||||
Body text.
|
||||
"""
|
||||
|
|
@ -73,46 +73,46 @@ Body text.
|
|||
|
||||
|
||||
def test_control_materialize_reads_top_level_title(tmp_path: Path) -> None:
|
||||
path = tmp_path / "vegnormal.md"
|
||||
path = tmp_path / "haandbok.md"
|
||||
path.write_text(FLAT, encoding="utf-8")
|
||||
assert parse_frontmatter(path)["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert parse_frontmatter(path)["title"] == "H100.2 Hytter og uthus"
|
||||
|
||||
|
||||
def test_control_structure_reads_top_level_title() -> None:
|
||||
declared, offset = _structure_split(FLAT)
|
||||
assert declared["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert declared["title"] == "H100.2 Hytter og uthus"
|
||||
assert offset > 0
|
||||
|
||||
|
||||
def test_control_profiles_reads_top_level_title() -> None:
|
||||
head, _body = _profiles_split(FLAT)
|
||||
assert head["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert head["title"] == "H100.2 Hytter og uthus"
|
||||
|
||||
|
||||
def test_control_derivation_reads_top_level_title() -> None:
|
||||
structure = derive_document_structure(FLAT, source_file="vegnormal.md")
|
||||
assert structure.title == "N100.2 Kryss og avkjoersler"
|
||||
assert structure.number == "N100.2"
|
||||
assert structure.parent_number == "N100"
|
||||
structure = derive_document_structure(FLAT, source_file="haandbok.md")
|
||||
assert structure.title == "H100.2 Hytter og uthus"
|
||||
assert structure.number == "H100.2"
|
||||
assert structure.parent_number == "H100"
|
||||
|
||||
|
||||
# --- the defect, once per parser copy ------------------------------------
|
||||
|
||||
|
||||
def test_nested_title_does_not_substitute_in_materialize(tmp_path: Path) -> None:
|
||||
path = tmp_path / "vegnormal.md"
|
||||
path = tmp_path / "haandbok.md"
|
||||
path.write_text(NESTED, encoding="utf-8")
|
||||
assert parse_frontmatter(path)["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert parse_frontmatter(path)["title"] == "H100.2 Hytter og uthus"
|
||||
|
||||
|
||||
def test_nested_title_does_not_substitute_in_structure() -> None:
|
||||
declared, _offset = _structure_split(NESTED)
|
||||
assert declared["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert declared["title"] == "H100.2 Hytter og uthus"
|
||||
|
||||
|
||||
def test_nested_title_does_not_substitute_in_profiles() -> None:
|
||||
head, _body = _profiles_split(NESTED)
|
||||
assert head["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert head["title"] == "H100.2 Hytter og uthus"
|
||||
|
||||
|
||||
# --- the propagation the order asks to be MEASURED, not assumed ----------
|
||||
|
|
@ -127,10 +127,10 @@ def test_substituted_title_moves_number_and_parent() -> None:
|
|||
question was whether `number` and `parent` move in PRACTICE or only in
|
||||
theory.
|
||||
"""
|
||||
structure = derive_document_structure(NESTED, source_file="vegnormal.md")
|
||||
assert structure.title == "N100.2 Kryss og avkjoersler"
|
||||
assert structure.number == "N100.2"
|
||||
assert structure.parent_number == "N100"
|
||||
structure = derive_document_structure(NESTED, source_file="haandbok.md")
|
||||
assert structure.title == "H100.2 Hytter og uthus"
|
||||
assert structure.number == "H100.2"
|
||||
assert structure.parent_number == "H100"
|
||||
|
||||
|
||||
def test_nested_key_does_not_invent_a_top_level_field() -> None:
|
||||
|
|
@ -141,7 +141,7 @@ def test_nested_key_does_not_invent_a_top_level_field() -> None:
|
|||
document never declared. `derive_document_structure` exposes `declared`
|
||||
directly, so this pins the namespace itself and not one lucky key.
|
||||
"""
|
||||
structure = derive_document_structure(NESTED, source_file="vegnormal.md")
|
||||
structure = derive_document_structure(NESTED, source_file="haandbok.md")
|
||||
assert "resource" not in structure.declared
|
||||
assert set(structure.declared) == {"title", "generated", "source_file", "sources"}
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ def test_flow_form_still_round_trips(tmp_path: Path) -> None:
|
|||
"""
|
||||
flow = (
|
||||
"---\n"
|
||||
"title: N100.2 Kryss og avkjoersler\n"
|
||||
"title: H100.2 Hytter og uthus\n"
|
||||
"generated: { by: process:okf-ingest, at: 2026-08-31T00:00:00Z }\n"
|
||||
"sources: [ a.pdf, b.pdf ]\n"
|
||||
"---\n\nBody.\n"
|
||||
|
|
@ -169,4 +169,4 @@ def test_flow_form_still_round_trips(tmp_path: Path) -> None:
|
|||
parsed = parse_frontmatter(path)
|
||||
assert parsed["generated"] == "{ by: process:okf-ingest, at: 2026-08-31T00:00:00Z }"
|
||||
assert parsed["sources"] == "[ a.pdf, b.pdf ]"
|
||||
assert parsed["title"] == "N100.2 Kryss og avkjoersler"
|
||||
assert parsed["title"] == "H100.2 Hytter og uthus"
|
||||
|
|
|
|||
|
|
@ -853,118 +853,6 @@ def test_the_empty_payload_still_passes_the_checker() -> None:
|
|||
assert okf_contract_check.check(_skill_declaring(payload), payload).findings == ()
|
||||
|
||||
|
||||
# --- Corpus-conditional arms --------------------------------------------------
|
||||
|
||||
K2_BUNDLE = Path.home() / "corpora" / "okf-telling-20260829" / "K2-bundle-20260903"
|
||||
K2_CONCEPTS = 629
|
||||
K2_PROPOSED = 618
|
||||
K2_KEYLESS = 11
|
||||
|
||||
requires_k2 = pytest.mark.skipif(
|
||||
not K2_BUNDLE.is_dir(),
|
||||
reason=(
|
||||
f"the K2 corpus is not present at {K2_BUNDLE}. NOT MEASURED, not zero: "
|
||||
f"this arm covers a denominator of {K2_CONCEPTS} concepts, of which "
|
||||
f"{K2_PROPOSED} carry `adjudication: proposed` and {K2_KEYLESS} carry no "
|
||||
"`adjudication` key at all. A skip here is an unmeasured denominator, "
|
||||
"never a pass."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@requires_k2
|
||||
def test_the_eleven_keyless_k2_concepts_come_back_unknown_over_a_stated_denominator() -> None:
|
||||
# SS 6.1's third state, on real data rather than on a fixture. The 11 are
|
||||
# asserted as ONE named set: measured, the concepts carrying no
|
||||
# `adjudication` are EXACTLY those carrying no `bundle_id`, so three
|
||||
# independent counts would share one blind spot.
|
||||
root_bundle_id = parse_frontmatter(K2_BUNDLE / "index.md")["bundle_id"]
|
||||
concepts = [
|
||||
okf_consume.read_concept(
|
||||
K2_BUNDLE / f"{concept_id}.md",
|
||||
bundle_root=K2_BUNDLE,
|
||||
root_bundle_id=root_bundle_id,
|
||||
)
|
||||
for concept_id in okf_consume.enumerate_concepts(K2_BUNDLE)
|
||||
]
|
||||
assert len(concepts) == K2_CONCEPTS
|
||||
unknown = {c.concept_id for c in concepts if c.adjudication == "unknown"}
|
||||
inherited = {c.concept_id for c in concepts if c.bundle_id_inherited}
|
||||
proposed = [c for c in concepts if c.adjudication == "proposed"]
|
||||
assert len(proposed) == K2_PROPOSED
|
||||
assert len(unknown) == K2_KEYLESS
|
||||
assert unknown == inherited, "the two sets diverged; the fallback is no longer one fact"
|
||||
assert all(c.bundle_id == root_bundle_id for c in concepts if c.bundle_id_inherited)
|
||||
# `adjudicated` has denominator ZERO on this corpus. Stated, not implied.
|
||||
assert [c for c in concepts if c.adjudication == "adjudicated"] == []
|
||||
|
||||
|
||||
@requires_k2
|
||||
def test_spent_is_the_delivered_set_where_the_whole_payload_reading_would_refuse() -> None:
|
||||
# The regression guard, with figures RE-MEASURED here rather than carried
|
||||
# from the plan: the plan predicted 101 576 B for this excerpt and 188 758 B
|
||||
# for the payload, both taken before per-line trailing-whitespace stripping
|
||||
# landed. What this build actually produces is recorded instead.
|
||||
# Run with `--withheld-full`, because the DEFAULT no longer reaches this
|
||||
# case and that is the point of `okf-consumption/2`: the whole payload of
|
||||
# this bundle came to 117 561 B under the default -- inside the limit --
|
||||
# where under the flat list it did not. The distinction between the two
|
||||
# readings of SS 7.2 is still real and still reachable, so it is measured
|
||||
# where it is reachable rather than deleted with the shape that produced
|
||||
# it. The control below fires if that stops being true.
|
||||
payload = okf_consume.build_payload(
|
||||
K2_BUNDLE, question="Hvordan skal prisene fylles ut?", withheld_full=True
|
||||
)
|
||||
budget, excerpts = payload["budget"], payload["excerpts"]
|
||||
assert isinstance(budget, dict) and isinstance(excerpts, list)
|
||||
whole_payload = len(okf_consume.serialise(payload).encode("utf-8"))
|
||||
assert whole_payload > int(budget["limit"]), (
|
||||
"the guard measures nothing: the whole payload already fits, so the two "
|
||||
"readings of SS 7.2 cannot be told apart on this case"
|
||||
)
|
||||
assert int(budget["spent"]) <= int(budget["limit"])
|
||||
|
||||
# And the default is now the other side of that line, on the same bundle
|
||||
# and the same question: the bookkeeping stopped dwarfing the content.
|
||||
default = okf_consume.build_payload(K2_BUNDLE, question="Hvordan skal prisene fylles ut?")
|
||||
assert len(okf_consume.serialise(default).encode("utf-8")) < whole_payload
|
||||
|
||||
|
||||
#: The gold set is LOCAL-ONLY: it names corpus documents, which never reach a
|
||||
#: tracked file here. The test reads it rather than restating it, so this file
|
||||
#: carries the assertion and not the answer key.
|
||||
GOLD_SET = PROJECT_ROOT / ".claude/projects/2026-09-07-okf-consume-prepass/hit-at-k-questions.json"
|
||||
|
||||
|
||||
@requires_k2
|
||||
@pytest.mark.skipif(not GOLD_SET.is_file(), reason=f"the local gold set is absent ({GOLD_SET})")
|
||||
def test_every_gold_document_in_the_local_set_is_reached_or_named_as_a_miss() -> None:
|
||||
# SC5 and SC6 together, run against the answer key rather than a literal.
|
||||
# Row 1's gold is the one confirmed by a signal from outside this
|
||||
# repository -- a live model reached that document unprompted in three
|
||||
# navigation steps on 2026-09-06 -- and its gold document holds exactly one
|
||||
# concept, so it is also the one concept-granularity row.
|
||||
spec = json.loads(GOLD_SET.read_text(encoding="utf-8"))
|
||||
questions = spec["questions"]
|
||||
assert len(questions) >= 5, "fewer than five questions is not the measurement"
|
||||
hits = 0
|
||||
for entry in questions:
|
||||
payload = okf_consume.build_payload(K2_BUNDLE, question=entry["question"], ranking="fusion")
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
if okf_consume_measure.hit_rank(excerpts, entry["gold_document"]) is not None:
|
||||
hits += 1
|
||||
# The published bar, and the published number. A regression that drops a
|
||||
# row goes red here rather than in a document nobody re-runs.
|
||||
#
|
||||
# 5 -> 6 ON 2026-09-10, with no bundle changing: `DEFAULT_SOURCE_QUOTA = 2`
|
||||
# reaches the one row that had missed everywhere. What that gain is not:
|
||||
# this metric asks whether the gold DOCUMENT was delivered, and a document
|
||||
# quota raises how many distinct documents a payload holds, so it is not
|
||||
# neutral with respect to the rule that moved it.
|
||||
assert hits == 6, f"hit@8 moved: {hits} of {len(questions)}"
|
||||
|
||||
|
||||
# --- Step 9: the CLI ----------------------------------------------------------
|
||||
|
||||
TOOL = PROJECT_ROOT / "tools" / "okf_consume.py"
|
||||
|
|
@ -1198,44 +1086,6 @@ def test_the_shipped_skill_is_the_generator_output_with_the_checkout_made_relati
|
|||
assert SKILL.read_text(encoding="utf-8") == generated.replace(prefix, "")
|
||||
|
||||
|
||||
@requires_k2
|
||||
def test_no_corpus_document_name_reaches_any_file_this_work_tracks() -> None:
|
||||
# CLAUDE.md's public-file rule. The pattern is DERIVED from the corpus's own
|
||||
# top-level document names at run time rather than hand-picked, so it covers
|
||||
# every document rather than the six someone thought of -- and so this
|
||||
# tracked file carries no corpus name of its own.
|
||||
documents = sorted(
|
||||
{concept_id.split("/", 1)[0] for concept_id in okf_consume.enumerate_concepts(K2_BUNDLE)}
|
||||
)
|
||||
assert len(documents) > 30, "too few documents to be the real corpus"
|
||||
leak = re.compile("|".join(re.escape(name) for name in documents), re.IGNORECASE)
|
||||
|
||||
# The known-positive, first: the pattern must be shown able to find before
|
||||
# its zero counts as a measurement.
|
||||
control = (K2_BUNDLE / "index.md").read_text(encoding="utf-8")
|
||||
assert leak.findall(control), "the pattern cannot find; the zeros below would mean nothing"
|
||||
|
||||
tracked = [
|
||||
SKILL,
|
||||
SKILL.parent / "references" / "README.md",
|
||||
SKILL.parent / "references" / "example-payload.json",
|
||||
PROJECT_ROOT / "tools" / "okf_consume.py",
|
||||
PROJECT_ROOT / "tools" / "okf_consume_measure.py",
|
||||
PROJECT_ROOT / "tests" / "test_okf_consume.py",
|
||||
PROJECT_ROOT / "docs" / "2026-09-07-okf-konsumskill-maaling.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-blindsone-below-k-k2.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-blindsone-laas2-budsjett-k2.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-prisform-og-loggen-k2.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-kravnummer-tokenisering.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-sjeldenhetsvekt.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-claude-code-skill-vilkaarlig-bundle.md",
|
||||
PROJECT_ROOT / "README.md",
|
||||
PROJECT_ROOT / "CLAUDE.md",
|
||||
]
|
||||
for path in tracked:
|
||||
assert leak.findall(path.read_text(encoding="utf-8")) == [], path
|
||||
|
||||
|
||||
def _quota_concept(concept_id: str, *, source_file: str) -> okf_consume.Concept:
|
||||
"""A minimal concept whose only interesting property is its source document."""
|
||||
return okf_consume.Concept(
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@ from __future__ import annotations
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf import quality
|
||||
|
||||
_FRONTMATTER = """---
|
||||
|
|
@ -248,11 +246,6 @@ def test_the_no_source_file_row_reports_no_document_count(tmp_path: Path) -> Non
|
|||
# `<title>` over into the concept id. The second match form -- the
|
||||
# (directory, residual title) pair -- is what takes it to 2 759 of 2 761.
|
||||
|
||||
_GOOD_ARM = Path.home() / "repos/vegnormal-okf/build/ferdig/r761-2025-generisk"
|
||||
_BAD_ARM = Path.home() / "repos/vegnormal-okf/build/860019-mdb-100"
|
||||
_OTHER_PRODUCT = Path.home() / "corpora/okf-telling-20260829/K2-bundle-default-20260912"
|
||||
_FASIT = Path.home() / "repos/vegnormal-okf/build/sk2-fasit-2761.json"
|
||||
|
||||
|
||||
def _fasit(path: Path, titles: list[str]) -> Path:
|
||||
"""A fasit file in the shipped shape: a list of rows carrying title and norm."""
|
||||
|
|
@ -449,48 +442,3 @@ def test_the_boundary_threshold_names_its_corpus_and_denominator() -> None:
|
|||
assert bar.limit_declared >= quality.MIN_DECLARED_FOR_A_THRESHOLD
|
||||
assert bar.corpora == 1
|
||||
assert bar.source
|
||||
|
||||
|
||||
@pytest.mark.skipif(not (_GOOD_ARM.is_dir() and _FASIT.is_file()), reason="local gold set absent")
|
||||
def test_the_known_good_arm_reproduces_its_published_recall() -> None:
|
||||
"""2 759 of 2 761, and the decomposition P1 asked for: 22 literal, 2 737 paired."""
|
||||
report = quality.measure_bundle(_GOOD_ARM, fasit=quality.load_fasit(_FASIT))
|
||||
assert report.boundaries is not None
|
||||
assert (report.boundaries.recovered, report.boundaries.declared) == (2759, 2761)
|
||||
assert (report.boundaries.literal, report.boundaries.paired) == (22, 2737)
|
||||
assert report.boundaries.verdict == "PASS"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not (_BAD_ARM.is_dir() and _FASIT.is_file()), reason="local gold set absent")
|
||||
def test_the_known_bad_arm_reproduces_its_published_recall() -> None:
|
||||
"""1 148 of 2 761 -- the 41.6 % the whole gate could not previously see."""
|
||||
report = quality.measure_bundle(_BAD_ARM, fasit=quality.load_fasit(_FASIT))
|
||||
assert report.boundaries is not None
|
||||
assert (report.boundaries.recovered, report.boundaries.declared) == (1148, 2761)
|
||||
assert report.boundaries.verdict == "FAIL"
|
||||
assert report.exit_code == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (_OTHER_PRODUCT.is_dir() and _FASIT.is_file()), reason="local gold set absent"
|
||||
)
|
||||
def test_a_bundle_of_another_product_scores_zero_and_that_is_the_assertion(tmp_path: Path) -> None:
|
||||
"""The adverse case, measured rather than argued.
|
||||
|
||||
The fasit describes ONE product. The pinned 43-document K2 bundle is a
|
||||
different corpus entirely and scores 0 of 2 761 -- a FAIL that says the
|
||||
caller's assertion was wrong, not that the bundle is bad. `--fasit` is an
|
||||
assertion by the caller, the way `okf consume --ref` is.
|
||||
"""
|
||||
report = quality.measure_bundle(_OTHER_PRODUCT, fasit=quality.load_fasit(_FASIT))
|
||||
assert report.boundaries is not None
|
||||
assert report.boundaries.recovered == 0
|
||||
assert report.boundaries.verdict == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _FASIT.is_file(), reason="local gold set absent")
|
||||
def test_the_shipped_fasit_reproduces_its_own_key_under_this_normalisation() -> None:
|
||||
"""2 761 of 2 761 rows: `norm` is exactly whitespace-stripped, lowercased title."""
|
||||
rows = quality.load_fasit(_FASIT)
|
||||
assert len(rows) == 2761
|
||||
assert sum(1 for row in rows if quality.normalise_title(row.title) == row.norm) == 2761
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@ Three rules this suite is written under, all of them the house pattern:
|
|||
- **Every zero carries a control.** The socket guard is fired before its
|
||||
silence during a run counts as evidence; the sha pin is shown to refuse a
|
||||
tampered set before its silence on the committed ones means anything.
|
||||
- **Nothing here touches a private corpus or the network.** The real sets in
|
||||
`claude-code-llm-wiki` and `vegnormal-okf` are never read by a test: the
|
||||
adapters are exercised against files written in their shapes, with invented
|
||||
content, inside `tmp_path`.
|
||||
- **Nothing here touches a private corpus or the network.** The real set in
|
||||
`claude-code-llm-wiki` is never read by a test: its adapter is exercised
|
||||
against files written in its shape, with invented content, inside
|
||||
`tmp_path`. The test track built on material tied to the operator's
|
||||
employer was retired 2026-09-21, and with it the gate's row 9 and two
|
||||
adapters; nothing below reads, needs or names that material.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -130,19 +132,6 @@ def test_row_one_is_green_when_the_ranker_delivers_every_fasit(tmp_path: Path) -
|
|||
assert (row.k, row.m, row.status) == (9, 9, gate.GREEN)
|
||||
|
||||
|
||||
#: v1.1 C1 moved the default ranking to BM25, and four of this gate's premises
|
||||
#: were built against the fusion it replaced: `set-miss` and B1/B2 force a
|
||||
#: concept BELOW k that BM25 now delivers, `set-quota` needs the source quota to
|
||||
#: decide a cut it no longer decides, and row 7's mutants patch fusion code the
|
||||
#: default no longer runs. The gate says so itself (`premise broken`, `NOT
|
||||
#: RUN`, `8 of 14`). STRICT, so the day the fixtures are re-measured for BM25
|
||||
#: these flip loudly instead of staying skipped.
|
||||
FUSION_PREMISE = pytest.mark.xfail(
|
||||
strict=True,
|
||||
reason="fixture premise built against the fusion ranking; re-measure for BM25",
|
||||
)
|
||||
|
||||
|
||||
def test_row_one_is_red_when_a_fasit_is_not_delivered(tmp_path: Path) -> None:
|
||||
path, sha = _set_file(
|
||||
tmp_path / "set.json",
|
||||
|
|
@ -180,7 +169,6 @@ def test_row_one_never_counts_a_question_that_declares_the_class_it_forces(
|
|||
# --- row 2 --------------------------------------------------------------------
|
||||
|
||||
|
||||
@FUSION_PREMISE
|
||||
def test_row_two_is_green_when_every_class_is_the_one_its_fixture_forces(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
|
@ -276,7 +264,6 @@ class _AlwaysTheQuota(dict[str, str]):
|
|||
return "source_quota_exceeded"
|
||||
|
||||
|
||||
@FUSION_PREMISE
|
||||
def test_row_three_is_green_on_the_shipped_code(tmp_path: Path) -> None:
|
||||
"""And the same three fixtures, unmutated, are the green direction.
|
||||
|
||||
|
|
@ -494,7 +481,6 @@ def _noop_mutant() -> gate.Mutant:
|
|||
return gate.Mutant("N01 nothing is changed", 1, lambda: gate._patched())
|
||||
|
||||
|
||||
@FUSION_PREMISE
|
||||
def test_row_seven_is_green_when_every_mutant_is_felled(tmp_path: Path) -> None:
|
||||
cases, _ = gate.synthetic_cases(tmp_path / "bundles", FIXTURES)
|
||||
baseline = gate.deterministic_rows(cases)
|
||||
|
|
@ -538,7 +524,7 @@ def test_a_mutant_is_felled_by_the_row_that_got_worse_and_never_by_one_that_did_
|
|||
assert after[3] >= before[3]
|
||||
|
||||
|
||||
# --- rows 8 and 9 -------------------------------------------------------------
|
||||
# --- row 8 --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_row_eight_is_never_green_when_it_did_not_run() -> None:
|
||||
|
|
@ -574,72 +560,21 @@ def test_the_wiki_adapter_reads_its_own_shape_and_hits_by_source_file(tmp_path:
|
|||
assert question_set.questions[0].fasit[0].by == "source_file"
|
||||
assert question_set.questions[0].fasit[0].value == "haandbok.md"
|
||||
row = gate.row_eight([(question_set, {"wiki": _bundles(tmp_path)["positive"]})])
|
||||
# J2: one set of three is NOT a measurement of the three -- this asserted
|
||||
# GREEN until 2026-09-19, which is the breakthrough PM measured.
|
||||
assert row.status == gate.NOT_RUN
|
||||
# The one required set, answered: row 8's own green direction. What
|
||||
# refuses a self-written file of this shape is the pin on the command-line
|
||||
# path (`test_a_wiki_set_of_one_question_is_refused_on_the_command_line`).
|
||||
assert (row.k, row.m, row.status) == (1, 1, gate.GREEN)
|
||||
assert any("citation granularity" in detail for detail in row.details)
|
||||
|
||||
|
||||
def test_the_r761_adapter_splits_the_known_negative_out_of_the_questions(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
path = tmp_path / "r761-shaped.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"fasit_form": "the normalised STS title",
|
||||
"sporsmal": [
|
||||
{"id": "S1", "sporsmal": "Hva sier punktet?", "fasit": "4.2 Vakthold"},
|
||||
{"id": "KN", "sporsmal": "Hvilken safran passer til fiskesuppe?", "fasit": ""},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
question_set = gate.read_real_set("r761", path, gate.sha256_of(path))
|
||||
assert [q.id for q in question_set.questions] == ["S1"]
|
||||
assert [c.id for c in question_set.controls] == ["KN"]
|
||||
assert question_set.questions[0].fasit[0].by == "title"
|
||||
# No quote in this set's fasit, so it is concept granularity and says so.
|
||||
assert not question_set.quoted
|
||||
|
||||
|
||||
def test_the_vegnormal_adapter_makes_one_question_per_standard(tmp_path: Path) -> None:
|
||||
path = tmp_path / "vegnormal-shaped.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"sporsmal": [
|
||||
{
|
||||
"id": "T1-1",
|
||||
"sporsmal": "Hva viser kravet til?",
|
||||
"must_cite": [
|
||||
{"normal": "N100:2023", "req_number": "2.3.2-3", "fil": "a"},
|
||||
{"normal": "N200:2024", "req_number": "1.6.3-3", "fil": "b"},
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
question_set = gate.read_real_set("vegnormal", path, gate.sha256_of(path))
|
||||
assert sorted(q.id for q in question_set.questions) == ["T1-1/N100:2023", "T1-1/N200:2024"]
|
||||
assert {q.bundle for q in question_set.questions} == {"N100:2023", "N200:2024"}
|
||||
assert question_set.units == 2
|
||||
|
||||
|
||||
def test_an_unknown_real_set_name_is_refused(tmp_path: Path) -> None:
|
||||
path = tmp_path / "x.json"
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
with pytest.raises(gate.GateUsage):
|
||||
gate.read_real_set("something-else", path, gate.sha256_of(path))
|
||||
|
||||
|
||||
def test_row_nine_states_k2_s_denominator_and_never_passes() -> None:
|
||||
row = gate.row_nine()
|
||||
assert (row.k, row.m, row.status) == (0, 6, gate.RED)
|
||||
assert any("the answer key does not" in detail for detail in row.details)
|
||||
# The two retired adapters are unknown names now, like any other.
|
||||
for name in ("something-else", "r761", "vegnormal"):
|
||||
with pytest.raises(gate.GateUsage) as refusal:
|
||||
gate.read_real_set(name, path, gate.sha256_of(path))
|
||||
assert "unknown real set" in str(refusal.value)
|
||||
|
||||
|
||||
# --- the verdict --------------------------------------------------------------
|
||||
|
|
@ -667,14 +602,16 @@ def test_the_verdict_names_every_failing_row() -> None:
|
|||
def test_the_gate_is_red_today_and_says_which_rows(tmp_path: Path) -> None:
|
||||
rows = gate.evaluate(tmp_path / "bundles")
|
||||
by_number = {row.number: row for row in rows}
|
||||
assert sorted(by_number) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
# Rows 2, 3 and 7 went red with v1.1 C1's BM25 default: see
|
||||
# `FUSION_PREMISE`. Row 1 holds every fasit it held.
|
||||
assert [row.number for row in rows if row.fails] == [2, 3, 5, 7, 8, 9]
|
||||
assert sorted(by_number) == [1, 2, 3, 4, 5, 6, 7, 8]
|
||||
# Rows 2 and 3 are green again since the synthetic corpus was re-measured
|
||||
# for BM25 (2026-09-21). Row 7 is red on three survivors, each printed with
|
||||
# what it moved.
|
||||
assert [row.number for row in rows if row.fails] == [5, 7, 8]
|
||||
assert (by_number[1].k, by_number[1].m) == (10, 10)
|
||||
assert (by_number[2].k, by_number[2].m) == (5, 7)
|
||||
assert (by_number[3].k, by_number[3].m) == (3, 5)
|
||||
assert (by_number[6].k, by_number[6].m) == (12, 12)
|
||||
assert (by_number[2].k, by_number[2].m) == (7, 7)
|
||||
assert (by_number[3].k, by_number[3].m) == (5, 5)
|
||||
assert (by_number[6].k, by_number[6].m) == (10, 10)
|
||||
assert (by_number[7].k, by_number[7].m) == (11, 14)
|
||||
|
||||
|
||||
def test_the_same_tree_measures_the_same_twice(tmp_path: Path) -> None:
|
||||
|
|
@ -688,15 +625,15 @@ def test_the_command_exits_one_and_prints_every_row(
|
|||
) -> None:
|
||||
assert gate.main([]) == 1
|
||||
printed = capsys.readouterr().out
|
||||
for number in range(1, 10):
|
||||
for number in range(1, 9):
|
||||
assert f"\n{number} " in f"\n{printed}"
|
||||
assert "GATE RED: rows 2, 3, 5, 7, 8, 9" in printed
|
||||
assert "GATE RED: rows 5, 7, 8" in printed
|
||||
|
||||
|
||||
def test_the_json_form_carries_the_same_rows(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
assert gate.main(["--json"]) == 1
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert [row["row"] for row in payload["rows"]] == list(range(1, 10))
|
||||
assert [row["row"] for row in payload["rows"]] == list(range(1, 9))
|
||||
assert payload["gate"] == gate.RED
|
||||
assert set(payload["classes"]) == {"a", "b", "c", "d", "e"}
|
||||
assert payload["limits"]
|
||||
|
|
@ -879,26 +816,25 @@ def _hitting_set(set_id: str, *, quote: str, entries: int = 1) -> gate.QuestionS
|
|||
|
||||
|
||||
def test_j2_row_eight_is_not_run_when_a_required_set_is_left_out(tmp_path: Path) -> None:
|
||||
"""PM's J2: one set of three came back `6 of 6 GREEN`."""
|
||||
"""PM's J2: one set of three came back `6 of 6 GREEN`. The rule outlived
|
||||
the two retired sets: a set that is not the required one measures
|
||||
something, and the row is still not a measurement of the required set."""
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
row = gate.row_eight([(_hitting_set("wiki-20", quote="innen 1. november"), bundles)])
|
||||
row = gate.row_eight([(_hitting_set("some-other-set", quote="innen 1. november"), bundles)])
|
||||
assert row.status == gate.NOT_RUN
|
||||
assert row.fails
|
||||
assert "r761-sk2" in row.reason and "vegnormal-32" in row.reason
|
||||
assert "wiki-20" in row.reason
|
||||
# The numbers it DID measure are still carried: a missing set must not cost
|
||||
# the reader the set that ran.
|
||||
assert any("wiki-20: " in detail for detail in row.details)
|
||||
assert any("some-other-set: " in detail for detail in row.details)
|
||||
|
||||
|
||||
def test_row_eight_is_green_only_with_all_three_named_sets(tmp_path: Path) -> None:
|
||||
def test_row_eight_is_green_only_with_every_named_set(tmp_path: Path) -> None:
|
||||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
real = [
|
||||
(_hitting_set(name, quote="innen 1. november"), bundles)
|
||||
for name in ("wiki-20", "r761-sk2", "vegnormal-32")
|
||||
]
|
||||
real = [(_hitting_set(name, quote="innen 1. november"), bundles) for name in ("wiki-20",)]
|
||||
row = gate.row_eight(real)
|
||||
assert row.status == gate.GREEN
|
||||
assert (row.k, row.m) == (3, 3)
|
||||
assert (row.k, row.m) == (1, 1)
|
||||
|
||||
|
||||
def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
|
||||
|
|
@ -909,17 +845,16 @@ def test_j2b_row_eight_never_sums_the_two_granularities_into_its_headline(
|
|||
bundles = {"positive": _bundles(tmp_path)["positive"]}
|
||||
real = [
|
||||
(_hitting_set("wiki-20", quote="innen 1. november", entries=2), bundles),
|
||||
(_hitting_set("r761-sk2", quote=""), bundles),
|
||||
(_hitting_set("vegnormal-32", quote=""), bundles),
|
||||
(_hitting_set("an-invented-concept-set", quote=""), bundles),
|
||||
]
|
||||
row = gate.row_eight(real)
|
||||
# Three questions, one per set: the headline is at QUESTION granularity.
|
||||
# The sum it must NOT be is 4 of 4 -- two citation units plus two concept
|
||||
# units -- which is why the first set carries two fasit entries.
|
||||
assert (row.k, row.m) == (3, 3)
|
||||
# Two questions, one per set: the headline is at QUESTION granularity.
|
||||
# The sum it must NOT be is 3 of 3 -- two citation units plus one concept
|
||||
# unit -- which is why the first set carries two fasit entries.
|
||||
assert (row.k, row.m) == (2, 2)
|
||||
assert "question" in row.reason
|
||||
assert any(
|
||||
"2 of 2 at citation granularity, 2 of 2 at concept granularity" in detail
|
||||
"2 of 2 at citation granularity, 1 of 1 at concept granularity" in detail
|
||||
for detail in row.details
|
||||
)
|
||||
|
||||
|
|
@ -1081,40 +1016,6 @@ def test_a_corpus_that_is_not_the_pinned_corpus_is_refused(
|
|||
assert "not the corpus that was pinned" in str(refusal.value)
|
||||
|
||||
|
||||
def _k2_shaped(tmp_path: Path, questions: int) -> tuple[Path, str]:
|
||||
"""A K2 gold set in this gate's own shape. `set-positive.json` carries
|
||||
exactly six questions the bundle answers, which is K2's denominator."""
|
||||
spec = json.loads((FIXTURES / "set-positive.json").read_text(encoding="utf-8"))
|
||||
spec["set_id"] = "k2-gold"
|
||||
spec["questions"] = spec["questions"][:questions]
|
||||
path = tmp_path / "k2.json"
|
||||
path.write_text(json.dumps(spec, ensure_ascii=False), encoding="utf-8")
|
||||
return path, gate.sha256_of(path)
|
||||
|
||||
|
||||
def test_row_nine_is_green_when_a_k2_gold_set_arrives(tmp_path: Path) -> None:
|
||||
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
|
||||
question_set = gate.load_set(path, sha)
|
||||
row = gate.row_nine((question_set, _bundles(tmp_path)))
|
||||
assert (row.k, row.m, row.status) == (6, 6, gate.GREEN)
|
||||
|
||||
|
||||
def test_row_nine_is_red_when_the_gold_set_is_not_answered(tmp_path: Path) -> None:
|
||||
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS)
|
||||
question_set = gate.load_set(path, sha)
|
||||
bundles = dict(_bundles(tmp_path))
|
||||
bundles["positive"] = bundles["miss"] # the same six questions, the wrong bundle
|
||||
row = gate.row_nine((question_set, bundles))
|
||||
assert (row.k, row.m, row.status) == (0, 6, gate.RED)
|
||||
|
||||
|
||||
def test_a_k2_set_of_another_size_is_another_set_and_is_refused(tmp_path: Path) -> None:
|
||||
path, sha = _k2_shaped(tmp_path, gate.K2_QUESTIONS - 1)
|
||||
with pytest.raises(gate.GateUsage) as refusal:
|
||||
gate._k2_set([str(path), sha, str(tmp_path)])
|
||||
assert "K2's denominator" in str(refusal.value)
|
||||
|
||||
|
||||
def test_j7_a_cut_that_delivers_nothing_is_exit_two_and_never_a_quiet_green_row(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
|
@ -1135,7 +1036,7 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
|
|||
) -> None:
|
||||
"""The row printed the SET's sha256 and never the bundle's identity.
|
||||
|
||||
Measured 2026-09-19 by PM with one pinned r761 set: three bundles gave
|
||||
Measured 2026-09-19 by PM with one pinned real set: three bundles gave
|
||||
4 of 7 and a fourth gave 0 of 7, while the session recorded 7 of 7 -- from
|
||||
a bundle no line of the output named. A number nobody can attach to a
|
||||
bundle cannot be reproduced and cannot be felled.
|
||||
|
|
@ -1158,15 +1059,15 @@ def test_row_eight_names_the_bundle_identity_of_every_set_it_measured(
|
|||
assert ref[:24] in block, f"{set_id}: the bundle ref is nowhere in the output"
|
||||
|
||||
|
||||
def test_all_three_real_sets_are_pinned_and_not_only_the_first(tmp_path: Path) -> None:
|
||||
"""Every one of the three, driven through the command-line path.
|
||||
def test_every_required_real_set_is_pinned(tmp_path: Path) -> None:
|
||||
"""Every required set, driven through the command-line path.
|
||||
|
||||
The names and the count are written out here rather than read from
|
||||
`REQUIRED_REAL_SETS`: a test taking its denominator from the tuple it is
|
||||
checking would stay green if a name were dropped from both.
|
||||
"""
|
||||
names = [("wiki", "wiki-20"), ("r761", "r761-sk2"), ("vegnormal", "vegnormal-32")]
|
||||
assert len(names) == 3
|
||||
names = [("wiki", "wiki-20")]
|
||||
assert len(names) == len(gate.REQUIRED_REAL_SETS) == 1
|
||||
bundle = _bundles(tmp_path)["positive"]
|
||||
for adapter, set_id in names:
|
||||
path = tmp_path / f"{adapter}-invented.json"
|
||||
|
|
@ -1178,29 +1079,14 @@ def test_all_three_real_sets_are_pinned_and_not_only_the_first(tmp_path: Path) -
|
|||
|
||||
def _invented_real_set(adapter: str) -> str:
|
||||
"""One question, in the adapter's own shape, with invented content."""
|
||||
if adapter == "wiki":
|
||||
return json.dumps(
|
||||
{
|
||||
"questions": [
|
||||
{
|
||||
"id": "W1",
|
||||
"question": "Naar kontrolleres vinterberedskapen?",
|
||||
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
if adapter == "r761":
|
||||
return json.dumps(
|
||||
{"sporsmal": [{"id": "S1", "sporsmal": "Hva sier punktet?", "fasit": "4.2 Vakthold"}]}
|
||||
)
|
||||
assert adapter == "wiki", adapter
|
||||
return json.dumps(
|
||||
{
|
||||
"sporsmal": [
|
||||
"questions": [
|
||||
{
|
||||
"id": "T1-1",
|
||||
"sporsmal": "Hva viser kravet til?",
|
||||
"must_cite": [{"normal": "N100:2023", "req_number": "2.3.2-3"}],
|
||||
"id": "W1",
|
||||
"question": "Naar kontrolleres vinterberedskapen?",
|
||||
"fasit": [{"doc": "haandbok", "quote": "innen 1. november"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1282,7 +1168,6 @@ def test_a_threshold_that_is_not_a_number_is_refused(tmp_path: Path) -> None:
|
|||
assert row.fails
|
||||
|
||||
|
||||
@FUSION_PREMISE
|
||||
def test_the_threshold_is_compared_with_the_measured_hold_out(tmp_path: Path) -> None:
|
||||
"""Both directions, from the same code path: a set the bundle answers
|
||||
clears a threshold under it, and a set it does not answer falls under one
|
||||
|
|
|
|||
|
|
@ -218,15 +218,3 @@ def test_the_pdf_page_table_is_measured_over_the_normalised_text(
|
|||
text, removed = extract.normalise_extracted(joined)
|
||||
assert removed == 3
|
||||
assert table.starts[1] == text.index("Asfaltbetong")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json").is_file(),
|
||||
reason="R761 is not on this machine",
|
||||
)
|
||||
def test_r761s_own_seventy_one_soft_hyphens_are_the_number_the_door_removes() -> None:
|
||||
"""PM's count, on the delivery the decision was taken for."""
|
||||
source = Path.home() / "repos/vegnormal-okf/data/raw/860019/R761-2025-860019.json"
|
||||
document = extract.extract_document(source.name, source.read_bytes())
|
||||
assert document.soft_hyphens == 71
|
||||
assert SHY not in document.text
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue