test(quality): the boundary bar is a round share that keeps every verdict — red
The bar was the known-good arm's own measured fraction, which names the document it was read off. The test asks for 999/1000 and for the verdict of every recorded build to stand: the known-good arm passes; an older build at 99.7 %, the known-bad arm at 41.6 % and five of six fail. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
88cf67f12e
commit
ea8590db82
1 changed files with 28 additions and 10 deletions
|
|
@ -240,12 +240,12 @@ def test_the_no_source_file_row_reports_no_document_count(tmp_path: Path) -> Non
|
|||
# (`docs/2026-09-12-g37-terskler.md` SS 7):
|
||||
#
|
||||
# 1. The fasit's `norm` key is reproduced from its own `title` by stripping all
|
||||
# whitespace and lowercasing -- 2 761 of 2 761 rows, so the normalisation is
|
||||
# whitespace and lowercasing -- N of N rows, so the normalisation is
|
||||
# not a guess.
|
||||
# 2. That normalisation ALONE reaches 22 of 2 761 on the known-good arm, not
|
||||
# 2. That normalisation ALONE reaches 22 of N on the known-good arm, not
|
||||
# 99 %, because okf's default route moves the numbering token STS glues into
|
||||
# `<title>` over into the concept id. The second match form -- the
|
||||
# (directory, residual title) pair -- is what takes it to 2 759 of 2 761.
|
||||
# (directory, residual title) pair -- is what takes it to all but two of N.
|
||||
|
||||
|
||||
def _fasit(path: Path, titles: list[str]) -> Path:
|
||||
|
|
@ -269,7 +269,7 @@ def _declaring(titles: list[str]) -> list[tuple[str, str, str]]:
|
|||
def test_the_normalisation_reproduces_the_fasit_key() -> None:
|
||||
"""`norm` is whitespace-stripped, lowercased `title` -- not a guess.
|
||||
|
||||
Measured over the shipped fasit before anything was written: 2 761 of 2 761
|
||||
Measured over the shipped fasit before anything was written: N of N
|
||||
rows reproduce, and the real file pins it below where it exists.
|
||||
"""
|
||||
assert quality.normalise_title(" 2.1 Hoved Kapitler\n") == "2.1hovedkapitler"
|
||||
|
|
@ -304,8 +304,8 @@ def test_a_bundle_recovering_every_declared_boundary_passes(tmp_path: Path) -> N
|
|||
def test_a_bundle_recovering_few_declared_boundaries_fails(tmp_path: Path) -> None:
|
||||
"""The known-negative for the same rule, and the arm it was built for.
|
||||
|
||||
`860019-mdb-100` recovers 1 148 of 2 761. Here five of six is already worse
|
||||
than the bar of 2 759/2 761 -- the bar is that tight, which is a property of
|
||||
The known-bad HTML arm recovers 41.6 %. Here five of six is already worse
|
||||
than the bar of 99.9 % -- the bar is that tight, which is a property of
|
||||
a regression bar set at its reference and is said out loud in the document.
|
||||
"""
|
||||
titles = [f"{n} Kapittel {n}" for n in range(1, 7)]
|
||||
|
|
@ -330,15 +330,15 @@ def test_a_bundle_recovering_few_declared_boundaries_fails(tmp_path: Path) -> No
|
|||
def test_the_numbering_token_is_matched_through_the_concept_directory(tmp_path: Path) -> None:
|
||||
"""The second match form, and the reason P1's literal reading is not enough.
|
||||
|
||||
STS glues the numbering into the title (`11.1Fastmerker`); okf's default
|
||||
STS glues the numbering into the title (`11.1Grunndeiger`); okf's default
|
||||
route moves it into the concept id (`11-1/...`) and keeps the residual as
|
||||
the title. Measured on the known-good arm: the literal form alone reaches
|
||||
22 of 2 761, the pair form 2 737, either 2 759.
|
||||
22 of N, the pair form N − 24, either N − 2.
|
||||
"""
|
||||
titles = ["11.1Fastmerker"]
|
||||
titles = ["11.1Grunndeiger"]
|
||||
bundle = _bundle(tmp_path / "paired", [("11-1/p1", "doc.pdf", "Body.")])
|
||||
(bundle / "11-1/p1.md").write_text(
|
||||
_FRONTMATTER.format(title="Fastmerker", source_file="doc.pdf", body="Body."),
|
||||
_FRONTMATTER.format(title="Grunndeiger", source_file="doc.pdf", body="Body."),
|
||||
encoding="utf-8",
|
||||
newline="",
|
||||
)
|
||||
|
|
@ -438,6 +438,24 @@ def test_the_boundary_row_says_its_threshold_rests_on_one_corpus(tmp_path: Path)
|
|||
assert "N = 1" in rendered
|
||||
|
||||
|
||||
def test_the_boundary_bar_is_a_round_share_that_keeps_every_recorded_verdict() -> None:
|
||||
"""99.9 %, just under the known-good arm's measured share.
|
||||
|
||||
The bar was the known-good arm's own fraction, which named the document it
|
||||
was read off. A round share just under it gives every build measured the
|
||||
verdict it had: the known-good arm (all but two of its declared sections)
|
||||
passes, an older build of the same product at 99.7 % and the known-bad arm
|
||||
at 41.6 % fail, and so does five of six.
|
||||
"""
|
||||
bar = quality.BOUNDARY_THRESHOLD
|
||||
assert bar.as_share() == "999/1000"
|
||||
assert not bar.undercut_by(999, 1000)
|
||||
assert not bar.undercut_by(1998, 2000)
|
||||
assert bar.undercut_by(997, 1000)
|
||||
assert bar.undercut_by(416, 1000)
|
||||
assert bar.undercut_by(5, 6)
|
||||
|
||||
|
||||
def test_the_boundary_threshold_names_its_corpus_and_denominator() -> None:
|
||||
bar = quality.BOUNDARY_THRESHOLD
|
||||
assert bar.limit_declared >= quality.MIN_DECLARED_FOR_A_THRESHOLD
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue