feat(quality): okf quality, a per-file-type verdict with the denominator
G37. `okf check` is a CONTRACT check and a green one is not a quality gate: measured 2026-09-10 by `vegnormal-okf`, three arms over one corpus all returned 0 findings and exit 0 while their hit@k ranged from 6 of 6 to 0 of 6. `okf quality <bundle>` asks the other question, per file type, with the denominator on every line. A separate command rather than `okf check --quality`, because the two answer different questions and a caller must not be able to read one as the other. `okf check` is untouched. Three verdicts and no fourth -- PASS, FAIL, UNMEASURED -- and a type with no measured threshold is never PASS. Exit 0 judged and clean, 1 at least one FAIL, 2 did not run, 3 nothing could be judged: exit 0 over a table of unmeasured rows would be the silent pass this command exists to stop. Two bars today, both `structure_null_share` (documents of a type yielding exactly one concept), read off the pinned 43-document reference bundle: .pdf 8/32, .docx 2/5. Plus one definitional bar for every type, taken from the harness's own degenerate-merge definition: 0 concepts with an empty body, measured 0 of 8 602 concepts over four bundles. A bar needs five documents on BOTH sides -- its own and the judged bundle's -- so .xlsx (2), .xml (1) and every type with no corpus class in `extract._EVIDENCE` are UNMEASURED and print their numbers without a verdict. The floor on the judged bundle was found by RUNNING the gate, not by reading it: one PDF cut into 2 182 concepts scored 0 of 1 against the 32-document reference and read as PASS. The gate walks the index tree and never a directory (SS 9.2; controlled against the listing on four bundles, 453 / 2 761 / 3 206 / 446 either way), and prints the bundle's own run log beside its counts -- a document rejected at extraction leaves no row in the bundle, so the pinned corpus's 33 PDFs show up as 32 and the two denominators must never be read as one. Three of the order's five premises moved when re-measured, and they are in the document rather than glossed: the four evidence corpora carry `source_file` on 0 of 446, 0 of 1 133, 0 of 270 and 0 of 2 756 concepts, so they name no file type and cannot PASS; "41,6 %" is `vegnormal-okf`'s number and not in this repository; and the same 828-document bundle carries two published hit@k figures from two question sets. Three candidate metrics measured and NOT shipped: duplicate titles within a document (0 of 3 206 on the known-bad arm against 349 of 2 761 on the known-good one) and short concepts (5.6 % against 14.6 %) order the two arms the wrong way round; duplicate titles across the whole bundle order all four correctly (37.8 / 16.3 / 12.6 / 5.7 %) and still ship without a bar, because any bar separating them is read off the two bundles it would judge. 19 new tests, each rule exercised in both directions; the three README pins were each driven red before being kept. Suite 1 850 passed, 1 skipped, 1 851 collected, run after `git add` -- +19 against a base of 1 832 collected, measured on the stashed tree (STATE's 1 831 is one short of that). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
d310dbb34e
commit
6ea8fcd3c7
8 changed files with 970 additions and 1 deletions
|
|
@ -207,3 +207,49 @@ def test_the_readme_carries_only_one_file_type_table() -> None:
|
|||
section = text.split("### Binary extraction", 1)[1].split("\n## ", 1)[0]
|
||||
rows = [line for line in section.splitlines() if line.strip().startswith("|")]
|
||||
assert not rows, f"a second file-type table is back under Binary extraction: {rows}"
|
||||
|
||||
|
||||
# The `okf quality` thresholds, in the one place the README writes them. A bar
|
||||
# published without a test goes false the way the format list did.
|
||||
_THRESHOLD_LINE = re.compile(r"^<!-- quality-thresholds: (.+) -->$", re.MULTILINE)
|
||||
|
||||
THRESHOLD_DOCUMENT = PROJECT_ROOT / "docs" / "2026-09-12-g37-terskler.md"
|
||||
|
||||
|
||||
def _declared_thresholds() -> dict[str, str]:
|
||||
match = _THRESHOLD_LINE.search(README.read_text(encoding="utf-8"))
|
||||
assert match is not None, (
|
||||
"README.md carries no `<!-- quality-thresholds: ... -->` marker; without "
|
||||
"it the published bars can drift from the ones the gate applies"
|
||||
)
|
||||
pairs = (token.strip().split("=") for token in match.group(1).split(","))
|
||||
return {extension: share for extension, share in pairs}
|
||||
|
||||
|
||||
def test_the_readme_names_exactly_the_thresholds_the_gate_applies() -> None:
|
||||
from llm_ingestion_okf.quality import THRESHOLDS
|
||||
|
||||
assert _declared_thresholds() == {
|
||||
extension: threshold.as_share() for extension, threshold in THRESHOLDS.items()
|
||||
}
|
||||
|
||||
|
||||
def test_the_threshold_document_carries_the_same_bars() -> None:
|
||||
"""Three copies, one measurement: the code, the README and the document.
|
||||
|
||||
The document is where a bar's N and corpus live, so a bar that moved in the
|
||||
code without moving there would publish a number nobody measured.
|
||||
"""
|
||||
from llm_ingestion_okf.quality import THRESHOLDS
|
||||
|
||||
text = THRESHOLD_DOCUMENT.read_text(encoding="utf-8")
|
||||
for extension, threshold in THRESHOLDS.items():
|
||||
row = f"| `{extension}` | `{threshold.metric}` | **{threshold.as_share()}** |"
|
||||
assert row in text, f"{THRESHOLD_DOCUMENT.name} carries no row {row}"
|
||||
|
||||
|
||||
def test_the_readme_quality_section_does_not_promise_a_quality_claim() -> None:
|
||||
"""The one sentence that must not come back: PASS as a statement of quality."""
|
||||
text = README.read_text(encoding="utf-8").lower()
|
||||
assert "okf quality" in text
|
||||
assert "regression bar against a pinned artifact" in text
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue