Kjell Tore Guttormsen
861c11e9f4
Order 20260904T172353Z-6290714297-from-.claude. Bilag 9.1's 95.1 % CID share (docs/2026-09-04-k3-arm-c.md) was found ad hoc, with no committed script -- the same gap this repo's own fidelity instrument criticizes in Arm A's uncommitted docx/xlsx figures. okf_cid_measure.py runs the exact extract_text call the door makes and reports per-document CID share and 4+-letter word count, denominator stated for files it cannot measure. Red-first: tests/test_cid_measure.py pins measure() against fixture text of known composition before the implementation existed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
50 lines
1.8 KiB
Python
50 lines
1.8 KiB
Python
"""The CID-share instrument: order 20260904T172353Z-6290714297-from-.claude.
|
|
|
|
Measures how much of a document's extracted text is undecoded `(cid:N)` glyph
|
|
codes -- the failure mode `docs/2026-09-04-k3-arm-c.md` found on Bilag 9.1
|
|
(95.1 %, 98 alphabetic words of four or more letters survive). This instrument
|
|
answers whether that document is alone or whether the corpus has more of them.
|
|
|
|
The negative control matters here as much as the positive one: an instrument
|
|
that reports a nonzero CID share on text that has none would inflate every
|
|
number it ever produces, the same reason `test_fidelity.py` pins its own
|
|
negative control.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
|
|
|
|
import okf_cid_measure # noqa: E402
|
|
|
|
|
|
def test_known_cid_fraction() -> None:
|
|
text = "abcd efgh " + "(cid:12)" * 2
|
|
result = okf_cid_measure.measure(text)
|
|
assert result.total_chars == len(text) == 26
|
|
assert result.cid_chars == len("(cid:12)") * 2 == 16
|
|
assert result.word_count == 2 # "abcd", "efgh" -- "cid" itself is 3 letters
|
|
assert abs(result.pct - (16 / 26 * 100)) < 1e-9
|
|
|
|
|
|
def test_no_cid_is_zero_with_nonzero_total() -> None:
|
|
result = okf_cid_measure.measure("plain readable text with no cid codes at all")
|
|
assert result.total_chars > 0
|
|
assert result.cid_chars == 0
|
|
assert result.pct == 0.0
|
|
|
|
|
|
def test_all_cid_is_full_share_and_zero_words() -> None:
|
|
result = okf_cid_measure.measure("(cid:1)(cid:2)(cid:3)")
|
|
assert result.pct == 100.0
|
|
assert result.word_count == 0
|
|
|
|
|
|
def test_empty_text_reports_zero_percent_not_a_crash() -> None:
|
|
result = okf_cid_measure.measure("")
|
|
assert result.total_chars == 0
|
|
assert result.cid_chars == 0
|
|
assert result.pct == 0.0
|