feat(consume): measure the below-k blind spot, add one flag-gated vocabulary bridge
The consumer report (portfolio-optimiser, S7 SS 2) found that a mandate-shaped
cost question withheld the corpus's one priced table under `below_k`. Measured
here, on a bundle proven byte-identical to a fresh HEAD rebuild:
- The mechanism is a VOCABULARY gap, not a `k` defect: two of three ranking
signals are exactly 0.0 and the concept is candidate 249 of 269.
- The k-sweep buys nothing: k in {8,12,16,24,32,64,128} all withhold it, at
+9.5 % tokens. It also found a regression -- for the question that WORKS,
k >= 16 EVICTS the gold concept, because one 67 838 B excerpt is 56.5 % of
the budget and the knapsack maximises a sum.
- Two proposed rules were falsified BEFORE any code: number/table density ranks
the priced table 178/165/46 of 269 (the form is unfilled, so it is
number-poor), and per-document spread puts its document 30th of 35.
Built instead, behind `--cost-vocabulary` (default OFF, DEFAULT byte-identical):
one declared cost/price/quantity vocabulary family that bridges a question and a
document naming money with different words. It moves the concept from candidate
rank 249 to 10 -- and does NOT close the blind spot: the budget still refuses
it, which is now a separately measured second lock.
Seven RED tests first; six mutations of the rule, six red (two survived the
first version of the tests and the tests were strengthened). Control: a question
with no cost term produces a byte-identical payload with the flag on, at every
k, on the real corpus. Known-positive: 164 987 B / 40 425 o200k tokens, equal to
the published pair.
Report: docs/2026-09-08-blindsone-below-k-k2.md
Suite 1268 green, mypy --strict clean over 28 files, both goldens unchanged.
Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
parent
5a0c8794af
commit
4c699fdbb1
5 changed files with 616 additions and 12 deletions
|
|
@ -1044,6 +1044,7 @@ def test_no_corpus_document_name_reaches_any_file_this_work_tracks() -> None:
|
|||
PROJECT_ROOT / "tools" / "okf_consume_measure.py",
|
||||
PROJECT_ROOT / "tests" / "test_okf_consume.py",
|
||||
PROJECT_ROOT / "docs" / "2026-09-07-okf-konsumskill-maaling.md",
|
||||
PROJECT_ROOT / "docs" / "2026-09-08-blindsone-below-k-k2.md",
|
||||
PROJECT_ROOT / "README.md",
|
||||
PROJECT_ROOT / "CLAUDE.md",
|
||||
]
|
||||
|
|
@ -1111,3 +1112,107 @@ def test_the_measurement_instrument_names_no_corpus_document() -> None:
|
|||
source = (PROJECT_ROOT / "tools" / "okf_consume_measure.py").read_text(encoding="utf-8")
|
||||
leak = re.compile(r"del-ii-bilag|del-i-vedlegg|del-i-konkurranse|prisskjema|stange", re.I)
|
||||
assert leak.findall(source) == []
|
||||
|
||||
|
||||
# --- Step 12: the declared cost vocabulary, behind a flag ---------------------
|
||||
#
|
||||
# Measured 2026-09-08 on the K2 corpus (`docs/2026-09-08-blindsone-below-k-k2.md`):
|
||||
# a mandate-shaped cost question ranks the corpus's one priced table 249th of
|
||||
# 269 lexical candidates, because its title, its id and its document index
|
||||
# entries carry none of the question's tokens. The gap is a VOCABULARY gap --
|
||||
# the question says `kostnadsbesparelser`, the document says `pris` -- and no
|
||||
# amount of `k` closes it. The fixture below reproduces that gap synthetically:
|
||||
# `krav/pristabell` is `no_lexical_match` for a question about `kostnader`.
|
||||
|
||||
|
||||
def test_a_cost_question_reaches_no_price_concept_without_the_flag() -> None:
|
||||
# The known-negative this whole step is measured against. Without it, the
|
||||
# flag's effect below would have no denominator.
|
||||
payload = _payload(question="Hvor kan vi kutte kostnader?")
|
||||
counts, withheld = payload["denominators"], payload["withheld"]
|
||||
assert isinstance(counts, dict) and isinstance(withheld, list)
|
||||
assert counts["delivered"] == 0
|
||||
assert {"concept_id": "krav/pristabell", "rule": "no_lexical_match"} in withheld
|
||||
|
||||
|
||||
def test_the_cost_vocabulary_flag_bridges_a_question_and_a_document_that_share_no_word() -> None:
|
||||
payload = okf_consume.build_payload(
|
||||
FIXTURE, question="Hvor kan vi kutte kostnader?", cost_vocabulary=True
|
||||
)
|
||||
excerpts = payload["excerpts"]
|
||||
assert isinstance(excerpts, list)
|
||||
assert "krav/pristabell" in [excerpt["concept_id"] for excerpt in excerpts]
|
||||
|
||||
|
||||
def test_the_flag_is_off_by_default_and_the_default_payload_is_byte_identical() -> None:
|
||||
# The library's standing promise to a consumer: a new parameter is
|
||||
# keyword-only with a default, and the default bytes do not move.
|
||||
question = "Hvor kan vi kutte kostnader?"
|
||||
off = okf_consume.serialise(okf_consume.build_payload(FIXTURE, question=question))
|
||||
explicit = okf_consume.serialise(
|
||||
okf_consume.build_payload(FIXTURE, question=question, cost_vocabulary=False)
|
||||
)
|
||||
assert off == explicit
|
||||
|
||||
|
||||
def test_the_flag_changes_nothing_when_the_question_names_no_such_term() -> None:
|
||||
# The GATE is the question, never the flag: a question with no cost term
|
||||
# gets byte-identical bytes whether the flag is set or not.
|
||||
question = "Hvor ofte er den årlige kontrollen?"
|
||||
off = okf_consume.serialise(okf_consume.build_payload(FIXTURE, question=question))
|
||||
on = okf_consume.serialise(
|
||||
okf_consume.build_payload(FIXTURE, question=question, cost_vocabulary=True)
|
||||
)
|
||||
assert off == on
|
||||
|
||||
|
||||
def test_the_bridge_needs_a_vocabulary_term_on_both_sides() -> None:
|
||||
# A one-sided bridge would make every cost question match every document,
|
||||
# which is the confident guess `no_lexical_match` exists to forbid.
|
||||
assert okf_consume.in_cost_vocabulary("kostnadsbesparelser")
|
||||
assert okf_consume.in_cost_vocabulary("prissammenstilling")
|
||||
assert not okf_consume.in_cost_vocabulary("kontrollen")
|
||||
tokens = okf_consume.normalise("kostnader")
|
||||
assert okf_consume._overlap(tokens, "aarlig kontroll", cost_vocabulary=True) == 0
|
||||
assert okf_consume._overlap(tokens, "prisene fylles ut", cost_vocabulary=True) == 1
|
||||
# And the bridge carries the vocabulary term ALONE: a question's unrelated
|
||||
# tokens do not ride along on it. Without this the widening would be
|
||||
# "everything matches a price document", not "cost words do".
|
||||
mixed = okf_consume.normalise("kostnader kontrollen")
|
||||
assert okf_consume._overlap(mixed, "prisene fylles ut", cost_vocabulary=True) == 1
|
||||
# The gate is the question. Asserted directly, because the per-token test
|
||||
# above holds even when the gate is stuck open.
|
||||
assert okf_consume.question_uses_cost_vocabulary("Hvor kan vi kutte kostnader?")
|
||||
assert not okf_consume.question_uses_cost_vocabulary("Hvor ofte er den årlige kontrollen?")
|
||||
|
||||
|
||||
def test_every_vocabulary_member_is_long_enough_to_ever_match() -> None:
|
||||
# `tokens_match` needs MIN_SHARED_PREFIX characters, so a shorter member is
|
||||
# dead code that reads as coverage. Measured: `sum` (3) never matches
|
||||
# `Summen` and was dropped for that reason.
|
||||
assert okf_consume.COST_VOCABULARY
|
||||
for member in okf_consume.COST_VOCABULARY:
|
||||
assert len(member) >= okf_consume.MIN_SHARED_PREFIX, member
|
||||
assert member == member.casefold(), member
|
||||
assert list(okf_consume.COST_VOCABULARY) == sorted(okf_consume.COST_VOCABULARY)
|
||||
|
||||
|
||||
def test_the_vocabulary_is_one_list_and_names_no_corpus_document() -> None:
|
||||
source = (PROJECT_ROOT / "tools" / "okf_consume.py").read_text(encoding="utf-8")
|
||||
assert source.count("COST_VOCABULARY = (") == 1
|
||||
leak = re.compile(r"del-ii-bilag|del-i-vedlegg|prisskjema|prissammenstilling|stange", re.I)
|
||||
assert leak.findall(source) == []
|
||||
|
||||
|
||||
def test_the_cli_exposes_the_flag_and_omitting_it_reproduces_the_default_bytes() -> None:
|
||||
question = "Hvordan skal prisene fylles ut?"
|
||||
plain = _run(str(FIXTURE), "--question", question)
|
||||
assert plain.returncode == 0
|
||||
flagged = _run(str(FIXTURE), "--question", question, "--cost-vocabulary")
|
||||
assert flagged.returncode == 0
|
||||
assert plain.stdout == okf_consume.serialise(
|
||||
okf_consume.build_payload(FIXTURE, question=question)
|
||||
)
|
||||
assert flagged.stdout == okf_consume.serialise(
|
||||
okf_consume.build_payload(FIXTURE, question=question, cost_vocabulary=True)
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue