feat(consume): the payload says when the bundle looks like it does not cover the question
C4. `coverage` gains two keys, and every key it had keeps its bytes: - `absent_terms`: the question's words the bundle holds in NO form -- not as written, and not through a relative it uses (`bm25.query_groups`, the same bridge the ranking reads through); - `weak`: true when one such word exists or nothing was delivered. A reading with its rule in the open, never a verdict about the bundle. It is computed for both rankings (`bm25.absent_terms` serves the fusion). The retrieval gate's `marked` -- the one reading both gates share -- reads `weak` beside its own bar, never instead of it; the known-negative that strips the payload's words now strips both readings. Words that only FRAME a question are stopwords in both languages (`how often`, `hvor ofte`, `hva står i`, `what does it say`), and so are the Norwegian function words spelled without their letters (`naar`, `paa`), the way ASCII-only text writes them. Read as topic words they would be "absent" from any collection that never uses them, which is what the synthetic sets showed on three answered questions before the list was extended. The working method says what to do with it, in one sentence each: the skill template's step 3 and the MCP server's instructions (1 253 bytes, under the 2 048 a client keeps) -- rephrase in the bundle's words, and if it stays weak, say the bundle does not cover the question. The search gate's table for this commit is kept in local state. Suite on a clean tree after `git add`: 2397 passed, 2 skipped, 4 xfailed. ruff, ruff format, mypy --strict clean. Retrieval gate unchanged at the rows the previous commit left red. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
735468f600
commit
ab6e24aa22
12 changed files with 183 additions and 8 deletions
|
|
@ -189,3 +189,17 @@ def test_a_concept_named_by_its_number_carries_a_lexical_match() -> None:
|
|||
ranked = bm25.rank(concepts, "Hva staar i punkt 4.2 om kontrollen av hytta?").ranked
|
||||
lexical = {concept.concept_id: count for concept, _, count in ranked}
|
||||
assert lexical["r/vakthold-4-2"] >= 1
|
||||
|
||||
|
||||
def test_a_norwegian_function_word_written_without_its_letters_is_a_stopword() -> None:
|
||||
# ASCII-only text writes `når` as `naar`; it is the same function word and
|
||||
# must not read as a content word the collection lacks.
|
||||
assert bm25.tokens("Naar skjer det paa hytta?") == bm25.tokens("Når skjer det på hytta?")
|
||||
|
||||
|
||||
def test_a_word_that_frames_a_question_is_not_a_topic() -> None:
|
||||
# `how often` / `hvor ofte` asks about a topic without naming one; read as
|
||||
# a topic word it would be "absent" from any collection that never says it.
|
||||
assert bm25.tokens("How often is the battery replaced?") == ["battery", "replac"]
|
||||
assert bm25.tokens("Hvor ofte byttes batteriet?") == ["bytt", "batteriet"]
|
||||
assert bm25.tokens("Hva står i punkt 4.2?") == bm25.tokens("punkt 4.2")
|
||||
|
|
|
|||
86
tests/test_coverage_signal.py
Normal file
86
tests/test_coverage_signal.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""The payload says when the collection looks like it does not cover a question
|
||||
(v1.1 order C, C4).
|
||||
|
||||
`coverage.absent_terms` is the question's words the collection holds in NO form
|
||||
-- not as written, and not through a relative it uses (`bm25.query_groups`).
|
||||
`coverage.weak` is the machine-readable reading: nothing was delivered, or at
|
||||
least one such word exists. A reader seeing `weak` rephrases in the
|
||||
collection's own words, or says the collection does not cover it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_ingestion_okf import consume
|
||||
|
||||
TOOLS = Path(__file__).resolve().parent.parent / "tools"
|
||||
if str(TOOLS) not in sys.path:
|
||||
sys.path.insert(0, str(TOOLS))
|
||||
|
||||
import okf_retrieval_gate as retrieval # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def bundle(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
spec = retrieval.BundleSpec(
|
||||
"coverage-synthetic",
|
||||
(
|
||||
retrieval.DocumentSpec(
|
||||
"cabin",
|
||||
"cabin.md",
|
||||
(
|
||||
retrieval.ConceptSpec(
|
||||
slug="heating",
|
||||
title="Heating",
|
||||
body="The cabin is heated by a wood stove. Vinterberedskap is checked.",
|
||||
),
|
||||
retrieval.ConceptSpec(
|
||||
slug="water",
|
||||
title="Water",
|
||||
body="Water comes from the well and is drained in autumn.",
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
return retrieval.build_bundle(tmp_path_factory.mktemp("coverage") / "bundle", spec)
|
||||
|
||||
|
||||
def _coverage(bundle: Path, question: str) -> dict[str, object]:
|
||||
coverage = consume.build_payload(bundle, question=question)["coverage"]
|
||||
assert isinstance(coverage, dict)
|
||||
return coverage
|
||||
|
||||
|
||||
def test_a_covered_question_is_not_weak(bundle: Path) -> None:
|
||||
coverage = _coverage(bundle, "How is the cabin heated?")
|
||||
assert coverage["absent_terms"] == []
|
||||
assert coverage["weak"] is False
|
||||
|
||||
|
||||
def test_a_word_held_in_no_form_is_named_and_makes_the_answer_weak(bundle: Path) -> None:
|
||||
coverage = _coverage(bundle, "How is the cabin sauna heated?")
|
||||
assert coverage["absent_terms"] == ["sauna"]
|
||||
assert coverage["weak"] is True
|
||||
|
||||
|
||||
def test_a_word_held_in_another_form_is_not_absent(bundle: Path) -> None:
|
||||
coverage = _coverage(bundle, "When is vinterberedskapen checked?")
|
||||
assert coverage["absent_terms"] == []
|
||||
assert coverage["weak"] is False
|
||||
|
||||
|
||||
def test_nothing_delivered_is_weak(bundle: Path) -> None:
|
||||
coverage = _coverage(bundle, "zzqx")
|
||||
assert coverage["weak"] is True
|
||||
|
||||
|
||||
def test_the_gates_shared_reading_reads_the_signal(bundle: Path) -> None:
|
||||
payload = consume.build_payload(bundle, question="How is the cabin sauna heated?")
|
||||
assert retrieval.marked(payload)
|
||||
payload = consume.build_payload(bundle, question="How is the cabin heated?")
|
||||
assert not retrieval.marked(payload)
|
||||
|
|
@ -32,7 +32,7 @@ sys.path.insert(0, str(PROJECT_ROOT / "tools"))
|
|||
|
||||
import okf_retrieval_gate as gate # noqa: E402
|
||||
|
||||
from llm_ingestion_okf import consume # noqa: E402
|
||||
from llm_ingestion_okf import bm25, consume # noqa: E402
|
||||
|
||||
FIXTURES = PROJECT_ROOT / "tests" / "fixtures" / "retrieval"
|
||||
|
||||
|
|
@ -326,6 +326,8 @@ def test_row_four_goes_red_again_when_the_payload_stops_saying_what_it_missed(
|
|||
detail line.
|
||||
"""
|
||||
monkeypatch.setattr(consume, "unanswered_terms", lambda *args, **kwargs: [])
|
||||
# And its second reading since v1.1 C4: no word is absent in any form.
|
||||
monkeypatch.setattr(bm25, "_absent", lambda *args, **kwargs: ())
|
||||
row = gate.row_four([_case(tmp_path, "set-controls.json")])
|
||||
assert row.status == gate.RED
|
||||
assert row.m == 6
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue