open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

feat(consume): several sub-questions in one call, merged by the product

Browse source 
C2. `consume.build_multi_payload` takes two or more questions, reads the
bundle ONCE (`bm25.prepare` builds the index a question does not depend on),
ranks and cuts each sub-question exactly as `build_payload` would alone, and
interleaves the deliveries: first excerpt of each sub-question in turn, then
the second, a concept already taken skipped, cut at the same `k` and `limit`
one question gets -- so asking four times does not buy a payload four times
the size.

Chose round-robin, not a merge by score, because two questions' BM25 totals
are not on one scale: a merge by score would let the wordiest sub-question
take every place. It is the rule the search gate measured with before the
product had it, moved unchanged.

Shape, and only for two or more questions (one question is `build_payload`'s
payload byte for byte):

- `questions` replaces `question`;
- every excerpt carries `subquestions`, the indices of every sub-question
  whose own delivery named it, the one whose text (passage) it carries first;
- `coverage` holds one block per sub-question (the single shape plus its
  `question`, `unanswered_in_payload` read against what the reader receives),
  `weak_subquestions`, and `weak` true only when EVERY sub-question is weak;
- `withheld` is every concept the merge did not deliver: `below_k` where a
  sub-question delivered it and the merge's cut did not, otherwise the rule
  of the sub-question that ranked it best. `nearest` walks the rankings in the
  delivery's turn order. The contract checker accepts it with 0 findings.

`okf consume --question A --question B` and `okf_ask` with `questions` (both
forms at once is `question_ambiguous`) reach it. A reservation or a fusion
widening acts on ONE cut and is refused with several questions
(`subquestions_flag_conflict`).

The search gate's series (e) and (f) now ask ONE call with every
sub-question; the gate's own merge is gone. Sets and thresholds untouched.
The gate's table for this commit is kept in local state.

Suite on a clean tree after `git add`: 2411 passed, 2 skipped, 4 xfailed.
ruff, ruff format, mypy --strict clean.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-21 08:04:29 +02:00
commit 80aac93b8f
6 changed files with 918 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

89
src/llm_ingestion_okf/bm25.py
View file

@ -273,34 +273,40 @@ def _fuse(fused: dict[str, float], scores: dict[str, float]) -> None:
start = stop
def rank(
concepts: Sequence[Concept],
question: str,
*,
bodies: Sequence[str] | None = None,
) -> Ranking:
"""Rank `concepts` for `question`.
@dataclass(frozen=True)
class Prepared:
"""Everything `rank` reads that does not depend on the question.
Built once per load of a bundle, so a call asking several sub-questions
(`consume.build_multi_payload`) tokenises and indexes the collection once
and ranks it once per sub-question. `rank` builds one itself when not
given one, so a single question pays exactly what it always paid.
"""
concepts: tuple[Concept, ...]
field_documents: tuple[tuple[str, ...], ...]
field_index: Index
vocabulary: frozenset[str]
owners: tuple[int, ...]
starts: tuple[int, ...]
passage_index: Index
def prepare(concepts: Sequence[Concept], *, bodies: Sequence[str] | None = None) -> Prepared:
"""Index `concepts` for ranking: the field documents and the passages.
`bodies` is the text searched per concept (defaults to each `body`); the
caller passes the body without the door's link line, so what is searched
is what the older ranking searched.
"""
texts = list(bodies) if bodies is not None else [concept.body for concept in concepts]
query = tokens(question)
shared = _shared_segments([concept.concept_id for concept in concepts])
own_source = len({concept.source_file for concept in concepts}) > 1
field_documents = [
tokens(field_text(concept, text, shared=shared, own_source=own_source))
field_documents = tuple(
tuple(tokens(field_text(concept, text, shared=shared, own_source=own_source)))
for concept, text in zip(concepts, texts, strict=True)
]
)
field_index = Index(field_documents)
vocabulary = frozenset(field_index.postings)
groups = query_groups(query, vocabulary)
field = {
concepts[position].concept_id: score
for position, score in field_index.scores(groups).items()
}
owners: list[int] = []
starts: list[int] = []
@ -311,25 +317,60 @@ def rank(
owners.append(position)
starts.append(start)
passages.append(tokens(chunk))
passage_index = Index(passages)
return Prepared(
concepts=tuple(concepts),
field_documents=field_documents,
field_index=field_index,
vocabulary=frozenset(field_index.postings),
owners=tuple(owners),
starts=tuple(starts),
passage_index=Index(passages),
)
def rank(
concepts: Sequence[Concept],
question: str,
*,
bodies: Sequence[str] | None = None,
prepared: Prepared | None = None,
) -> Ranking:
"""Rank `concepts` for `question`.
`bodies` is the text searched per concept (defaults to each `body`); the
caller passes the body without the door's link line, so what is searched
is what the older ranking searched. `prepared` is `prepare`'s result for
the same `concepts` and `bodies`, given when one load answers several
questions; the ranking is the same either way.
"""
if prepared is None:
prepared = prepare(concepts, bodies=bodies)
concepts = prepared.concepts
query = tokens(question)
groups = query_groups(query, prepared.vocabulary)
field = {
concepts[position].concept_id: score
for position, score in prepared.field_index.scores(groups).items()
}
passage: dict[str, float] = {}
best_window: dict[str, int] = {}
for window, score in sorted(passage_index.scores(groups).items()):
concept_id = concepts[owners[window]].concept_id
for window, score in sorted(prepared.passage_index.scores(groups).items()):
concept_id = concepts[prepared.owners[window]].concept_id
if score > passage.get(concept_id, 0.0):
passage[concept_id] = score
best_window[concept_id] = starts[window]
best_window[concept_id] = prepared.starts[window]
fused = {concept.concept_id: 0.0 for concept in concepts}
_fuse(fused, passage)
_fuse(fused, field)
asked = [group for group in groups if field_index.idf(group) > 0.0]
asked = [group for group in groups if prepared.field_index.idf(group) > 0.0]
lexical = {
concept.concept_id: sum(1 for group in asked if group & held)
for concept, held in (
(concept, set(document))
for concept, document in zip(concepts, field_documents, strict=True)
for concept, document in zip(concepts, prepared.field_documents, strict=True)
)
}
by_id = {concept.concept_id: concept for concept in concepts}

782
src/llm_ingestion_okf/consume.py
View file

@ -2388,6 +2388,276 @@ def withheld_block(
}
@dataclass(frozen=True)
class _Loaded:
"""One read of a bundle: everything a question is ranked against.
Read ONCE per call, so several sub-questions (`build_multi_payload`) pay
for one walk, one tokenisation and one index and not one per sub-question.
"""
root_bundle_id: str
concepts: list[Concept]
tokenised: list[tuple[str, ...]]
texts: list[str]
stems: frozenset[str] | None
prepared: bm25.Prepared | None
known_positive: tuple[str, int, int]
def _load(
bundle_root: Path,
*,
profile: BundleProfile,
cost_vocabulary: bool,
rarity_weight: bool,
stem_prefix: bool,
link_in_signal: bool,
ranking: str,
) -> _Loaded:
case, expected, measured = known_positive()
if expected != measured:
# SS 7.4: the instrument reports NONE of its own numbers until it has
# reproduced a known figure. Refusing here rather than emitting a
# payload with a failing known-positive is the difference between an
# instrument that has been shown to count and one that merely says so.
raise ConsumeError(
f"the budget instrument's known-positive ({case}) expected {expected} "
f"and measured {measured}; refusing to report any figure until the "
"two agree",
code="instrument_unvalidated",
)
root_bundle_id = root_bundle_id_of(bundle_root, profile=profile)
concept_ids = enumerate_concepts(bundle_root, profile=profile)
concepts = link_parents(
[
read_concept(
read_path_in_bundle(bundle_root, f"{concept_id}{profile.paths.concept_suffix}"),
bundle_root=bundle_root,
root_bundle_id=root_bundle_id,
)
for concept_id in concept_ids
]
)
# The text the two lexical signals read, tokenised ONCE: the stem
# vocabulary, the rarity `df` and the coverage report below all count over
# the same strings, so none of them can weigh a token by how rare it is
# somewhere it is not read.
texts = searchable_text(concepts, link_in_signal=link_in_signal)
tokenised = [normalise(text) for text in texts]
# The bundle's OWN vocabulary, and the reason the rule is a set rather than
# a threshold: `pris` is a word here and `bila` is not, which is what
# separates a Norwegian compound from four coincidental characters.
stems = frozenset(token for tokens in tokenised for token in tokens) if stem_prefix else None
if ranking not in RANKINGS:
raise ConsumeError(
f"unknown ranking {ranking!r}; expected one of {', '.join(RANKINGS)}",
code="ranking_invalid",
)
if ranking == "bm25" and (cost_vocabulary or rarity_weight):
# Both widen a signal only the fusion has. Accepting them here would be
# a flag that silently does nothing, which reads as a measurement.
raise ConsumeError(
"--cost-vocabulary and --rarity-weight are widenings of the fusion "
"ranking; ask for it with ranking='fusion'",
code="ranking_flag_conflict",
)
prepared = (
bm25.prepare(
concepts,
# The DELIVERED form of each body, so a window's offset points at
# the same characters `as_passage` cuts from.
bodies=[
delivered_text(
concept.body if link_in_signal else body_without_link_line(concept.body)
)
for concept in concepts
],
)
if ranking == "bm25"
else None
)
return _Loaded(
root_bundle_id=root_bundle_id,
concepts=concepts,
tokenised=tokenised,
texts=texts,
stems=stems,
prepared=prepared,
known_positive=(case, expected, measured),
)
@dataclass(frozen=True)
class _Answer:
"""One question's ranking and cut over a `_Loaded` bundle."""
ranked: list[tuple[Concept, float, int]]
delivered: tuple[dict[str, object], ...]
withheld: tuple[tuple[str, str], ...]
reserved: tuple[str, int] | None
absent: tuple[str, ...]
def _answer(
loaded: _Loaded,
bundle_root: Path,
question: str,
*,
k: int,
limit: int,
profile: BundleProfile,
cost_vocabulary: bool,
reserve_top_rank: bool,
rarity_weight: bool,
tie_shared_rank: bool,
title_covered: bool,
source_quota: int | None,
link_in_signal: bool,
) -> _Answer:
concepts = loaded.concepts
stems = loaded.stems
if loaded.prepared is not None:
result = bm25.rank(concepts, question, prepared=loaded.prepared)
by_id = {concept.concept_id: concept for concept in concepts}
scored = {concept.concept_id: (score, lexical) for concept, score, lexical in result.ranked}
windows: Mapping[str, int] | None = result.best_window
absent = result.absent
ranked = [
(by_id[concept_id], *scored[concept_id])
for concept_id in _partitioned(
[concept.concept_id for concept, _, _ in result.ranked],
concepts,
question,
title_covered=title_covered,
lookup=True,
)
]
else:
windows = None
absent = bm25.absent_terms(
concepts,
question,
bodies=[delivered_text(body_without_link_line(concept.body)) for concept in concepts],
)
weights = (
rarity_weights(normalise(question), loaded.texts, stems=stems)
if rarity_weight
else None
)
ranked = concept_scores(
concepts,
question,
document_scores(
bundle_root,
question,
profile=profile,
cost_vocabulary=cost_vocabulary,
weights=weights,
stems=stems,
),
cost_vocabulary=cost_vocabulary,
weights=weights,
tie_shared_rank=tie_shared_rank,
title_covered=title_covered,
stems=stems,
link_in_signal=link_in_signal,
)
matched = sum(1 for _, _, lexical in ranked if lexical > 0)
delivered, withheld, reserved = cut(
ranked,
k=k,
limit=limit,
reserve_top_rank=reserve_top_rank,
source_quota=source_quota,
windows=windows,
)
if matched and not delivered:
# SS 7.3: a finding requiring a decision, never something to retry
# narrower. Distinguished from the honest empty result BY THE
# DENOMINATOR: concepts answered this question and the budget admitted
# none of them, which is a statement about the limit, not about the
# bundle.
raise ConsumeError(
f"{matched} concept(s) answered this question and the {limit}-byte "
f"budget admitted none of them; the cut strategy is wrong for this "
"bundle at this limit -- refusing rather than emitting an empty "
"payload that would read as 'nothing was found'",
code="budget_admits_nothing",
)
return _Answer(
ranked=ranked, delivered=delivered, withheld=withheld, reserved=reserved, absent=absent
)
def _question_coverage(
loaded: _Loaded,
question: str,
absent: Sequence[str],
*,
own_delivery: Sequence[Mapping[str, object]],
received: Sequence[Mapping[str, object]],
) -> dict[str, object]:
"""SS 8: what of ONE question the payload reaches. Facts, and no verdict
-- see `unanswered_terms` for the two readings that were measured and
felled.
`own_delivery` is what the question's own cut delivered and `received`
what the reader actually gets; for a single question they are the same
list, and for a sub-question the second is the merged payload.
"""
question_terms = list(dict.fromkeys(normalise(question)))
return {
"question_terms": question_terms,
"unanswered_in_bundle": unanswered_terms(
question_terms, loaded.tokenised, stems=loaded.stems
),
"unanswered_in_payload": unanswered_terms(
question_terms,
[normalise(excerpt_text(excerpt)) for excerpt in received],
stems=loaded.stems,
),
# v1.1 C4: a READING, with its rule in the open, never a verdict.
# `absent_terms` are the question's words this bundle holds in no
# form -- not as written, not through a relative it uses -- and
# `weak` is true when nothing was delivered or one such word
# exists. A reader seeing it rephrases in the bundle's own words
# or says the bundle does not cover the question.
"absent_terms": list(absent),
"weak": not own_delivery or bool(absent),
}
def _budget_block(loaded: _Loaded, *, limit: int, spent: int) -> dict[str, object]:
case, expected, measured = loaded.known_positive
return {
"unit": BUDGET_UNIT,
"instrument": BUDGET_INSTRUMENT,
"limit": limit,
"spent": spent,
"known_positive": {
"case": case,
"expected": expected,
"measured": measured,
# The second, independent route (`wc -c` on the same file), so
# `expected == measured` is not the only thing standing between
# a broken instrument and a green gate.
"raw_bytes": len(known_positive_path().read_bytes()),
"encoding_delta": KNOWN_POSITIVE_ENCODING_DELTA,
},
}
def _checked_spent(delivered: Sequence[Mapping[str, object]], limit: int) -> int:
spent = sum(excerpt_weight(excerpt) for excerpt in delivered)
if spent > limit:
# Structurally unreachable while the knapsack bucket arithmetic is
# one-sided, and kept because SS 7.3 is a MUST about the emitted
# payload rather than about the algorithm that produced it.
raise ConsumeError(f"spent ({spent}) exceeds limit ({limit})", code="budget_exceeded")
return spent
def build_payload(
bundle_root: Path,
*,
@ -2447,207 +2717,299 @@ def build_payload(
a bundle changing: **0 of 5** bundles anyone ships carries the line, so **5
of 5** payloads are byte-identical across the move.
"""
case, expected, measured = known_positive()
if expected != measured:
# SS 7.4: the instrument reports NONE of its own numbers until it has
# reproduced a known figure. Refusing here rather than emitting a
# payload with a failing known-positive is the difference between an
# instrument that has been shown to count and one that merely says so.
raise ConsumeError(
f"the budget instrument's known-positive ({case}) expected {expected} "
f"and measured {measured}; refusing to report any figure until the "
"two agree",
code="instrument_unvalidated",
)
root_bundle_id = root_bundle_id_of(bundle_root, profile=profile)
concept_ids = enumerate_concepts(bundle_root, profile=profile)
concepts = link_parents(
[
read_concept(
read_path_in_bundle(bundle_root, f"{concept_id}{profile.paths.concept_suffix}"),
bundle_root=bundle_root,
root_bundle_id=root_bundle_id,
)
for concept_id in concept_ids
]
loaded = _load(
bundle_root,
profile=profile,
cost_vocabulary=cost_vocabulary,
rarity_weight=rarity_weight,
stem_prefix=stem_prefix,
link_in_signal=link_in_signal,
ranking=ranking,
)
# The text the two lexical signals read, tokenised ONCE: the stem
# vocabulary, the rarity `df` and the coverage report below all count over
# the same strings, so none of them can weigh a token by how rare it is
# somewhere it is not read.
texts = searchable_text(concepts, link_in_signal=link_in_signal)
tokenised = [normalise(text) for text in texts]
# The bundle's OWN vocabulary, and the reason the rule is a set rather than
# a threshold: `pris` is a word here and `bila` is not, which is what
# separates a Norwegian compound from four coincidental characters.
stems = frozenset(token for tokens in tokenised for token in tokens) if stem_prefix else None
if ranking not in RANKINGS:
raise ConsumeError(
f"unknown ranking {ranking!r}; expected one of {', '.join(RANKINGS)}",
code="ranking_invalid",
)
if ranking == "bm25" and (cost_vocabulary or rarity_weight):
# Both widen a signal only the fusion has. Accepting them here would be
# a flag that silently does nothing, which reads as a measurement.
raise ConsumeError(
"--cost-vocabulary and --rarity-weight are widenings of the fusion "
"ranking; ask for it with ranking='fusion'",
code="ranking_flag_conflict",
)
if ranking == "bm25":
result = bm25.rank(
concepts,
question,
# The DELIVERED form of each body, so a window's offset points at
# the same characters `as_passage` cuts from.
bodies=[
delivered_text(
concept.body if link_in_signal else body_without_link_line(concept.body)
)
for concept in concepts
],
)
by_id = {concept.concept_id: concept for concept in concepts}
scored = {concept.concept_id: (score, lexical) for concept, score, lexical in result.ranked}
windows: Mapping[str, int] | None = result.best_window
absent = result.absent
ranked = [
(by_id[concept_id], *scored[concept_id])
for concept_id in _partitioned(
[concept.concept_id for concept, _, _ in result.ranked],
concepts,
question,
title_covered=title_covered,
lookup=True,
)
]
else:
windows = None
absent = bm25.absent_terms(
concepts,
question,
bodies=[delivered_text(body_without_link_line(concept.body)) for concept in concepts],
)
weights = rarity_weights(normalise(question), texts, stems=stems) if rarity_weight else None
ranked = concept_scores(
concepts,
question,
document_scores(
bundle_root,
question,
profile=profile,
cost_vocabulary=cost_vocabulary,
weights=weights,
stems=stems,
),
cost_vocabulary=cost_vocabulary,
weights=weights,
tie_shared_rank=tie_shared_rank,
title_covered=title_covered,
stems=stems,
link_in_signal=link_in_signal,
)
titles_by_id = {concept.concept_id: concept.title for concept in concepts}
matched = sum(1 for _, _, lexical in ranked if lexical > 0)
delivered, withheld, reserved = cut(
ranked,
answer = _answer(
loaded,
bundle_root,
question,
k=k,
limit=limit,
profile=profile,
cost_vocabulary=cost_vocabulary,
reserve_top_rank=reserve_top_rank,
rarity_weight=rarity_weight,
tie_shared_rank=tie_shared_rank,
title_covered=title_covered,
source_quota=source_quota,
windows=windows,
link_in_signal=link_in_signal,
)
concepts = loaded.concepts
delivered = answer.delivered
if follow_parent:
# After the cut and never inside it: see `attach_parent_text`.
delivered = attach_parent_text(
delivered, {concept.concept_id: concept for concept in concepts}, limit=limit
)
spent = sum(excerpt_weight(excerpt) for excerpt in delivered)
if matched and not delivered:
# SS 7.3: a finding requiring a decision, never something to retry
# narrower. Distinguished from the honest empty result BY THE
# DENOMINATOR: concepts answered this question and the budget admitted
# none of them, which is a statement about the limit, not about the
# bundle.
raise ConsumeError(
f"{matched} concept(s) answered this question and the {limit}-byte "
f"budget admitted none of them; the cut strategy is wrong for this "
"bundle at this limit -- refusing rather than emitting an empty "
"payload that would read as 'nothing was found'",
code="budget_admits_nothing",
)
if spent > limit:
# Structurally unreachable while the knapsack bucket arithmetic is
# one-sided, and kept because SS 7.3 is a MUST about the emitted
# payload rather than about the algorithm that produced it.
raise ConsumeError(f"spent ({spent}) exceeds limit ({limit})", code="budget_exceeded")
question_terms = list(dict.fromkeys(normalise(question)))
nearest_cap = len(withheld) if withheld_full else max(withheld_nearest, 0)
spent = _checked_spent(delivered, limit)
budget = _budget_block(loaded, limit=limit, spent=spent)
if answer.reserved is not None:
# Present only when a reservation was made, because a cut whose
# strategy changed without saying so is the silent cut SS 5.3
# forbids -- and absent otherwise, so the default payload keeps
# every byte it had.
budget["reserved"] = {"concept_id": answer.reserved[0], "bytes": answer.reserved[1]}
nearest_cap = len(answer.withheld) if withheld_full else max(withheld_nearest, 0)
return {
"contract": CONTRACT_REVISION,
"bundle": {
"bundle_id": root_bundle_id,
"bundle_id": loaded.root_bundle_id,
"ref": bundle_ref(bundle_root, profile=profile),
},
"budget": {
"unit": BUDGET_UNIT,
"instrument": BUDGET_INSTRUMENT,
"limit": limit,
"spent": spent,
"known_positive": {
"case": case,
"expected": expected,
"measured": measured,
# The second, independent route (`wc -c` on the same file), so
# `expected == measured` is not the only thing standing between
# a broken instrument and a green gate.
"raw_bytes": len(known_positive_path().read_bytes()),
"encoding_delta": KNOWN_POSITIVE_ENCODING_DELTA,
},
# Present only when a reservation was made, because a cut whose
# strategy changed without saying so is the silent cut SS 5.3
# forbids -- and absent otherwise, so the default payload keeps
# every byte it had.
**(
{"reserved": {"concept_id": reserved[0], "bytes": reserved[1]}}
if reserved is not None
else {}
),
},
"budget": budget,
"denominators": {
"considered": len(concepts),
"withheld": len(withheld),
"withheld": len(answer.withheld),
"delivered": len(delivered),
},
"question": question,
# SS 8: what of the QUESTION this payload reaches. Facts, and no
# verdict -- see `unanswered_terms` for the two readings that were
# measured and felled.
"coverage": {
"question_terms": question_terms,
"unanswered_in_bundle": unanswered_terms(question_terms, tokenised, stems=stems),
"unanswered_in_payload": unanswered_terms(
question_terms,
[normalise(excerpt_text(excerpt)) for excerpt in delivered],
stems=stems,
),
# v1.1 C4: a READING, with its rule in the open, never a verdict.
# `absent_terms` are the question's words this bundle holds in no
# form -- not as written, not through a relative it uses -- and
# `weak` is true when nothing was delivered or one such word
# exists. A reader seeing it rephrases in the bundle's own words
# or says the bundle does not cover the question.
"absent_terms": list(absent),
"weak": not delivered or bool(absent),
},
"coverage": _question_coverage(
loaded, question, answer.absent, own_delivery=delivered, received=delivered
),
"excerpts": list(delivered),
# SS 5.1/5.2 on the NUMBERS and SS 5.3 on the names. See this
# function's docstring and `WITHHELD_NEAREST_DEFAULT` for why the flat
# list is not here.
"withheld": withheld_block(
answer.withheld,
answer.ranked,
titles_by_id={concept.concept_id: concept.title for concept in concepts},
sources_by_id={concept.concept_id: concept.source_file for concept in concepts},
nearest=nearest_cap,
),
}
def interleave(
deliveries: Sequence[Sequence[Mapping[str, object]]], *, k: int, limit: int
) -> list[tuple[Mapping[str, object], list[int]]]:
"""Several sub-questions' deliveries as ONE list: position by position,
each sub-question in turn, a concept already taken skipped, cut at `k`
excerpts and at `limit` bytes.
Each entry is the excerpt as its PLACING sub-question delivered it --
for a large concept that is the passage that sub-question found -- and
the indices of every sub-question whose own delivery named the concept,
the placing one first.
Round-robin rather than by score because the scores of two questions are
not on one scale: a narrow sub-question's BM25 total is not comparable to
a broad one's, and a merge by score would let the wordiest sub-question
take every place. The rule is the one the search gate measured with
before the product had it (series (e) and (f)), moved here unchanged.
"""
named: dict[str, list[int]] = {}
for index, delivery in enumerate(deliveries):
for excerpt in delivery:
named.setdefault(str(excerpt.get("concept_id")), []).append(index)
merged: list[tuple[Mapping[str, object], list[int]]] = []
taken: set[str] = set()
spent = 0
for position in range(max((len(delivery) for delivery in deliveries), default=0)):
for index, delivery in enumerate(deliveries):
if position >= len(delivery) or len(merged) >= k:
continue
excerpt = delivery[position]
concept_id = str(excerpt.get("concept_id"))
if concept_id in taken:
continue
weight = excerpt_weight(excerpt)
if spent + weight > limit:
continue
taken.add(concept_id)
spent += weight
placing = [index, *(other for other in named[concept_id] if other != index)]
merged.append((excerpt, placing))
return merged
def build_multi_payload(
bundle_root: Path,
*,
questions: Sequence[str],
k: int = DEFAULT_K,
limit: int = DEFAULT_LIMIT,
profile: BundleProfile = DEFAULT_PROFILE,
tie_shared_rank: bool = DEFAULT_TIE_SHARED_RANK,
title_covered: bool = DEFAULT_TITLE_COVERED,
withheld_nearest: int = WITHHELD_NEAREST_DEFAULT,
withheld_full: bool = False,
stem_prefix: bool = DEFAULT_STEM_PREFIX,
source_quota: int | None = DEFAULT_SOURCE_QUOTA,
follow_parent: bool = DEFAULT_FOLLOW_PARENT,
link_in_signal: bool = DEFAULT_LINK_IN_SIGNAL,
ranking: str = DEFAULT_RANKING,
) -> dict[str, object]:
"""Several sub-questions, one load of the bundle, one payload (v1.1 C2).
A broad question is asked best as two to four narrow ones in the bundle's
own words, and a caller should not have to make four calls and merge four
payloads to do it. Each sub-question is ranked and cut exactly as
`build_payload` would rank and cut it alone; the deliveries are then
merged by `interleave`, and the merged set is cut at the SAME `k` and
`limit` one question gets -- so asking four times does not buy a payload
four times the size.
**ONE question is `build_payload`'s payload, byte for byte**, so a caller
passing a list of one gets exactly what it always got.
What differs from the single shape, and only for two or more:
- `questions` (the list, in the caller's order) replaces `question`;
- every excerpt carries `subquestions`, the indices into `questions` of
every sub-question whose own delivery named it, the one whose text it
carries FIRST;
- `coverage` holds one block per sub-question under `subquestions`, each
in the single shape plus its `question`, with `unanswered_in_payload`
read against what the reader actually receives; `weak_subquestions`
lists the weak ones and `weak` is true only when EVERY sub-question is
-- one covered sub-question means the bundle does cover part of it;
- `withheld` is every concept the merge did not deliver. A concept a
sub-question delivered and the merge cut is `below_k`; any other takes
the rule its best-placed sub-question gave it. `nearest` walks the
sub-questions' rankings in the same turn order as the delivery.
`reserve_top_rank` and the fusion's widenings are not taken here: a
reservation is a statement about ONE cut, and a merged payload has
several.
"""
cleaned = list(questions)
if not cleaned or any(not question.strip() for question in cleaned):
raise ConsumeError(
"at least one question is required, and none may be empty",
code="question_missing",
)
if len(cleaned) == 1:
return build_payload(
bundle_root,
question=cleaned[0],
k=k,
limit=limit,
profile=profile,
tie_shared_rank=tie_shared_rank,
title_covered=title_covered,
withheld_nearest=withheld_nearest,
withheld_full=withheld_full,
stem_prefix=stem_prefix,
source_quota=source_quota,
follow_parent=follow_parent,
link_in_signal=link_in_signal,
ranking=ranking,
)
loaded = _load(
bundle_root,
profile=profile,
cost_vocabulary=False,
rarity_weight=False,
stem_prefix=stem_prefix,
link_in_signal=link_in_signal,
ranking=ranking,
)
answers = [
_answer(
loaded,
bundle_root,
question,
k=k,
limit=limit,
profile=profile,
cost_vocabulary=False,
reserve_top_rank=False,
rarity_weight=False,
tie_shared_rank=tie_shared_rank,
title_covered=title_covered,
source_quota=source_quota,
link_in_signal=link_in_signal,
)
for question in cleaned
]
concepts = loaded.concepts
merged = interleave([answer.delivered for answer in answers], k=k, limit=limit)
delivered: tuple[dict[str, object], ...] = tuple(
{**excerpt, "subquestions": placing} for excerpt, placing in merged
)
if follow_parent:
delivered = attach_parent_text(
delivered, {concept.concept_id: concept for concept in concepts}, limit=limit
)
spent = _checked_spent(delivered, limit)
# Every concept not delivered, with ONE rule: `below_k` where a
# sub-question delivered it and the merge's cut did not, otherwise the
# rule of the sub-question that ranked it best (ties to the earlier one).
taken = {str(excerpt["concept_id"]) for excerpt in delivered}
delivered_by_some = {
str(excerpt["concept_id"]) for answer in answers for excerpt in answer.delivered
}
positions = [
{concept.concept_id: place for place, (concept, _, _) in enumerate(answer.ranked)}
for answer in answers
]
rules = [dict(answer.withheld) for answer in answers]
withheld: list[tuple[str, str]] = []
for concept in sorted(concepts, key=lambda each: each.concept_id):
concept_id = concept.concept_id
if concept_id in taken:
continue
if concept_id in delivered_by_some:
withheld.append((concept_id, "below_k"))
continue
best = min(range(len(answers)), key=lambda index: (positions[index][concept_id], index))
withheld.append((concept_id, rules[best][concept_id]))
by_id = {concept.concept_id: concept for concept in concepts}
turn_order: list[tuple[Concept, float, int]] = []
seen: set[str] = set()
for place in range(len(concepts)):
for answer in answers:
concept_id = answer.ranked[place][0].concept_id
if concept_id not in seen:
seen.add(concept_id)
turn_order.append((by_id[concept_id], 0.0, 0))
nearest_cap = len(withheld) if withheld_full else max(withheld_nearest, 0)
subquestion_coverage = [
{
"question": question,
**_question_coverage(
loaded,
question,
answer.absent,
own_delivery=answer.delivered,
received=delivered,
),
}
for question, answer in zip(cleaned, answers, strict=True)
]
weak = [index for index, block in enumerate(subquestion_coverage) if block["weak"] is True]
return {
"contract": CONTRACT_REVISION,
"bundle": {
"bundle_id": loaded.root_bundle_id,
"ref": bundle_ref(bundle_root, profile=profile),
},
"budget": _budget_block(loaded, limit=limit, spent=spent),
"denominators": {
"considered": len(concepts),
"withheld": len(withheld),
"delivered": len(delivered),
},
"questions": cleaned,
"coverage": {
"weak": len(weak) == len(cleaned),
"weak_subquestions": weak,
"subquestions": subquestion_coverage,
},
"excerpts": list(delivered),
"withheld": withheld_block(
withheld,
ranked,
titles_by_id=titles_by_id,
turn_order,
titles_by_id={concept.concept_id: concept.title for concept in concepts},
sources_by_id={concept.concept_id: concept.source_file for concept in concepts},
nearest=nearest_cap,
),
@ -2673,7 +3035,17 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace:
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument("bundle", type=Path, help="the OKF bundle directory to read")
parser.add_argument("--question", required=True, help="the question to cut the bundle for")
parser.add_argument(
"--question",
required=True,
action="append",
help=(
"the question to cut the bundle for. Give it more than once to ask "
"several sub-questions in one call: each is ranked alone and the "
"deliveries are interleaved into one payload of the same `--k` and "
"`--limit`, every excerpt naming the sub-questions it answered"
),
)
parser.add_argument(
"--k", type=int, default=DEFAULT_K, help=f"cap on delivered excerpts (default {DEFAULT_K})"
)
@ -2870,23 +3242,45 @@ def main(argv: list[str] | None = None) -> int:
print(f"okf_consume: FAILED - {args.bundle} is not a directory", file=sys.stderr)
return 2
try:
payload = build_payload(
args.bundle,
question=args.question,
k=args.k,
limit=args.limit,
cost_vocabulary=args.cost_vocabulary,
reserve_top_rank=args.reserve_top_rank,
rarity_weight=args.rarity_weight,
tie_shared_rank=args.tie_shared_rank,
title_covered=args.title_covered,
stem_prefix=args.stem_prefix,
source_quota=args.source_quota,
withheld_nearest=args.withheld_nearest,
withheld_full=args.withheld_full,
follow_parent=args.follow_parent,
ranking=args.ranking,
)
if len(args.question) > 1:
if args.cost_vocabulary or args.rarity_weight or args.reserve_top_rank:
raise ConsumeError(
"--cost-vocabulary, --rarity-weight and --reserve-top-rank "
"act on one question's cut; ask one question to use them",
code="subquestions_flag_conflict",
)
payload = build_multi_payload(
args.bundle,
questions=args.question,
k=args.k,
limit=args.limit,
tie_shared_rank=args.tie_shared_rank,
title_covered=args.title_covered,
stem_prefix=args.stem_prefix,
source_quota=args.source_quota,
withheld_nearest=args.withheld_nearest,
withheld_full=args.withheld_full,
follow_parent=args.follow_parent,
ranking=args.ranking,
)
else:
payload = build_payload(
args.bundle,
question=args.question[0],
k=args.k,
limit=args.limit,
cost_vocabulary=args.cost_vocabulary,
reserve_top_rank=args.reserve_top_rank,
rarity_weight=args.rarity_weight,
tie_shared_rank=args.tie_shared_rank,
title_covered=args.title_covered,
stem_prefix=args.stem_prefix,
source_quota=args.source_quota,
withheld_nearest=args.withheld_nearest,
withheld_full=args.withheld_full,
follow_parent=args.follow_parent,
ranking=args.ranking,
)
except ConsumeError as error:
print(f"okf_consume: FAILED - {error}", file=sys.stderr)
return 1
@ -2896,6 +3290,10 @@ def main(argv: list[str] | None = None) -> int:
except UnicodeDecodeError as error:
print(f"okf_consume: FAILED - the bundle is not utf-8: {error}", file=sys.stderr)
return 2
return _emit(payload, args)
def _emit(payload: Mapping[str, object], args: argparse.Namespace) -> int:
computed = payload["bundle"]
assert isinstance(computed, dict)
if args.ref is not None and args.ref != computed["ref"]:

69
src/llm_ingestion_okf/mcp_server.py
View file

@ -408,13 +408,15 @@ def tools(surface: Surface) -> tuple[Tool, ...]:
),
Tool(
"okf_ask",
"One question, one bounded payload of excerpts, each carrying its "
"bundle id, concept id, title and provenance locators, plus what was "
"withheld and why. This is the library's only reading direction and "
"it calls no model. On a multi-bundle server, omitting `bundle_id` "
"asks every served bundle and splits the budget between them. "
"ASK IT MORE THAN ONCE: one call answers one wording of one "
"sub-question, and `withheld.nearest` names the best-ranked "
"One question -- or two to four sub-questions in `questions` -- and "
"one bounded payload of excerpts, each carrying its bundle id, "
"concept id, title and provenance locators, plus what was withheld "
"and why. With `questions` each sub-question is ranked alone and "
"the answers are interleaved, every excerpt naming the "
"sub-questions it answered. This is the library's only reading "
"direction and it calls no model. On a multi-bundle server, "
"omitting `bundle_id` asks every served bundle and splits the "
"budget between them. `withheld.nearest` names the best-ranked "
"concepts that just missed, with their titles -- if one of those is "
"what you wanted, ask again in that concept's own words, or fetch "
"it by name. Exists "
@ -425,6 +427,15 @@ def tools(surface: Surface) -> tuple[Tool, ...]:
"type": "object",
"properties": {
"question": {"type": "string", "description": "the question, in prose"},
"questions": {
"type": "array",
"items": {"type": "string"},
"description": (
"two to four sub-questions in the bundle's own words, asked "
"in ONE call instead of `question`; the answers are "
"interleaved and each excerpt names its sub-questions"
),
},
**bundle,
"k": {
"type": "integer",
@ -432,7 +443,6 @@ def tools(surface: Surface) -> tuple[Tool, ...]:
},
"limit": {"type": "integer", "description": "payload budget in utf-8 bytes"},
},
"required": ["question"],
"additionalProperties": False,
},
),
@ -510,10 +520,37 @@ def call_describe(surface: Surface, arguments: Mapping[str, Any]) -> dict[str, A
}
def _questions(arguments: Mapping[str, Any]) -> list[str]:
"""`question` (one string) or `questions` (a list), never both.
Both at once is refused rather than merged: a caller that sent both has
two ideas of what it asked, and the payload would name only one of them.
"""
single = _string(arguments, "question")
many = arguments.get("questions")
if single and many is not None:
raise ToolError("give `question` or `questions`, not both", code="question_ambiguous")
if many is None:
if not single:
raise ToolError(
"`question` or `questions` is required and may not be empty",
code="question_missing",
)
return [single]
if (
not isinstance(many, list)
or not many
or not all(isinstance(each, str) and each.strip() for each in many)
):
raise ToolError(
"`questions` must be a non-empty list of non-empty strings",
code="question_missing",
)
return [str(each) for each in many]
def call_ask(surface: Surface, arguments: Mapping[str, Any]) -> dict[str, Any]:
question = _string(arguments, "question")
if not question:
raise ToolError("`question` is required and may not be empty", code="question_missing")
questions = _questions(arguments)
k = int(arguments.get("k") or DEFAULT_K)
limit = int(arguments.get("limit") or okf_consume.DEFAULT_LIMIT)
named = _string(arguments, "bundle_id")
@ -533,16 +570,20 @@ def call_ask(surface: Surface, arguments: Mapping[str, Any]) -> dict[str, Any]:
answers = []
for served in targets:
try:
payload = okf_consume.build_payload(
served.root, question=question, k=k, limit=share, profile=surface.profile
payload = okf_consume.build_multi_payload(
served.root, questions=questions, k=k, limit=share, profile=surface.profile
)
except okf_consume.ConsumeError as error:
raise ToolError(
f"{served.bundle_id}: {error}", code=getattr(error, "code", "consume_refused")
) from error
answers.append({"bundle_id": served.bundle_id, "payload": payload})
# ONE question keeps the reply it always had; several name the list.
asked: dict[str, Any] = (
{"question": questions[0]} if len(questions) == 1 else {"questions": questions}
)
return {
"question": question,
**asked,
"asked": [served.bundle_id for served in targets],
"budget_per_bundle": share,
"answers": answers,