open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 4 Activity

test(fixtures): replace sector-specific example material with generic, fictitious examples — green

Browse source 
Every fixture, test document, tool example and document now uses an invented
kitchen-and-baking handbook series, written in this repository. The package's
behaviour is unchanged; src/ changes are comments and help text only.

- Generated fixtures are regenerated from their generators. Their structural
  counts are identical before and after: elements, images, rows, cells,
  headings, bookmarks and the witness inventory's per-document totals. The
  image-inbox and accounting documents are renamed kapittel-84-*.
- tools/okf_accounting_gate.py: the two options that named one real corpus
  each are replaced by a generic, repeatable --corpus PATH with no default.
  Row 5 compares the PDF pair alone. Gate verdict unchanged: RED rows 2, 3, 6.
- tools/okf_witness.py: the STS JSON reader for one publisher's delivery is
  removed, along with its three twins and five tests. The mutation harness
  loses W09.
- docs/: 13 dated reports that documented runs on a retired reference corpus
  are removed, and 40 are neutralized. Dead links are removed, and no new
  dangling path is introduced.
- The synthetic MCP-gate corpus and the residual probe words are neutral.

Valgt: keep the `okf quality --fasit` bar value (the measured fraction, one corpus) and
rewrite only its provenance, because the verdict stays unchanged and the
number names nothing.

Term check with the local list: 0 of 411 tracked files, 0 file names, 0 of
27 binary fixtures. Suite after git add: 2457 passed, 1 skipped. The base
tree had 2460 passed and 2 skipped; five tests went with the JSON reader and
four were added by the term check. ruff, ruff format and mypy --strict src/
are clean.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 13:54:57 +02:00
commit 9d1f4b14ed
174 changed files with 1889 additions and 6512 deletions
Download patch file Download diff file Expand all files Collapse all files

60
tools/okf_mcp_gate.py
View file

@ -390,12 +390,12 @@ def write_bundle(root: Path, bundle_id: str, concepts: Sequence[tuple[str, str,
def corpus(scratch: Path) -> dict[str, Path]:
"""Two bundles that share no vocabulary, so a cross-bundle answer cannot
be produced by one of them alone."""
bridges = write_bundle(
scratch / "bundles" / "bridge-notes",
"bridge-notes",
gardens = write_bundle(
scratch / "bundles" / "garden-notes",
"garden-notes",
[
("spennvidde", "Spennvidde for gangbru", "En gangbru med spennvidde 24 meter."),
("rekkverk", "Rekkverk paa gangbru", "Rekkverket skal vaere 1,2 meter hoeyt."),
("bedbredde", "Bedbredde for drivhus", "Et drivhus med bedbredde 24 meter."),
("gelender", "Gelender paa terrasse", "Gelenderet skal vaere 1,2 meter hoeyt."),
],
)
kitchens = write_bundle(
@ -406,7 +406,7 @@ def corpus(scratch: Path) -> dict[str, Path]:
("ovn", "Ovnstemperatur", "Ovnen forvarmes til 250 grader."),
],
)
return {"bridge-notes": bridges, "kitchen-notes": kitchens}
return {"garden-notes": gardens, "kitchen-notes": kitchens}
# --------------------------------------------------------------------------
@ -522,7 +522,7 @@ def _probe_arguments(tool: str, bundle_id: str, concept_id: str, *, named: bool)
if tool == "okf_describe":
return bundle
if tool == "okf_ask":
return {**bundle, "question": "spennvidde for gangbru"}
return {**bundle, "question": "bedbredde for drivhus"}
return {**bundle, "concept_id": concept_id}
@ -559,21 +559,21 @@ def row_one(bundles: Mapping[str, Path], reachable: tuple[bool, str]) -> Row:
k = 0
for variant, required in REQUIRED_TOOLS.items():
named = variant == "one-to-many"
target = bundles["bridge-notes"] if not named else bundles["bridge-notes"].parent
target = bundles["garden-notes"] if not named else bundles["garden-notes"].parent
try:
with server(variant_argv(variant, target)) as client:
client.handshake()
listed = set(client.tool_names())
concept = ""
with suppress(Exception):
concept = _first_concept_id(client, "bridge-notes", named=named)
concept = _first_concept_id(client, "garden-notes", named=named)
for tool in required:
if tool not in listed:
details.append(f"{variant}: {tool} not in tools/list")
continue
try:
answer = client.call(
tool, _probe_arguments(tool, "bridge-notes", concept, named=named)
tool, _probe_arguments(tool, "garden-notes", concept, named=named)
)
except (RpcError, ServerGone) as error:
details.append(f"{variant}: {tool} did not answer: {error}")
@ -667,8 +667,8 @@ def read_anchor_set(questions: Path, freeze: Path, *, want_version: int) -> Anch
# surface must be able to hand back verbatim. Without it, a row reporting
# `0 of M` could be reporting that the lookup was never able to find anything.
SYNTHETIC_ANCHORS: tuple[tuple[str, str, str], ...] = (
("bridge-notes", "spennvidde", "En gangbru med spennvidde 24 meter."),
("bridge-notes", "rekkverk", "Rekkverket skal vaere 1,2 meter hoeyt."),
("garden-notes", "bedbredde", "Et drivhus med bedbredde 24 meter."),
("garden-notes", "gelender", "Gelenderet skal vaere 1,2 meter hoeyt."),
("kitchen-notes", "surdeig", "Surdeigen hever i 14 timer ved 22 grader."),
("kitchen-notes", "ovn", "Ovnen forvarmes til 250 grader."),
)
@ -854,8 +854,8 @@ def row_two(
def _served_name(target: Path, served: set[str]) -> str | None:
"""The id the server calls this directory, read from the bundle itself.
Never guessed from the directory name: the frozen set names `n100-2023`
and the bundle declares `vegnormal-n100-2023`, so a rule matching names
Never guessed from the directory name: the frozen set names `q100-2023`
and the bundle declares `kokebok-q100-2023`, so a rule matching names
would have to invent an alias rule of its own.
"""
from llm_ingestion_okf import consume as okf_consume
@ -924,7 +924,7 @@ def _drill_skill_one_to_one(bundle: Path, scratch: Path) -> DrillResult:
return DrillResult("skill-one-to-one", loud=False, recreate=(), note=str(error))
out = scratch / "skill-1-1"
try:
skill.generate(bundle, out=out, question="spennvidde for gangbru", force=True)
skill.generate(bundle, out=out, question="bedbredde for drivhus", force=True)
except Exception as error:
return DrillResult(
"skill-one-to-one", loud=False, recreate=(), note=f"not generated: {error}"
@ -968,14 +968,14 @@ def _drill_skill_one_to_many(bundles: Mapping[str, Path], scratch: Path) -> Dril
recreate=(),
note="`skill.render_generic` does not exist: the candidate has not been built",
)
first, second = (bundles["bridge-notes"], bundles["kitchen-notes"])
first, second = (bundles["garden-notes"], bundles["kitchen-notes"])
text_first = render_generic()
text_second = render_generic()
identical = text_first == text_second
leaked = [
token
for token in (
"bridge-notes",
"garden-notes",
"kitchen-notes",
okf_consume.bundle_ref(first),
okf_consume.bundle_ref(second),
@ -1007,11 +1007,11 @@ def row_three(scratch: Path) -> Row:
stage.mkdir(parents=True, exist_ok=True)
bundles = corpus(stage)
if artefact == "mcp-one-to-one":
results.append(_drill_mcp("one-to-one", bundles["bridge-notes"], "bridge-notes"))
results.append(_drill_mcp("one-to-one", bundles["garden-notes"], "garden-notes"))
elif artefact == "mcp-one-to-many":
results.append(_drill_mcp("one-to-many", bundles["bridge-notes"], "bridge-notes"))
results.append(_drill_mcp("one-to-many", bundles["garden-notes"], "garden-notes"))
elif artefact == "skill-one-to-one":
results.append(_drill_skill_one_to_one(bundles["bridge-notes"], stage))
results.append(_drill_skill_one_to_one(bundles["garden-notes"], stage))
else:
results.append(_drill_skill_one_to_many(bundles, stage))
for result in results:
@ -1037,7 +1037,7 @@ def row_four(scratch: Path) -> Row:
stage = scratch / "discovery"
stage.mkdir(parents=True, exist_ok=True)
bundles = corpus(stage)
root = bundles["bridge-notes"].parent
root = bundles["garden-notes"].parent
newcomers = {
"ferry-notes": ("avgang", "Avgangstider", "Fergen gaar hver time fra klokken 06."),
"lichen-notes": ("vekst", "Vekstrate", "Lav vokser under en millimeter i aaret."),
@ -1092,18 +1092,18 @@ def row_five(scratch: Path) -> Row:
stage = scratch / "cross"
stage.mkdir(parents=True, exist_ok=True)
bundles = corpus(stage)
root = bundles["bridge-notes"].parent
root = bundles["garden-notes"].parent
details: list[str] = []
k = 0
try:
with server(variant_argv("one-to-many", root)) as client:
client.handshake()
calls = 0
answer = client.call("okf_ask", {"question": "spennvidde og surdeig"})
answer = client.call("okf_ask", {"question": "bedbredde og surdeig"})
calls += 1
named = _bundle_ids(answer)
sources = _source_ids(answer)
if {"bridge-notes", "kitchen-notes"} <= named:
if {"garden-notes", "kitchen-notes"} <= named:
k += 1
else:
details.append(f"one answer named {sorted(named) or 'no bundle'}")
@ -1135,7 +1135,7 @@ def row_six(scratch: Path) -> Row:
stage = scratch / "hostile"
stage.mkdir(parents=True, exist_ok=True)
bundles = corpus(stage)
root = bundles["bridge-notes"].parent
root = bundles["garden-notes"].parent
outside = stage / "outside"
outside.mkdir(parents=True, exist_ok=True)
@ -1147,12 +1147,12 @@ def row_six(scratch: Path) -> Row:
write_bundle(root / "broken-notes", "broken-notes", [("x", "X", "y")])
(root / "broken-notes" / "index.md").write_bytes(b"\xff\xfe not a manifest at all\x00")
huge = root / "bridge-notes" / "svulmende.md"
huge = root / "garden-notes" / "svulmende.md"
huge.write_text(
CONCEPT.format(
title="Svulmende",
source="svulmende.txt",
bundle_id="bridge-notes",
bundle_id="garden-notes",
segment="s9",
body="A" * OVERSIZED_BYTES,
),
@ -1160,7 +1160,7 @@ def row_six(scratch: Path) -> Row:
)
# Named in the index, or it is not a concept and the ceiling never runs:
# the bundle's own index is what makes a file reachable at all.
index = root / "bridge-notes" / "index.md"
index = root / "garden-notes" / "index.md"
index.write_text(
index.read_text(encoding="utf-8").rstrip("\n")
+ "\n- [Svulmende](svulmende.md) \u2014 adjudication: proposed\n",
@ -1174,11 +1174,11 @@ def row_six(scratch: Path) -> Row:
(
"traversal-in-concept-id",
"okf_fetch",
{BUNDLE_KEY: "bridge-notes", "concept_id": "../../outside/secret-notes/hemmelig"},
{BUNDLE_KEY: "garden-notes", "concept_id": "../../outside/secret-notes/hemmelig"},
),
("symlink-out-of-root", "okf_describe", {BUNDLE_KEY: "linked-notes"}),
("broken-manifest", "okf_describe", {BUNDLE_KEY: "broken-notes"}),
("oversized-concept", "okf_fetch", {BUNDLE_KEY: "bridge-notes", "concept_id": "svulmende"}),
("oversized-concept", "okf_fetch", {BUNDLE_KEY: "garden-notes", "concept_id": "svulmende"}),
("unknown-bundle-id", "okf_describe", {BUNDLE_KEY: "does-not-exist"}),
]
try: