Kjell Tore Guttormsen
37547fe292
The context sets, the packaged knowledge bases and the example bundles are replaced by one fictitious example set about IT operations in an invented organisation: three context sets (serverrom-2027, driftsavtale-2027 and the two-base drift-og-avtale-2027), two synthetic knowledge bases under src/portfolio_optimiser/data/kunnskapsbaser and two example bundles under src/portfolio_optimiser/data/bundles. Numbers, codes and structural values in tests and fixtures are kept; names, ids and wording change. Dated measurement documents that only recorded runs on the replaced material are deleted. Gate figures measured on the new set are not comparable with earlier ones. The exclusion gate from the previous commit is green: 0 tracked files hit outside the shared/ subtree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
304 lines
14 KiB
Python
304 lines
14 KiB
Python
"""P18/A — a listing is a WINDOW, and a path the caller invented is refused by name.
|
|
|
|
P16 (the first stress round; the ledger is ``docs/invarianter.md``) measured the ladder S7a-3
|
|
built against a DELIVERED corpus for the first time, and found two things the fixture bases could
|
|
not show:
|
|
|
|
* **one level is not bounded by being one level.** ``okf.directory_listing`` on one requirements
|
|
level was 69 250 characters over 445 documents, on another 169 974 over 1 132, and a process
|
|
catalogue's own root 110 912 over 2 728 SUBDIRECTORIES — 27-113x the 1 500-character ceiling
|
|
S7a-3 set, riding in every later prompt. Three of seven paid runs died on the token cap.
|
|
* **0 of 26 fasit concepts were opened** in 24 ``read_file`` calls, and 10 of those calls named a
|
|
path the base does not hold. Each reached the model as MAF's opaque ``"Error: Function failed."``
|
|
(``agent_framework/_tools.py:1410-1432``) while counting toward the three consecutive tool errors
|
|
that end a request — so the one thing the caller could have acted on, the level that DOES hold
|
|
documents, never reached it.
|
|
|
|
Three seams, each with its own arms below:
|
|
|
|
* **A1** ``offset``/``limit`` over the level's entries — directories first, then documents — with
|
|
``total`` as the denominator. Clamped, never refused.
|
|
* **A2** ``filter``: a case-insensitive substring over a document's title and reference number and
|
|
over a directory's path, answering with ``total_matches``. A filter that matches nothing is an
|
|
ANSWER, not a refusal.
|
|
* **A3** an absent path becomes ``BundlePathNotFound`` in the funn-99 returned form, naming the
|
|
nearest directory that actually holds documents. Narrowness is arm (h) and lives in
|
|
``test_read_file_directory_refusal_loadbearing.py`` alongside the tripwire it replaced.
|
|
|
|
The bases this measures are the package's two example bases (``data/kunnskapsbaser/``, pinned
|
|
in ``frozen_bundles.json``), so the arms that need them run wherever the package does; they SKIP,
|
|
with the store named, only when a user's own store (``PORTFOLIO_FROZEN_BUNDLES``) lacks them. The
|
|
arms that do NOT need them (the window's own algebra, the filter's negative, the refusal) are
|
|
UNCONDITIONAL and run over a synthetic base, so this file can never be silently absent in full.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from portfolio_optimiser import frozen_bundles, okf
|
|
from portfolio_optimiser.explore import navigator_tools
|
|
from portfolio_optimiser.mandate import load_mandate
|
|
from portfolio_optimiser.stress import read_bundle_declarations
|
|
|
|
#: S7a-3's ceiling, restated here rather than imported: a gate that imported the implementation's
|
|
#: own budget would move with it, and raising the budget is exactly the regression it guards.
|
|
_CEILING_CHARS = 1_500
|
|
|
|
|
|
def _delivered(name: str) -> Path:
|
|
"""The FROZEN copy this repository pins, resolved at call time.
|
|
|
|
Absence SKIPS (a user's own store, named by ``PORTFOLIO_FROZEN_BUNDLES``, may not hold it),
|
|
drift is allowed to propagate and FAIL — a measurement of the wrong corpus is not a missing one.
|
|
"""
|
|
try:
|
|
return frozen_bundles.bundle_dir(name)
|
|
except frozen_bundles.FrozenBundleMissing as exc:
|
|
pytest.skip(str(exc))
|
|
|
|
|
|
def _tools(bundle_dir: Path) -> dict[str, Any]:
|
|
return {t.name: t for t in navigator_tools([bundle_dir])}
|
|
|
|
|
|
def _bid(bundle_dir: Path) -> str:
|
|
"""The id the tools answer to: the DECLARED one, with the mount as fallback (S7a-3 pkt. 1)."""
|
|
return okf.reconcile_bundle_id(str(bundle_dir)).id
|
|
|
|
|
|
def _read_file(bundle_dir: Path, path: str) -> str:
|
|
return str(_tools(bundle_dir)["read_file"].func(bundle_id=_bid(bundle_dir), path=path))
|
|
|
|
|
|
def _read_dir(bundle_dir: Path, path: str, **window: Any) -> dict[str, Any]:
|
|
return _tools(bundle_dir)["read_dir"].func(bundle_id=_bid(bundle_dir), path=path, **window)
|
|
|
|
|
|
def _chars(payload: Any) -> int:
|
|
return len(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
|
def _synthetic(tmp_path: Path, *, dirs: int, per_dir: int) -> Path:
|
|
"""A base wide enough that a window is visibly a window, and deterministic."""
|
|
base = tmp_path / "syntetisk-korpus"
|
|
(base).mkdir(parents=True)
|
|
links = []
|
|
for d in range(dirs):
|
|
(base / f"seksjon-{d:02d}").mkdir()
|
|
for n in range(per_dir):
|
|
rel = f"seksjon-{d:02d}/dok-{n:02d}.md"
|
|
(base / rel).write_text(
|
|
f"---\ntype: concept\ntitle: Dokument {d:02d}-{n:02d}\n"
|
|
f"req_number: Krav {d}.{n}\n---\n\nkort tekst.\n",
|
|
encoding="utf-8",
|
|
)
|
|
links.append(f"- [{rel}]({rel})")
|
|
(base / "index.md").write_text("# Korpus\n\n" + "\n".join(links) + "\n", encoding="utf-8")
|
|
return base
|
|
|
|
|
|
# --- A1: the window ------------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("name", "level", "before"),
|
|
[
|
|
("driftskrav-2027", "krav/D200", 29_453),
|
|
("prosesskatalog-2027", "P900", 12_345),
|
|
],
|
|
)
|
|
def test_a_default_listing_of_a_delivered_level_is_bounded(
|
|
name: str, level: str, before: int
|
|
) -> None:
|
|
"""(a) The two example levels that are far wider than one window: the requirements level
|
|
``krav/D200`` (220 documents) and the process catalogue's root ``P900``, whose 300 entries are
|
|
DIRECTORIES — which is why the window covers both kinds and not only documents.
|
|
|
|
``before`` is what reading the WHOLE level costs through the window (every page at the maximum
|
|
limit, summed), measured on the pinned base and carried so the arm cannot pass by the level
|
|
having shrunk. The control below proves the fixture is the thing that did not fit."""
|
|
base = _delivered(name)
|
|
listing = _read_dir(base, level)
|
|
|
|
assert _chars(listing) <= _CEILING_CHARS, (
|
|
f"{name}/{level} default listing is {_chars(listing)} chars, over {_CEILING_CHARS}"
|
|
)
|
|
assert _chars(listing) * 20 < before, "the bound must be a fall, not a rounding"
|
|
assert int(listing["total"]) > 10 * int(listing["limit"]), (
|
|
"the CONTROL is inert: this level must hold far more than one window, or the bound above "
|
|
"is measuring a small level rather than the window"
|
|
)
|
|
whole = 0
|
|
for offset in range(0, int(listing["total"]), okf._DIRECTORY_PAGE_MAX):
|
|
whole += _chars(_read_dir(base, level, offset=offset, limit=okf._DIRECTORY_PAGE_MAX))
|
|
assert whole == before, f"{name}/{level}: the whole level now costs {whole}, pinned {before}"
|
|
|
|
|
|
def test_the_window_pages_the_whole_level_exactly_once(tmp_path: Path) -> None:
|
|
"""(b) The anti-vacuity arm for A1. ``{"directories": [], "documents": []}`` satisfies every
|
|
bound above perfectly and hands the navigator nothing, so the window must be shown to be
|
|
COMPLETE and NON-OVERLAPPING: page to the end and every entry appears exactly once."""
|
|
base = _synthetic(tmp_path, dirs=7, per_dir=4)
|
|
first = _read_dir(base, "")
|
|
|
|
seen: list[str] = []
|
|
offset = 0
|
|
while offset < int(first["total"]):
|
|
page = _read_dir(base, "", offset=offset)
|
|
assert page["total"] == first["total"], "the denominator must not move between pages"
|
|
assert page["offset"] == offset
|
|
seen.extend(str(e["path"]) for e in page["directories"])
|
|
seen.extend(str(e["name"]) for e in page["documents"])
|
|
offset += int(page["limit"])
|
|
|
|
assert len(seen) == len(set(seen)) == int(first["total"]) == 7
|
|
assert set(seen) == {f"seksjon-{d:02d}" for d in range(7)}
|
|
|
|
|
|
def test_a_caller_cannot_ask_for_an_unbounded_window(tmp_path: Path) -> None:
|
|
"""(c) The bound is this rung's to keep, not the caller's to remember. A limit past the maximum
|
|
is CLAMPED rather than refused: the caller asked for a listing, and refusing it would send a
|
|
model that asked for too much away with nothing."""
|
|
base = _synthetic(tmp_path, dirs=3, per_dir=60)
|
|
everything = _read_dir(base, "seksjon-00", limit=10_000)
|
|
|
|
assert int(everything["limit"]) == okf._DIRECTORY_PAGE_MAX < int(everything["total"])
|
|
assert len(everything["documents"]) == okf._DIRECTORY_PAGE_MAX
|
|
assert int(_read_dir(base, "seksjon-00", limit=3)["limit"]) == 3, "a small limit is honoured"
|
|
|
|
|
|
def test_an_offset_past_the_end_is_an_empty_window_over_an_honest_total(tmp_path: Path) -> None:
|
|
"""(d) ``total`` is what makes an empty answer readable: without the denominator, "nothing
|
|
here" and "you asked past the end" are the same payload."""
|
|
base = _synthetic(tmp_path, dirs=2, per_dir=3)
|
|
page = _read_dir(base, "seksjon-00", offset=999)
|
|
|
|
assert page["documents"] == [] and page["directories"] == []
|
|
assert int(page["total"]) == 3
|
|
|
|
|
|
# --- A2: the filter ------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_known_positive_filter_finds_the_fasit_concepts_and_not_the_level() -> None:
|
|
"""(e) The known positive: ``filter="autonomitid"`` on ``krav/D200`` must answer with the
|
|
two concepts serverrom/a2 must cite, and NOT with 220 rows."""
|
|
base = _delivered("driftskrav-2027")
|
|
fasit = json.loads(Path("contexts/serverrom-2027/fasit.json").read_text(encoding="utf-8"))
|
|
wanted = {
|
|
c["path"]
|
|
for m in fasit["must_cite"]
|
|
if m["approach_id"] == "a2-ups-autonomi"
|
|
for c in m["concepts"]
|
|
}
|
|
assert wanted, "the fixture must name concepts, or this arm proves nothing"
|
|
|
|
hits = _read_dir(base, "krav/D200", filter="autonomitid", limit=50)
|
|
|
|
assert wanted <= {str(d["name"]) for d in hits["documents"]}
|
|
assert int(hits["total_matches"]) < int(hits["total"]) / 50
|
|
assert _chars(hits) <= _CEILING_CHARS
|
|
|
|
|
|
def test_a_filter_that_matches_nothing_is_an_answer_and_not_a_refusal(tmp_path: Path) -> None:
|
|
"""(f) The order's known NEGATIVE. "No document here is about X" is a finding; refusing it
|
|
would make an honest negative indistinguishable from a path that does not exist — which is the
|
|
very confusion ``BundlePathNotFound`` exists to prevent one line below."""
|
|
base = _synthetic(tmp_path, dirs=2, per_dir=3)
|
|
empty = _read_dir(base, "seksjon-00", filter="finnes-ikke-noe-sted")
|
|
|
|
assert "refused" not in empty
|
|
assert empty["documents"] == [] and int(empty["total_matches"]) == 0
|
|
assert int(empty["total"]) == 3, "the denominator survives the filter"
|
|
|
|
|
|
def test_the_filter_reads_the_reference_number_and_not_only_the_title(tmp_path: Path) -> None:
|
|
"""(g) The second field is load-bearing: on a process catalogue the thing a navigator knows
|
|
is the process number, which is not in the title. Written over a synthetic base so it is
|
|
unconditional."""
|
|
base = _synthetic(tmp_path, dirs=2, per_dir=3)
|
|
by_ref = _read_dir(base, "seksjon-01", filter="krav 1.2")
|
|
|
|
assert [str(d["name"]) for d in by_ref["documents"]] == ["seksjon-01/dok-02.md"]
|
|
assert int(by_ref["total_matches"]) == 1
|
|
|
|
|
|
def test_a_filter_narrows_directories_too(tmp_path: Path) -> None:
|
|
"""(h) A process catalogue's root is all DIRECTORIES (P900's: 300; the largest ever measured
|
|
here: 2 728): a filter that only narrowed documents would leave the biggest level
|
|
unnarrowable."""
|
|
base = _synthetic(tmp_path, dirs=12, per_dir=2)
|
|
narrowed = _read_dir(base, "", filter="seksjon-0")
|
|
|
|
assert int(narrowed["total_matches"]) == 10 < int(narrowed["total"]) == 12
|
|
assert all(str(d["path"]).startswith("seksjon-0") for d in narrowed["directories"])
|
|
|
|
|
|
# --- A3: the invented path -----------------------------------------------------------------------
|
|
|
|
|
|
def test_an_invented_path_is_refused_by_name_over_a_delivered_base() -> None:
|
|
"""(i) The measured live shape: a one-character slip in a UUID. Before, this reached the model
|
|
as ``"Error: Function failed."``; the refusal now names the path AND the level that holds
|
|
documents, which is the only thing the caller can act on."""
|
|
base = _delivered("driftskrav-2027")
|
|
real = "krav/D200/id-b1e2ba25-9825-57b0-a358-569be457d2c8.md"
|
|
slip = _slip(real)
|
|
assert (base / real).is_file() and not (base / slip).exists(), "control: one real, one invented"
|
|
answer = _read_file(base, slip)
|
|
|
|
assert answer.startswith(f"REFUSED ({okf.BundlePathNotFound.__name__})")
|
|
assert slip in answer and "krav/D200" in answer and "read_dir" in answer
|
|
# The named level resolves, and it is the one the caller was already in.
|
|
assert int(_read_dir(base, "krav/D200")["total"]) > 0
|
|
|
|
|
|
def _slip(path: str) -> str:
|
|
"""The measured live shape of an invented path: ONE character of the document id changed."""
|
|
stem, suffix = path[:-3], path[-3:]
|
|
return stem[:-1] + ("b" if stem[-1] == "a" else "a") + suffix
|
|
|
|
|
|
def test_every_invented_path_answers_by_name_and_every_real_one_is_served() -> None:
|
|
"""(j) The nevner arm (ansikt 4). P16 replayed its own population — 32 ``read_file`` calls from
|
|
four paid runs, 10 of them naming a path the base does not hold — but those runs read corpora
|
|
this repository no longer carries, so the population here is CONSTRUCTED from the three context
|
|
sets' own fasit: every cited path (routed at the base its approach names), and the same path
|
|
with one id character changed. The denominator is re-measured from the fasit, not quoted.
|
|
|
|
Every invented path must come back as a NAMED refusal, and every real one must still return
|
|
the document — a gate that refused everything would pass the first half on its own, which is
|
|
the failure this file's own A2 negative arm is written against."""
|
|
calls: list[tuple[str, str]] = []
|
|
for fasit_path in sorted(Path("contexts").glob("*/fasit.json")):
|
|
set_dir = fasit_path.parent
|
|
names = {
|
|
b["bundle_id"]: b["name"] for b in read_bundle_declarations(set_dir / "bundle.txt")
|
|
}
|
|
routed = {
|
|
a.id: names[a.bundle_id] for a in load_mandate(set_dir / "mandate.json").approaches
|
|
}
|
|
for row in json.loads(fasit_path.read_text(encoding="utf-8"))["must_cite"]:
|
|
for concept in row["concepts"]:
|
|
calls.append((routed[row["approach_id"]], concept["path"]))
|
|
calls.append((routed[row["approach_id"]], _slip(concept["path"])))
|
|
refused = served = 0
|
|
cache: dict[str, Any] = {}
|
|
for name, path in calls:
|
|
base = _delivered(name)
|
|
if name not in cache:
|
|
cache[name] = (_tools(base)["read_file"], _bid(base))
|
|
tool, bundle_id = cache[name]
|
|
answer = str(tool.func(bundle_id=bundle_id, path=path))
|
|
if answer.startswith("REFUSED ("):
|
|
refused += 1
|
|
assert okf.BundlePathNotFound.__name__ in answer and "read_dir" in answer
|
|
else:
|
|
served += 1
|
|
|
|
assert (refused, served) == (18, 18), f"population moved: {refused} refused, {served} served"
|