First public release, published as one snapshot. What changed between versions is in CHANGELOG.md. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
609 lines
30 KiB
Python
609 lines
30 KiB
Python
"""Tests for the P1 method trial's build side (TRIAL, branch p1-metodeprove, not v1 code).
|
||
|
||
Covers the two trial agent types, the splitter (evals/p1/split.py), the D map and register (dmap.py), K packing
|
||
(kpack.py) and assembly with K+ (assemble.py). None of these call a model. The fixtures are synthetic text about
|
||
gardening and bread (CLAUDE.md rule 3: no text from the Claude Code documentation is committed).
|
||
Run: python3 -m unittest discover -s evals/tests -p 'test_p1_build.py'
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
sys.path.insert(0, str(ROOT))
|
||
sys.path.insert(0, str(ROOT / "evals"))
|
||
|
||
from okfwiki import audit, claims, view # noqa: E402
|
||
from p1 import assemble, dmap, kpack, split # noqa: E402
|
||
|
||
READER_SHA256 = "13a93bc5a9e87a7282752316ac670664cbdca321b0dac9cdaf92d9e74c383445"
|
||
BANNED = ("anthropic", "openai", "urllib", "http.client", "requests", "subprocess", "socket")
|
||
|
||
DOC_TOMATO = """# Growing tomatoes
|
||
Source: https://example.org/garden/tomatoes
|
||
|
||
How to grow tomatoes in a small garden.
|
||
|
||
## Soil
|
||
|
||
Tomatoes need deep, loose soil with `compost` mixed in.
|
||
Water the soil every morning.
|
||
|
||
* **Sun**: full sun for eight hours.
|
||
* **Space**: forty centimetres between plants,
|
||
more in a wet summer.
|
||
|
||
```bash
|
||
# not a heading
|
||
mix compost
|
||
```
|
||
|
||
| Month | Task |
|
||
|---|---|
|
||
| May | plant |
|
||
|
||
## Overview
|
||
|
||
The Kitchen Garden keeps tomatoes and basil together.
|
||
|
||
### Pruning
|
||
|
||
Remove side shoots once a week.
|
||
"""
|
||
|
||
DOC_BASIL = """# Growing basil
|
||
Source: https://example.org/garden/basil
|
||
|
||
## Soil
|
||
|
||
Basil likes warm soil and the Kitchen Garden.
|
||
Plant Sweet Genovese early.
|
||
|
||
## Watering
|
||
|
||
Water basil with `compost` tea when the leaves droop.
|
||
"""
|
||
|
||
REL_1 = """## 1.2.3
|
||
|
||
- Added the `compost` command to the Kitchen Garden planner.
|
||
- Fixed watering reminders.
|
||
"""
|
||
|
||
REL_2 = """## 1.10.0
|
||
|
||
- The Kitchen Garden planner now shows `compost` levels.
|
||
"""
|
||
|
||
REL_3 = """## 1.9.2
|
||
|
||
- Basil seedlings get their own tray.
|
||
"""
|
||
|
||
|
||
def runs8(text: str) -> set[tuple[str, ...]]:
|
||
words = re.findall(r"[a-z0-9]+", text.lower())
|
||
return {tuple(words[i:i + 8]) for i in range(len(words) - 7)}
|
||
|
||
|
||
def before_link_rule(found: list[audit.Finding]) -> list[audit.Finding]:
|
||
"""The P1 trial predates criterion 6 as structure (operator 04.10.2026), so its link-rule findings are left out."""
|
||
return [f for f in found if f.check != "link-rule"]
|
||
|
||
|
||
class Fixture(unittest.TestCase):
|
||
def setUp(self) -> None:
|
||
self._tmp = tempfile.TemporaryDirectory()
|
||
self.tmp = Path(self._tmp.name)
|
||
self.kilde = self.tmp / "wiki" / "kilde"
|
||
self.kilde.mkdir(parents=True)
|
||
files = {
|
||
"docs-en-tomatoes.md": DOC_TOMATO,
|
||
"docs-en-basil.md": DOC_BASIL,
|
||
"release-v1.2.3.md": REL_1,
|
||
"release-v1.10.0.md": REL_2,
|
||
"release-v1.9.2.md": REL_3,
|
||
}
|
||
for name, text in files.items():
|
||
(self.kilde / name).write_text(text, encoding="utf-8")
|
||
|
||
def tearDown(self) -> None:
|
||
self._tmp.cleanup()
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- step 1: agents
|
||
|
||
|
||
class AgentFilesTest(unittest.TestCase):
|
||
def frontmatter(self, text: str) -> dict[str, str]:
|
||
self.assertTrue(text.startswith("---\n"))
|
||
head = text[4:text.index("\n---\n", 4)]
|
||
return dict(line.split(": ", 1) for line in head.splitlines())
|
||
|
||
def test_two_trial_agents_have_narrow_tools_and_a_trial_marker(self) -> None:
|
||
for name in ("p1-compiler", "p1-consolidator"):
|
||
text = (ROOT / ".claude" / "agents" / f"{name}.md").read_text(encoding="utf-8")
|
||
fm = self.frontmatter(text)
|
||
self.assertEqual(fm["name"], name)
|
||
self.assertEqual(fm["tools"], "Read, Write")
|
||
self.assertEqual(fm["model"], "opus")
|
||
body = text[text.index("\n---\n", 4) + 5:].strip()
|
||
self.assertTrue(body.startswith("TRIAL"), name)
|
||
fixture = DOC_TOMATO + DOC_BASIL + REL_1 + REL_2 + REL_3
|
||
self.assertEqual(runs8(body) & runs8(fixture), set(), name)
|
||
|
||
def test_the_reader_is_unchanged(self) -> None:
|
||
shipped = (ROOT / "agents" / "okf-wiki-reader.md").read_bytes()
|
||
self.assertEqual(hashlib.sha256(shipped).hexdigest(), READER_SHA256)
|
||
self.assertEqual(shipped, (ROOT / ".claude" / "agents" / "okf-wiki-reader.md").read_bytes())
|
||
|
||
def test_compiler_names_the_rules_it_must_keep(self) -> None:
|
||
body = (ROOT / ".claude" / "agents" / "p1-compiler.md").read_text(encoding="utf-8").lower()
|
||
for idea in ("first turn", "never read", "release note", "four tests", "json"):
|
||
self.assertIn(idea, body, idea)
|
||
|
||
def test_consolidator_keeps_every_label(self) -> None:
|
||
body = (ROOT / ".claude" / "agents" / "p1-consolidator.md").read_text(encoding="utf-8").lower()
|
||
for idea in ("every footnote label", "no new label", "json"):
|
||
self.assertIn(idea, body, idea)
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- step 1: split
|
||
|
||
|
||
class OrderTest(unittest.TestCase):
|
||
def test_docs_first_then_release_notes_by_version_tuple(self) -> None:
|
||
names = ["release-v0.2.34.md", "docs-en-b.md", "release-v0.2.9.md", "docs-en-a.md", "release-v1.0.0.md"]
|
||
self.assertEqual(split.order(names), ["docs-en-a.md", "docs-en-b.md", "release-v0.2.9.md",
|
||
"release-v0.2.34.md", "release-v1.0.0.md"])
|
||
|
||
def test_an_unknown_name_is_refused(self) -> None:
|
||
with self.assertRaises(split.RunError):
|
||
split.order(["notes.md"])
|
||
|
||
|
||
class BlocksTest(Fixture):
|
||
def test_fence_aware_blocks_with_spans(self) -> None:
|
||
got = split.blocks(self.kilde / "docs-en-tomatoes.md")
|
||
kinds = [(b.kind, b.start, b.end) for b in got]
|
||
# line 2 "Source:" and line 4 are paragraphs; headings are not blocks
|
||
self.assertIn(("paragraph", 2, 2), kinds)
|
||
self.assertIn(("paragraph", 8, 9), kinds)
|
||
self.assertIn(("item", 11, 11), kinds)
|
||
self.assertIn(("item", 12, 13), kinds) # a continuation line belongs to its item
|
||
self.assertIn(("code", 15, 18), kinds) # the "#" inside the fence is not a heading
|
||
self.assertIn(("table", 20, 22), kinds)
|
||
heads = {b.section for b in got}
|
||
self.assertIn("Growing tomatoes › Soil", heads)
|
||
self.assertIn("Growing tomatoes › Overview › Pruning", heads)
|
||
self.assertNotIn("not a heading", " ".join(heads))
|
||
|
||
def test_every_non_blank_non_heading_line_is_in_exactly_one_block(self) -> None:
|
||
path = self.kilde / "docs-en-tomatoes.md"
|
||
lines = path.read_text(encoding="utf-8").split("\n")
|
||
got = split.blocks(path)
|
||
covered: dict[int, int] = {}
|
||
for b in got:
|
||
for n in range(b.start, b.end + 1):
|
||
covered[n] = covered.get(n, 0) + 1
|
||
for n, raw in enumerate(lines, 1):
|
||
heading = raw.startswith("#") and not (15 <= n <= 18)
|
||
if raw.strip() and not heading:
|
||
self.assertEqual(covered.get(n), 1, (n, raw))
|
||
if heading:
|
||
self.assertNotIn(n, covered, (n, raw))
|
||
|
||
def test_release_note_blocks_carry_the_version(self) -> None:
|
||
got = split.blocks(self.kilde / "release-v1.10.0.md")
|
||
self.assertEqual({b.version for b in got}, {"1.10.0"})
|
||
self.assertEqual(split.blocks(self.kilde / "docs-en-basil.md")[0].version, None)
|
||
|
||
|
||
class DocsHeadingsTest(Fixture):
|
||
def test_titles_and_second_and_third_level_headings_of_docs_pages_only(self) -> None:
|
||
got = split.docs_headings(self.kilde)
|
||
texts = [(h.file, h.level, h.text) for h in got]
|
||
self.assertIn(("docs-en-tomatoes.md", 1, "Growing tomatoes"), texts)
|
||
self.assertIn(("docs-en-tomatoes.md", 3, "Pruning"), texts)
|
||
self.assertFalse(any(f.startswith("release-") for f, _, _ in texts))
|
||
self.assertFalse(any(t == "not a heading" for _, _, t in texts))
|
||
|
||
|
||
class PickTest(Fixture):
|
||
def blocks_doc(self, order: list[str]) -> dict[str, object]:
|
||
return split.blocks_document(self.kilde, order)
|
||
|
||
def test_three_concepts_by_the_stated_rules(self) -> None:
|
||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||
picked = split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")
|
||
names = [p.name for p in picked]
|
||
self.assertEqual(len(names), 3)
|
||
self.assertEqual(len(set(names)), 3)
|
||
self.assertEqual(names[0], "Soil") # the focus page's heading with the most blocks
|
||
# "Kitchen Garden" and "compost" are both in 2 release notes; the tie is broken by name
|
||
self.assertEqual(names[1], "Kitchen Garden")
|
||
self.assertEqual(names[2], "compost") # in 2 docs pages and 2 release notes, distinct from 1 and 2
|
||
|
||
def test_a_document_counts_for_a_term_only_where_it_stands_as_a_term(self) -> None:
|
||
# Found in session 2: the word "and", backticked once, was counted in every document where it is prose.
|
||
(self.kilde / "docs-en-zz.md").write_text("# Joining\n\n## Words\n\nUse `and` to join two beds.\n",
|
||
encoding="utf-8")
|
||
for v in ("2.0.0", "2.0.1", "2.0.2"):
|
||
(self.kilde / f"release-v{v}.md").write_text(f"## {v}\n\n- Tomatoes and basil and mint.\n",
|
||
encoding="utf-8")
|
||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||
names = [p.name for p in split.pick(self.blocks_doc(order), self.kilde, focus="docs-en-tomatoes.md")]
|
||
self.assertNotIn("and", names)
|
||
self.assertEqual(names[1:], ["Kitchen Garden", "compost"])
|
||
|
||
def test_a_backticked_term_counts_its_length_after_trimming(self) -> None:
|
||
# Found in session 2: "` or `" is four characters with its spaces, and became the two-letter term "or".
|
||
got = split.terms("Use `compost` ` or ` ` ab ` ` mulch ` here.")
|
||
self.assertEqual(got, ["compost", "mulch"])
|
||
|
||
def test_the_focus_page_must_be_in_the_order(self) -> None:
|
||
with self.assertRaises(split.RunError):
|
||
split.pick(self.blocks_doc(["docs-en-basil.md"]), self.kilde, focus="docs-en-tomatoes.md")
|
||
|
||
|
||
class CliTest(Fixture):
|
||
def test_order_refuses_names_that_differ_from_the_l4_cut(self) -> None:
|
||
manifest = self.tmp / "m.json"
|
||
manifest.write_text(json.dumps({"cut": {"names": ["docs-en-basil.md", "release-v1.2.3.md"]}}), encoding="utf-8")
|
||
l4 = self.tmp / "l4"
|
||
l4.mkdir()
|
||
(l4 / "a.md").write_text("---\ntype: Concept\nsources:\n - id: a\n resource: /kilde/docs-en-basil.md\n"
|
||
"---\n\nx\n", encoding="utf-8")
|
||
out = self.tmp / "rekkefolge.txt"
|
||
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 2)
|
||
self.assertFalse(out.exists())
|
||
(l4 / "b.md").write_text("---\ntype: Concept\nsources:\n - id: b\n resource: /kilde/release-v1.2.3.md\n"
|
||
"---\n\nx\n", encoding="utf-8")
|
||
self.assertEqual(split.main(["order", "--manifest", str(manifest), "--l4", str(l4), "--out", str(out)]), 0)
|
||
self.assertEqual(out.read_text(encoding="utf-8").split(), ["docs-en-basil.md", "release-v1.2.3.md"])
|
||
|
||
def test_blocks_and_pick_refuse_to_overwrite(self) -> None:
|
||
order = self.tmp / "order.txt"
|
||
order.write_text("\n".join(split.order([p.name for p in self.kilde.iterdir()])) + "\n", encoding="utf-8")
|
||
out = self.tmp / "blocks.json"
|
||
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 0)
|
||
data = json.loads(out.read_text(encoding="utf-8"))
|
||
self.assertEqual(data["schema"], "okf-wiki/p1-blocks/1")
|
||
self.assertEqual(split.main(["blocks", "--kilde", str(self.kilde), "--order", str(order), "--out", str(out)]), 2)
|
||
picked = self.tmp / "begreper.txt"
|
||
args = ["pick", "--blocks", str(out), "--kilde", str(self.kilde), "--focus", "docs-en-tomatoes.md",
|
||
"--out", str(picked)]
|
||
self.assertEqual(split.main(args), 0)
|
||
self.assertEqual(len(picked.read_text(encoding="utf-8").strip().splitlines()), 3)
|
||
self.assertEqual(split.main(args), 2)
|
||
|
||
def test_total_counts_blocks_in_every_file(self) -> None:
|
||
expected = sum(len(split.blocks(p)) for p in self.kilde.iterdir())
|
||
self.assertEqual(split.blocks_total(self.kilde), expected)
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- step 2: D
|
||
|
||
|
||
class DMapTest(Fixture):
|
||
def build(self) -> Path:
|
||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||
out = self.tmp / "D"
|
||
dmap.build(split.blocks_document(self.kilde, order), self.kilde.parent, out, at="2026-10-02T12:00:00Z")
|
||
return out
|
||
|
||
def test_d_passes_the_auditor_with_no_claims(self) -> None:
|
||
out = self.build()
|
||
self.assertEqual(before_link_rule(audit.audit(out, self.kilde.parent)), [])
|
||
found = claims.extract(out, self.kilde.parent)
|
||
self.assertEqual(found.claims, [])
|
||
self.assertTrue(found.exceptions)
|
||
|
||
def test_register_links_point_to_map_sections(self) -> None:
|
||
out = self.build()
|
||
page = (out / "r-kitchen-garden.md").read_text(encoding="utf-8")
|
||
self.assertIn("(k-docs-en-basil.md#soil)", page)
|
||
self.assertIn("(k-release-v1-2-3.md#1-2-3)", page)
|
||
|
||
def test_register_keeps_docs_headings_and_terms_in_two_documents_only(self) -> None:
|
||
out = self.build()
|
||
names = {p.name for p in out.glob("r-*.md")}
|
||
self.assertIn("r-pruning.md", names) # a docs heading (i), in one document
|
||
self.assertIn("r-compost.md", names) # a backticked term in several documents
|
||
self.assertNotIn("r-sweet-genovese.md", names) # a capitalised phrase in one document only
|
||
self.assertNotIn("r-overview.md", names) # a stop-listed heading fails test 2
|
||
|
||
def test_quotes_stand_on_their_lines_and_stay_strings(self) -> None:
|
||
out = self.build()
|
||
doc = audit.read_document(out / "r-compost.md", "r-compost.md")
|
||
assert doc.frontmatter is not None
|
||
for s in doc.frontmatter["sources"]:
|
||
self.assertIsInstance(s["x-quote"], str)
|
||
line = (self.kilde / Path(s["resource"]).name).read_text(encoding="utf-8").split("\n")[s["x-locator"]["line"] - 1]
|
||
self.assertIn(s["x-quote"], line)
|
||
self.assertIsInstance(doc.frontmatter["generated"]["at"], str)
|
||
|
||
def test_a_quote_like_no_survives_a_yaml_round_trip(self) -> None:
|
||
text = dmap.frontmatter({"type": "Concept", "sources": [{"id": "a", "x-quote": "no"}, {"id": "b", "x-quote": "1.10"}]})
|
||
path = self.tmp / "x.md"
|
||
path.write_text(text + "\n# x\n", encoding="utf-8")
|
||
fm = audit.read_document(path, "x.md").frontmatter
|
||
assert fm is not None
|
||
self.assertEqual([s["x-quote"] for s in fm["sources"]], ["no", "1.10"])
|
||
|
||
def test_the_log_records_the_thresholds_and_build_refuses_an_existing_target(self) -> None:
|
||
out = self.build()
|
||
log = (out / "log.md").read_text(encoding="utf-8")
|
||
self.assertIn("2 of the", log)
|
||
with self.assertRaises(dmap.RunError):
|
||
dmap.build({"order": [], "blocks": []}, self.kilde.parent, out, at="2026-10-02T12:00:00Z")
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- step 3: K packing
|
||
|
||
|
||
def entry(eid: str, title: str, aliases: tuple[str, ...] = ()) -> dict[str, object]:
|
||
return {"id": eid, "title": title, "aliases": [title, *aliases], "kind": "skeleton", "merge_of": None,
|
||
"origin": None}
|
||
|
||
|
||
class KSkeletonTest(Fixture):
|
||
def test_the_skeleton_is_an_index_of_docs_headings_that_pass_test_2(self) -> None:
|
||
index = kpack.skeleton(self.kilde)
|
||
titles = {e["title"] for e in index["entries"]}
|
||
self.assertIn("Growing tomatoes", titles)
|
||
self.assertIn("Pruning", titles)
|
||
self.assertIn("Soil", titles)
|
||
self.assertNotIn("Overview", titles)
|
||
soil = [e for e in index["entries"] if e["title"] == "Soil"]
|
||
self.assertEqual(len(soil), 1) # the same heading in two pages is one entry
|
||
self.assertEqual(soil[0]["origin"]["file"], "docs-en-basil.md") # pages in name order, first one wins
|
||
self.assertFalse(any(p.suffix == ".md" for p in self.tmp.rglob("*") if p.parent.name != "kilde"))
|
||
|
||
def test_norm(self) -> None:
|
||
self.assertEqual(kpack.norm("The `Compost` bins"), "compost bin")
|
||
self.assertEqual(kpack.norm("settings.json"), "settings.json")
|
||
self.assertEqual(kpack.norm(" Kitchen Garden! "), "kitchen garden")
|
||
self.assertEqual(kpack.norm("CLI flags"), "cli flag")
|
||
|
||
|
||
class KIdentityTest(unittest.TestCase):
|
||
def setUp(self) -> None:
|
||
self.index = {"entries": [entry("kitchen-garden-planner", "Kitchen Garden planner"),
|
||
entry("compost", "compost", ("compost heap",))]}
|
||
|
||
def frag(self, concept: str, status: str = "new", file: str = "docs-en-tomatoes.md", eid: str | None = None
|
||
) -> dict[str, object]:
|
||
f: dict[str, object] = {"concept": concept, "status": status, "anchor": {"file": file, "start": 8, "end": 9}}
|
||
if eid:
|
||
f["id"] = eid
|
||
return f
|
||
|
||
def test_outcomes(self) -> None:
|
||
r = kpack.resolve
|
||
self.assertEqual(r(self.frag("x", "existing", eid="compost"), self.index), ("existing", "compost"))
|
||
self.assertEqual(r(self.frag("Kitchen Garden planners"), self.index), ("existing", "kitchen-garden-planner"))
|
||
self.assertEqual(r(self.frag("Compost heap"), self.index), ("existing", "compost"))
|
||
self.assertEqual(r(self.frag("Kitchen Garden planner tool"), self.index),
|
||
("merge-candidate", "kitchen-garden-planner"))
|
||
self.assertEqual(r(self.frag("Garden hose"), self.index), ("new", None))
|
||
self.assertEqual(r(self.frag("Garden hose", file="release-v1.2.3.md"), self.index), ("rejected", "release-note-new"))
|
||
self.assertEqual(r(self.frag("Kitchen Garden planner tool", file="release-v1.2.3.md"), self.index),
|
||
("rejected", "release-note-new"))
|
||
self.assertEqual(r(self.frag("Troubleshooting"), self.index), ("rejected", "meta"))
|
||
|
||
def test_relevant_lines_by_phrase(self) -> None:
|
||
got = kpack.relevant_lines(self.index, "We keep the compost heaps behind the shed.")
|
||
self.assertEqual([e["id"] for e in got], ["compost"])
|
||
self.assertEqual(kpack.relevant_lines(self.index, "Nothing about gardens here."), [])
|
||
|
||
|
||
class KPackTest(Fixture):
|
||
def blocks_doc(self) -> dict[str, object]:
|
||
return split.blocks_document(self.kilde, split.order([p.name for p in self.kilde.iterdir()]))
|
||
|
||
def test_packs_of_five(self) -> None:
|
||
self.assertEqual([len(p) for p in kpack.packs([f"n{i}" for i in range(19)], 5)], [5, 5, 5, 4])
|
||
|
||
def test_a_large_pack_is_split_into_parts_under_the_read_cap(self) -> None:
|
||
big = "".join(f"Line {i} says the tomatoes are fine today and tomorrow, and the basil too.\n\n"
|
||
for i in range(1110))
|
||
(self.kilde / "docs-en-big.md").write_text("# Big\n\n" + big, encoding="utf-8")
|
||
doc = split.blocks_document(self.kilde, ["docs-en-big.md"])
|
||
parts = kpack.render_input(1, ["docs-en-big.md"], doc, [], self.kilde)
|
||
self.assertGreater(sum(len(p) for p in parts), 130_000)
|
||
self.assertEqual(len(parts), 3)
|
||
self.assertTrue(all(len(p) <= kpack.PART_CHARS for p in parts))
|
||
self.assertIn("docs-en-big.md › Big › 3–3", parts[0])
|
||
|
||
def test_validate(self) -> None:
|
||
doc = self.blocks_doc()
|
||
pack = ["docs-en-tomatoes.md", "release-v1.2.3.md"]
|
||
good = {"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
|
||
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}}
|
||
cases = {
|
||
"anchor-crosses-blocks": {**good, "anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 11}},
|
||
"quote-missing": {**good, "text": "Plants stand 40 cm apart."},
|
||
"quote-not-in-span": {**good, "text": "Keep 40 cm.", "quote": "forty centimetres"},
|
||
"sentence-count": {**good, "text": "One. Two. Three. Four."},
|
||
"tests-missing": {**good, "status": "new", "concept": "Loose soil"},
|
||
"anchor-outside-pack": {**good, "anchor": {"file": "docs-en-basil.md", "start": 7, "end": 7}},
|
||
"malformed": {"concept": "Soil"},
|
||
}
|
||
no_text = {**good, "text": ""}
|
||
accepted, rejected = kpack.validate([good, no_text, *cases.values()], pack, doc, self.kilde)
|
||
self.assertEqual(len(accepted), 1)
|
||
self.assertEqual(rejected["sentence-count"], 2) # the empty text and the four sentences
|
||
for reason in cases:
|
||
self.assertGreaterEqual(rejected[reason], 1, reason)
|
||
ok = {**good, "text": "Plants stand forty centimetres apart.", "anchor": {"file": "docs-en-tomatoes.md",
|
||
"start": 12, "end": 13}, "quote": "forty centimetres"}
|
||
accepted, rejected = kpack.validate([ok], pack, doc, self.kilde)
|
||
self.assertEqual((len(accepted), sum(rejected.values())), (1, 0))
|
||
|
||
|
||
class KCliTest(Fixture):
|
||
def test_skeleton_pack_and_accept(self) -> None:
|
||
work = self.tmp / "work"
|
||
blocks = self.tmp / "blocks.json"
|
||
order = split.order([p.name for p in self.kilde.iterdir()])
|
||
blocks.write_text(json.dumps(split.blocks_document(self.kilde, order)), encoding="utf-8")
|
||
index = work / "K-work" / "index.json"
|
||
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 0)
|
||
self.assertEqual(kpack.main(["skeleton", "--kilde", str(self.kilde), "--out", str(index)]), 2)
|
||
root = str(self.kilde.parent)
|
||
self.assertEqual(kpack.main(["pack", "1", "--blocks", str(blocks), "--index", str(index), "--kilde-root", root,
|
||
"--work", str(work)]), 0)
|
||
self.assertTrue((work / "K-oppgaver" / "pack-1-part-1.md").is_file())
|
||
fragments = {"fragments": [
|
||
{"concept": "Soil", "status": "existing", "id": "soil", "text": "Tomatoes need deep, loose soil.",
|
||
"anchor": {"file": "docs-en-tomatoes.md", "start": 8, "end": 9}},
|
||
{"concept": "Kitchen Garden", "status": "new", "text": "The Kitchen Garden keeps tomatoes and basil.",
|
||
"anchor": {"file": "docs-en-tomatoes.md", "start": 26, "end": 26},
|
||
"tests": {"topic_shape": "a named planner", "citation": "See the Kitchen Garden reference"}},
|
||
{"concept": "Kitchen Garden", "status": "existing", "text": "The planner gained a command.",
|
||
"anchor": {"file": "release-v1.2.3.md", "start": 3, "end": 3}},
|
||
{"concept": "Watering reminders", "status": "new", "text": "Reminders were fixed.",
|
||
"anchor": {"file": "release-v1.2.3.md", "start": 4, "end": 4},
|
||
"tests": {"topic_shape": "x", "citation": "y"}},
|
||
]}
|
||
out = work / "K-fragments" / "pack-1.json"
|
||
out.parent.mkdir(parents=True)
|
||
out.write_text(json.dumps(fragments), encoding="utf-8")
|
||
self.assertEqual(kpack.main(["accept", "1", "--work", str(work), "--blocks", str(blocks), "--kilde-root", root]), 0)
|
||
acc = json.loads((work / "K-work" / "accepted.json").read_text(encoding="utf-8"))["fragments"]
|
||
self.assertEqual([a["outcome"] for a in acc], ["existing", "new", "existing"])
|
||
self.assertEqual(acc[2]["concept_id"], acc[1]["concept_id"]) # the release note enriches the new concept
|
||
self.assertEqual(acc[2]["version"], "1.2.3")
|
||
log = json.loads((work / "K-work" / "packs.json").read_text(encoding="utf-8"))
|
||
self.assertEqual(log["packs"][0]["rejected"], {"release-note-new": 1})
|
||
self.assertEqual(log["packs"][0]["proposed"], 4)
|
||
# PM 02.10: the ids of the index lines sent, and the accepted fragments whose concept was not among them
|
||
sent = log["packs"][0]["index_ids"]
|
||
self.assertIn("soil", sent)
|
||
self.assertNotIn(acc[1]["concept_id"], sent)
|
||
self.assertEqual(log["packs"][0]["not_sent"], {"total": 2, "by_outcome": {"existing": 1, "new": 1}})
|
||
|
||
|
||
# ---------------------------------------------------------------------------------------------- step 4: assembly
|
||
|
||
|
||
def accepted(cid: str, file: str, start: int, end: int, text: str, pack: int, version: str | None = None,
|
||
quote: str | None = None, quote_line: int | None = None) -> dict[str, object]:
|
||
return {"pack": pack, "position": 1, "file": file, "start": start, "end": end, "section": "-", "text": text,
|
||
"quote": quote, "quote_line": quote_line, "version": version, "concept_id": cid,
|
||
"outcome": "existing"}
|
||
|
||
|
||
class AssembleTest(Fixture):
|
||
def setUp(self) -> None:
|
||
super().setUp()
|
||
self.work = self.tmp / "work"
|
||
(self.work / "K-work").mkdir(parents=True)
|
||
index = {"entries": [
|
||
{**entry("soil", "Soil"), "origin": {"file": "docs-en-basil.md", "line": 4}},
|
||
{**entry("pruning", "Pruning"), "origin": {"file": "docs-en-tomatoes.md", "line": 28}},
|
||
{**entry("kitchen-garden", "Kitchen Garden"), "kind": "new"},
|
||
{**entry("kitchen-garden-planner", "Kitchen Garden planner"), "kind": "merge-candidate",
|
||
"merge_of": "kitchen-garden"},
|
||
]}
|
||
frags = [
|
||
accepted("soil", "docs-en-tomatoes.md", 8, 9, "Tomatoes need deep, loose soil.", 1),
|
||
accepted("kitchen-garden", "docs-en-tomatoes.md", 26, 26,
|
||
"The Kitchen Garden keeps tomatoes and basil together.", 1, quote="Kitchen Garden", quote_line=26),
|
||
accepted("kitchen-garden", "release-v1.2.3.md", 3, 3, "The planner gained a compost command.", 1, "1.2.3"),
|
||
accepted("soil", "docs-en-basil.md", 6, 7, "Basil likes warm soil near the Kitchen Garden.", 1),
|
||
accepted("kitchen-garden", "release-v1.10.0.md", 3, 3, "The planner shows compost levels.", 2, "1.10.0"),
|
||
accepted("kitchen-garden-planner", "release-v1.9.2.md", 3, 3, "Seedlings get their own tray.", 2, "1.9.2"),
|
||
]
|
||
(self.work / "K-work" / "index.json").write_text(json.dumps(index), encoding="utf-8")
|
||
(self.work / "K-work" / "accepted.json").write_text(json.dumps({"fragments": frags}), encoding="utf-8")
|
||
self.out = self.tmp / "K"
|
||
assemble.assemble(self.work, self.kilde.parent, self.out, at="2026-10-02T12:00:00Z")
|
||
|
||
def page(self, name: str) -> audit.Document:
|
||
return audit.read_document(self.out / name, name)
|
||
|
||
def test_the_bundle_passes_the_auditor(self) -> None:
|
||
self.assertEqual(before_link_rule(audit.audit(self.out, self.kilde.parent)), [])
|
||
found = claims.extract(self.out, self.kilde.parent)
|
||
self.assertEqual(claims.criterion_1(found)["total"], (6, 6))
|
||
|
||
def test_only_enriched_concepts_get_pages(self) -> None:
|
||
names = {p.name for p in self.out.glob("*.md")} - {"index.md", "log.md"}
|
||
self.assertEqual(names, {"soil.md", "kitchen-garden.md", "kitchen-garden-planner.md"})
|
||
fm = self.page("kitchen-garden-planner.md").frontmatter
|
||
assert fm is not None
|
||
self.assertEqual(fm["x-merge-candidate"], "kitchen-garden")
|
||
|
||
def test_history_in_version_order_without_version_numbers_in_the_text(self) -> None:
|
||
doc = self.page("kitchen-garden.md")
|
||
body = doc.body
|
||
self.assertLess(body.index("gained a compost command"), body.index("shows compost levels"))
|
||
prose = " ".join(b.text for b in audit.prose_blocks(doc))
|
||
self.assertEqual(view.version_numbers(audit.FOOTNOTE_REF.sub("", prose)), [])
|
||
fm = doc.frontmatter
|
||
assert fm is not None
|
||
versions = [s for s in fm["sources"] if view.is_version_heading(s)]
|
||
self.assertEqual(sorted(s["x-quote"] for s in versions), ["1.10.0", "1.2.3"])
|
||
self.assertEqual(fm["x-fragments"], 3)
|
||
|
||
def test_code_quotes_skip_structural_lines_and_are_marked(self) -> None:
|
||
fm = self.page("soil.md").frontmatter
|
||
assert fm is not None
|
||
first = fm["sources"][0]
|
||
self.assertEqual(first["x-locator"], {"line": 8})
|
||
self.assertEqual(first["x-quote-by"], "code")
|
||
self.assertEqual(first["x-span"], [8, 9])
|
||
kg = self.page("kitchen-garden.md").frontmatter
|
||
assert kg is not None
|
||
self.assertNotIn("x-quote-by", kg["sources"][0])
|
||
self.assertEqual(assemble.code_quote(["```bash", "<Note>", "|---|---|", "", " real line "], 10),
|
||
("real line", 14))
|
||
|
||
def test_a_concept_title_in_another_page_becomes_one_link(self) -> None:
|
||
self.assertIn("[Kitchen Garden](kitchen-garden.md)", self.page("soil.md").body)
|
||
|
||
|
||
class KPlusTest(AssembleTest):
|
||
def test_copy_tasks_and_apply(self) -> None:
|
||
kplus = self.tmp / "K+"
|
||
assemble.copy(self.out, kplus)
|
||
with self.assertRaises(assemble.RunError):
|
||
assemble.copy(self.out, kplus)
|
||
runs = assemble.consolidation_tasks(kplus, self.work, min_fragments=3, per_run=8)
|
||
self.assertEqual([r.pages for r in runs], [["kitchen-garden"]])
|
||
self.assertTrue(runs[0].task.is_file())
|
||
old = self.page("kitchen-garden.md").body
|
||
labels = sorted(set(audit.FOOTNOTE_REF.findall(old)))
|
||
good = "The Kitchen Garden keeps tomatoes and basil together, and its planner tracks compost " + "".join(
|
||
f"[^{x}]" for x in labels) + "."
|
||
dropped = "The Kitchen Garden keeps tomatoes and basil together[^" + labels[0] + "]."
|
||
added = good.replace(".", "[^zz].", 1)
|
||
bodies = self.tmp / "bodies.json"
|
||
bodies.write_text(json.dumps({"kitchen-garden": good, "soil": dropped}), encoding="utf-8")
|
||
self.assertEqual(assemble.apply(kplus, bodies), (1, 1))
|
||
self.assertIn("planner tracks compost", (kplus / "kitchen-garden.md").read_text(encoding="utf-8"))
|
||
self.assertEqual(before_link_rule(audit.audit(kplus, self.kilde.parent)), [])
|
||
bodies.write_text(json.dumps({"kitchen-garden": added}), encoding="utf-8")
|
||
self.assertEqual(assemble.apply(kplus, bodies), (0, 1))
|
||
|
||
|
||
class NoModelTest(unittest.TestCase):
|
||
def test_trial_modules_import_no_network_or_model_client(self) -> None:
|
||
for path in sorted((ROOT / "evals" / "p1").glob("*.py")):
|
||
source = path.read_text(encoding="utf-8")
|
||
for banned in BANNED:
|
||
self.assertNotIn(f"import {banned}", source, path.name)
|
||
self.assertNotIn(f"from {banned}", source, path.name)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|