open/llm-ingestion-okf
1
0
Fork 0
Code Issues Releases 1 Activity

feat(tools): the corpus harness replays plans and writes the bundle's log

Browse source 
Two defects a consumer measured on 2026-09-03 against the bundle built by
this harness, both with one cause: `measure` passed `profile=STRUCTURED_V1`
and no plans at all, though Step 17 of the plan says the harness reuses
`process_inbox` "with the per-document plan mapping from Step 15".

- `adjudication` was in 0 of 39 concepts, because the key is written only
  inside the plan-covered branch and no plan was ever passed. `--plans-dir`
  replays proposals produced per document first; the profile follows from
  the flag rather than being something the harness may choose on its own.
  `--bundle-id` and `--okf-version` are arguments, never constants: a
  profile names a key and the caller owns its value (decision E1).
- `log.md` did not exist, so `merged` was countable from the bundle and `N`
  was not -- K1b could only be taken on trust from a report that does not
  travel with the artifact. Written in SPEC section 9 form and dated from
  `ingested_at`, never the wall clock.

Additive: without `--plans-dir` the run stays the flat `STRUCTURED_V1` run
that produced the published K1/K2 numbers.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-03 03:50:28 +02:00
commit c859d9bbfe
2 changed files with 352 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

183
tests/test_corpus_run.py
View file

@ -138,3 +138,186 @@ def test_wall_time_per_file_is_reported(tmp_path: Path) -> None:
report = okf_corpus_run.measure(root, tmp_path / "bundle", ingested_at=INGESTED_AT)
assert report.seconds_total >= 0.0
assert "per file" in report.render()
def test_the_bundle_carries_a_log_md_that_makes_k1b_checkable_from_the_bundle(
tmp_path: Path,
) -> None:
"""K1b was verifiable only from the harness's stdout, which no bundle carries.
Measured by a consumer 2026-09-03: given the bundle alone, `N` could not be
recovered, so `merged + Sigma(coded rejections) == N` was not checkable from
the artifact -- only `merged` was. SPEC section 9 already reserves `log.md`
for exactly this, so the run path writes the denominator and every rejection
code into it. Existence is not the property under test: an empty stub would
pass that and recover nothing.
"""
root = corpus(tmp_path, {"a.md": SUBSTANTIVE, "b.md": DEGENERATE, "c.bin": "x"})
bundle = tmp_path / "bundle"
code = okf_corpus_run.main(
[
"--corpus",
str(root),
"--report",
str(tmp_path / "r.md"),
"--bundle",
str(bundle),
"--ingested-at",
INGESTED_AT,
]
)
assert code == 0
log = (bundle / "log.md").read_text(encoding="utf-8")
# Section 9 form: reserved frontmatter type, a heading, ISO date headings.
assert log.startswith("---\ntype: Log\n")
assert f"\n## {INGESTED_AT[:10]}\n" in log
# The numbers, not merely the file. N is the denominator that was missing.
assert "N = 3" in log
assert "merged = 2" in log
assert "`extractor_unknown`: 1" in log
# The identity itself, so a reader does not have to re-derive it.
assert "2 + 1 = 3" in log
def test_the_log_is_dated_from_ingested_at_and_never_the_wall_clock(tmp_path: Path) -> None:
"""Determinism is bit-exact here as everywhere: two runs of the same corpus
at the same `ingested_at` produce the same `log.md` bytes."""
root = corpus(tmp_path, {"a.md": SUBSTANTIVE})
first = tmp_path / "b1"
second = tmp_path / "b2"
for bundle in (first, second):
okf_corpus_run.main(
[
"--corpus",
str(root),
"--report",
str(tmp_path / "r.md"),
"--bundle",
str(bundle),
"--ingested-at",
INGESTED_AT,
]
)
assert (first / "log.md").read_bytes() == (second / "log.md").read_bytes()
SEGMENTABLE = (
"# 1 Innledning\n\nBakgrunnen for anskaffelsen er beskrevet her, med nok\n"
"tekst til at seksjonen baerer innhold.\n\n"
"# 2 Kravspesifikasjon\n\nKravene til leveransen er listet i dette\n"
"avsnittet, ogsaa med en andre setning.\n"
)
def _propose(source: Path, out: Path) -> None:
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
import okf_propose_segments
assert okf_propose_segments.run(source, out, okf_type="reference", proposed_at=INGESTED_AT) == 0
def test_a_plan_directory_makes_the_run_segment_and_mark_every_concept_proposed(
tmp_path: Path,
) -> None:
"""The defect this closes: the harness ran under `STRUCTURED_V1` and passed
no plans at all, so a corpus arrived as one flat concept per file with no
`adjudication` key anywhere -- measured by a consumer as 0 of 39.
Segmentation is not a flag the harness may set on its own: the plans are
produced per document by the proposer first, and the harness replays them.
So the wiring under test is `--plans-dir`, and the profile follows from it.
"""
root = corpus(tmp_path, {"doc.md": SEGMENTABLE})
plans = tmp_path / "plans"
plans.mkdir()
_propose(root / "doc.md", plans / "doc.json")
bundle = tmp_path / "bundle"
code = okf_corpus_run.main(
[
"--corpus",
str(root),
"--report",
str(tmp_path / "r.md"),
"--bundle",
str(bundle),
"--ingested-at",
INGESTED_AT,
"--plans-dir",
str(plans),
"--bundle-id",
"k2-trinn1",
"--okf-version",
"0.2",
]
)
assert code == 0
concepts = sorted(
path for path in bundle.rglob("*.md") if path.name not in ("index.md", "log.md")
)
# 1-to-N: one dropped file, more than one concept.
assert len(concepts) > 1
bodies = [path.read_text(encoding="utf-8") for path in concepts]
# Every concept carries the marker, and it says PROPOSED -- the proposer
# never adjudicates, so an `adjudicated` here would be a machine's guess
# standing where a human's judgement belongs.
assert all("\nadjudication: proposed\n" in body for body in bodies)
assert not any("adjudication: adjudicated" in body for body in bodies)
def test_a_plan_directory_without_the_caller_owned_root_values_is_refused(
tmp_path: Path,
) -> None:
"""`okf_version`'s value belongs to the catalog (decision E1) and `bundle_id`
to whoever delimits the bundle. A literal for either in this harness would
claim a decision this repository does not own, so both are arguments and
their absence is a refusal rather than a default."""
root = corpus(tmp_path, {"doc.md": SEGMENTABLE})
plans = tmp_path / "plans"
plans.mkdir()
_propose(root / "doc.md", plans / "doc.json")
code = okf_corpus_run.main(
[
"--corpus",
str(root),
"--report",
str(tmp_path / "r.md"),
"--bundle",
str(tmp_path / "bundle"),
"--ingested-at",
INGESTED_AT,
"--plans-dir",
str(plans),
]
)
assert code == 2
assert not (tmp_path / "bundle").exists()
def test_without_a_plan_directory_the_run_is_unchanged(tmp_path: Path) -> None:
"""The wiring is additive. A run with no plans stays exactly the flat,
`STRUCTURED_V1` run it was, so the K1/K2 numbers already published remain
reproducible from the same command."""
root = corpus(tmp_path, {"doc.md": SEGMENTABLE})
bundle = tmp_path / "bundle"
assert (
okf_corpus_run.main(
[
"--corpus",
str(root),
"--report",
str(tmp_path / "r.md"),
"--bundle",
str(bundle),
"--ingested-at",
INGESTED_AT,
]
)
== 0
)
concepts = [path for path in bundle.rglob("*.md") if path.name not in ("index.md", "log.md")]
assert len(concepts) == 1
assert "adjudication:" not in concepts[0].read_text(encoding="utf-8")