fix(inbox): a segment title is declared, never marked derived

A segment's title comes from the plan, so a human adjudicated it. But
structure derivation runs over the segment body, finds no title key and
no usable heading, falls back to a stem, and adds "title" to derived.
The concept then emitted a stated fact under an inferred marker, and a
consumer that distrusts derived fields would distrust exactly the thing
a human decided. An over-marked field is the same defect class as an
unmarked heuristic: the marker is only worth something if it is
accurate in both directions.

Scoped to title alone, and pinned that way by test: number stays in
derived on a segment, because nothing about segmentation makes an
inferred document number declared. Without a segment a derived title is
still marked, so only a plan makes a title declared.

The SEGMENTED_V1 golden moves, which is the intended consequence and
the only golden that may. The four existing goldens are byte-identical
to baseline 770d8d4, measured against the sha rather than inspected.

Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-01 19:29:46 +02:00
commit f65f5fc342
8 changed files with 80 additions and 8 deletions

View file

@ -2,5 +2,5 @@
bundle_id: b-golden-segmented
---
- [Veiledning for eksempelbundel](veiledning.md) — derived: [title]
- [Veiledning for eksempelbundel](veiledning.md)
- [krav (index)](krav/index.md)

View file

@ -6,7 +6,6 @@ source_sha256: 6906ec0acbcfc246e825bda9863c716eb5611b465020e8204eeb448c32343f7d
ingested_at: 2026-08-30T09:00:00Z
generated: true
parent: s0
derived: [title]
bundle_id: b-golden-segmented
segment_id: s1
source_offset: [94, 176]

View file

@ -1 +1 @@
- [Foerste krav](foerste-krav.md) — parent: s0?; derived: [title]
- [Foerste krav](foerste-krav.md) — parent: s0?

View file

@ -6,7 +6,6 @@ source_sha256: 6906ec0acbcfc246e825bda9863c716eb5611b465020e8204eeb448c32343f7d
ingested_at: 2026-08-30T09:00:00Z
generated: true
parent: s0
derived: [title]
bundle_id: b-golden-segmented
segment_id: s2
source_offset: [176, 253]

View file

@ -1 +1 @@
- [Andre krav](andre-krav.md) — parent: s0?; derived: [title]
- [Andre krav](andre-krav.md) — parent: s0?

View file

@ -5,7 +5,6 @@ source_file: veiledning.md
source_sha256: 6906ec0acbcfc246e825bda9863c716eb5611b465020e8204eeb448c32343f7d
ingested_at: 2026-08-30T09:00:00Z
generated: true
derived: [title]
bundle_id: b-golden-segmented
segment_id: s0
source_offset: [0, 94]

View file

@ -22,7 +22,7 @@ from __future__ import annotations
import hashlib
import unicodedata
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path, PurePosixPath
from .errors import IngestError, MaterializationError, SegmentationError, SourceError
@ -168,6 +168,16 @@ def render_inbox_concept(
"generated": "true",
}
if structure is not None and profile.index.facets is not None:
if segmented:
# A segment's title comes from the PLAN, so it is DECLARED by the
# adjudicator -- but derivation runs over the segment BODY, finds
# no title key and no usable heading, falls back, and marks it
# derived. The concept then emits a stated fact under an inferred
# marker, and a consumer that distrusts derived fields distrusts
# exactly the thing a human decided. Scoped to `title` alone: every
# other field here really was inferred from the body, and blunting
# the marker would be the opposite defect.
structure = replace(structure, derived=structure.derived - {"title"})
frontmatter.update(structure_frontmatter(structure, profile.index.facets.keys))
if segmented:
assert segment is not None

View file

@ -29,6 +29,7 @@ from llm_ingestion_okf.errors import MaterializationError
from llm_ingestion_okf.inbox import GateDecision, process_inbox, render_inbox_concept
from llm_ingestion_okf.profiles import DEFAULT, SEGMENTED_V1, STRUCTURED_V1
from llm_ingestion_okf.segmentation import SegmentEntry, parse_segmentation_plan
from llm_ingestion_okf.structure import derive_document_structure
INGESTED_AT = "2026-07-25T12:00:00Z"
@ -213,7 +214,8 @@ def render(**overrides: object) -> str:
"profile": SEGMENTED_V1,
}
arguments.update(overrides)
return render_inbox_concept(DOCUMENT, **arguments) # type: ignore[arg-type]
text = arguments.pop("text", DOCUMENT)
return render_inbox_concept(text, **arguments) # type: ignore[arg-type]
def frontmatter_of(document: str) -> dict[str, str]:
@ -336,3 +338,66 @@ def test_two_bundles_hold_colliding_paths_and_disjoint_identity_values() -> None
assert [frontmatter_of(document)["bundle_id"] for document in one] == ["b-1", "b-1"]
assert [frontmatter_of(document)["bundle_id"] for document in other] == ["b-2", "b-2"]
assert set(one).isdisjoint(set(other))
# --- `derived` never marks a title the adjudicator declared ---------------
#
# The second measured defect. A segment's `title` comes from the PLAN, which a
# human adjudicated; but structure derivation runs over the segment BODY, finds
# no `title` key and no usable heading, falls back, and adds `title` to
# `derived`. The concept then emits the declared title and marks it inferred.
# The error points the wrong way: a consumer that distrusts derived fields ends
# up distrusting a fact the producer stated. Marking is only worth anything if
# it is accurate in both directions, so an over-marked field is the same defect
# class as an unmarked heuristic.
def structure_of(text: str, source_file: str = "veiledning.md"):
return derive_document_structure(text, source_file=source_file)
def test_a_segment_title_is_never_marked_derived() -> None:
body = "## 1.1 Foerste krav\n\nEt krav.\n"
assert "title" in structure_of(body).derived # the premise, measured
keys = frontmatter_of(
render(
text=body,
title="Foerste krav",
structure=structure_of(body),
segment=segment_entry(),
bundle_id="b-1",
)
)
assert keys["title"] == "Foerste krav"
assert "derived" not in keys
def test_a_genuinely_derived_field_is_still_marked_on_a_segment() -> None:
"""Scoped to `title` alone -- the fix must not blunt the marker.
`number` here IS inferred from the body, and nothing about segmentation
makes it declared, so it must survive in `derived`.
"""
body = "# N500 Vegtunneler\n\nSe N100.\n"
derived = structure_of(body, "n500.md").derived
assert {"title", "number"} <= derived # the premise, measured
keys = frontmatter_of(
render(
text=body,
title="Vegtunneler",
structure=structure_of(body, "n500.md"),
segment=segment_entry(),
bundle_id="b-1",
)
)
assert "title" not in keys["derived"]
assert "number" in keys["derived"]
def test_without_a_segment_a_derived_title_is_still_marked() -> None:
"""The negative control. Only a PLAN makes a title declared."""
body = "## 1.1 Foerste krav\n\nEt krav.\n"
keys = frontmatter_of(
render(text=body, title="Foerste krav", structure=structure_of(body), profile=STRUCTURED_V1)
)
assert keys["derived"] == "[title]"