llm-ingestion-okf/tests/test_frontmatter_nesting.py

"""A nested frontmatter key must never substitute for a top-level one.

This library's line-oriented frontmatter grammar exists in three copies, by
design and with the duplication documented at each site: `materialize`
reads a path, `structure` needs a character offset, `profiles` returns body
lines. All three split a line on its first colon and key the result on
`key.strip()` — which discards the indentation that is the ONLY thing
distinguishing a nested key from a top-level one. An indented `title:` under
a `sources:` block therefore lands in the same flat namespace as the
document's own `title:` and, arriving later, wins.

The failure this closes is not a dropped value but a SUBSTITUTED one. A
missing title is visible to whoever reads the concept; a substituted title
is not — the document carries a title that looks entirely right and belongs
to something else. And because `number` derives from `title` and `parent`
derives from `number`, one substitution walks the whole hierarchy. That is
`P4` (a concept ID is a path, and a path is a promise) broken in the one
property no later run can repair.

Scope: this is the SUBSTITUTION defect only. The block-list family — a
`- item` line carrying no colon and being dropped — is the same family and a
different defect, and is deliberately not addressed here. Nor is this a
structured YAML reader; that is D1b. Flow form stays the emission rule, and
the round-trip tests below pin that the fix does not disturb it.
"""

from pathlib import Path

from llm_ingestion_okf.materialize import parse_frontmatter
from llm_ingestion_okf.profiles import _split_frontmatter as _profiles_split
from llm_ingestion_okf.structure import _split_frontmatter as _structure_split
from llm_ingestion_okf.structure import derive_document_structure

# A `sources:` block whose entry carries its own `title:`. The top-level title
# and the nested one are both well-formed document titles, and both bear a
# number — which is what lets the test measure `number` and `parent` moving
# rather than only asserting that they could.
NESTED = """\
---
title: H100.2 Hytter og uthus
generated: true
source_file: haandbok.md
sources:
  - resource: https://example.test/broeyting.pdf
    title: H200.7 Broeyting
---

# Hytter og uthus

Body text.
"""

# The same document with the nested block removed. Nothing else differs.
FLAT = """\
---
title: H100.2 Hytter og uthus
generated: true
source_file: haandbok.md
---

# Hytter og uthus

Body text.
"""


# --- known-positive controls ---------------------------------------------
#
# Without these, a parser that returned nothing at all would pass every
# assertion below: "the nested title did not win" is satisfied by a parser
# that reads no title whatsoever. Each control proves the SAME parser, on the
# SAME shape, actually finds the top-level key when no nested key competes.


def test_control_materialize_reads_top_level_title(tmp_path: Path) -> None:
    path = tmp_path / "haandbok.md"
    path.write_text(FLAT, encoding="utf-8")
    assert parse_frontmatter(path)["title"] == "H100.2 Hytter og uthus"


def test_control_structure_reads_top_level_title() -> None:
    declared, offset = _structure_split(FLAT)
    assert declared["title"] == "H100.2 Hytter og uthus"
    assert offset > 0


def test_control_profiles_reads_top_level_title() -> None:
    head, _body = _profiles_split(FLAT)
    assert head["title"] == "H100.2 Hytter og uthus"


def test_control_derivation_reads_top_level_title() -> None:
    structure = derive_document_structure(FLAT, source_file="haandbok.md")
    assert structure.title == "H100.2 Hytter og uthus"
    assert structure.number == "H100.2"
    assert structure.parent_number == "H100"


# --- the defect, once per parser copy ------------------------------------


def test_nested_title_does_not_substitute_in_materialize(tmp_path: Path) -> None:
    path = tmp_path / "haandbok.md"
    path.write_text(NESTED, encoding="utf-8")
    assert parse_frontmatter(path)["title"] == "H100.2 Hytter og uthus"


def test_nested_title_does_not_substitute_in_structure() -> None:
    declared, _offset = _structure_split(NESTED)
    assert declared["title"] == "H100.2 Hytter og uthus"


def test_nested_title_does_not_substitute_in_profiles() -> None:
    head, _body = _profiles_split(NESTED)
    assert head["title"] == "H100.2 Hytter og uthus"


# --- the propagation the order asks to be MEASURED, not assumed ----------


def test_substituted_title_moves_number_and_parent() -> None:
    """`title` -> `number` -> `parent`, all three, on a real derivation.

    The source filename carries no number, so `number` falls through to the
    title — which is precisely the path a substituted title travels. This
    asserts the whole chain rather than the first link, because the order's
    question was whether `number` and `parent` move in PRACTICE or only in
    theory.
    """
    structure = derive_document_structure(NESTED, source_file="haandbok.md")
    assert structure.title == "H100.2 Hytter og uthus"
    assert structure.number == "H100.2"
    assert structure.parent_number == "H100"


def test_nested_key_does_not_invent_a_top_level_field() -> None:
    """A nested `resource:` must not appear as a top-level `resource`.

    The substitution has a quieter twin: a nested key with NO top-level
    counterpart is not overwriting anything, it is fabricating a field the
    document never declared. `derive_document_structure` exposes `declared`
    directly, so this pins the namespace itself and not one lucky key.
    """
    structure = derive_document_structure(NESTED, source_file="haandbok.md")
    assert "resource" not in structure.declared
    assert set(structure.declared) == {"title", "generated", "source_file", "sources"}


# --- the emission rule the fix must not disturb --------------------------


def test_flow_form_still_round_trips(tmp_path: Path) -> None:
    """Flow form is a single top-level line and must be untouched by the fix.

    This library pins its own emission to flow form precisely because the
    line-oriented parser round-trips it as one opaque string. A fix that
    keyed on anything other than indentation could break that, so it is
    pinned here rather than assumed.
    """
    flow = (
        "---\n"
        "title: H100.2 Hytter og uthus\n"
        "generated: { by: process:okf-ingest, at: 2026-08-31T00:00:00Z }\n"
        "sources: [ a.pdf, b.pdf ]\n"
        "---\n\nBody.\n"
    )
    path = tmp_path / "flow.md"
    path.write_text(flow, encoding="utf-8")
    parsed = parse_frontmatter(path)
    assert parsed["generated"] == "{ by: process:okf-ingest, at: 2026-08-31T00:00:00Z }"
    assert parsed["sources"] == "[ a.pdf, b.pdf ]"
    assert parsed["title"] == "H100.2 Hytter og uthus"