test(ingest-spec): make the §12 detach-proofs permanent tests
Mirrors the form already solved in test_method_spec_loadbearing.py: every predicate now takes the spec TEXT as an argument, so last session's four manual detach measurements become tests in the suite instead of a one-off spot-check that died with the session. Red-proofs (mutated COPY in tmp_path, never shared/): - M1 row removed from §12 -> the field reads as undocumented - M2 row renamed -> detach-proof is not a value-proof - M3 §12 heading renamed -> ValueError, fail-closed (no wider fallback) - M4 anchor degenerates -> the slice guard is red on the whole spec Measured, not asserted: mutating _cross_check_table to return the whole text turns 2 tests red; reverting the row assertion to substring-anywhere turns 14 red. 16 -> 61 tests in this file; suite 638 -> 683. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011t6M7qfsYgpZyNvRySixNj
This commit is contained in:
parent
e2eb31675d
commit
e70de95afb
1 changed files with 101 additions and 12 deletions
|
|
@ -5,6 +5,11 @@ UNCHANGED from commons, and this test keeps the contract honest — it goes RED
|
|||
spec goes missing, names a concrete agent toolkit (the framework-neutrality rule), or
|
||||
stops documenting a contract field. It is the load-bearing guard the ingest layer relies
|
||||
on to keep being implementable "from this spec alone".
|
||||
|
||||
Form mirrored from the sibling ``test_method_spec_loadbearing.py``: every predicate takes
|
||||
the spec TEXT as an argument, so the detach-proofs are tests in the suite rather than a
|
||||
one-off spot-check that dies with the session. Red-proofs run against a mutated COPY of
|
||||
the spec in ``tmp_path`` — never against ``shared/`` itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -62,39 +67,123 @@ _CONTRACT_FIELDS = (
|
|||
)
|
||||
|
||||
|
||||
def _cross_check_table() -> str:
|
||||
def _cross_check_table(text: str) -> str:
|
||||
"""The §12 section body — heading to end-of-spec or the next section, whichever first."""
|
||||
text = SPEC.read_text(encoding="utf-8")
|
||||
start = text.index(_CROSS_CHECK_HEADING) # RED (ValueError) if §12 is renamed or dropped
|
||||
end = text.find("\n## ", start + len(_CROSS_CHECK_HEADING))
|
||||
return text[start:] if end == -1 else text[start:end]
|
||||
|
||||
|
||||
def _named_toolkits(text: str) -> list[str]:
|
||||
low = text.lower()
|
||||
return [tok for tok in _FORBIDDEN_TOOLKITS if tok in low]
|
||||
|
||||
|
||||
def _undocumented_fields(text: str) -> list[str]:
|
||||
# The row's FIRST column is the documented-field claim; a field named only in another
|
||||
# row's "Contract" prose does not count.
|
||||
table = _cross_check_table(text)
|
||||
return [field for field in _CONTRACT_FIELDS if f"| `{field}` |" not in table]
|
||||
|
||||
|
||||
def _slice_defects(table: str) -> list[str]:
|
||||
"""Defects in the anchor itself — a slice that widened into the surrounding spec."""
|
||||
defects = []
|
||||
if not table.startswith(_CROSS_CHECK_HEADING):
|
||||
defects.append("does not start at the §12 heading")
|
||||
if "Honesty rule" in table:
|
||||
defects.append("leaked §1 prose — the anchor is not a slice")
|
||||
if "\n## " in table:
|
||||
defects.append("leaked a following section")
|
||||
return defects
|
||||
|
||||
|
||||
def _row_removed(text: str, field: str) -> str:
|
||||
table = _cross_check_table(text)
|
||||
kept = [ln for ln in table.splitlines(keepends=True) if not ln.startswith(f"| `{field}` |")]
|
||||
return text.replace(table, "".join(kept))
|
||||
|
||||
|
||||
def _row_renamed(text: str, field: str) -> str:
|
||||
table = _cross_check_table(text)
|
||||
return text.replace(table, table.replace(f"| `{field}` |", f"| `{field}_renamed` |"))
|
||||
|
||||
|
||||
# --- The guard itself (against the real spec) ---------------------------------------
|
||||
|
||||
|
||||
def test_spec_is_present() -> None:
|
||||
# RED if the spec goes missing (the layer stops being implementable from spec alone).
|
||||
assert SPEC.is_file(), "ingest-spec.md missing — subtree pull the commons contract"
|
||||
|
||||
|
||||
def test_spec_names_no_agent_toolkit() -> None:
|
||||
text = SPEC.read_text(encoding="utf-8").lower()
|
||||
present = [tok for tok in _FORBIDDEN_TOOLKITS if tok in text]
|
||||
present = _named_toolkits(SPEC.read_text(encoding="utf-8"))
|
||||
assert not present, f"framework-neutral spec names a concrete toolkit: {present}"
|
||||
|
||||
|
||||
def test_cross_check_slice_is_a_slice_and_not_the_whole_spec() -> None:
|
||||
# Guards the anchor itself: a slice that degenerated into the full text would make
|
||||
# every row assertion below green-but-dead again, silently. RED if it widens.
|
||||
table = _cross_check_table()
|
||||
assert table.startswith(_CROSS_CHECK_HEADING)
|
||||
assert "Honesty rule" not in table, "slice leaked §1 prose — the anchor is not a slice"
|
||||
assert "\n## " not in table, "slice leaked a following section"
|
||||
defects = _slice_defects(_cross_check_table(SPEC.read_text(encoding="utf-8")))
|
||||
assert not defects, f"the §12 anchor degenerated: {defects}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", _CONTRACT_FIELDS)
|
||||
def test_spec_documents_contract_field(field: str) -> None:
|
||||
# The row's FIRST column is the documented-field claim; a field named only in another
|
||||
# row's "Contract" prose does not count. RED when an amendment drops or renames a row.
|
||||
table = _cross_check_table()
|
||||
assert f"| `{field}` |" in table, (
|
||||
# RED when an amendment drops or renames a §12 row.
|
||||
undocumented = _undocumented_fields(SPEC.read_text(encoding="utf-8"))
|
||||
assert field not in undocumented, (
|
||||
f"contract field {field!r} is no longer a row in the §12 cross-check table"
|
||||
)
|
||||
|
||||
|
||||
# --- Red-proofs: the guard MUST fail on a detached spec (mutated copy, never shared/) --
|
||||
|
||||
|
||||
def test_guard_red_when_spec_missing(tmp_path: Path) -> None:
|
||||
assert not (tmp_path / "ingest-spec.md").is_file()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("toolkit", _FORBIDDEN_TOOLKITS)
|
||||
def test_guard_red_when_toolkit_injected(tmp_path: Path, toolkit: str) -> None:
|
||||
mutated = SPEC.read_text(encoding="utf-8") + f"\n\nBuilt on {toolkit}.\n"
|
||||
copy = tmp_path / "ingest-spec.md"
|
||||
copy.write_text(mutated, encoding="utf-8")
|
||||
assert toolkit in _named_toolkits(copy.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", _CONTRACT_FIELDS)
|
||||
def test_guard_red_when_row_removed_from_cross_check_table(tmp_path: Path, field: str) -> None:
|
||||
# M1: the row is gone from §12 — even though prose elsewhere may still name the field.
|
||||
copy = tmp_path / "ingest-spec.md"
|
||||
copy.write_text(_row_removed(SPEC.read_text(encoding="utf-8"), field), encoding="utf-8")
|
||||
assert field in _undocumented_fields(copy.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", _CONTRACT_FIELDS)
|
||||
def test_guard_red_when_row_renamed_in_cross_check_table(tmp_path: Path, field: str) -> None:
|
||||
# M2: a detach-proof is not a value-proof — the row still exists, under another name.
|
||||
copy = tmp_path / "ingest-spec.md"
|
||||
copy.write_text(_row_renamed(SPEC.read_text(encoding="utf-8"), field), encoding="utf-8")
|
||||
assert field in _undocumented_fields(copy.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_guard_red_when_section12_heading_renamed(tmp_path: Path) -> None:
|
||||
# M3: fail-closed — no §12 heading means no anchor, and the guard must raise, not
|
||||
# silently fall back to a wider (green-but-dead) slice.
|
||||
mutated = SPEC.read_text(encoding="utf-8").replace(
|
||||
_CROSS_CHECK_HEADING, "## 12. Field reference"
|
||||
)
|
||||
copy = tmp_path / "ingest-spec.md"
|
||||
copy.write_text(mutated, encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
_cross_check_table(copy.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_slice_guard_red_when_anchor_degenerates_to_whole_spec() -> None:
|
||||
# M4: the anchor can degenerate. If _cross_check_table ever returned the full text,
|
||||
# the row assertions would go green-but-dead again — this proves the slice guard is
|
||||
# what catches that, and that it is not itself green by accident.
|
||||
defects = _slice_defects(SPEC.read_text(encoding="utf-8"))
|
||||
assert defects, "the slice guard accepts the whole spec as the §12 table — it is dead"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue