test(spec): C1.1 — method-spec integrity guard (§11 'Spec integrity', closes C-N1)
Presence-only guard over shared/method-spec.md, mirroring the ingest-spec guard: file presence, structure markers (##1–##12, Step 1–8, MUST), framework-neutrality (same forbidden-toolkit list), and §12 coverage of all 31 consumed contract fields — scoped to the §12 block in backticked form so a pure table-row removal detaches (precision lesson from the ingest guard's detach spot-check). Red-proofs (a: missing file, b: injected toolkit name, c: removed §12 field) run parametrized against a mutated copy in tmp_path, never against shared/. Full gate green: 347 passed, ruff + format + mypy clean; no src changes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
d746891f06
commit
d8f4e13bfa
1 changed files with 163 additions and 0 deletions
163
tests/test_method_spec_loadbearing.py
Normal file
163
tests/test_method_spec_loadbearing.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""Spec-integrity seam for the method spec (method-spec §11, row "Spec integrity").
|
||||
|
||||
The sibling of ``test_ingest_spec_loadbearing.py``: this repo is built from
|
||||
``shared/method-spec.md`` alone (pulled unchanged from commons), and this test keeps that
|
||||
contract honest — it goes RED when the spec goes missing, names a concrete agent toolkit
|
||||
(the framework-neutrality rule), loses a structural section/step marker, or stops
|
||||
documenting a contract field this implementation consumes (§12 completeness).
|
||||
|
||||
Precision note (lesson from the ingest guard's detach spot-check): a substring-anywhere
|
||||
assertion does not detach when only the §12 table row is removed but the field is still
|
||||
mentioned in running text. The field assertions here are therefore scoped to the §12
|
||||
cross-check block and require the backticked form, so removing a table row alone turns
|
||||
the guard red. All assertions are presence-only (no forbidden-new-sections semantics),
|
||||
so a commons amendment that ADDS sections or fields keeps the guard green.
|
||||
|
||||
Red-proofs run against a mutated COPY of the spec in ``tmp_path`` — never against
|
||||
``shared/`` itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SPEC = Path(__file__).resolve().parents[1] / "shared" / "method-spec.md"
|
||||
|
||||
# Concrete agent toolkits / vendor stacks the framework-neutral spec MUST NOT name
|
||||
# (same list as the ingest-spec guard).
|
||||
_FORBIDDEN_TOOLKITS = (
|
||||
"claude",
|
||||
"anthropic",
|
||||
"openai",
|
||||
"gpt",
|
||||
"gemini",
|
||||
"llama",
|
||||
"langchain",
|
||||
"autogen",
|
||||
"crewai",
|
||||
"semantic kernel",
|
||||
"microsoft agent framework",
|
||||
"agent sdk",
|
||||
"bedrock",
|
||||
"vertex",
|
||||
"foundry",
|
||||
"maf",
|
||||
)
|
||||
|
||||
# Structural markers the spec must keep: the twelve normative sections, the eight loop
|
||||
# steps, and RFC 2119 normative language.
|
||||
_STRUCTURE_MARKERS = tuple(
|
||||
[f"## {n}." for n in range(1, 13)] + [f"### Step {n}" for n in range(1, 9)] + ["MUST"]
|
||||
)
|
||||
|
||||
# Every §12 contract field this implementation consumes (src-consumed per the 2026-07-16
|
||||
# review's consume-list; the last four — outcome, modelled_saving_nok, gap_source,
|
||||
# context_key — are golden/test-consumed only, guarded all the same).
|
||||
_CONTRACT_FIELDS = (
|
||||
"decision",
|
||||
"marker",
|
||||
"rationale",
|
||||
"id",
|
||||
"proposal_features",
|
||||
"affected_codes",
|
||||
"measure_type",
|
||||
"claimed_saving_nok",
|
||||
"description",
|
||||
"project_id",
|
||||
"measure",
|
||||
"affected_items",
|
||||
"code",
|
||||
"quantity",
|
||||
"unit_cost",
|
||||
"assumptions",
|
||||
"validates",
|
||||
"nominal_feasible",
|
||||
"p10",
|
||||
"p50",
|
||||
"p90",
|
||||
"realization_rate",
|
||||
"expected_actual_saving_nok",
|
||||
"type",
|
||||
"approved",
|
||||
"rejected",
|
||||
"approved_with_adjustment",
|
||||
"outcome",
|
||||
"modelled_saving_nok",
|
||||
"gap_source",
|
||||
"context_key",
|
||||
)
|
||||
|
||||
|
||||
def _named_toolkits(text: str) -> list[str]:
|
||||
low = text.lower()
|
||||
return [tok for tok in _FORBIDDEN_TOOLKITS if tok in low]
|
||||
|
||||
|
||||
def _missing_markers(text: str) -> list[str]:
|
||||
return [marker for marker in _STRUCTURE_MARKERS if marker not in text]
|
||||
|
||||
|
||||
def _section12_block(text: str) -> str:
|
||||
start = text.find("## 12.")
|
||||
if start == -1:
|
||||
return ""
|
||||
end = text.find("\n## ", start + 1)
|
||||
return text[start:] if end == -1 else text[start:end]
|
||||
|
||||
|
||||
def _undocumented_fields(text: str) -> list[str]:
|
||||
block = _section12_block(text)
|
||||
return [field for field in _CONTRACT_FIELDS if f"`{field}`" not in block]
|
||||
|
||||
|
||||
# --- The guard itself (against the real spec) ---------------------------------------
|
||||
|
||||
|
||||
def test_spec_is_present() -> None:
|
||||
# RED if the spec goes missing (the method stops being implementable from spec alone).
|
||||
assert SPEC.is_file(), "method-spec.md missing — subtree pull the commons contract"
|
||||
|
||||
|
||||
def test_spec_keeps_structure_markers() -> None:
|
||||
missing = _missing_markers(SPEC.read_text(encoding="utf-8"))
|
||||
assert not missing, f"structural markers gone from the spec: {missing}"
|
||||
|
||||
|
||||
def test_spec_names_no_agent_toolkit() -> None:
|
||||
present = _named_toolkits(SPEC.read_text(encoding="utf-8"))
|
||||
assert not present, f"framework-neutral spec names a concrete toolkit: {present}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", _CONTRACT_FIELDS)
|
||||
def test_spec_documents_contract_field(field: str) -> None:
|
||||
undocumented = _undocumented_fields(SPEC.read_text(encoding="utf-8"))
|
||||
assert field not in undocumented, (
|
||||
f"contract field {field!r} is no longer documented (backticked) in the §12 block"
|
||||
)
|
||||
|
||||
|
||||
# --- Red-proofs: the guard MUST fail on a detached spec (mutated copy, never shared/) --
|
||||
|
||||
|
||||
def test_guard_red_when_spec_missing(tmp_path: Path) -> None:
|
||||
assert not (tmp_path / "method-spec.md").is_file()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("toolkit", _FORBIDDEN_TOOLKITS)
|
||||
def test_guard_red_when_toolkit_injected(tmp_path: Path, toolkit: str) -> None:
|
||||
mutated = SPEC.read_text(encoding="utf-8") + f"\n\nBuilt on {toolkit}.\n"
|
||||
copy = tmp_path / "method-spec.md"
|
||||
copy.write_text(mutated, encoding="utf-8")
|
||||
assert toolkit in _named_toolkits(copy.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", _CONTRACT_FIELDS)
|
||||
def test_guard_red_when_field_removed_from_section12(tmp_path: Path, field: str) -> None:
|
||||
text = SPEC.read_text(encoding="utf-8")
|
||||
block = _section12_block(text)
|
||||
mutated = text.replace(block, block.replace(f"`{field}`", ""))
|
||||
copy = tmp_path / "method-spec.md"
|
||||
copy.write_text(mutated, encoding="utf-8")
|
||||
assert field in _undocumented_fields(copy.read_text(encoding="utf-8"))
|
||||
Loading…
Add table
Add a link
Reference in a new issue