test(loadbearing): close the vacuous-negative class across the whole suite
Oekt 17 found the class on four named files. This sweep ENUMERATES it: 42 negative substring assertions across 21 test files (STATE's "~34 across 23" was a premise -- measured, it is 42/21). Sixteen of them measured an absence without ever having shown presence; all sixteen now carry a positive control asserting the searched-for string PRESENT in the source artifact, in EXACTLY the form the negative looks for. Files touched: test_costsim, test_loop, test_okf (3 sites), test_preflight, test_run_entrance, test_s10_run_layer, test_sdk_version_guard, test_simulation (2 sites), test_step1_expel, test_step5_refine, test_step7_async_loop, test_step8_promotion, test_valuereport. VALUE-PROOF (green-without / red-with, per the oekt-17 rule that a detach proof is not a value proof). Seven source/fixture mutations, each making the negative vacuous: M1 verdict fixture loses the realization signal VALUE-PROVEN M2 decoy fixture loses its text VALUE-PROVEN M3 renderer stops emitting typed section headings VALUE-PROVEN M4 promotion stops writing the marker VALUE-PROVEN (pass 2) M5 fold stops rendering the realization surface VALUE-PROVEN M6 report stops labelling the cost section VALUE-PROVEN M7 preflight stops importing the SDK VALUE-PROVEN M4 needed pass 2: a PRECEDING assertion caught the same mutation, hiding the new control behind it -- the oekt-17 lesson reproduced. The remaining nine controls are vacuity guards (non-emptiness / form-presence) whose mutation would have to break the source artificially; they are stated as guards, not claimed as value-proven. MEASURED FINDING (test_loop): the FIRST-RUN-MARKER negative cannot be given a positive control at all. Within a run only the CHECKER's critique is fed back -- the proposer's own prior reasoning crosses no prompt boundary, not even within a run. So that negative holds trivially. Left in place with the limitation stated in the test rather than dressed up as a controlled seam; the CRITIQUE negative beside it IS controlled and is the real seam. Mutations were in-place on src/ and shared/ with original bytes restored and sha-verified; git status clean before and after. Suite 688 -> 688 (assertions added inside existing tests, no new test cases). ruff + mypy --strict green. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Vc5PmZGjwuJypdhzKnJa5
This commit is contained in:
parent
123ecc3113
commit
30ba68a703
13 changed files with 98 additions and 5 deletions
|
|
@ -128,6 +128,11 @@ class TestNoPriceLiteralInSource:
|
||||||
def test_no_example_price_appears_as_a_literal(self) -> None:
|
def test_no_example_price_appears_as_a_literal(self) -> None:
|
||||||
source = (SRC_PKG / "costsim.py").read_text(encoding="utf-8")
|
source = (SRC_PKG / "costsim.py").read_text(encoding="utf-8")
|
||||||
pricing = load_pricing() # the bundled data/pricing.example.json
|
pricing = load_pricing() # the bundled data/pricing.example.json
|
||||||
|
# Positive controls: an empty price config would make the loop below iterate
|
||||||
|
# zero times, and an unreadable/empty source would make every substring miss —
|
||||||
|
# both green without guarding anything.
|
||||||
|
assert pricing.prices
|
||||||
|
assert "usd_per_mtok" in source
|
||||||
for model_id, price in pricing.prices.items():
|
for model_id, price in pricing.prices.items():
|
||||||
assert str(price.usd_per_mtok) not in source, (
|
assert str(price.usd_per_mtok) not in source, (
|
||||||
f"price for {model_id} is hardcoded in costsim.py — must come from config"
|
f"price for {model_id} is hardcoded in costsim.py — must come from config"
|
||||||
|
|
|
||||||
|
|
@ -162,6 +162,19 @@ class TestRunDebate:
|
||||||
)
|
)
|
||||||
run_debate(client, "context A", max_rounds=2, meter=_meter())
|
run_debate(client, "context A", max_rounds=2, meter=_meter())
|
||||||
run_debate(client, "context B", max_rounds=2, meter=_meter())
|
run_debate(client, "context B", max_rounds=2, meter=_meter())
|
||||||
|
# Positive control for the CRITIQUE negative: within the first run the critique
|
||||||
|
# DOES reach the proposer's second prompt, in EXACTLY the form the negative
|
||||||
|
# below searches for. That is what makes "absent from run B" a measured seam
|
||||||
|
# rather than a string that never travels anywhere.
|
||||||
|
first_run_second_prompt = client.prompts("proposer")[1]
|
||||||
|
assert "FIRST-RUN-CRITIQUE" in first_run_second_prompt
|
||||||
|
# MEASURED (this session): no such control exists for FIRST-RUN-MARKER — the
|
||||||
|
# proposer's own prior reasoning crosses NO prompt boundary, not even within a
|
||||||
|
# run (only the checker's critique is fed back). So the marker negative below is
|
||||||
|
# a weaker, complementary check: it cannot go red by state leaking through the
|
||||||
|
# transcript, because that channel carries the critique alone. Stated, not
|
||||||
|
# dressed up as a controlled seam.
|
||||||
|
assert "FIRST-RUN-MARKER" not in first_run_second_prompt
|
||||||
second_run_opening = client.prompts("proposer")[2]
|
second_run_opening = client.prompts("proposer")[2]
|
||||||
assert "FIRST-RUN-MARKER" not in second_run_opening
|
assert "FIRST-RUN-MARKER" not in second_run_opening
|
||||||
assert "FIRST-RUN-CRITIQUE" not in second_run_opening
|
assert "FIRST-RUN-CRITIQUE" not in second_run_opening
|
||||||
|
|
|
||||||
|
|
@ -195,6 +195,15 @@ class TestNavigationBoundary:
|
||||||
# one level above ``bundle/``: a ``..`` escape would succeed, so its absence
|
# one level above ``bundle/``: a ``..`` escape would succeed, so its absence
|
||||||
# is a boundary proof rather than a missing-file accident.
|
# is a boundary proof rather than a missing-file accident.
|
||||||
rendered, _ = self._read_context("nav-golden-escape")
|
rendered, _ = self._read_context("nav-golden-escape")
|
||||||
|
# Positive control: the decoy file REALLY carries both strings, in EXACTLY the
|
||||||
|
# form the negatives search for. The comment above asserted this in prose; a
|
||||||
|
# fixture that lost the file (or reworded it) would satisfy both `not in`
|
||||||
|
# checks without the boundary doing anything.
|
||||||
|
decoy = (NAV_GOLDENS / "nav-golden-escape" / "SHOULD-NOT-BE-READ.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert "MUST never be reached" in decoy
|
||||||
|
assert "Decoy" in decoy
|
||||||
assert "MUST never be reached" not in rendered
|
assert "MUST never be reached" not in rendered
|
||||||
assert "Decoy" not in rendered
|
assert "Decoy" not in rendered
|
||||||
|
|
||||||
|
|
@ -276,10 +285,17 @@ class TestRendering:
|
||||||
def test_empty_sections_are_dropped(self, tmp_path: Path) -> None:
|
def test_empty_sections_are_dropped(self, tmp_path: Path) -> None:
|
||||||
bundle = _make_bundle(
|
bundle = _make_bundle(
|
||||||
tmp_path,
|
tmp_path,
|
||||||
"Summary. See [empty](empty.md).",
|
"Summary. See [empty](empty.md) and [full](full.md).",
|
||||||
{"empty.md": "---\ntype: reference\ntitle: Empty\n---\n"},
|
{
|
||||||
|
"empty.md": "---\ntype: reference\ntitle: Empty\n---\n",
|
||||||
|
"full.md": "---\ntype: reference\ntitle: Full\n---\nBody.",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
context = bundle_context(bundle)
|
context = bundle_context(bundle)
|
||||||
|
# Positive control: a reference WITH a body renders in EXACTLY the heading form
|
||||||
|
# the negative searches for — so the absence below is the empty-drop, not a
|
||||||
|
# renamed heading or an unnavigated file.
|
||||||
|
assert "## reference: Full" in context
|
||||||
assert "## reference: Empty" not in context
|
assert "## reference: Empty" not in context
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -311,8 +327,15 @@ class TestVerdictLayerExclusion:
|
||||||
# actual 24 600 NOK) must be absent from the rendered context — it may
|
# actual 24 600 NOK) must be absent from the rendered context — it may
|
||||||
# reach the prompt ONLY via the gated experience fold.
|
# reach the prompt ONLY via the gated experience fold.
|
||||||
context = bundle_context(BUNDLE)
|
context = bundle_context(BUNDLE)
|
||||||
|
# Positive control: every leak string IS live in the bundle's verdict file, in
|
||||||
|
# EXACTLY the form the loop below searches for. Without it "absent from the
|
||||||
|
# context" would also hold for a bundle that never carried the signal at all.
|
||||||
|
leaks = ("0.82", "0,82", "24600", "24 600", "realization_rate")
|
||||||
|
verdict_text = (BUNDLE / "verdict-led-fro.md").read_text(encoding="utf-8")
|
||||||
|
for leak in leaks:
|
||||||
|
assert leak in verdict_text
|
||||||
assert "## verdict" not in context
|
assert "## verdict" not in context
|
||||||
for leak in ("0.82", "0,82", "24600", "24 600", "realization_rate"):
|
for leak in leaks:
|
||||||
assert leak not in context
|
assert leak not in context
|
||||||
# ...while the non-verdict concept layers ARE rendered:
|
# ...while the non-verdict concept layers ARE rendered:
|
||||||
assert "## project:" in context
|
assert "## project:" in context
|
||||||
|
|
|
||||||
|
|
@ -256,5 +256,9 @@ class TestPreflightIsOffline:
|
||||||
# AST-based, so a docstring mentioning query() stays green; only a real
|
# AST-based, so a docstring mentioning query() stays green; only a real
|
||||||
# call trips it (add ``query(prompt=...)`` → RED).
|
# call trips it (add ``query(prompt=...)`` → RED).
|
||||||
called = _called_names(SRC_PKG / "preflight.py")
|
called = _called_names(SRC_PKG / "preflight.py")
|
||||||
|
# Positive control: the AST walk really does find calls (a parse that yielded an
|
||||||
|
# empty set would satisfy both negatives vacuously), and specifically the import
|
||||||
|
# the comment above permits — so this measures IMPORT-yes/INVOKE-no, not silence.
|
||||||
|
assert "import_module" in called
|
||||||
assert "query" not in called
|
assert "query" not in called
|
||||||
assert "ClaudeSDKClient" not in called
|
assert "ClaudeSDKClient" not in called
|
||||||
|
|
|
||||||
|
|
@ -76,6 +76,10 @@ class TestComposeRunContext:
|
||||||
with_empty = compose_run_context(BUNDLE, inbox, k=3)
|
with_empty = compose_run_context(BUNDLE, inbox, k=3)
|
||||||
without = compose_run_context(BUNDLE, None, k=3)
|
without = compose_run_context(BUNDLE, None, k=3)
|
||||||
assert with_empty.context == without.context
|
assert with_empty.context == without.context
|
||||||
|
# Positive control: the base composition is non-empty — an empty context would
|
||||||
|
# satisfy the marker negative vacuously. That the marker CAN reach a context is
|
||||||
|
# pinned by the seeded-inbox test above; here it must not.
|
||||||
|
assert without.context
|
||||||
assert MARKER not in without.context
|
assert MARKER not in without.context
|
||||||
assert with_empty.inbox_merged == 0
|
assert with_empty.inbox_merged == 0
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -83,7 +83,13 @@ def test_build_citations_from_micro_bundle_are_exact_spans() -> None:
|
||||||
|
|
||||||
|
|
||||||
def test_build_citations_excludes_verdict_files() -> None:
|
def test_build_citations_excludes_verdict_files() -> None:
|
||||||
cited_files = {citation.file for citation in build_citations(navigate_bundle(BUNDLE_DIR))}
|
concepts = navigate_bundle(BUNDLE_DIR)
|
||||||
|
# Positive controls: the verdict file IS navigated (so the exclusion below does real
|
||||||
|
# work rather than never seeing the file), and citations ARE produced — an empty set
|
||||||
|
# would satisfy the negative without excluding anything.
|
||||||
|
assert "verdict-led-fro.md" in {concept.path.name for concept in concepts}
|
||||||
|
cited_files = {citation.file for citation in build_citations(concepts)}
|
||||||
|
assert cited_files
|
||||||
assert "verdict-led-fro.md" not in cited_files
|
assert "verdict-led-fro.md" not in cited_files
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -168,5 +168,9 @@ class TestTheSeamGoesRedWhenDetached:
|
||||||
# pyproject would re-admit the substring-anywhere defect unnoticed.
|
# pyproject would re-admit the substring-anywhere defect unnoticed.
|
||||||
text = _PYPROJECT.read_text(encoding="utf-8")
|
text = _PYPROJECT.read_text(encoding="utf-8")
|
||||||
block = _dependencies_block(text)
|
block = _dependencies_block(text)
|
||||||
|
# Positive control: the marker EXISTS in the whole file, in exactly the form the
|
||||||
|
# negative searches for — otherwise "absent from the slice" would hold for a
|
||||||
|
# pyproject that never had a [project] table to be excluded.
|
||||||
|
assert "[project]" in text
|
||||||
assert len(block) < len(text)
|
assert len(block) < len(text)
|
||||||
assert "[project]" not in block
|
assert "[project]" not in block
|
||||||
|
|
|
||||||
|
|
@ -146,6 +146,12 @@ class TestMarkerNeverCrossesWithoutPromotion:
|
||||||
assert result.promoted_path is None
|
assert result.promoted_path is None
|
||||||
assert result.promotion_refusal is not None
|
assert result.promotion_refusal is not None
|
||||||
assert "rejected" in result.promotion_refusal
|
assert "rejected" in result.promotion_refusal
|
||||||
|
# Positive controls: the docstring's claim that the marker IS authored into the
|
||||||
|
# artifact the run reads, asserted rather than stated — and prompts really were
|
||||||
|
# issued, since `all()` over an empty list is vacuously True. Without both, this
|
||||||
|
# control would pass for a marker that was never written anywhere.
|
||||||
|
assert marker in artifact.read_text(encoding="utf-8")
|
||||||
|
assert client_b.prompts("proposer")
|
||||||
assert marker not in result.run_b.composed.context
|
assert marker not in result.run_b.composed.context
|
||||||
assert all(marker not in prompt for prompt in client_b.prompts("proposer"))
|
assert all(marker not in prompt for prompt in client_b.prompts("proposer"))
|
||||||
|
|
||||||
|
|
@ -175,8 +181,16 @@ class TestGatedFoldIsTheOnlyChannel:
|
||||||
# ...but NO verdict-typed concept is rendered into the read-context —
|
# ...but NO verdict-typed concept is rendered into the read-context —
|
||||||
# a `## verdict` section in either run's context means the seed's (or
|
# a `## verdict` section in either run's context means the seed's (or
|
||||||
# a promoted file's) body leaked around the gated fold (§3 Step 1).
|
# a promoted file's) body leaked around the gated fold (§3 Step 1).
|
||||||
|
# Positive control: `## `-typed sections ARE rendered into both contexts, so the
|
||||||
|
# two negatives measure a GATED type and not a context without sections at all.
|
||||||
|
assert "## project:" in result.run_a.composed.context
|
||||||
|
assert "## project:" in result.run_b.composed.context
|
||||||
assert "## verdict" not in result.run_a.composed.context
|
assert "## verdict" not in result.run_a.composed.context
|
||||||
assert "## verdict" not in result.run_b.composed.context
|
assert "## verdict" not in result.run_b.composed.context
|
||||||
# Belt and braces: the promoted marker itself never surfaces via
|
# Belt and braces: the promoted marker itself never surfaces via
|
||||||
# rendering (promoted files are frontmatter-only by design, §6).
|
# rendering (promoted files are frontmatter-only by design, §6).
|
||||||
|
# Positive control: the marker IS live on disk in the promoted file, so the
|
||||||
|
# absence below measures the RENDERING boundary, not a marker never written.
|
||||||
|
assert result.promoted_path is not None
|
||||||
|
assert MARKER in result.promoted_path.read_text(encoding="utf-8")
|
||||||
assert MARKER not in bundle_context(bundle)
|
assert MARKER not in bundle_context(bundle)
|
||||||
|
|
|
||||||
|
|
@ -223,8 +223,13 @@ class TestExperienceFold:
|
||||||
features = CandidateFeatures.from_proposal(load_validator_input(BUNDLE))
|
features = CandidateFeatures.from_proposal(load_validator_input(BUNDLE))
|
||||||
unfolded = fold_experience(VerdictStore(), features, BASE_PROMPT, k=3)
|
unfolded = fold_experience(VerdictStore(), features, BASE_PROMPT, k=3)
|
||||||
folded = fold_experience(seeded_store, features, BASE_PROMPT, k=3)
|
folded = fold_experience(seeded_store, features, BASE_PROMPT, k=3)
|
||||||
|
surface = f"realiseringsgrad={golden_surface['realization_rate']}"
|
||||||
assert unfolded == BASE_PROMPT
|
assert unfolded == BASE_PROMPT
|
||||||
assert f"realiseringsgrad={golden_surface['realization_rate']}" not in unfolded
|
# Positive control: the FOLDED prompt carries the surface in EXACTLY the form the
|
||||||
|
# negative searches for — so its absence from `unfolded` is the empty store's
|
||||||
|
# doing, not a surface that is never rendered in that shape at all.
|
||||||
|
assert surface in folded
|
||||||
|
assert surface not in unfolded
|
||||||
assert folded != unfolded
|
assert folded != unfolded
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -89,6 +89,10 @@ class TestInformedRefinement:
|
||||||
client = ScriptedClient(replies=[reply(json.dumps(BAD_90K)), reply(json.dumps(GOOD))])
|
client = ScriptedClient(replies=[reply(json.dumps(BAD_90K)), reply(json.dumps(GOOD))])
|
||||||
run_candidate_loop(client, "base prompt", meter=_meter(), max_attempts=3)
|
run_candidate_loop(client, "base prompt", meter=_meter(), max_attempts=3)
|
||||||
second_prompt = client.prompts("proposer")[1]
|
second_prompt = client.prompts("proposer")[1]
|
||||||
|
# Positive control: the REASON did cross into the retry prompt — so the two
|
||||||
|
# negatives measure what was deliberately withheld, not an empty or misindexed
|
||||||
|
# prompt that would carry nothing either way.
|
||||||
|
assert _rejection_reason(BAD_90K) in second_prompt
|
||||||
assert json.dumps(BAD_90K) not in second_prompt
|
assert json.dumps(BAD_90K) not in second_prompt
|
||||||
assert '"affected_items"' not in second_prompt
|
assert '"affected_items"' not in second_prompt
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -354,6 +354,10 @@ class TestAsyncFileLoopLoadBearing:
|
||||||
inbox = tmp_path / "inbox"
|
inbox = tmp_path / "inbox"
|
||||||
inbox.mkdir()
|
inbox.mkdir()
|
||||||
client_b, claimed_b = _run_b(inbox)
|
client_b, claimed_b = _run_b(inbox)
|
||||||
|
# Positive control: prompts were actually issued. `all()` over an empty list is
|
||||||
|
# vacuously True, so without this the control would pass for a run that never
|
||||||
|
# prompted the proposer at all.
|
||||||
|
assert client_b.prompts("proposer")
|
||||||
assert all(MARKER not in prompt for prompt in client_b.prompts("proposer"))
|
assert all(MARKER not in prompt for prompt in client_b.prompts("proposer"))
|
||||||
assert claimed_b == 25000
|
assert claimed_b == 25000
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,9 @@ class TestPromotedFile:
|
||||||
|
|
||||||
def test_no_structured_learning_fields_are_reproduced(self, bundle: Path) -> None:
|
def test_no_structured_learning_fields_are_reproduced(self, bundle: Path) -> None:
|
||||||
concept = parse_concept_file(_promote(_document(), bundle))
|
concept = parse_concept_file(_promote(_document(), bundle))
|
||||||
|
# Positive control: the frontmatter IS populated, so the absences below are
|
||||||
|
# withheld fields rather than an empty mapping that excludes everything.
|
||||||
|
assert concept.frontmatter
|
||||||
for field in _SEED_ONLY_FIELDS:
|
for field in _SEED_ONLY_FIELDS:
|
||||||
assert field not in concept.frontmatter
|
assert field not in concept.frontmatter
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -414,6 +414,10 @@ def test_markdown_render_marks_every_unmarked_figure(layers: Layers) -> None:
|
||||||
|
|
||||||
assert "FV42-P2" in rendered
|
assert "FV42-P2" in rendered
|
||||||
assert "UNMARKED" in rendered # the pending/unquantified projects are labelled
|
assert "UNMARKED" in rendered # the pending/unquantified projects are labelled
|
||||||
|
# Positive control: WITH an estimate the cost section renders under exactly the label
|
||||||
|
# the negative searches for — so its absence here is the missing estimate, not a
|
||||||
|
# renamed or removed label that would make the check green forever.
|
||||||
|
assert "ESTIMAT" in render_report(_build(layers, estimated_cost_usd=0.5))
|
||||||
assert "ESTIMAT" not in rendered # no cost estimate was passed in → no cost section
|
assert "ESTIMAT" not in rendered # no cost estimate was passed in → no cost section
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue