refactor(examples): replace sector-specific example material with generic, fictitious examples
The context sets, the packaged knowledge bases and the example bundles are replaced by one fictitious example set about IT operations in an invented organisation: three context sets (serverrom-2027, driftsavtale-2027 and the two-base drift-og-avtale-2027), two synthetic knowledge bases under src/portfolio_optimiser/data/kunnskapsbaser and two example bundles under src/portfolio_optimiser/data/bundles. Numbers, codes and structural values in tests and fixtures are kept; names, ids and wording change. Dated measurement documents that only recorded runs on the replaced material are deleted. Gate figures measured on the new set are not comparable with earlier ones. The exclusion gate from the previous commit is green: 0 tracked files hit outside the shared/ subtree. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
058dd25570
commit
37547fe292
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
1147 changed files with 24138 additions and 9503 deletions
|
|
@ -9,9 +9,11 @@ cost codes — ``M-04-01`` / ``M-04-03`` — that appear in NO prompt of that ru
|
|||
tied to whether a cost baseline happened to exist. **The input always exists; the baseline does
|
||||
not.**
|
||||
|
||||
Every fixture under ``tests/fixtures/p7-grounding/`` is the VERBATIM generation prompt of a free
|
||||
recording — the entire input the proposer saw on the attempt that produced its candidate. They are
|
||||
tracked test data on purpose: the recordings live in ``scratchpad/``, which ``git archive HEAD``
|
||||
Every fixture under ``tests/fixtures/p7-grounding/`` is a generation prompt in the exact form a
|
||||
free recording captured — the entire input the proposer saw on the attempt that produced its
|
||||
candidate. The two K2 fixtures are verbatim; the ``p4-driftskrav`` fixture is a fictional rewrite of
|
||||
a recorded prompt that keeps its structure and the two facts its tests read (the known positive is
|
||||
present verbatim, the control identifier is absent). They are tracked test data on purpose: the recordings live in ``scratchpad/``, which ``git archive HEAD``
|
||||
does not carry, so a test reading them would pass here and fail the handover gate.
|
||||
"""
|
||||
|
||||
|
|
@ -36,7 +38,7 @@ from portfolio_optimiser.validator import (
|
|||
|
||||
FIXTURES = Path(__file__).parent / "fixtures" / "p7-grounding"
|
||||
|
||||
#: The known positive (premiss (vii), `docs/2026-09-08-n-bundlene-hypoteseform.md` § 11.4): the
|
||||
#: The known positive (premiss (vii), measured on a requirements corpus during development): the
|
||||
#: model quoted this requirement number VERBATIM in 6 of 6 replies, and it stands in 6 of 6
|
||||
#: prompts. Note the EM-DASH (U+2014): the hyphen variant scores 0 of 6 in both.
|
||||
KNOWN_POSITIVE = "Krav 3.3.1—13"
|
||||
|
|
@ -51,7 +53,7 @@ def _proposal(*codes: str, unit_cost: float = 100.0) -> SavingsProposal:
|
|||
so a rejection can only have come from the grounding stage."""
|
||||
return SavingsProposal(
|
||||
project_id="K2",
|
||||
measure="reduce marking scope",
|
||||
measure="reduce cabling scope",
|
||||
affected_items=[AffectedItem(code=c, quantity=10.0, unit_cost=unit_cost) for c in codes],
|
||||
claimed_saving_nok=1.0,
|
||||
)
|
||||
|
|
@ -112,10 +114,10 @@ def test_b_the_reason_names_every_ungrounded_identifier_in_the_proposal_s_own_or
|
|||
|
||||
|
||||
def test_c_the_known_positive_is_not_flagged() -> None:
|
||||
"""``Krav 3.3.1—13`` stands VERBATIM in the N100 generation prompt, so a proposal that cites it
|
||||
is grounded and the grounding stage must stay silent. A rule that flags everything code-shaped
|
||||
"""``Krav 3.3.1—13`` stands VERBATIM in the driftskrav generation prompt, so a proposal that
|
||||
cites it is grounded and the grounding stage must stay silent. A rule that flags everything code-shaped
|
||||
is red here."""
|
||||
text = _prompt("p4-n100-generation-prompt.txt")
|
||||
text = _prompt("p4-driftskrav-generation-prompt.txt")
|
||||
assert KNOWN_POSITIVE in text, "fixture drifted: the known positive is not in the prompt"
|
||||
ruling = validate_proposal(
|
||||
_proposal(KNOWN_POSITIVE), baseline=None, grounding=Grounding.of(text)
|
||||
|
|
@ -125,9 +127,9 @@ def test_c_the_known_positive_is_not_flagged() -> None:
|
|||
|
||||
def test_c_control_the_rule_can_still_flag_on_that_same_recording() -> None:
|
||||
"""THE CONTROL that keeps the arm above from proving nothing. ``CRS-01`` is the one identifier
|
||||
the model produced on the N100 recording that no prompt carries (1 of 2 ungrounded). Same
|
||||
the model produced on the requirements recording that no prompt carries (1 of 2 ungrounded). Same
|
||||
fixture, same call — only the identifier differs, and this one must fall."""
|
||||
text = _prompt("p4-n100-generation-prompt.txt")
|
||||
text = _prompt("p4-driftskrav-generation-prompt.txt")
|
||||
assert "CRS-01" not in text
|
||||
ruling = validate_proposal(_proposal("CRS-01"), baseline=None, grounding=Grounding.of(text))
|
||||
assert isinstance(ruling, Rejection)
|
||||
|
|
@ -142,7 +144,7 @@ def test_c_control_the_rule_can_still_flag_on_that_same_recording() -> None:
|
|||
def test_d_a_code_quoted_verbatim_from_the_input_is_not_flagged() -> None:
|
||||
"""The discriminator between this rule and "flag anything that looks like a code". The token is
|
||||
deliberately shaped like the fabrications above; the ONLY difference is that the input says it."""
|
||||
text = "Context:\nPrice schedule line ZZZ-999-01 covers technical marking.\n"
|
||||
text = "Context:\nPrice schedule line ZZZ-999-01 covers network cabling.\n"
|
||||
ruling = validate_proposal(_proposal("ZZZ-999-01"), baseline=None, grounding=Grounding.of(text))
|
||||
assert isinstance(ruling, ValidatedProposal), getattr(ruling, "reason", "")
|
||||
|
||||
|
|
@ -211,7 +213,7 @@ async def test_f_generate_via_llm_grounds_the_candidate_in_the_prompt_it_sent()
|
|||
result = await generate_via_llm(
|
||||
ScriptedChatClient(reply=reply, tokens_per_reply=8),
|
||||
project,
|
||||
"The price schedule carries line REAL-11-02 for technical marking.",
|
||||
"The price schedule carries line REAL-11-02 for network cabling.",
|
||||
meter,
|
||||
max_attempts=1,
|
||||
)
|
||||
|
|
@ -233,7 +235,7 @@ async def test_f_control_a_code_the_context_names_survives_the_same_call() -> No
|
|||
result = await generate_via_llm(
|
||||
ScriptedChatClient(reply=reply, tokens_per_reply=8),
|
||||
project,
|
||||
"The price schedule carries line REAL-11-02 for technical marking.",
|
||||
"The price schedule carries line REAL-11-02 for network cabling.",
|
||||
meter,
|
||||
max_attempts=1,
|
||||
)
|
||||
|
|
@ -367,7 +369,7 @@ async def test_h_the_refusal_does_not_ground_the_next_attempt_that_repeats_the_c
|
|||
result = await generate_via_llm(
|
||||
client,
|
||||
_project(),
|
||||
"The price schedule carries line REAL-11-02 for technical marking.",
|
||||
"The price schedule carries line REAL-11-02 for network cabling.",
|
||||
TokenMeter(Budget(max_tokens=10**9, max_rounds=20)),
|
||||
max_attempts=2,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue