open/portfolio-optimiser
1
0
Fork 0
Code Issues Releases 3 Activity

feat(persona): load the falsification skill from commons at call time

Browse source 
[skip-docs] — the invariant row for this plan lands in Step 13, after the mutations.

Amendment 2 in the same commit: the framework-neutrality sweep covered only
expert-reviewer, so the skill that arrived by subtree pull had no framework guard
anywhere. GUARDED_SKILL_DIRS names both, and a fail-closed coverage arm turns red
when a future pull brings a third skill that is not listed.

Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-02 21:31:32 +02:00
commit 488a0f2b6c
3 changed files with 276 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

155
tests/test_falsification_skill_loadbearing.py Normal file
View file

@ -0,0 +1,155 @@
"""Step 12 (Session 5) - the falsification method as a SHARED Agent Skill, authored in commons.
**Operator decision D4, and why the skill is not written here.** ``shared/`` is a PULL-ONLY git
subtree of ``portfolio-optimiser-commons``. Authoring the skill locally would produce a wheel the
sibling never inherits, and **no test in this suite detects that divergence** --
``tests/test_shared_packaged_data_loadbearing.py`` compares the working tree against the wheel,
never against commons. The natural "fix" is ``git subtree push``, which leaked consumer history
once already, on 2026-07-03. So the two files under ``shared/skills/falsification-reviewer/``
arrive by ``git subtree pull``, and nothing in this file writes there.
**What is guarded HERE, and what deliberately is not.** The framework-neutrality rule has ONE copy
and it is NOT in this file: ``test_method_spec_loadbearing.test_method_spec_is_framework_neutral``
sweeps every shared spec and every skill tree, and Amendment 2 extended it to cover this skill.
Two copies of one rule is the ko-(p) drift class, and the plan concedes the framework guard is
close to trivially green on pure prose anyway. **The TERMINOLOGY guard is the one that bites**, and
it lives here: the customer-facing wording is "knowledge base" / "knowledge bundle", never the
internal format name, so the literal string ``OKF bundle`` appearing in the skill's prose turns
this file red.
**Amendment 3 decides the round trip's input, and it is not a preference.** ``frontmatter_verbatim``
is AUTHORITATIVE: it is the bytes a test materialises into a throwaway concept file before reading
them back. The sibling ``frontmatter`` object is a line-oriented PROJECTION and informative only --
by construction it cannot carry a block form, which is why concept 2 has no ``sources`` key there.
Materialising from ``frontmatter`` would derive ``state: absent`` and silently lose the
``unreadable`` case the example exists to demonstrate: the arm would be GREEN while proving the
opposite of its docstring. The example's own judgement is ``undecided`` -- not ``survived`` -- for
the same reason, because a claim whose one possible refuter was unreadable was never attacked.
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import pytest
from portfolio_optimiser import okf, persona
REPO_ROOT = Path(__file__).resolve().parents[1]
SKILL_DIR = REPO_ROOT / "shared" / "skills" / "falsification-reviewer"
EXAMPLE = SKILL_DIR / "references" / "example-evidence.json"
#: The internal format name. Customer-facing prose says "knowledge base" / "knowledge bundle".
_INTERNAL_NAME = re.compile(r"OKF bundle")
def test_the_skill_ships_its_two_files_with_usable_frontmatter() -> None:
"""(a) Structure. RED before the subtree pull has landed -- which is exactly Step 12's
On-failure clause: if the files are absent, STOP; do not author under ``shared/`` locally."""
skill_md = SKILL_DIR / "SKILL.md"
assert skill_md.is_file(), "shared/skills/falsification-reviewer/SKILL.md missing"
assert EXAMPLE.is_file(), "the skill's worked example is missing"
fm = okf.parse_frontmatter(skill_md)
assert fm.get("name") == "falsification-reviewer"
assert fm.get("description", "").strip(), "SKILL.md description must be non-empty"
json.loads(EXAMPLE.read_text(encoding="utf-8"))
def test_the_skill_never_uses_the_internal_format_name() -> None:
"""(b) The DISCRIMINATING guard. Paired with a KNOWN-POSITIVE control: a guard that searched
for something no file could ever contain would be green by construction, so the same regex is
first shown to fire on text that does carry the string."""
assert _INTERNAL_NAME.search("read the OKF bundle at that path") is not None, (
"the guard cannot find the string it is supposed to forbid -- it proves nothing"
)
for f in sorted(p for p in SKILL_DIR.rglob("*") if p.is_file()):
hit = _INTERNAL_NAME.search(f.read_text(encoding="utf-8"))
assert hit is None, f"internal format name {hit.group(0)!r} in customer-facing prose: {f}"
def test_the_example_resolves_at_call_time_not_at_import(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""(c) The loader seam, mirroring ``persona.load_persona_example``: resolved INSIDE the call
(never frozen into a default argument), so both the test seam and ``PORTFOLIO_SHARED_ROOT``
stay live. A loader that bound its path at import would answer from the tree that existed when
the module was first imported."""
real = persona.load_falsification_example()
assert real.judgement == "undecided"
stand_in = tmp_path / "example-evidence.json"
stand_in.write_text(
json.dumps(
{
"claim": "c",
"judgement": "survived",
"refuter": None,
"concepts": [],
}
),
encoding="utf-8",
)
monkeypatch.setattr(persona, "_FALSIFICATION_EXAMPLE_PATH", stand_in)
assert persona.load_falsification_example().judgement == "survived"
def test_a_missing_example_fails_fast_rather_than_degrading(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""(d) The example is REQUIRED input (contrast the tolerant Step-7 inbox). A broken shared
artefact must surface, not pass silently."""
monkeypatch.setattr(persona, "_FALSIFICATION_EXAMPLE_PATH", tmp_path / "nope.json")
with pytest.raises(FileNotFoundError):
persona.load_falsification_example()
def test_the_worked_example_round_trips_through_the_real_readers(tmp_path: Path) -> None:
"""(e) The example is not decoration: every concept's declared ``evidence`` and ``derived``
values are re-derived by the SHIPPED readers from the SHIPPED bytes.
``read_provenance`` takes a MARKDOWN path, not JSON, so the fields are materialised into a
throwaway concept file first (Amendment 3) -- ``frontmatter_verbatim`` VERBATIM, never
reassembled from the ``frontmatter`` projection. Reassembling would emit concept 2's
``sources`` as nothing at all, and the ``unreadable``/``block-sequence`` arm would pass as
``absent`` while claiming to prove the opposite.
Nothing git-tracked is touched: each concept is written under ``tmp_path``.
"""
example = persona.load_falsification_example()
assert len(example.concepts) == 2, "the example must keep both the readable and unreadable case"
seen_states = set()
for concept in example.concepts:
path = tmp_path / f"{concept.concept_id}.md"
path.write_text(concept.frontmatter_verbatim, encoding="utf-8")
# The plan's letter: ``read_provenance`` + ``trust_tier``, never ``evidence_for`` with a
# non-default key. That is not a stylistic choice — the example itself splits the two,
# putting state/reason/items_seen under ``evidence`` and the tier under ``derived``,
# because ``FalsificationEvidence.tier`` is only meaningful for the ONE key SPEC §5.3
# tiers. Branching on the primitive's own documented three-way return is test-level
# dispatch, not a second copy of production logic.
result = okf.read_provenance(path, "sources")
if concept.state == "present":
assert isinstance(result, tuple), concept.concept_id
assert len(result) == concept.items_seen, concept.concept_id
assert concept.reason is None, concept.concept_id
else:
assert isinstance(result, okf.UnreadableProvenance), concept.concept_id
assert result.reason == concept.reason, concept.concept_id
assert result.items_seen == concept.items_seen, concept.concept_id
# The verified half goes through the SHIPPED ``evidence_for`` on its DEFAULT key — the
# path where the tier derivation is the one §5.3 defines.
verified = okf.evidence_for(path)
assert okf.trust_tier(verified.entries) == concept.trust_tier, concept.concept_id
assert okf.adjudication_for(path) == concept.adjudication, concept.concept_id
seen_states.add(concept.state)
assert seen_states == {"present", "unreadable"}, (
"the example must exercise BOTH a readable and an unreadable concept -- one of each is what "
f"makes the round trip discriminating, got {sorted(seen_states)}"
)

40
tests/test_method_spec_loadbearing.py
View file

@ -26,8 +26,15 @@ from portfolio_optimiser.verdicts import ProposalFeatures, capture_verdict, verd
REPO_ROOT = Path(__file__).resolve().parents[1]
SPEC_PATH = REPO_ROOT / "shared" / "method-spec.md"
INGEST_SPEC_PATH = REPO_ROOT / "shared" / "ingest-spec.md"
SKILL_DIR = REPO_ROOT / "shared" / "skills" / "expert-reviewer"
SKILLS_ROOT = REPO_ROOT / "shared" / "skills"
SKILL_DIR = SKILLS_ROOT / "expert-reviewer"
EXAMPLE_VERDICT = SKILL_DIR / "references" / "example-verdict.json"
#: Every shared Agent Skill the framework-neutrality sweep covers, ENUMERATED rather than globbed —
#: "a new spec file is guarded explicitly, never implicitly", the rule this file already states for
#: the two specs. The enumeration is kept honest by the coverage arm at the bottom of this module:
#: a third skill arriving in a subtree pull turns THAT red until it is listed here, so explicit
#: never degrades into stale (the ``_LIVE_DOCS`` coverage pattern).
GUARDED_SKILL_DIRS = (SKILL_DIR, SKILLS_ROOT / "falsification-reviewer")
BUNDLE_DIR = REPO_ROOT / "shared" / "examples" / "bygg-energi-mikro"
# Name-shaped framework guard (stricter than the persona test's import-shaped guard): the spec's
@ -190,8 +197,15 @@ def test_method_spec_is_framework_neutral() -> None:
NAME a concrete agent framework or vendor stack. Mirrors the persona SKILL.md rule; stricter
than the import-shaped guard because the specs are pure prose (no AST to parse). RED the moment
a framework name leaks into the shared method prose. Covers BOTH shared specs: the method spec
and the ingest spec (I1) — a new spec file is guarded explicitly, never implicitly."""
for f in [SPEC_PATH, INGEST_SPEC_PATH, *sorted(p for p in SKILL_DIR.rglob("*") if p.is_file())]:
and the ingest spec (I1) — a new spec file is guarded explicitly, never implicitly.
Amendment 2 (2026-09-02) widened the skill half from ``expert-reviewer`` alone to
``GUARDED_SKILL_DIRS``: the falsification-reviewer skill arrived by subtree pull with NO
framework guard anywhere, and a sweep pinned to one skill silently stops covering the tree it
claims to cover."""
guarded = [f for d in GUARDED_SKILL_DIRS for f in sorted(d.rglob("*")) if f.is_file()]
assert guarded, "the sweep found no skill files — it would be green by construction"
for f in [SPEC_PATH, INGEST_SPEC_PATH, *guarded]:
assert f.is_file(), f"expected shared artifact missing: {f}"
hit = _FRAMEWORK_NAMES.search(f.read_text(encoding="utf-8"))
assert hit is None, f"framework name {hit.group(0)!r} in shared method prose: {f}"
@ -342,3 +356,23 @@ def test_ingest_spec_documents_every_contract_field() -> None:
# The §11 golden extraction case layout.
for entry in ("manifest.json", "fixture/", "ingested-at.txt", "expected-bundle/"):
documented(entry, "golden extraction case")
def test_every_shared_skill_is_covered_by_the_framework_sweep() -> None:
"""Fail-closed coverage for Amendment 2's enumeration.
``GUARDED_SKILL_DIRS`` is written out by hand, following this file's own rule that a new shared
artefact is guarded explicitly rather than by a glob that quietly absorbs it. The cost of an
explicit list is that it goes stale, and ``shared/`` is a PULL-ONLY subtree — a third skill can
arrive here without anyone in this repo deciding to add it. This arm makes staleness LOUD: the
enumeration must name every directory under ``shared/skills/``, so the next pull that brings a
skill turns this red until it is listed. Mirrors
``test_every_document_is_classified_live_or_archive``.
"""
on_disk = {d for d in SKILLS_ROOT.iterdir() if d.is_dir()}
assert on_disk, "no shared skills found — the control cannot distinguish covered from empty"
missing = sorted(d.name for d in on_disk - set(GUARDED_SKILL_DIRS))
assert not missing, (
"shared skills not named in GUARDED_SKILL_DIRS, so the framework sweep does not reach "
f"them: {missing}"
)