feat(persona): load the falsification skill from commons at call time
[skip-docs] — the invariant row for this plan lands in Step 13, after the mutations. Amendment 2 in the same commit: the framework-neutrality sweep covered only expert-reviewer, so the skill that arrived by subtree pull had no framework guard anywhere. GUARDED_SKILL_DIRS names both, and a fail-closed coverage arm turns red when a future pull brings a third skill that is not listed. Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
parent
21c476bbe7
commit
488a0f2b6c
3 changed files with 276 additions and 3 deletions
|
|
@ -54,3 +54,87 @@ def load_persona_example() -> PersonaExample:
|
|||
rationale=data["rationale"],
|
||||
marker=data["marker"],
|
||||
)
|
||||
|
||||
|
||||
# --- The falsification-reviewer skill (Step 12, D4) ----------------------------------------------
|
||||
# A SECOND shared Agent Skill, read the same way and for the same reason: it is authored in commons
|
||||
# and arrives by ``git subtree pull``, so a loader that consumes it is what keeps it from rotting
|
||||
# silently. Same call-time seam, same fail-fast discipline as the persona example above.
|
||||
|
||||
_FALSIFICATION_EXAMPLE_PATH: Path | None = None
|
||||
|
||||
_FALSIFICATION_SUBPATH = (
|
||||
Path("skills") / "falsification-reviewer" / "references" / "example-evidence.json"
|
||||
)
|
||||
|
||||
|
||||
def _falsification_example_path() -> Path:
|
||||
"""The worked example's location: the test seam if set, else resolved under ``shared_root()``."""
|
||||
if _FALSIFICATION_EXAMPLE_PATH is not None:
|
||||
return _FALSIFICATION_EXAMPLE_PATH
|
||||
return shared_root() / _FALSIFICATION_SUBPATH
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FalsificationConcept:
|
||||
"""One concept the worked example judged, with what the readers should derive from its bytes.
|
||||
|
||||
``frontmatter_verbatim`` is AUTHORITATIVE and is the only field a consumer should materialise
|
||||
from. The example also ships a line-oriented ``frontmatter`` projection, which is informative
|
||||
only: by construction it cannot carry a block form, so materialising from it would turn the
|
||||
unreadable case into an absent one — the tolerant read that widens the answer.
|
||||
"""
|
||||
|
||||
concept_id: str
|
||||
#: The document's frontmatter block exactly as written, newlines and indentation included.
|
||||
frontmatter_verbatim: str
|
||||
state: str
|
||||
reason: str | None
|
||||
items_seen: int
|
||||
trust_tier: str
|
||||
adjudication: str
|
||||
#: Whether the verdict was allowed to rest on this concept.
|
||||
relied_on: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FalsificationExample:
|
||||
"""The falsification reviewer's worked example: one claim, the concepts consulted, the verdict.
|
||||
|
||||
``judgement`` is ``undecided`` in the shipped example, and deliberately not ``survived``: the
|
||||
one concept that could have carried a refutation was unreadable, so the claim was never
|
||||
actually attacked. Returning ``survived`` there would convert a gap in the knowledge base into
|
||||
support for the claim — the inversion this role exists to prevent.
|
||||
"""
|
||||
|
||||
claim: str
|
||||
judgement: str
|
||||
#: Who refuted the claim, or ``None`` when nobody did. A refutation that names no refuter is
|
||||
#: not a refutation (the ``verified``-without-an-actor rule, one layer up).
|
||||
refuter: str | None
|
||||
concepts: tuple[FalsificationConcept, ...]
|
||||
|
||||
|
||||
def load_falsification_example() -> FalsificationExample:
|
||||
"""Read the falsification skill's worked example. Fail-fast: a missing file raises
|
||||
``FileNotFoundError`` and a missing key raises ``KeyError`` (required input, contrast the
|
||||
tolerant Step-7 verdict inbox). Resolves the path at CALL time."""
|
||||
data = json.loads(_falsification_example_path().read_text(encoding="utf-8"))
|
||||
return FalsificationExample(
|
||||
claim=data["claim"],
|
||||
judgement=data["judgement"],
|
||||
refuter=data["refuter"],
|
||||
concepts=tuple(
|
||||
FalsificationConcept(
|
||||
concept_id=c["concept_id"],
|
||||
frontmatter_verbatim=c["frontmatter_verbatim"],
|
||||
state=c["evidence"]["state"],
|
||||
reason=c["evidence"]["reason"],
|
||||
items_seen=c["evidence"]["items_seen"],
|
||||
trust_tier=c["derived"]["trust_tier"],
|
||||
adjudication=c["derived"]["adjudication"],
|
||||
relied_on=c["relied_on"],
|
||||
)
|
||||
for c in data["concepts"]
|
||||
),
|
||||
)
|
||||
|
|
|
|||
155
tests/test_falsification_skill_loadbearing.py
Normal file
155
tests/test_falsification_skill_loadbearing.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"""Step 12 (Session 5) - the falsification method as a SHARED Agent Skill, authored in commons.
|
||||
|
||||
**Operator decision D4, and why the skill is not written here.** ``shared/`` is a PULL-ONLY git
|
||||
subtree of ``portfolio-optimiser-commons``. Authoring the skill locally would produce a wheel the
|
||||
sibling never inherits, and **no test in this suite detects that divergence** --
|
||||
``tests/test_shared_packaged_data_loadbearing.py`` compares the working tree against the wheel,
|
||||
never against commons. The natural "fix" is ``git subtree push``, which leaked consumer history
|
||||
once already, on 2026-07-03. So the two files under ``shared/skills/falsification-reviewer/``
|
||||
arrive by ``git subtree pull``, and nothing in this file writes there.
|
||||
|
||||
**What is guarded HERE, and what deliberately is not.** The framework-neutrality rule has ONE copy
|
||||
and it is NOT in this file: ``test_method_spec_loadbearing.test_method_spec_is_framework_neutral``
|
||||
sweeps every shared spec and every skill tree, and Amendment 2 extended it to cover this skill.
|
||||
Two copies of one rule is the ko-(p) drift class, and the plan concedes the framework guard is
|
||||
close to trivially green on pure prose anyway. **The TERMINOLOGY guard is the one that bites**, and
|
||||
it lives here: the customer-facing wording is "knowledge base" / "knowledge bundle", never the
|
||||
internal format name, so the literal string ``OKF bundle`` appearing in the skill's prose turns
|
||||
this file red.
|
||||
|
||||
**Amendment 3 decides the round trip's input, and it is not a preference.** ``frontmatter_verbatim``
|
||||
is AUTHORITATIVE: it is the bytes a test materialises into a throwaway concept file before reading
|
||||
them back. The sibling ``frontmatter`` object is a line-oriented PROJECTION and informative only --
|
||||
by construction it cannot carry a block form, which is why concept 2 has no ``sources`` key there.
|
||||
Materialising from ``frontmatter`` would derive ``state: absent`` and silently lose the
|
||||
``unreadable`` case the example exists to demonstrate: the arm would be GREEN while proving the
|
||||
opposite of its docstring. The example's own judgement is ``undecided`` -- not ``survived`` -- for
|
||||
the same reason, because a claim whose one possible refuter was unreadable was never attacked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from portfolio_optimiser import okf, persona
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL_DIR = REPO_ROOT / "shared" / "skills" / "falsification-reviewer"
|
||||
EXAMPLE = SKILL_DIR / "references" / "example-evidence.json"
|
||||
|
||||
#: The internal format name. Customer-facing prose says "knowledge base" / "knowledge bundle".
|
||||
_INTERNAL_NAME = re.compile(r"OKF bundle")
|
||||
|
||||
|
||||
def test_the_skill_ships_its_two_files_with_usable_frontmatter() -> None:
|
||||
"""(a) Structure. RED before the subtree pull has landed -- which is exactly Step 12's
|
||||
On-failure clause: if the files are absent, STOP; do not author under ``shared/`` locally."""
|
||||
skill_md = SKILL_DIR / "SKILL.md"
|
||||
assert skill_md.is_file(), "shared/skills/falsification-reviewer/SKILL.md missing"
|
||||
assert EXAMPLE.is_file(), "the skill's worked example is missing"
|
||||
|
||||
fm = okf.parse_frontmatter(skill_md)
|
||||
assert fm.get("name") == "falsification-reviewer"
|
||||
assert fm.get("description", "").strip(), "SKILL.md description must be non-empty"
|
||||
json.loads(EXAMPLE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_the_skill_never_uses_the_internal_format_name() -> None:
|
||||
"""(b) The DISCRIMINATING guard. Paired with a KNOWN-POSITIVE control: a guard that searched
|
||||
for something no file could ever contain would be green by construction, so the same regex is
|
||||
first shown to fire on text that does carry the string."""
|
||||
assert _INTERNAL_NAME.search("read the OKF bundle at that path") is not None, (
|
||||
"the guard cannot find the string it is supposed to forbid -- it proves nothing"
|
||||
)
|
||||
for f in sorted(p for p in SKILL_DIR.rglob("*") if p.is_file()):
|
||||
hit = _INTERNAL_NAME.search(f.read_text(encoding="utf-8"))
|
||||
assert hit is None, f"internal format name {hit.group(0)!r} in customer-facing prose: {f}"
|
||||
|
||||
|
||||
def test_the_example_resolves_at_call_time_not_at_import(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""(c) The loader seam, mirroring ``persona.load_persona_example``: resolved INSIDE the call
|
||||
(never frozen into a default argument), so both the test seam and ``PORTFOLIO_SHARED_ROOT``
|
||||
stay live. A loader that bound its path at import would answer from the tree that existed when
|
||||
the module was first imported."""
|
||||
real = persona.load_falsification_example()
|
||||
assert real.judgement == "undecided"
|
||||
|
||||
stand_in = tmp_path / "example-evidence.json"
|
||||
stand_in.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"claim": "c",
|
||||
"judgement": "survived",
|
||||
"refuter": None,
|
||||
"concepts": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(persona, "_FALSIFICATION_EXAMPLE_PATH", stand_in)
|
||||
assert persona.load_falsification_example().judgement == "survived"
|
||||
|
||||
|
||||
def test_a_missing_example_fails_fast_rather_than_degrading(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""(d) The example is REQUIRED input (contrast the tolerant Step-7 inbox). A broken shared
|
||||
artefact must surface, not pass silently."""
|
||||
monkeypatch.setattr(persona, "_FALSIFICATION_EXAMPLE_PATH", tmp_path / "nope.json")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
persona.load_falsification_example()
|
||||
|
||||
|
||||
def test_the_worked_example_round_trips_through_the_real_readers(tmp_path: Path) -> None:
|
||||
"""(e) The example is not decoration: every concept's declared ``evidence`` and ``derived``
|
||||
values are re-derived by the SHIPPED readers from the SHIPPED bytes.
|
||||
|
||||
``read_provenance`` takes a MARKDOWN path, not JSON, so the fields are materialised into a
|
||||
throwaway concept file first (Amendment 3) -- ``frontmatter_verbatim`` VERBATIM, never
|
||||
reassembled from the ``frontmatter`` projection. Reassembling would emit concept 2's
|
||||
``sources`` as nothing at all, and the ``unreadable``/``block-sequence`` arm would pass as
|
||||
``absent`` while claiming to prove the opposite.
|
||||
|
||||
Nothing git-tracked is touched: each concept is written under ``tmp_path``.
|
||||
"""
|
||||
example = persona.load_falsification_example()
|
||||
assert len(example.concepts) == 2, "the example must keep both the readable and unreadable case"
|
||||
|
||||
seen_states = set()
|
||||
for concept in example.concepts:
|
||||
path = tmp_path / f"{concept.concept_id}.md"
|
||||
path.write_text(concept.frontmatter_verbatim, encoding="utf-8")
|
||||
|
||||
# The plan's letter: ``read_provenance`` + ``trust_tier``, never ``evidence_for`` with a
|
||||
# non-default key. That is not a stylistic choice — the example itself splits the two,
|
||||
# putting state/reason/items_seen under ``evidence`` and the tier under ``derived``,
|
||||
# because ``FalsificationEvidence.tier`` is only meaningful for the ONE key SPEC §5.3
|
||||
# tiers. Branching on the primitive's own documented three-way return is test-level
|
||||
# dispatch, not a second copy of production logic.
|
||||
result = okf.read_provenance(path, "sources")
|
||||
if concept.state == "present":
|
||||
assert isinstance(result, tuple), concept.concept_id
|
||||
assert len(result) == concept.items_seen, concept.concept_id
|
||||
assert concept.reason is None, concept.concept_id
|
||||
else:
|
||||
assert isinstance(result, okf.UnreadableProvenance), concept.concept_id
|
||||
assert result.reason == concept.reason, concept.concept_id
|
||||
assert result.items_seen == concept.items_seen, concept.concept_id
|
||||
|
||||
# The verified half goes through the SHIPPED ``evidence_for`` on its DEFAULT key — the
|
||||
# path where the tier derivation is the one §5.3 defines.
|
||||
verified = okf.evidence_for(path)
|
||||
assert okf.trust_tier(verified.entries) == concept.trust_tier, concept.concept_id
|
||||
assert okf.adjudication_for(path) == concept.adjudication, concept.concept_id
|
||||
seen_states.add(concept.state)
|
||||
|
||||
assert seen_states == {"present", "unreadable"}, (
|
||||
"the example must exercise BOTH a readable and an unreadable concept -- one of each is what "
|
||||
f"makes the round trip discriminating, got {sorted(seen_states)}"
|
||||
)
|
||||
|
|
@ -26,8 +26,15 @@ from portfolio_optimiser.verdicts import ProposalFeatures, capture_verdict, verd
|
|||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SPEC_PATH = REPO_ROOT / "shared" / "method-spec.md"
|
||||
INGEST_SPEC_PATH = REPO_ROOT / "shared" / "ingest-spec.md"
|
||||
SKILL_DIR = REPO_ROOT / "shared" / "skills" / "expert-reviewer"
|
||||
SKILLS_ROOT = REPO_ROOT / "shared" / "skills"
|
||||
SKILL_DIR = SKILLS_ROOT / "expert-reviewer"
|
||||
EXAMPLE_VERDICT = SKILL_DIR / "references" / "example-verdict.json"
|
||||
#: Every shared Agent Skill the framework-neutrality sweep covers, ENUMERATED rather than globbed —
|
||||
#: "a new spec file is guarded explicitly, never implicitly", the rule this file already states for
|
||||
#: the two specs. The enumeration is kept honest by the coverage arm at the bottom of this module:
|
||||
#: a third skill arriving in a subtree pull turns THAT red until it is listed here, so explicit
|
||||
#: never degrades into stale (the ``_LIVE_DOCS`` coverage pattern).
|
||||
GUARDED_SKILL_DIRS = (SKILL_DIR, SKILLS_ROOT / "falsification-reviewer")
|
||||
BUNDLE_DIR = REPO_ROOT / "shared" / "examples" / "bygg-energi-mikro"
|
||||
|
||||
# Name-shaped framework guard (stricter than the persona test's import-shaped guard): the spec's
|
||||
|
|
@ -190,8 +197,15 @@ def test_method_spec_is_framework_neutral() -> None:
|
|||
NAME a concrete agent framework or vendor stack. Mirrors the persona SKILL.md rule; stricter
|
||||
than the import-shaped guard because the specs are pure prose (no AST to parse). RED the moment
|
||||
a framework name leaks into the shared method prose. Covers BOTH shared specs: the method spec
|
||||
and the ingest spec (I1) — a new spec file is guarded explicitly, never implicitly."""
|
||||
for f in [SPEC_PATH, INGEST_SPEC_PATH, *sorted(p for p in SKILL_DIR.rglob("*") if p.is_file())]:
|
||||
and the ingest spec (I1) — a new spec file is guarded explicitly, never implicitly.
|
||||
|
||||
Amendment 2 (2026-09-02) widened the skill half from ``expert-reviewer`` alone to
|
||||
``GUARDED_SKILL_DIRS``: the falsification-reviewer skill arrived by subtree pull with NO
|
||||
framework guard anywhere, and a sweep pinned to one skill silently stops covering the tree it
|
||||
claims to cover."""
|
||||
guarded = [f for d in GUARDED_SKILL_DIRS for f in sorted(d.rglob("*")) if f.is_file()]
|
||||
assert guarded, "the sweep found no skill files — it would be green by construction"
|
||||
for f in [SPEC_PATH, INGEST_SPEC_PATH, *guarded]:
|
||||
assert f.is_file(), f"expected shared artifact missing: {f}"
|
||||
hit = _FRAMEWORK_NAMES.search(f.read_text(encoding="utf-8"))
|
||||
assert hit is None, f"framework name {hit.group(0)!r} in shared method prose: {f}"
|
||||
|
|
@ -342,3 +356,23 @@ def test_ingest_spec_documents_every_contract_field() -> None:
|
|||
# The §11 golden extraction case layout.
|
||||
for entry in ("manifest.json", "fixture/", "ingested-at.txt", "expected-bundle/"):
|
||||
documented(entry, "golden extraction case")
|
||||
|
||||
|
||||
def test_every_shared_skill_is_covered_by_the_framework_sweep() -> None:
|
||||
"""Fail-closed coverage for Amendment 2's enumeration.
|
||||
|
||||
``GUARDED_SKILL_DIRS`` is written out by hand, following this file's own rule that a new shared
|
||||
artefact is guarded explicitly rather than by a glob that quietly absorbs it. The cost of an
|
||||
explicit list is that it goes stale, and ``shared/`` is a PULL-ONLY subtree — a third skill can
|
||||
arrive here without anyone in this repo deciding to add it. This arm makes staleness LOUD: the
|
||||
enumeration must name every directory under ``shared/skills/``, so the next pull that brings a
|
||||
skill turns this red until it is listed. Mirrors
|
||||
``test_every_document_is_classified_live_or_archive``.
|
||||
"""
|
||||
on_disk = {d for d in SKILLS_ROOT.iterdir() if d.is_dir()}
|
||||
assert on_disk, "no shared skills found — the control cannot distinguish covered from empty"
|
||||
missing = sorted(d.name for d in on_disk - set(GUARDED_SKILL_DIRS))
|
||||
assert not missing, (
|
||||
"shared skills not named in GUARDED_SKILL_DIRS, so the framework sweep does not reach "
|
||||
f"them: {missing}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue