portfolio-optimiser/src/portfolio_optimiser/verdicts.py

575 lines
24 KiB
Python

"""VerdictStore + ExpeL retrieval + Layer-2 out-of-band verdict capture (B2).
The learning substrate: a structurally-ranked store of historical expert verdicts, surfaced
as ExpeL few-shots for the next run. **Similarity is structural, not textual** — a weighted
score over the affected cost-code set (Jaccard) + measure-type match + a magnitude bucket;
raw ``description`` text is deliberately excluded, so a true match with different wording
beats surface-text decoys.
Two corrections vs the Fase 1 spike:
1. **Two-arg ``extend_instructions``** (the Critical Fase 1 bug): ``before_run`` calls
``context.extend_instructions(self.source_id, [...])`` — the genuine GA signature
(``_sessions.py:253``), exercised against a REAL ``SessionContext`` in the tests.
2. **Layer-2 capture** (``capture_verdict``): mints a stable content-hash ``Verdict.id`` so a
structurally identical proposal maps to the same id (the learning-loop key). The store is
**in-memory only** for the MVP — durable persistence is deferred to Fase 3.
"""
from __future__ import annotations
import hashlib
import json
import logging
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from agent_framework import ContextProvider, SessionContext
from portfolio_optimiser import okf, semretrieval
_log = logging.getLogger(__name__)
# Weights: the affected cost-code overlap dominates, then measure type, then magnitude.
_W_CODES, _W_MEASURE, _W_MAGNITUDE = 0.60, 0.25, 0.15
_MAGNITUDE_BUCKETS = [(0.0, 1e5), (1e5, 5e5), (5e5, 1e6), (1e6, float("inf"))]
@dataclass(frozen=True)
class ProposalFeatures:
"""The *structured* features retrieval ranks on. ``description`` is surface text and is
deliberately NOT part of the similarity score (nor the minted id)."""
affected_codes: frozenset[str]
measure_type: str
claimed_saving_nok: float
description: str = ""
@dataclass(frozen=True)
class Verdict:
"""One historical expert verdict in the store."""
id: str
proposal_features: ProposalFeatures
decision: str # "approved" | "rejected"
rationale: str
def _magnitude_bucket(value: float) -> int:
for i, (low, high) in enumerate(_MAGNITUDE_BUCKETS):
if low <= value < high:
return i
return len(_MAGNITUDE_BUCKETS) - 1
def _jaccard(a: frozenset[str], b: frozenset[str]) -> float:
union = a | b
return len(a & b) / len(union) if union else 1.0
def similarity(query: ProposalFeatures, candidate: ProposalFeatures) -> float:
"""Weighted structural similarity in [0, 1] — text is ignored by design."""
codes = _jaccard(query.affected_codes, candidate.affected_codes)
measure = 1.0 if query.measure_type == candidate.measure_type else 0.0
magnitude = (
1.0
if _magnitude_bucket(query.claimed_saving_nok)
== _magnitude_bucket(candidate.claimed_saving_nok)
else 0.0
)
return _W_CODES * codes + _W_MEASURE * measure + _W_MAGNITUDE * magnitude
def _mint_id(features: ProposalFeatures) -> str:
"""Stable content-hash id over the STRUCTURAL fields (not description), so a
structurally identical proposal maps to the same id."""
canonical = json.dumps(
{
"affected_codes": sorted(features.affected_codes),
"measure_type": features.measure_type,
"claimed_saving_nok": features.claimed_saving_nok,
},
sort_keys=True,
separators=(",", ":"),
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16]
def capture_verdict(features: ProposalFeatures, decision: str, rationale: str) -> Verdict:
"""Layer-2 out-of-band verdict constructor: mint a stable content-hash id (the
learning-loop key) and build the ``Verdict`` to persist in the store."""
return Verdict(
id=_mint_id(features),
proposal_features=features,
decision=decision,
rationale=rationale,
)
# --- Async file inbox (Fase 5, Steg 7): plain-JSON verdict serialization + tolerant folder load ---
# The raw output layer is a plain JSON store (R2), NOT OKF — OKF is reserved for the promoted
# (Step-8) layer. One verdict per file lets an expert/persona drop verdicts into a folder out of
# band, and a separate later run pick them up (målbilde §3 long loop).
_REQUIRED_VERDICT_KEYS = {"id", "decision", "rationale", "proposal_features"}
# The binary run-path decision vocabulary (Verdict.decision domain + FeedbackContract). An inbox file
# with any other decision is SKIPPED by load_verdicts_from_dir — ``approved_with_adjustment`` is
# deliberately EXCLUDED here (it lives only in bundle-seed frontmatter + the promotion gate).
_INBOX_DECISION_VOCABULARY = frozenset({"approved", "rejected"})
def verdict_to_dict(verdict: Verdict) -> dict[str, Any]:
"""Serialize a ``Verdict`` to a JSON-ready dict. ``affected_codes`` (a frozenset) is emitted as
a SORTED list — matching ``_mint_id``'s canonical form — so the round-trip is lossless and the
on-disk form is deterministic."""
f = verdict.proposal_features
return {
"id": verdict.id,
"decision": verdict.decision,
"rationale": verdict.rationale,
"proposal_features": {
"affected_codes": sorted(f.affected_codes),
"measure_type": f.measure_type,
"claimed_saving_nok": f.claimed_saving_nok,
"description": f.description,
},
}
def verdict_from_dict(data: dict[str, Any]) -> Verdict:
"""Reconstruct a ``Verdict`` from its serialized dict. The ``id`` is read VERBATIM — never
re-minted — so a verdict authored elsewhere keeps its identity and the int/float magnitudes in
its features survive the JSON round-trip untouched (re-minting would re-hash and could diverge,
since ``_mint_id`` hashes the raw value, e.g. ``30000`` vs ``30000.0``)."""
pf = data["proposal_features"]
return Verdict(
id=data["id"],
proposal_features=ProposalFeatures(
affected_codes=frozenset(pf["affected_codes"]),
measure_type=pf["measure_type"],
claimed_saving_nok=pf["claimed_saving_nok"],
description=pf.get("description", ""),
),
decision=data["decision"],
rationale=data["rationale"],
)
def write_verdict(directory: str, verdict: Verdict) -> Path:
"""Write one verdict as ``{id}.json`` into ``directory`` (created if needed) and return the path.
This is the public authoring primitive the expert/persona (sim) or human (prod) uses to drop a
verdict into the async inbox via the SAME folder interface — and the writer a future Step-8
promotion would reuse. It is deliberately NOT called inside ``run_project``: the system READS the
inbox, it does not write to it (målbilde §3 role split).
Limitation: ``id`` is a content hash of the FEATURES only (``_mint_id``), so two verdicts with
identical features but different decisions share a filename — last write wins on disk (mirroring
``VerdictStore.add``'s first-wins in memory). Acceptable for the raw MVP layer."""
path = Path(directory)
path.mkdir(parents=True, exist_ok=True)
target = path / f"{verdict.id}.json"
target.write_text(
json.dumps(verdict_to_dict(verdict), sort_keys=True, indent=2), encoding="utf-8"
)
return target
def load_verdicts_from_dir(
directory: str,
*,
max_rationale_len: int | None = None,
max_files: int | None = None,
) -> list[Verdict]:
"""Load every well-formed verdict JSON file from an async inbox folder, TOLERANTLY (OKF §4
spirit): a missing folder yields ``[]``; files that are not ``.json``, fail to parse, lack a
required key, carry a decision outside the binary vocabulary ``{approved, rejected}``, or exceed
``max_rationale_len`` are SKIPPED, never raised. The folder is written out of band by an external
party, so half-written or foreign files (e.g. a bundle's ``golden.json``) are realistic — this is
the raw layer, not required input (contrast ``okf.load_ir_projection``'s fail-fast). Order is
deterministic (sorted by filename).
Herding (S2.5), two distinct axes by design:
- ``max_rationale_len`` (per-file, TOLERANT): an over-long rationale is skipped + logged, never
raised — matching the per-file skip contract the Step-7 loop relies on.
- ``max_files`` (aggregate, FAIL-FAST): more files than the cap RAISES ``ValueError`` before any
parsing — an oversized inbox is an aggregate integrity signal, not a per-file anomaly.
Both default to ``None`` (no cap), so existing callers are byte-for-byte unaffected."""
path = Path(directory)
if not path.is_dir():
return []
files = sorted(path.glob("*.json"))
if max_files is not None and len(files) > max_files:
raise ValueError(
f"verdict inbox {directory!r} has {len(files)} files, over the max_files cap "
f"({max_files}) — an oversized inbox is a fail-fast aggregate guard (contrast the "
"per-file tolerant skips)"
)
verdicts: list[Verdict] = []
for file in files:
try:
data = json.loads(file.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
# UnicodeDecodeError: a *.json file hand-saved in Latin-1 (Norwegian æ/ø/å) is invalid
# UTF-8 — a per-file tolerant skip, not a raise (a ValueError subclass, caught by neither
# OSError nor JSONDecodeError). Kept in lockstep with hitl._load_json_dict.
continue
if not isinstance(data, dict) or not _REQUIRED_VERDICT_KEYS <= data.keys():
continue
if data.get("decision") not in _INBOX_DECISION_VOCABULARY:
_log.debug(
"skipping inbox verdict %s: decision %r outside vocabulary",
file.name,
data.get("decision"),
)
continue
if (
max_rationale_len is not None
and len(str(data.get("rationale", ""))) > max_rationale_len
):
_log.warning(
"skipping inbox verdict %s: rationale length %d exceeds max_rationale_len %d",
file.name,
len(str(data.get("rationale", ""))),
max_rationale_len,
)
continue
try:
verdicts.append(verdict_from_dict(data))
except (KeyError, TypeError):
continue
return verdicts
@dataclass
class VerdictStore:
"""Minimal store of historical verdicts. In-memory by default; ``from_dir`` / the
``load_verdicts_from_dir`` merge primitive back it with the async file inbox (Steg 7)."""
verdicts: list[Verdict]
# S3.1 opt-in seam: ``None`` means the structural default, which is byte-identical to the
# pre-seam inline sort. Only an explicit caller (``run.py --semantic-retrieval``) installs a
# different ranker, so default retrieval behaviour is unchanged.
retriever: semretrieval.Retriever | None = None
@classmethod
def from_dir(cls, directory: str) -> VerdictStore:
"""Build a store from an async inbox folder (convenience constructor). ``run_project`` uses
the merge primitive (``add`` per loaded verdict) instead, to preserve a passed store's
existing verdicts (the cross-project threading in ``run_portfolio``); this is for callers
that want a fresh store straight from a folder."""
return cls(verdicts=load_verdicts_from_dir(directory))
def retrieve(self, query: ProposalFeatures, k: int) -> list[Verdict]:
"""Return the top-``k`` verdicts. Deterministic: ties break by verdict id, so ordering
is stable across runs.
Ranking is delegated to ``self.retriever``, defaulting to ``StructuralRetriever`` — the
same weighted structural score and ``(-similarity, id)`` key as before the seam existed.
A caller that installs a ``HybridRanker`` opts into an additional semantic term."""
if k <= 0:
raise ValueError(f"k must be positive, got {k}")
ranker = (
self.retriever
if self.retriever is not None
else semretrieval.StructuralRetriever(similarity)
)
return ranker.rank(query, self.verdicts, k)
def add(self, verdict: Verdict) -> None:
"""Persist a captured verdict in-memory.
Conflict semantics (chosen, documented): the store is FIRST-write-wins per ``id`` — a
later verdict with the same content-hash id is dropped, which makes repeated Step-7 inbox
merges idempotent. The disk layers are the opposite: ``write_verdict`` (inbox) and
``promote_verdict`` (wiki) are last-write-wins per file. Because ids hash the candidate
FEATURES, "same id" means "same candidate measure", not "same verdict event". A full
verdict-conflict taxonomy (B10: rejection categories + a rule for conflicting expert
verdicts) is deliberately deferred until real experts produce conflicting verdicts."""
if all(v.id != verdict.id for v in self.verdicts):
self.verdicts.append(verdict)
class ExpeLContextProvider(ContextProvider):
"""Wraps ``VerdictStore.retrieve`` for ExpeL few-shot injection via the GA
``ContextProvider`` hook."""
def __init__(self, store: VerdictStore, query: ProposalFeatures, *, k: int = 3) -> None:
super().__init__(source_id="expel-verdictstore")
self._store = store
self._query = query
self._k = k
def format_fewshot(self) -> str:
hits = self._store.retrieve(self._query, self._k)
body = "\n".join(f"- [{v.id}] {v.decision}: {v.rationale}" for v in hits)
return f"Relevant prior verdicts (ExpeL few-shot):\n{body}"
async def before_run(
self,
*,
agent: object,
session: object,
context: SessionContext,
state: dict,
) -> None:
# Two-arg GA signature (_sessions.py:253) — the Fase 1 single-arg bug is fixed.
context.extend_instructions(self.source_id, [self.format_fewshot()])
def seed_store() -> VerdictStore:
"""Seed 12 synthetic verdicts spanning the reference domain's cost codes, measure types,
magnitudes, and decisions (B2 store of 10-20)."""
rows = [
(
"V01",
{"05.2", "03.1"},
"scope_reduction",
180_000,
"approved",
"asphalt + base course trimmed within feasible range",
),
(
"V02",
{"05.2"},
"rate_renegotiation",
60_000,
"approved",
"renegotiated asphalt unit rate",
),
(
"V03",
{"07.4"},
"material_substitution",
120_000,
"rejected",
"granite kerb substitution unsafe",
),
(
"V04",
{"09.1"},
"scope_reduction",
240_000,
"approved",
"fewer LED masts on low-traffic stretch",
),
(
"V05",
{"02.3", "03.1"},
"scope_reduction",
350_000,
"rejected",
"soil replacement is load-bearing, cannot cut",
),
("V06", {"21.2"}, "rate_renegotiation", 90_000, "approved", "blasting rate renegotiated"),
(
"V07",
{"22.4"},
"material_substitution",
700_000,
"rejected",
"fiber shotcrete spec is mandated",
),
(
"V08",
{"88.2"},
"scope_reduction",
150_000,
"approved",
"concrete repair area re-measured smaller",
),
(
"V09",
{"87.3"},
"material_substitution",
130_000,
"approved",
"alternative membrane qualified",
),
(
"V10",
{"05.2", "03.1"},
"rate_renegotiation",
200_000,
"approved",
"combined paving rate discount",
),
(
"V11",
{"01.1"},
"scope_reduction",
95_000,
"rejected",
"rigging is fixed cost, no scope to cut",
),
(
"V12",
{"31.3"},
"scope_reduction",
110_000,
"approved",
"drainage length reduced after survey",
),
]
return VerdictStore(
verdicts=[
Verdict(
id=vid,
proposal_features=ProposalFeatures(
affected_codes=frozenset(codes),
measure_type=mtype,
claimed_saving_nok=saving,
description=desc,
),
decision=decision,
rationale=desc,
)
for vid, codes, mtype, saving, decision, desc in rows
]
)
# --- OKF-bundle seeding (Fase 2a): turn a project's bundle into the ExpeL substrate ---
def _features_from_ir(ir: dict[str, Any]) -> ProposalFeatures:
"""Map a bundle's IR projection (``validator-input.json``) to the structural features the
store ranks on: the affected cost-code set, the measure string, and the claimed magnitude."""
return ProposalFeatures(
affected_codes=frozenset(item["code"] for item in ir["affected_items"]),
measure_type=ir["measure"],
claimed_saving_nok=ir["claimed_saving_nok"],
description=ir.get("measure", ""),
)
def bundle_candidate_features(bundle_dir: str) -> ProposalFeatures:
"""The pre-hypothesis ExpeL query key: the candidate measure's structural features, read from
the OKF bundle's IR projection. Available BEFORE any proposal is generated — which is what lets
Step-1 retrieve prior verdicts and fold them into the hypothesis prompt (målbilde §2 step 1)."""
return _features_from_ir(okf.load_ir_projection(bundle_dir))
def _verdict_rationale(fm: dict[str, str]) -> str:
"""Build the few-shot rationale from a ``type: verdict`` file's frontmatter, carrying the
learning signal the deterministic validator cannot compute (the realization rate + expected
actual). This is the ExpeL signal that must reach the next hypothesis."""
base = fm.get("description", "")
signal = [
f"{label}={fm[key]}"
for key, label in (
("realization_rate", "realiseringsgrad"),
("expected_actual_saving_nok", "forventet_faktisk_NOK"),
)
if fm.get(key)
]
return f"{base} [{'; '.join(signal)}]" if signal else base
def seed_store_from_bundle(bundle_dir: str) -> VerdictStore:
"""Build a ``VerdictStore`` from an OKF bundle's ``type: verdict`` files. Each verdict is keyed
on the bundle's candidate features (so it retrieves for that measure) and carries the
realization signal in its rationale. The seed verdict stands in for the durable HITL verdict a
real expert would supply via the same folder interface (målbilde §3)."""
features = bundle_candidate_features(bundle_dir)
bundle = okf.navigate_bundle(bundle_dir)
verdicts = [
capture_verdict(
features,
vf.frontmatter.get("decision", "approved"),
_verdict_rationale(vf.frontmatter),
)
for vf in bundle.verdicts
]
return VerdictStore(verdicts=verdicts)
# --- Gated wiki-promotion (Fase 6, Steg 8): output layer -> context layer, HITL-gated ------------
# målbilde §3 (promoterings-gate) / §6 (kun godkjent kunnskap, aldri rå agent-output; provenance) /
# §7 (load-bearing: a non-approved verdict must NOT reach the wiki). R4 = optional+gated: this is a
# PUBLIC opt-in primitive, deliberately NOT wired into run_project — the system reads context; the
# gate/persona promotes (mirrors write_verdict's role split).
_APPROVED_DECISIONS = frozenset({"approved", "approved_with_adjustment"})
# A FIXED neutral index label carrying NO verdict signal. link_in_index folds it into index.md ->
# index_summary -> bundle_context verbatim, so passing the rationale here would leak the realization
# signal into the read-context on a path that bypasses the gate (målbilde §3/§6). Load-bearing:
# test_step8 Test C goes red if this is replaced by the rationale.
_PROMOTED_LINK_LABEL = "Promotert ekspert-vurdering (gated)"
class PromotionRefused(Exception):
"""The gate (målbilde §6): a non-approved verdict was offered for promotion. Fail-closed — the
wiki receives ONLY human/persona-approved knowledge, never raw agent output (self-contamination).
"""
def _safe_filename_token(verdict_id: str) -> str:
"""Turn a verbatim ``Verdict.id`` (arbitrary author string — sentinels, hashes, anything) into a
filename/link-safe token: keep ``[A-Za-z0-9._-]``, replace the rest with ``-``. A token that is
only separators/dots (degenerate, e.g. ``".."``) falls back to a content hash. This prevents an
id with ``/`` (an unnavigable link) or ``..`` from steering the written path — defense beside
``write_concept_file``'s fail-closed ``safe_resolve``. The original id is kept in frontmatter."""
token = re.sub(r"[^A-Za-z0-9._-]", "-", verdict_id)
if not token.strip(".-_"):
return hashlib.sha256(verdict_id.encode("utf-8")).hexdigest()[:16]
return token
def promote_verdict(
bundle_dir: str,
verdict: Verdict,
*,
approver: str,
experiment: str,
timestamp: str,
) -> Path:
"""Promote an APPROVED verdict from the raw output layer into the OKF context layer (the wiki) as
a ``type: verdict`` concept file, navigable by the next run's ``seed_store_from_bundle`` (Steg 8,
målbilde §3/§6/§7). GATE (fail-closed): a verdict whose ``decision`` is not an approval raises
``PromotionRefused`` and writes/links NOTHING — only human/persona-approved knowledge enters the
wiki. Provenance-stamped (who/which-experiment/when). ``timestamp`` is a required keyword (no
wall-clock default) so promotion is deterministic and the stamp reproducible.
The promoted file is MINIMAL: it does NOT reproduce the hand-authored seed's structured fields
(``realization_rate`` etc.) — the raw ``Verdict`` model carries the learning signal only as
``rationale`` prose, which becomes the ``description`` frontmatter ``seed_store_from_bundle``
folds into ExpeL. The index link uses a NEUTRAL label (``_PROMOTED_LINK_LABEL``), so the signal
reaches a prompt only via the gated fold, never via ``bundle_context`` (§3/§6).
Known limitation (mirrors ``write_verdict``): ``_mint_id`` keys on the candidate features, so two
approved verdicts about the SAME candidate share an id -> share a filename -> last-write-wins;
the wiki grows one curated verdict file per distinct candidate measure, not per verdict event.
Returns the written path."""
if verdict.decision not in _APPROVED_DECISIONS:
raise PromotionRefused(
f"refusing to promote a non-approved verdict (decision={verdict.decision!r}); "
"only human/persona-approved knowledge enters the wiki (målbilde §6)"
)
f = verdict.proposal_features
frontmatter = {
"type": "verdict",
"decision": verdict.decision,
"description": verdict.rationale,
"verdict_id": verdict.id,
"provenance": f"godkjent av {approver}; eksperiment {experiment}; {timestamp}",
"timestamp": timestamp,
"tags": "[verdict, promoted, HITL]",
}
codes = ", ".join(sorted(f.affected_codes))
body = (
"# Promotert ekspert-vurdering\n\n"
f"{verdict.rationale}\n\n"
f"- Tiltak: {f.measure_type}\n"
f"- Berørte koder: {codes}\n"
f"- Beslutning: {verdict.decision}\n"
f"- Provenance: {frontmatter['provenance']}\n"
)
filename = f"promoted-verdict-{_safe_filename_token(verdict.id)}.md"
path = okf.write_concept_file(bundle_dir, filename, frontmatter, body)
okf.link_in_index(bundle_dir, filename, _PROMOTED_LINK_LABEL)
return path