docs(sim): the console trace walks all eight steps, one labelled line each
The simulation proved the loop but printed only four of its eight steps, so a
listener could not follow what they were looking at without narration. This is
presentation only: every value printed is read off the RunResult the run already
returned -- nothing is recomputed against the bundle, nothing is inferred, and
the run path is untouched. The week plan's assumption ("seven of eight steps are
pure presentation") therefore held; no new logic was needed.
Run A walks steps 1-7, the promotion between the runs IS step 8, and Run B is
not re-numbered -- it shows only what changed, which is the marker reaching the
hypothesis prompt. Two honesty limits are visible in what is printed rather than
papered over: `retrieved` is the post-hoc proposal-keyed retrieval, not the
Step-1 fold (the marker line is what evidences the fold reaching the prompt),
and the run carries the checker's DECISION, not its prose -- the decision is
what gates, so it is what is shown.
The working-copy path moves to stderr: mkdtemp is the one non-deterministic
value in the output, and stdout must be byte-identical across runs for the dress
rehearsal's diff check. Status tokens stay VALIDATED/REJECTED in English on
purpose -- the same vocabulary as provenance.validator_decision, which the
Step-6 line prints verbatim.
Verified: `... | grep -cE "^ *Steg [1-8]"` -> 8; two runs byte-identical on
stdout; both a REJECTED and a VALIDATED line for the same candidate; suite
759 passed / 4 skipped; ruff + mypy clean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XoHJCKBTjFKcjsfEQyGbzh
This commit is contained in:
parent
d6f3359fae
commit
e93e921b1f
1 changed files with 119 additions and 35 deletions
|
|
@ -36,6 +36,7 @@ from agent_framework import (
|
||||||
)
|
)
|
||||||
from agent_framework_openai import OpenAIChatCompletionClient
|
from agent_framework_openai import OpenAIChatCompletionClient
|
||||||
|
|
||||||
|
from portfolio_optimiser.ir import SavingsProposal
|
||||||
from portfolio_optimiser.persona import load_persona_example
|
from portfolio_optimiser.persona import load_persona_example
|
||||||
from portfolio_optimiser.run import RunResult, run_project
|
from portfolio_optimiser.run import RunResult, run_project
|
||||||
from portfolio_optimiser.shared_root import shared_root
|
from portfolio_optimiser.shared_root import shared_root
|
||||||
|
|
@ -312,24 +313,101 @@ def _outcome_line(result: RunResult) -> str:
|
||||||
o = result.outcome
|
o = result.outcome
|
||||||
if isinstance(o, ValidatedProposal):
|
if isinstance(o, ValidatedProposal):
|
||||||
return (
|
return (
|
||||||
f"VALIDATED (claimed {o.proposal.claimed_saving_nok:.0f} <= P90 {o.p90:.0f} NOK; "
|
f"VALIDATED (påstått {o.proposal.claimed_saving_nok:.0f} <= P90 {o.p90:.0f} NOK; "
|
||||||
f"measure: {o.proposal.measure})"
|
f"tiltak: {o.proposal.measure})"
|
||||||
)
|
)
|
||||||
return f"REJECTED ({o.reason})"
|
return f"REJECTED ({o.reason})"
|
||||||
|
|
||||||
|
|
||||||
def _refinement_lines(result: RunResult) -> list[str]:
|
def _clip(text: str, limit: int = 92) -> str:
|
||||||
"""Step 5 made visible: every falsification that was fed back into a further hypothesis. Empty
|
"""One-line, length-capped rendering of a captured agent text. The trace is a walkthrough a
|
||||||
when the first candidate validated — printing nothing is the honest output there."""
|
listener can follow, not a transcript dump."""
|
||||||
lines = []
|
flat = " ".join(text.split())
|
||||||
for n, rejected in enumerate(result.refinements, start=1):
|
return flat if len(flat) <= limit else flat[: limit - 1] + "…"
|
||||||
lines.append(
|
|
||||||
f" steg 5 #{n} : REJECTED (claimed "
|
|
||||||
f"{rejected.proposal.claimed_saving_nok:.0f} NOK) — {rejected.reason}"
|
def _items_line(proposal: SavingsProposal) -> str:
|
||||||
)
|
return ", ".join(f"{i.code} {i.quantity:g} x {i.unit_cost:g}" for i in proposal.affected_items)
|
||||||
lines.append(
|
|
||||||
" -> grunnen mates tilbake i neste hypotese (bundet av max_attempts)"
|
|
||||||
|
def _first_hypothesis(result: RunResult) -> SavingsProposal:
|
||||||
|
"""Step 2 made visible: the FIRST candidate the run produced. When Step 5 corrected the run,
|
||||||
|
that first candidate is the one the validator falsified (``refinements[0]``); with no
|
||||||
|
refinement, the candidate that left the run IS the first one."""
|
||||||
|
if result.refinements:
|
||||||
|
return result.refinements[0].proposal
|
||||||
|
return result.outcome.proposal
|
||||||
|
|
||||||
|
|
||||||
|
def _step5_lines(result: RunResult) -> list[str]:
|
||||||
|
"""Step 5 made visible: every falsification that was fed back into a further hypothesis, then
|
||||||
|
what the bounded loop ended on. No refinement means the first hypothesis validated — saying so
|
||||||
|
is the honest output there, not printing nothing."""
|
||||||
|
if not result.refinements:
|
||||||
|
return [" (ingen — første hypotese validerte; ingen forbedring var nødvendig)"]
|
||||||
|
lines = [
|
||||||
|
f" #{n}: grunnen fra {rejected.proposal.claimed_saving_nok:.0f} NOK-hypotesen mates "
|
||||||
|
"tilbake i neste forsøk (bundet av max_attempts)"
|
||||||
|
for n, rejected in enumerate(result.refinements, start=1)
|
||||||
|
]
|
||||||
|
lines.append(f" etter forbedring: {_outcome_line(result)}")
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_line(result: RunResult) -> str:
|
||||||
|
"""Step 6 made visible: the TYPED outcome that leaves the run, with BOTH falsifiers named —
|
||||||
|
the deterministic validator gated the numbers, the checker gated the reasoning."""
|
||||||
|
o = result.outcome
|
||||||
|
if isinstance(o, ValidatedProposal):
|
||||||
|
return (
|
||||||
|
f"FORESLÅTT — {o.proposal.measure}: {o.proposal.claimed_saving_nok:.0f} NOK "
|
||||||
|
f"(validator={result.provenance.validator_decision}, checker={result.checker_verdict})"
|
||||||
)
|
)
|
||||||
|
return f"FORKASTET — {o.reason}"
|
||||||
|
|
||||||
|
|
||||||
|
def _run_trace_lines(result: RunResult, *, marker: str, marker_in_prompt: bool) -> list[str]:
|
||||||
|
"""Steps 1-7 of one run, one labelled line per step (``method-spec`` §3), for the walkthrough.
|
||||||
|
|
||||||
|
PRESENTATION ONLY: every value is read off the ``RunResult`` the run already returned — nothing
|
||||||
|
is recomputed against the bundle and nothing is inferred. Two honesty limits are visible in what
|
||||||
|
is printed rather than papered over:
|
||||||
|
|
||||||
|
* ``retrieved`` is the POST-hoc, proposal-keyed retrieval (``run.py`` step 7), not the Step-1
|
||||||
|
fold itself. The marker line is the evidence that a prior verdict reached the hypothesis
|
||||||
|
PROMPT, which is the property Step 1 actually claims.
|
||||||
|
* the run carries the checker's DECISION, not its prose (``_checker_verdict`` parses the gate
|
||||||
|
marker and keeps the decision). The decision is what gates, so it is what is shown.
|
||||||
|
"""
|
||||||
|
hypothesis = _first_hypothesis(result)
|
||||||
|
files = [c.file for c in result.provenance.citations]
|
||||||
|
lines = [
|
||||||
|
" Steg 1 — FORSTÅ KONTEKSTEN (navigert kunnskapsbase + tidligere dommer)",
|
||||||
|
f" navigerte konseptfiler ({len(files)}): {', '.join(files)}",
|
||||||
|
f" tidligere dommer hentet for kandidaten: {len(result.retrieved)}",
|
||||||
|
f" markør '{marker}' i hypotese-prompten: {marker_in_prompt}",
|
||||||
|
" Steg 2 — HYPOTESE (kandidat med parametere)",
|
||||||
|
f" tiltak: {hypothesis.measure}",
|
||||||
|
f" kostlinjer: {_items_line(hypothesis)}",
|
||||||
|
f" påstått besparelse: {hypothesis.claimed_saving_nok:.0f} NOK",
|
||||||
|
" Steg 3 — DEBATT (maker-checker, Group Chat)",
|
||||||
|
f" proposer (konvergert): {_clip(result.debate_output)}",
|
||||||
|
f" checker (gate på resonnementet): VERDICT={result.checker_verdict.upper()}",
|
||||||
|
" Steg 4 — VALIDER / FALSIFISER (deterministisk, blokkerende)",
|
||||||
|
]
|
||||||
|
if result.refinements:
|
||||||
|
# The status tokens stay VALIDATED/REJECTED (English) on purpose: they are the same
|
||||||
|
# vocabulary as ``provenance.validator_decision``, which the Step-6 line prints verbatim.
|
||||||
|
lines.append(f" hypotese #1: REJECTED ({result.refinements[0].reason})")
|
||||||
|
else:
|
||||||
|
lines.append(f" {_outcome_line(result)}")
|
||||||
|
lines.append(" Steg 5 — FORBEDRE, INFORMERT OG BUNDET")
|
||||||
|
lines.extend(_step5_lines(result))
|
||||||
|
lines.append(" Steg 6 — FORKAST ELLER FORESLÅ (typet utfall forlater kjøringen)")
|
||||||
|
lines.append(f" {_decision_line(result)}")
|
||||||
|
lines.append(" Steg 7 — SVAR PÅ TILBAKEMELDING (ekspert-persona, lang fil-løkke)")
|
||||||
|
lines.append(f" dom: {result.verdict.decision}")
|
||||||
|
lines.append(f" begrunnelse: {_clip(result.verdict.rationale, 300)}")
|
||||||
return lines
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -337,47 +415,53 @@ def main(argv: list[str] | None = None) -> int: # pragma: no cover - console tr
|
||||||
"""Run the simulation against the energi bundle in a throwaway temp dir and print an honest,
|
"""Run the simulation against the energi bundle in a throwaway temp dir and print an honest,
|
||||||
readable trace. Invoke: ``uv run python -m portfolio_optimiser.simulation``."""
|
readable trace. Invoke: ``uv run python -m portfolio_optimiser.simulation``."""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
work = tempfile.mkdtemp(prefix="po-sim-")
|
work = tempfile.mkdtemp(prefix="po-sim-")
|
||||||
result = asyncio.run(simulate_learning_loop(str(_default_bundle_dir()), work))
|
result = asyncio.run(simulate_learning_loop(str(_default_bundle_dir()), work))
|
||||||
|
|
||||||
print("=" * 78)
|
print("=" * 78)
|
||||||
print("OFFLINE SIMULATION — scripted agent replies, NO real model.")
|
print("OFFLINE SIMULERING — skriptede agent-svar, INGEN ekte modell.")
|
||||||
print("Proves the loop's dataflow + deterministic spine + that the learning loop closes.")
|
print("Beviser dataflyten, den deterministiske ryggraden og at læringssløyfa lukkes.")
|
||||||
print("Does NOT prove a live LLM would produce these — proposal/verdict are scripted.")
|
print("Beviser IKKE at en levende modell ville produsert dette — forslag og dom er skriptet.")
|
||||||
print("=" * 78)
|
print("=" * 78)
|
||||||
|
|
||||||
print("\nRUN A (fresh wiki — no prior verdicts)")
|
# Run A walks steps 1-7 of the method; the promotion between the runs IS step 8. Run B is not
|
||||||
for line in _refinement_lines(result.run_a):
|
# re-numbered — it re-runs the same eight steps, and what the demo needs from it is the ONE
|
||||||
|
# thing that changed: the marker now reaches the hypothesis prompt.
|
||||||
|
print(f"\nKJØRING A ({_PROJECT_ID} — fersk kunnskapsbase, ingen tidligere dommer)")
|
||||||
|
for line in _run_trace_lines(
|
||||||
|
result.run_a, marker=result.marker, marker_in_prompt=result.marker_in_run_a_prompt
|
||||||
|
):
|
||||||
print(line)
|
print(line)
|
||||||
print(f" validator : {_outcome_line(result.run_a)}")
|
print(" (forventet: markøren er FRAVÆRENDE her — dommen finnes ikke i wikien ennå)")
|
||||||
print(f" checker : VERDICT={result.run_a.checker_verdict.upper()}")
|
|
||||||
print(f" persona : {result.run_a.verdict.decision} -> {result.run_a.verdict.rationale}")
|
|
||||||
print(
|
|
||||||
f" prompt has marker '{result.marker}': {result.marker_in_run_a_prompt} (expected False)"
|
|
||||||
)
|
|
||||||
|
|
||||||
print("\nPROMOTE (gated wiki-promotion, Steg 8)")
|
print("\n Steg 8 — PROMOTER GODKJENT KUNNSKAP (gatet wiki-promotering)")
|
||||||
print(f" wrote : {result.promoted_path.name} (linked into index.md, neutral label)")
|
print(f" skrev: {result.promoted_path.name} (lenket i index.md, nøytral etikett)")
|
||||||
|
print(" gaten er fail-closed: kun en godkjent dom promoteres — rå agent-output aldri")
|
||||||
|
|
||||||
print("\nRUN B (re-seeded wiki — reads the promoted verdict)")
|
print("\nKJØRING B (re-seedet kunnskapsbase — leser den promoterte dommen)")
|
||||||
for line in _refinement_lines(result.run_b):
|
print(" samme åtte steg kjøres igjen; her vises kun det som ENDRET seg:")
|
||||||
print(line)
|
print(f" tidligere dommer hentet for kandidaten: {len(result.run_b.retrieved)}")
|
||||||
print(f" validator : {_outcome_line(result.run_b)}")
|
|
||||||
print(
|
print(
|
||||||
f" prompt has marker '{result.marker}': {result.marker_in_run_b_prompt} (expected True)"
|
f" markør '{result.marker}' i hypotese-prompten: "
|
||||||
|
f"{result.marker_in_run_b_prompt} (forventet True)"
|
||||||
)
|
)
|
||||||
|
print(f" utfall: {_decision_line(result.run_b)}")
|
||||||
|
|
||||||
closed = result.marker_in_run_b_prompt and not result.marker_in_run_a_prompt
|
closed = result.marker_in_run_b_prompt and not result.marker_in_run_a_prompt
|
||||||
print("\n" + "-" * 78)
|
print("\n" + "-" * 78)
|
||||||
if closed:
|
if closed:
|
||||||
print("LEARNING LOOP CLOSED: the persona knowledge approved in Run A reached Run B's")
|
print("LÆRINGSSLØYFA ER LUKKET: kunnskapen eksperten godkjente i kjøring A nådde kjøring")
|
||||||
print("hypothesis purely via the file-backed OKF wiki (promote -> re-seed -> ExpeL fold).")
|
print("B's hypotese utelukkende via den fil-baserte wikien (promoter -> re-seed -> fold).")
|
||||||
else:
|
else:
|
||||||
print("LEARNING LOOP NOT CLOSED — the marker did not cross runs as expected.")
|
print("LÆRINGSSLØYFA ER IKKE LUKKET — markøren krysset ikke kjøringene som forventet.")
|
||||||
print("-" * 78)
|
print("-" * 78)
|
||||||
print(f"\n(working copy: {work})")
|
# The throwaway copy's path is the ONE non-deterministic value here (``mkdtemp``), so it goes to
|
||||||
|
# stderr: stdout is then byte-identical across runs, which is what the dress rehearsal's
|
||||||
|
# ``diff <(run1) <(run2)`` check compares. It stays visible on a terminal either way.
|
||||||
|
print(f"\n(arbeidskopi: {work})", file=sys.stderr)
|
||||||
return 0 if closed else 1
|
return 0 if closed else 1
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue