feat(portfolio): K12 — CLI parity, doc sync, knowledge-base recipe (parity row 24) [skip-docs]
The last ungated build session: the operator now drives the whole build from the command line, and the documents claim exactly what the code does (§1). run.py becomes the collecting entrance. Exactly one of --bundle (one project) or --portfolio (N projects from a schema-validated reference config, with --verdict-dir as the portfolio-level expert inbox) is required; both and neither are refused. --goals loads a goal contract and checks it against --ledger's realized sum BEFORE the first model call: the §8 caps bound spend, the goal bounds achievement, so a hard target the book already meets stops the run at exit 4 without constructing a client. A soft target reached is a flag and the run continues; an absent ledger is an empty book, so the goal is still evaluated, never skipped. The one declared goal also drives --value-report's goal progress — one contract, never two figures that can disagree. The portfolio path persists nothing (K3 returns typed results; the outbox names pairs by run_id, which a portfolio pass has none of). Rather than accept --out/--outbox/--run-id/--value-report/--inbox/--live-dry-run there and silently ignore them, the entrance refuses them and says why. run_portfolio is imported lazily — portfolio.py imports this module, so a module-level import is circular. Three seams, each detach-proven RED: - unwire the goal check → the run proceeds and spends → red - unwire the portfolio branch → the configured projects never run → red - document a flag no CLI offers → the README honesty grep goes red That last one is the doc-sync made load-bearing: the test reads README.md, collects every --flag it documents (excluding third-party dev-tooling lines) and asserts each exists in the --help of a CLI the README names. The drift it exists to close was real — README claimed 562 tests, CHANGELOG claimed 265, actual 597. Docs synced to the code: README gains an operator-CLI section and honest goal/ portfolio descriptions, CHANGELOG is rewritten to what actually shipped, and docs/oppskrift-kunnskapsbase.md delivers D-H point 1 — the documented team process for building a knowledge base, with the honest 1–2 week expectation and every factory-dependent step (verdict translation, demo path) marked NOT BUILT. 597 passed · ruff clean · mypy strict clean. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MQu2xxwedckjU56byu1aUG
This commit is contained in:
parent
4dcdd8017a
commit
da93a68ce7
6 changed files with 956 additions and 28 deletions
|
|
@ -13,7 +13,17 @@ constructed only by ``default_client_factory`` on the CLI path (wired, never
|
|||
executed by the suite — honesty rule §1); the navigated docs dir comes from
|
||||
the validated startup contract, never straight from the raw argument (§10).
|
||||
|
||||
K12 makes the whole build drivable from here: ``--bundle`` runs ONE project,
|
||||
``--portfolio`` runs N from a schema-validated config (with ``--verdict-dir`` as
|
||||
the portfolio-level expert inbox), ``--goals`` + ``--ledger`` bound ACHIEVEMENT
|
||||
where the §8 caps bound spend, and ``--value-report`` projects what the run
|
||||
delivered. A flag the chosen path cannot honour is REFUSED, never silently
|
||||
ignored (§1).
|
||||
|
||||
Run: uv run python -m portfolio_optimiser_claude.run --bundle <dir> [--inbox <dir>]
|
||||
# N projects, sequential, one shared meter + one shared learning store:
|
||||
uv run python -m portfolio_optimiser_claude.run --portfolio <file> \\
|
||||
[--verdict-dir <dir>]
|
||||
# K8 live-run drill (builds all, captures artifacts, STOPS before the first call):
|
||||
uv run python -m portfolio_optimiser_claude.run --bundle <dir> \\
|
||||
--outbox <dir> --run-id <id> --live-dry-run
|
||||
|
|
@ -22,6 +32,7 @@ Run: uv run python -m portfolio_optimiser_claude.run --bundle <dir> [--inbox <d
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Sequence
|
||||
|
|
@ -34,7 +45,14 @@ from portfolio_optimiser_claude.artifacts import (
|
|||
)
|
||||
from portfolio_optimiser_claude.outbox import persist_outbox
|
||||
from portfolio_optimiser_claude.budget import BudgetExceeded, BudgetMeter
|
||||
from portfolio_optimiser_claude.contracts import Contracts, load_contracts, resolve_model
|
||||
from portfolio_optimiser_claude.contracts import (
|
||||
Contracts,
|
||||
ReferenceProjectsContract,
|
||||
load_contracts,
|
||||
load_reference_projects,
|
||||
resolve_model,
|
||||
)
|
||||
from portfolio_optimiser_claude.goals import GoalContract, GoalReached
|
||||
from portfolio_optimiser_claude.preflight import Refusal, run_preflight
|
||||
from portfolio_optimiser_claude.experience import (
|
||||
CandidateFeatures,
|
||||
|
|
@ -58,12 +76,17 @@ from portfolio_optimiser_claude.notify import (
|
|||
notify_config_from_args,
|
||||
)
|
||||
from portfolio_optimiser_claude.validator import Rejection
|
||||
from portfolio_optimiser_claude.valuereport import build_value_report, report_to_json
|
||||
from portfolio_optimiser_claude.valuereport import build_value_report, load_ledger, report_to_json
|
||||
|
||||
_PROPOSER_ROLE = "proposer"
|
||||
_CHECKER_ROLE = "checker"
|
||||
# The one backend profile the run resolves against (mirrors SdkModelClient's default).
|
||||
_DEFAULT_PROFILE = "anthropic"
|
||||
# Exit codes for the two STRUCTURED stops the entrance can reach. They are
|
||||
# distinct because the criteria are: 3 bounds SPEND (§8), 4 bounds ACHIEVEMENT
|
||||
# (K2). Neither is an error — both are outcomes, and both say so in the output.
|
||||
_BUDGET_STOP_EXIT = 3
|
||||
_GOAL_STOP_EXIT = 4
|
||||
|
||||
# The injected client seam of the entrance: (contracts, max_budget_usd_per_call).
|
||||
ClientFactory = Callable[[Contracts, float], ModelClient]
|
||||
|
|
@ -194,7 +217,7 @@ def execute_run(
|
|||
fields={"kind": stop.kind, "observed": stop.observed, "limit": stop.limit},
|
||||
),
|
||||
)
|
||||
return 3
|
||||
return _BUDGET_STOP_EXIT
|
||||
|
||||
provenance = Provenance(
|
||||
citations=composed.citations,
|
||||
|
|
@ -241,12 +264,145 @@ def execute_run(
|
|||
return 0
|
||||
|
||||
|
||||
def load_goal(path: Path) -> GoalContract:
|
||||
"""Load + validate the goal contract fail-fast (§10) — a startup contract like the rest.
|
||||
|
||||
A percent goal is D-E-gated and refuses loudly here (``NotImplementedError``
|
||||
from ``GoalContract``), before a model client exists — never silent semantics.
|
||||
"""
|
||||
return GoalContract(**json.loads(path.read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
def check_goal_before_spend(
|
||||
goal: GoalContract,
|
||||
ledger_path: Path | None,
|
||||
*,
|
||||
notifiers: Sequence[Notifier] = (),
|
||||
) -> bool:
|
||||
"""Evaluate the goal against the book BEFORE any model call; True ⇒ stop (K12).
|
||||
|
||||
The §8 caps bound SPEND; the goal bounds ACHIEVEMENT — so the honest place
|
||||
to ask "is the target already met?" is ahead of the first call, not after
|
||||
paying for one. Observed value is the ledger's dimension-free realized sum
|
||||
(K1); an absent ``--ledger`` is an EMPTY book (0 realized), so the goal is
|
||||
still evaluated, simply not reached — never a skipped check.
|
||||
|
||||
A HARD goal reached returns True after printing the structured stop and
|
||||
notifying the configured sinks (a stop is an outcome, exactly as a budget
|
||||
stop is). A SOFT goal reached is a flag: it prints and the run continues.
|
||||
"""
|
||||
realized = load_ledger(ledger_path).total_realized_nok()
|
||||
try:
|
||||
reached = goal.check(realized)
|
||||
except GoalReached as stop:
|
||||
print(
|
||||
f"GOAL REACHED (hard): realized {stop.observed_nok} NOK >= target "
|
||||
f"{stop.target_nok} NOK — stopping BEFORE any model call "
|
||||
"(the §8 caps bound spend; the goal bounds achievement)."
|
||||
)
|
||||
emit(
|
||||
notifiers,
|
||||
Notification(
|
||||
event="run.goal_reached",
|
||||
summary=(
|
||||
f"hard goal reached: realized {stop.observed_nok} NOK >= "
|
||||
f"target {stop.target_nok} NOK — run not started"
|
||||
),
|
||||
fields={
|
||||
"mode": goal.mode,
|
||||
"target_nok": stop.target_nok,
|
||||
"observed_nok": stop.observed_nok,
|
||||
},
|
||||
),
|
||||
)
|
||||
return True
|
||||
if reached:
|
||||
print(
|
||||
f"GOAL REACHED (soft): realized {realized} NOK >= target {goal.target_nok} NOK "
|
||||
"— flagged, the run continues."
|
||||
)
|
||||
else:
|
||||
print(f"goal ({goal.mode}): realized {realized} of {goal.target_nok} NOK — not reached.")
|
||||
return False
|
||||
|
||||
|
||||
def execute_portfolio(
|
||||
client: ModelClient,
|
||||
projects: ReferenceProjectsContract,
|
||||
*,
|
||||
contracts: Contracts,
|
||||
top_k: int,
|
||||
max_debate_rounds: int,
|
||||
max_attempts: int,
|
||||
verdict_dir: Path | None,
|
||||
notifiers: Sequence[Notifier] = (),
|
||||
) -> int:
|
||||
"""Drive N projects sequentially under ONE §8 meter; print one line per project (K12).
|
||||
|
||||
This is the CLI reach to ``run_portfolio`` — the capability itself is K3's
|
||||
and is not extended here: the pass returns typed results and persists
|
||||
NOTHING, so nothing is filed and the summary IS the output. A budget stop
|
||||
propagates out of the shared meter and is reported as the structured stop it
|
||||
is (exit 3); because this path files no artifacts, none are claimed.
|
||||
|
||||
``run_portfolio`` is imported lazily: ``portfolio.py`` imports this module
|
||||
for ``compose_run_context``, so a module-level import would be circular.
|
||||
"""
|
||||
from portfolio_optimiser_claude.portfolio import run_portfolio
|
||||
|
||||
meter = BudgetMeter(contracts.termination)
|
||||
try:
|
||||
result = run_portfolio(
|
||||
projects,
|
||||
client,
|
||||
meter,
|
||||
top_k=top_k,
|
||||
max_debate_rounds=max_debate_rounds,
|
||||
max_attempts=max_attempts,
|
||||
verdict_dir=verdict_dir,
|
||||
)
|
||||
except BudgetExceeded as stop:
|
||||
print(f"STOPPED by budget: {stop.kind} observed {stop.observed} > limit {stop.limit}")
|
||||
print("portfolio: no artifacts written — this path persists nothing (§1).")
|
||||
emit(
|
||||
notifiers,
|
||||
Notification(
|
||||
event="run.stopped",
|
||||
summary=f"portfolio stopped by budget: {stop.kind}",
|
||||
fields={"kind": stop.kind, "observed": stop.observed, "limit": stop.limit},
|
||||
),
|
||||
)
|
||||
return _BUDGET_STOP_EXIT
|
||||
|
||||
for project in result.results:
|
||||
outcome_kind = "rejected" if isinstance(project.run.outcome, Rejection) else "validated"
|
||||
print(
|
||||
f"project: {project.project_id} validator={project.run.validator_decision} "
|
||||
f"checker={project.run.checker_decision} attempts={project.run.attempts} "
|
||||
f"outcome={outcome_kind}"
|
||||
)
|
||||
print(
|
||||
f"portfolio: {len(result.results)} project(s) completed — results are returned, "
|
||||
"not filed (this path persists no artifacts)."
|
||||
)
|
||||
emit(
|
||||
notifiers,
|
||||
Notification(
|
||||
event="portfolio.completed",
|
||||
summary=f"portfolio: {len(result.results)} project(s) completed",
|
||||
fields={"projects": [project.project_id for project in result.results]},
|
||||
),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def write_value_report(
|
||||
*,
|
||||
outbox_dir: Path,
|
||||
inbox_dir: Path | None,
|
||||
ledger_path: Path | None,
|
||||
destination: Path,
|
||||
goal: GoalContract | None = None,
|
||||
) -> bool:
|
||||
"""K11 opt-in: project the persisted layers into a value report AFTER the run.
|
||||
|
||||
|
|
@ -256,10 +412,14 @@ def write_value_report(
|
|||
and makes the COMMAND non-zero (the operator asked for a report and did not
|
||||
get one), but a budget stop stays a budget stop: reporting never rewrites a
|
||||
run's own verdict. Returns whether the report was written.
|
||||
|
||||
A ``goal`` (K12's ``--goals``) is passed straight through, so the operator's
|
||||
one declared target drives BOTH the pre-spend stop and the report's goal
|
||||
progress — one contract, never two figures that can disagree.
|
||||
"""
|
||||
try:
|
||||
report = build_value_report(
|
||||
outbox_dir=outbox_dir, inbox_dir=inbox_dir, ledger_path=ledger_path
|
||||
outbox_dir=outbox_dir, inbox_dir=inbox_dir, ledger_path=ledger_path, goal=goal
|
||||
)
|
||||
except (OSError, TypeError, ValueError) as exc:
|
||||
print(f"VALUE REPORT FAILED — refusing to project a malformed layer (§10): {exc}")
|
||||
|
|
@ -374,11 +534,33 @@ def main(
|
|||
canned transport so no socket is opened.
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run one project through the loop (merge inbox → seed → fold → run)."
|
||||
description=(
|
||||
"Run one project (--bundle) or a whole portfolio (--portfolio) through the "
|
||||
"loop (merge inbox → seed → fold → run), under the §8 caps and an optional "
|
||||
"savings goal."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bundle", type=Path, default=None, help="OKF bundle dir of ONE project to run."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--portfolio",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="reference-projects config (JSON): run N projects sequentially under one "
|
||||
"shared §8 meter and one shared learning store (K3).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verdict-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="portfolio-level expert inbox, READ before each project's fold (requires "
|
||||
"--portfolio; a single run uses --inbox).",
|
||||
)
|
||||
parser.add_argument("--bundle", type=Path, required=True)
|
||||
parser.add_argument("--inbox", type=Path, default=None)
|
||||
parser.add_argument("--out", type=Path, default=Path("runs") / "run")
|
||||
parser.add_argument(
|
||||
"--out", type=Path, default=None, help="run-artifact dir (default runs/run)."
|
||||
)
|
||||
parser.add_argument("--outbox", type=Path, default=None)
|
||||
parser.add_argument("--run-id", type=str, default=None)
|
||||
parser.add_argument("--max-rounds", type=int, default=12)
|
||||
|
|
@ -398,7 +580,15 @@ def main(
|
|||
"--ledger",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="realized-savings ledger read by --value-report; absent = an empty book.",
|
||||
help="realized-savings ledger (K1) read by --goals and --value-report; "
|
||||
"absent = an empty book.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--goals",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="goal contract (JSON: target_nok + hard/soft) checked against --ledger "
|
||||
"BEFORE any model call; a hard target already reached stops the run (exit 4).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--live-dry-run",
|
||||
|
|
@ -409,6 +599,42 @@ def main(
|
|||
add_notify_args(parser)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# K12: exactly ONE run shape. Refusing both the empty and the ambiguous call
|
||||
# keeps the entrance honest — neither flag can silently win over the other.
|
||||
if (args.bundle is None) == (args.portfolio is None):
|
||||
parser.error(
|
||||
"give exactly one of --bundle (one project) or --portfolio (a config of N projects)"
|
||||
)
|
||||
if args.verdict_dir is not None and args.portfolio is None:
|
||||
parser.error(
|
||||
"--verdict-dir is the PORTFOLIO-level expert inbox and requires --portfolio "
|
||||
"(a single run reads its inbox with --inbox)"
|
||||
)
|
||||
if args.portfolio is not None:
|
||||
# The portfolio pass returns typed results and persists NOTHING (K3), and
|
||||
# the outbox names its pairs by run_id, which a portfolio pass has none of.
|
||||
# Every flag that would therefore do nothing is refused rather than
|
||||
# silently ignored (§1) — a flag that quietly no-ops is a false claim.
|
||||
unsupported = [
|
||||
name
|
||||
for name, value in (
|
||||
("--inbox", args.inbox),
|
||||
("--out", args.out),
|
||||
("--outbox", args.outbox),
|
||||
("--run-id", args.run_id),
|
||||
("--value-report", args.value_report),
|
||||
)
|
||||
if value is not None
|
||||
]
|
||||
if args.live_dry_run:
|
||||
unsupported.append("--live-dry-run")
|
||||
if unsupported:
|
||||
parser.error(
|
||||
f"--portfolio does not support {', '.join(sorted(unsupported))}: the "
|
||||
"portfolio pass persists nothing and has no run_id of its own. Run the "
|
||||
"projects individually with --bundle to file per-run artifacts."
|
||||
)
|
||||
|
||||
# fail-fast (§10 spirit): a run persisted to the outbox MUST carry an explicit
|
||||
# run_id — reject BEFORE composing or constructing a client, so no spend rides
|
||||
# on a run that cannot be filed (no wall-clock default fills the gap). The dry
|
||||
|
|
@ -438,12 +664,61 @@ def main(
|
|||
except EgressNotPermitted as exc:
|
||||
parser.error(str(exc))
|
||||
|
||||
# §10: ALL startup contracts schema-validated BEFORE any model client exists.
|
||||
# §10: ALL startup contracts schema-validated BEFORE any model client exists —
|
||||
# the portfolio config and the goal contract included.
|
||||
projects: ReferenceProjectsContract | None = None
|
||||
if args.portfolio is not None:
|
||||
try:
|
||||
projects = load_reference_projects(
|
||||
json.loads(args.portfolio.read_text(encoding="utf-8"))
|
||||
)
|
||||
except (OSError, TypeError, ValueError) as exc:
|
||||
parser.error(f"--portfolio config is not a valid reference-projects file: {exc}")
|
||||
goal: GoalContract | None = None
|
||||
if args.goals is not None:
|
||||
try:
|
||||
goal = load_goal(args.goals)
|
||||
except (OSError, TypeError, ValueError) as exc:
|
||||
parser.error(f"--goals is not a valid goal contract: {exc}")
|
||||
except NotImplementedError as exc: # the D-E-gated percent goal, refused loudly
|
||||
parser.error(f"--goals: {exc}")
|
||||
|
||||
# The §10 contract carries ONE data source. A portfolio pass carries one per
|
||||
# project, so the contract records the first and every project still navigates
|
||||
# its OWN bundle_dir inside run_portfolio — never the contract's.
|
||||
docs_dir = str(args.bundle) if projects is None else projects.projects[0].bundle_dir
|
||||
contracts = load_contracts(
|
||||
data_source={"docs_dir": str(args.bundle), "top_k": args.top_k},
|
||||
data_source={"docs_dir": docs_dir, "top_k": args.top_k},
|
||||
termination={"max_rounds": args.max_rounds, "max_tokens": args.max_tokens},
|
||||
feedback={"decision": "approved", "rationale": "startup shape check (§10)"},
|
||||
)
|
||||
|
||||
# K12: achievement is checked BEFORE spend. A hard target already met by the
|
||||
# book means the run has nothing left to buy — it stops here, structured,
|
||||
# with no client ever constructed.
|
||||
if goal is not None and check_goal_before_spend(goal, args.ledger, notifiers=notifiers):
|
||||
return _GOAL_STOP_EXIT
|
||||
|
||||
factory = default_client_factory if client_factory is None else client_factory
|
||||
|
||||
if projects is not None:
|
||||
print(
|
||||
f"portfolio: {len(projects.projects)} project(s) caps: "
|
||||
f"max_rounds={args.max_rounds} max_tokens={args.max_tokens} "
|
||||
f"max_budget_usd_per_call={args.max_budget_usd_per_call}"
|
||||
)
|
||||
return execute_portfolio(
|
||||
factory(contracts, args.max_budget_usd_per_call),
|
||||
projects,
|
||||
contracts=contracts,
|
||||
top_k=contracts.data_source.top_k,
|
||||
max_debate_rounds=args.max_debate_rounds,
|
||||
max_attempts=args.max_attempts,
|
||||
verdict_dir=args.verdict_dir,
|
||||
notifiers=notifiers,
|
||||
)
|
||||
|
||||
assert args.bundle is not None # narrowed by the exactly-one check above
|
||||
# The navigated dir is the CONTRACT's, so the validated config is load-bearing.
|
||||
composed = compose_run_context(
|
||||
Path(contracts.data_source.docs_dir), args.inbox, k=contracts.data_source.top_k
|
||||
|
|
@ -454,7 +729,6 @@ def main(
|
|||
f"max_tokens={args.max_tokens} "
|
||||
f"max_budget_usd_per_call={args.max_budget_usd_per_call}"
|
||||
)
|
||||
factory = default_client_factory if client_factory is None else client_factory
|
||||
client = factory(contracts, args.max_budget_usd_per_call)
|
||||
|
||||
# K8: the live-run drill builds the client (the key-free SDK construction
|
||||
|
|
@ -494,7 +768,7 @@ def main(
|
|||
client,
|
||||
composed,
|
||||
contracts=contracts,
|
||||
out_dir=args.out,
|
||||
out_dir=args.out if args.out is not None else Path("runs") / "run",
|
||||
max_debate_rounds=args.max_debate_rounds,
|
||||
max_attempts=args.max_attempts,
|
||||
outbox_dir=args.outbox,
|
||||
|
|
@ -511,6 +785,7 @@ def main(
|
|||
inbox_dir=args.inbox,
|
||||
ledger_path=args.ledger,
|
||||
destination=args.value_report,
|
||||
goal=goal,
|
||||
)
|
||||
if not written and code == 0:
|
||||
code = 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue