feat(prepass): one renderer and one artefact carry the declaration out of the run [skip-docs]
[skip-docs]: CLI-flagget og README-blokka kommer i neste commit.
`prepass_notice` er ENESTE renderer, tar den ALT OPPLOESTE deklarasjonen (aldri en
payload-sti -- en renderer som leste fila paa nytt ville vaert en andre oppløsning fri
til aa vaere uenig med kjoeringen den beskriver), og returnerer `None` uten payload.
Omisjonen er entydig her paa en maate `proposal_review_notice`s bevisst ikke er: en
kjoering sier ingenting om et kutt fordi operatoeren ikke ga noe, og det finnes
noeyaktig EN maate aa gi et paa. Golden-transkriptet er det uavhengige, eksisterende
vitnet for den halvdelen. To kallsteder: dry-run og full kjoering.
`outbox.write_prepass` skriver `{run_id}-prepass.json` IFF et payload ble gitt --
`write_proposal_reviews`-regelen, og her gjoer den en andre jobb: siden et payload
TREKKER navigatoerverktoeyene er `{run_id}-debate.json`s `tool_calls` tom ved
konstruksjon paa denne stien, og DEN fila skrives ubetinget nettopp fordi et tomt spor
ER S2c-regresjonen. Uten dette artefaktet ved siden av ville "trukket med vilje" og
"regrert" lest likt. TILSTEDEVAERELSEN er det som skiller dem, og en egen arm
observerer BEGGE filene paa SAMME kjoering.
Skrevet fra `finally` (`write_parse_failures`-presedensen) ved et DIREKTE kall, ikke
via `_write_or_report`: den helperens paakrevde `in_flight` bindes foerst inne i
genererings-blokka, og `None` der ville latt en `OSError` fortrenge en `BudgetExceeded`
i luften -- noeyaktig defekten helperen finnes for. En egen arm driver et budsjettstopp
midt i debatten og krever at deklarasjonen likevel ligger der.
Regel -> ANTALL i baade rendereren og artefaktet; en arm beviser at 20 tilbakeholdte
konsept-ider naar HVERKEN stdout eller artefaktet mens antallet gjoer det.
1446 passed / 5 skipped (fra 1440/5, +6, 0 fjernet). ruff + mypy rene. Golden
`shasum -a 1` av INNHOLDET = ea8c534773acdbe41ae68f2c55724d69aaf8be4f, BYTE-UENDRET.
Co-Authored-By: Claude <claude-opus-5>
This commit is contained in:
parent
7aa06d581f
commit
d3476d55ee
3 changed files with 154 additions and 0 deletions
|
|
@ -219,6 +219,31 @@ def write_debate_tools(
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def write_prepass(
|
||||||
|
outbox_dir: str,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
declaration: Mapping[str, Any],
|
||||||
|
) -> Path:
|
||||||
|
"""Write ``{run_id}-prepass.json`` — the CUT this run was given (order 20260907T080223Z).
|
||||||
|
|
||||||
|
Written IFF a payload was supplied, including when the cut withheld nothing —
|
||||||
|
``write_proposal_reviews``' rule, and here it does a second job. Since a payload WITHDRAWS the
|
||||||
|
navigator tools, ``{run_id}-debate.json``'s ``tool_calls`` is empty by construction on this
|
||||||
|
path; and that file is written unconditionally precisely because an empty trace IS the S2c
|
||||||
|
regression (a debate that navigated nothing looks exactly like a cheap one). Without this
|
||||||
|
artefact beside it, "withdrawn by design" and "regressed" would read identically. Its PRESENCE
|
||||||
|
is what tells them apart.
|
||||||
|
|
||||||
|
Takes an already-rendered plain mapping (``prepass.declaration_payload``) so the RAW output
|
||||||
|
layer stays framework-free — ``write_exploration``'s rule, same reason."""
|
||||||
|
directory = Path(outbox_dir)
|
||||||
|
directory.mkdir(parents=True, exist_ok=True)
|
||||||
|
path = directory / f"{run_id}-prepass.json"
|
||||||
|
path.write_text(_dump({"run_id": run_id, "prepass": dict(declaration)}), encoding="utf-8")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
def write_plan_review(
|
def write_plan_review(
|
||||||
outbox_dir: str,
|
outbox_dir: str,
|
||||||
run_id: str,
|
run_id: str,
|
||||||
|
|
|
||||||
|
|
@ -838,6 +838,36 @@ def unkeyed_verdicts_notice(unkeyed: int) -> str | None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def prepass_notice(declaration: prepass.PrepassDeclaration | None) -> str | None:
|
||||||
|
"""Render the CUT this run was given, or ``None`` when it was given none.
|
||||||
|
|
||||||
|
ONE renderer, N callsites (kø-(p)), taking the ALREADY-RESOLVED declaration rather than a
|
||||||
|
payload path: a renderer that re-read the file would be a second resolution free to disagree
|
||||||
|
with the run it describes.
|
||||||
|
|
||||||
|
``None`` without a payload — omission, never an empty row (``mandate.announce``'s rule, which
|
||||||
|
``cost_baseline_notice`` and ``skipped_links_notice`` both follow). Here the omission is
|
||||||
|
unambiguous in a way ``proposal_review_notice``'s deliberately is not: a run says nothing about
|
||||||
|
a cut because the operator supplied no payload, and there is exactly one way to supply one. The
|
||||||
|
golden transcript is an independent, pre-existing witness for that half — a renderer that
|
||||||
|
always returned a line would print in the demo and go red there.
|
||||||
|
|
||||||
|
The withheld concepts appear as rule -> COUNT, never as ids: the same rule the rendering
|
||||||
|
follows, for the same measured reason (34 451 o200k tokens of ids on a real corpus), and
|
||||||
|
because an operator reading a run summary wants to know WHAT was dropped and HOW MUCH, not
|
||||||
|
which. The full list is in the payload the operator already holds."""
|
||||||
|
if declaration is None:
|
||||||
|
return None
|
||||||
|
rules = ", ".join(f"{rule} ({count})" for rule, count in declaration.withheld_rules)
|
||||||
|
return (
|
||||||
|
f" Knowledge base: a DECLARED CUT was used — {declaration.delivered} of "
|
||||||
|
f"{declaration.considered} concept(s) delivered, {declaration.withheld} withheld"
|
||||||
|
f"{' by rule: ' + rules if rules else ''}. "
|
||||||
|
f"Base {declaration.bundle_id} at ref {declaration.ref}; "
|
||||||
|
f"cut computed for: {declaration.question}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def skipped_links_notice(skipped: tuple[okf.SkippedLink, ...]) -> str | None:
|
def skipped_links_notice(skipped: tuple[okf.SkippedLink, ...]) -> str | None:
|
||||||
"""Render what the run could NOT read, or ``None`` when every cross-link was followed.
|
"""Render what the run could NOT read, or ``None`` when every cross-link was followed.
|
||||||
|
|
||||||
|
|
@ -1182,6 +1212,18 @@ async def run_project(
|
||||||
# the S2c regression itself, so it must be readable rather than inferred from an absence.
|
# the S2c regression itself, so it must be readable rather than inferred from an absence.
|
||||||
if outbox_dir is not None:
|
if outbox_dir is not None:
|
||||||
assert run_id is not None # narrowed by the step-0 guard (no wall-clock default)
|
assert run_id is not None # narrowed by the step-0 guard (no wall-clock default)
|
||||||
|
if prepass_declaration is not None:
|
||||||
|
# BEFORE the debate trace, and by a DIRECT call rather than ``_write_or_report``:
|
||||||
|
# that helper's required ``in_flight`` is bound only inside the generation block
|
||||||
|
# below, and passing ``None`` here would let an OSError displace an in-flight
|
||||||
|
# ``BudgetExceeded`` — the very defect it exists to prevent. Written from the
|
||||||
|
# ``finally`` so a budget stop mid-debate still leaves the declaration, and built
|
||||||
|
# from the SAME object ``RunResult.prepass`` carries, never a second load.
|
||||||
|
outbox.write_prepass(
|
||||||
|
outbox_dir,
|
||||||
|
run_id,
|
||||||
|
declaration=prepass.declaration_payload(prepass_declaration),
|
||||||
|
)
|
||||||
outbox.write_debate_tools(
|
outbox.write_debate_tools(
|
||||||
outbox_dir, run_id, tool_calls=tool_call_payload(debate_tool_calls)
|
outbox_dir, run_id, tool_calls=tool_call_payload(debate_tool_calls)
|
||||||
)
|
)
|
||||||
|
|
@ -3498,6 +3540,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||||
# of its own now OPENS, so the disagreement has to be said out loud or nothing ever reports
|
# of its own now OPENS, so the disagreement has to be said out loud or nothing ever reports
|
||||||
# that the run's artefacts name something other than the path the operator typed.
|
# that the run's artefacts name something other than the path the operator typed.
|
||||||
id_notice = bundle_id_notice(report.bundle_id_source)
|
id_notice = bundle_id_notice(report.bundle_id_source)
|
||||||
|
cut_notice = prepass_notice(report.prepass)
|
||||||
|
if cut_notice is not None:
|
||||||
|
print(cut_notice)
|
||||||
if id_notice is not None:
|
if id_notice is not None:
|
||||||
print(id_notice)
|
print(id_notice)
|
||||||
return 0
|
return 0
|
||||||
|
|
@ -3575,6 +3620,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||||
id_notice = bundle_id_notice(result.provenance.bundle_id_source)
|
id_notice = bundle_id_notice(result.provenance.bundle_id_source)
|
||||||
if id_notice is not None:
|
if id_notice is not None:
|
||||||
print(id_notice)
|
print(id_notice)
|
||||||
|
# The CUT this run was given, read off the run's OWN declaration rather than off argv, for
|
||||||
|
# the reason above: stdout and ``{run_id}-prepass.json`` are built from the same object.
|
||||||
|
cut_notice = prepass_notice(result.prepass)
|
||||||
|
if cut_notice is not None:
|
||||||
|
print(cut_notice)
|
||||||
# Full run only, and structurally so: the fold happens BELOW the ``--live-dry-run`` cut, so a
|
# Full run only, and structurally so: the fold happens BELOW the ``--live-dry-run`` cut, so a
|
||||||
# dry run has nothing to report here (contrast the three notices above, all resolved above it).
|
# dry run has nothing to report here (contrast the three notices above, all resolved above it).
|
||||||
fold_notice = unkeyed_verdicts_notice(result.unkeyed_verdicts)
|
fold_notice = unkeyed_verdicts_notice(result.unkeyed_verdicts)
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ from typing import Any
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
import portfolio_optimiser.run as run_module
|
import portfolio_optimiser.run as run_module
|
||||||
|
from portfolio_optimiser.budget import BudgetExceeded
|
||||||
from portfolio_optimiser import okf, prepass
|
from portfolio_optimiser import okf, prepass
|
||||||
from portfolio_optimiser.run import RunResult, run_project
|
from portfolio_optimiser.run import RunResult, run_project
|
||||||
from portfolio_optimiser.mcp_tools import McpServerConfig
|
from portfolio_optimiser.mcp_tools import McpServerConfig
|
||||||
|
|
@ -378,3 +379,81 @@ async def test_the_dry_run_report_carries_the_declaration(tmp_path: Path) -> Non
|
||||||
assert isinstance(report, run_module.DryRunReport)
|
assert isinstance(report, run_module.DryRunReport)
|
||||||
assert report.prepass is not None
|
assert report.prepass is not None
|
||||||
assert report.prepass.considered == payload.denominators.considered
|
assert report.prepass.considered == payload.denominators.considered
|
||||||
|
|
||||||
|
|
||||||
|
# --- the declaration leaves the run ---------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_notice_is_omitted_without_a_payload() -> None:
|
||||||
|
"""Omission, never an empty row. The golden transcript is the independent, pre-existing
|
||||||
|
witness: a renderer that always returned a line would print in the demo and go red there."""
|
||||||
|
assert run_module.prepass_notice(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_notice_reports_the_cut_when_there_was_one() -> None:
|
||||||
|
declaration = prepass.declaration_of(prepass.load_prepass_payload(str(FIXTURE)))
|
||||||
|
line = run_module.prepass_notice(declaration)
|
||||||
|
assert line is not None
|
||||||
|
assert "4 of 5" in line
|
||||||
|
assert "verdict_layer_excluded (1)" in line
|
||||||
|
assert declaration.ref in line
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_withheld_rule_is_counted_and_the_concept_is_never_named() -> None:
|
||||||
|
""" "We held 620 back under this rule" is the fact; naming them is 34 451 tokens of cost."""
|
||||||
|
raw = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||||
|
raw["withheld"] = [
|
||||||
|
{"concept_id": f"hemmelig-konsept-{n}", "rule": "no_lexical_match"} for n in range(20)
|
||||||
|
] + raw["withheld"]
|
||||||
|
raw["denominators"]["withheld"] = len(raw["withheld"])
|
||||||
|
raw["denominators"]["considered"] = len(raw["withheld"]) + raw["denominators"]["delivered"]
|
||||||
|
declaration = prepass.declaration_of(prepass.PrepassPayload.model_validate(raw))
|
||||||
|
line = run_module.prepass_notice(declaration)
|
||||||
|
assert line is not None and "no_lexical_match (20)" in line
|
||||||
|
assert "hemmelig-konsept-0" not in line
|
||||||
|
body = prepass.declaration_payload(declaration)
|
||||||
|
assert "hemmelig-konsept-0" not in json.dumps(body)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_payload_run_writes_the_artefact_beside_the_debate_trace(tmp_path: Path) -> None:
|
||||||
|
"""SC15's actual qualification, observed on ONE run: an empty ``tool_calls`` next to a
|
||||||
|
prepass artefact is a withdrawal; an empty one alone is the S2c regression."""
|
||||||
|
bundle_dir, payload = _base(tmp_path)
|
||||||
|
outbox_dir = tmp_path / "outbox"
|
||||||
|
result, _, _ = await _run(
|
||||||
|
bundle_dir, prepass_payload=payload, outbox_dir=str(outbox_dir), run_id="r1"
|
||||||
|
)
|
||||||
|
debate = json.loads((outbox_dir / "r1-debate.json").read_text(encoding="utf-8"))
|
||||||
|
cut = json.loads((outbox_dir / "r1-prepass.json").read_text(encoding="utf-8"))
|
||||||
|
assert debate["tool_calls"] == []
|
||||||
|
assert cut["prepass"]["ref"] == payload.bundle.ref
|
||||||
|
assert cut["prepass"]["question"] == payload.question
|
||||||
|
assert isinstance(result, RunResult) and result.prepass is not None
|
||||||
|
assert cut["prepass"]["delivered"] == result.prepass.delivered
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_run_without_a_payload_writes_no_prepass_artefact(tmp_path: Path) -> None:
|
||||||
|
"""The write-iff-offered rule, with three pre-existing witnesses for the same shape."""
|
||||||
|
bundle_dir, _ = _base(tmp_path)
|
||||||
|
outbox_dir = tmp_path / "outbox"
|
||||||
|
await _run(bundle_dir, outbox_dir=str(outbox_dir), run_id="r1")
|
||||||
|
assert not (outbox_dir / "r1-prepass.json").exists()
|
||||||
|
assert (outbox_dir / "r1-debate.json").exists()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_budget_stop_inside_the_debate_still_leaves_the_declaration(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""The ``finally`` is the point: the run that most needs the evidence is the one a cap cut."""
|
||||||
|
bundle_dir, payload = _base(tmp_path)
|
||||||
|
outbox_dir = tmp_path / "outbox"
|
||||||
|
with pytest.raises(BudgetExceeded):
|
||||||
|
await _run(
|
||||||
|
bundle_dir,
|
||||||
|
prepass_payload=payload,
|
||||||
|
outbox_dir=str(outbox_dir),
|
||||||
|
run_id="r1",
|
||||||
|
max_tokens=1,
|
||||||
|
)
|
||||||
|
cut = json.loads((outbox_dir / "r1-prepass.json").read_text(encoding="utf-8"))
|
||||||
|
assert cut["prepass"]["delivered"] == payload.denominators.delivered
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue