feat(p19): the trace says HOW, and a run says what it spent and why it stopped
DEL C. P18 gave read_dir a window (filter/offset/limit) and then measured its
own paid round without being able to see it used: five of 31 documents read
lay outside the default window, so the window HAD been widened and the trace
could not say with which knob. ToolCall now carries the three arguments,
always present and empty/zero when not passed -- an absent key and "not
narrowed" must not read the same -- and the judge counts filter_calls and
paged_calls. _number_argument is a SIBLING of _string_argument, not a widening
of it: a model may send limit as 10 or as "10", and a reader that knew one
shape would report a paged call as unpaged.
DEL D. P18's finding 4 was WRONG AS WRITTEN. provenance.token_usage has been
stamped on every proposal artefact since S3.4 and stands in every one of round
2's; what was missing is a READER. The judge reads it now (round 2 measured:
289 054 tokens against round 1's 2 679 305, -89 %), and the P18 report gets a
dated correction UNDER its original paragraph rather than instead of it.
What was genuinely absent is {run_id}-coverage.json. settle prints the
coverage report and ApproachOutcome has carried not_evaluated since Trekk A3,
but neither ever reached a file, so a judge could see an approach had no
artefact and could not tell a budget stop from an approach nobody ordered.
Written from the finally IFF a mandate was given. stop_reason comes from a
CALLER-OWNED sink rather than from in_flight, and that is a measurement:
_evaluate_mandate SWALLOWS BudgetExceeded once something has been produced, so
run_project's own in_flight never sees it.
Load-bearing measured (10 arms), four mutations all red against the whole
suite, green control 1744/5 and the golden byte-unchanged. D-i stood GREEN
first -- the vacuous-gate class, 25th time: the arm called write_coverage
itself and therefore chose the reason it then asserted.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
d74f32dc1c
commit
4c6084e5df
12 changed files with 684 additions and 11 deletions
|
|
@ -343,6 +343,15 @@ class ToolCall:
|
|||
name: str
|
||||
bundle_id: str
|
||||
path: str
|
||||
#: P19 DEL C — HOW the listing was asked for, not just WHICH one. P18 gave ``read_dir`` a
|
||||
#: window (``filter``/``offset``/``limit``) and the trace could not say whether a model used
|
||||
#: it: five of 31 documents read in round 2 lay outside the default window, so the window HAD
|
||||
#: been widened, and nothing said with which knob. Empty/zero mean "not passed" — the
|
||||
#: ``bundle_id``/``path`` rule one field over, and unambiguous here because ``limit`` is
|
||||
#: clamped to at least one wherever it is given.
|
||||
filter: str = ""
|
||||
offset: int = 0
|
||||
limit: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -393,6 +402,28 @@ def _string_argument(arguments: Any, key: str) -> str:
|
|||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _number_argument(arguments: Any, key: str) -> int:
|
||||
"""One NUMERIC argument of a call — ``_string_argument``'s sibling, and deliberately separate.
|
||||
|
||||
A model may send ``limit`` as ``10`` or as ``"10"`` (both reach a tool through the same wire),
|
||||
so a recorder that read only the first shape would say a paging call was unpaged. Anything that
|
||||
is neither is ``0``: the recorder describes the call, and inventing a number for an argument
|
||||
nobody passed would be the false attribution ``_string_argument`` refuses for its own field.
|
||||
``bool`` is excluded explicitly because it is an ``int`` in Python and a flag is not a window.
|
||||
"""
|
||||
if isinstance(arguments, Mapping):
|
||||
value: Any = arguments.get(key)
|
||||
else:
|
||||
value = getattr(arguments, key, None)
|
||||
if isinstance(value, bool):
|
||||
return 0
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, str) and value.strip().lstrip("-").isdigit():
|
||||
return int(value.strip())
|
||||
return 0
|
||||
|
||||
|
||||
class ExplorationToolRecorder(FunctionMiddleware):
|
||||
"""Records WHICH exploration tool an agent actually called, in the order it called them.
|
||||
|
||||
|
|
@ -424,6 +455,9 @@ class ExplorationToolRecorder(FunctionMiddleware):
|
|||
name=name,
|
||||
bundle_id=_string_argument(arguments, "bundle_id"),
|
||||
path=_string_argument(arguments, "path"),
|
||||
filter=_string_argument(arguments, "filter"),
|
||||
offset=_number_argument(arguments, "offset"),
|
||||
limit=_number_argument(arguments, "limit"),
|
||||
)
|
||||
)
|
||||
await call_next()
|
||||
|
|
@ -479,7 +513,20 @@ def tool_call_payload(calls: Sequence[ToolCall]) -> list[dict[str, Any]]:
|
|||
Plain mappings only, so the RAW output layer stays MAF-free (``outbox.py`` may not import
|
||||
this module).
|
||||
"""
|
||||
return [{"name": call.name, "bundle_id": call.bundle_id, "path": call.path} for call in calls]
|
||||
return [
|
||||
{
|
||||
"name": call.name,
|
||||
"bundle_id": call.bundle_id,
|
||||
"path": call.path,
|
||||
# P19 DEL C: HOW the level was asked for. Always present, zero/empty when not passed —
|
||||
# the ``write_debate_tools`` rule one field down: an absent key and "not narrowed" must
|
||||
# not be the same reading.
|
||||
"filter": call.filter,
|
||||
"offset": call.offset,
|
||||
"limit": call.limit,
|
||||
}
|
||||
for call in calls
|
||||
]
|
||||
|
||||
|
||||
def trace_payload(
|
||||
|
|
|
|||
|
|
@ -312,6 +312,43 @@ def write_proposal_reviews(
|
|||
return path
|
||||
|
||||
|
||||
def write_coverage(
|
||||
outbox_dir: str,
|
||||
run_id: str,
|
||||
*,
|
||||
rows: Sequence[Mapping[str, Any]],
|
||||
stop_reason: str,
|
||||
) -> Path:
|
||||
"""Write ``{run_id}-coverage.json`` — WHY each commissioned approach ended as it did (P19 D2).
|
||||
|
||||
``settle`` prints the coverage report and ``ApproachOutcome`` has carried ``not_evaluated``
|
||||
since Trekk A3, but neither ever reached a FILE: measured 14.09, a judge reading an outbox could
|
||||
see that an approach had no artefact and could not tell a budget stop from an approach nobody
|
||||
ordered. That is the very silence ``ApproachOutcome`` exists to remove, one layer out.
|
||||
|
||||
``stop_reason`` is ``BudgetExceeded.kind`` when a cap cut the run short (``tokens`` /
|
||||
``rounds`` / the portfolio's own kinds) and ``""`` when nothing did. A REQUIRED argument rather
|
||||
than an inferred one, for ``cost_baseline_anchored``'s reason: "the run finished" and "we never
|
||||
found out" must not be the same value.
|
||||
|
||||
Byte-deterministic and wall-clock-free, mirroring ``write_run_config``; plain data only, so the
|
||||
RAW output layer stays MAF-free."""
|
||||
directory = Path(outbox_dir)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / f"{run_id}-coverage.json"
|
||||
path.write_text(
|
||||
_dump(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"stop_reason": stop_reason,
|
||||
"rows": [dict(row) for row in rows],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def write_run_config(
|
||||
config_dir: str,
|
||||
run_id: str,
|
||||
|
|
|
|||
|
|
@ -509,6 +509,7 @@ def _select_outcome(
|
|||
async def _evaluate_mandate(
|
||||
mandate: Mandate,
|
||||
evaluate: Callable[[Approach | None], Awaitable[ValidatedProposal | Rejection]],
|
||||
budget_stops: list[str] | None = None,
|
||||
) -> tuple[
|
||||
ValidatedProposal | Rejection,
|
||||
tuple[ApproachOutcome, ...],
|
||||
|
|
@ -541,9 +542,16 @@ async def _evaluate_mandate(
|
|||
for index, (row_id, label, approach) in enumerate(plan):
|
||||
try:
|
||||
outcome = await evaluate(approach)
|
||||
except BudgetExceeded:
|
||||
except BudgetExceeded as stop:
|
||||
if not produced:
|
||||
raise
|
||||
# P19 D2: WHICH cap bound, recorded on a caller-owned sink before the rows are built.
|
||||
# The exception is SWALLOWED here (the approaches that were reached are a real result),
|
||||
# so ``run_project``'s own ``in_flight`` never sees it — and a coverage artefact that
|
||||
# said "nothing stopped this run" about a commission cut in half would be the silence
|
||||
# the artefact exists to remove.
|
||||
if budget_stops is not None:
|
||||
budget_stops.append(stop.kind)
|
||||
rows.extend(
|
||||
ApproachOutcome(
|
||||
id=rid,
|
||||
|
|
@ -1452,6 +1460,10 @@ async def run_project(
|
|||
# on the very attempt a revise bought, and on that path ``generate_via_llm`` returns nothing —
|
||||
# so the run whose record matters most is exactly the one a return value cannot reach.
|
||||
expert_reviews: list[ProposalReview] = []
|
||||
# P19 D2: which cap, if any, cut the commission short. Caller-owned for the reason every other
|
||||
# sink here is: ``_evaluate_mandate`` SWALLOWS the stop once something has been produced, so a
|
||||
# return value would not reach the ``finally`` that writes the artefact.
|
||||
budget_stops: list[str] = []
|
||||
|
||||
async def _evaluate(approach: Approach | None) -> ValidatedProposal | Rejection:
|
||||
# Which candidate the expert is being asked about. With a mandate every entry is keyed —
|
||||
|
|
@ -1496,7 +1508,9 @@ async def run_project(
|
|||
if mandate is None:
|
||||
validator_outcome = await _evaluate(None)
|
||||
else:
|
||||
validator_outcome, coverage, evaluated = await _evaluate_mandate(mandate, _evaluate)
|
||||
validator_outcome, coverage, evaluated = await _evaluate_mandate(
|
||||
mandate, _evaluate, budget_stops
|
||||
)
|
||||
except BaseException as stop:
|
||||
# Recorded and re-raised UNTOUCHED. This arm decides nothing about the exception itself —
|
||||
# only what the two writers in the ``finally`` are allowed to do to it (``_write_or_report``).
|
||||
|
|
@ -1518,6 +1532,39 @@ async def run_project(
|
|||
what=f"{run_id}-parse-failures.json",
|
||||
in_flight=in_flight,
|
||||
)
|
||||
# Same ``finally``, a THIRD write rule: IFF a mandate was given, including when the run
|
||||
# stopped before a single approach was evaluated (P19 D2). Coverage is the MANDATE's
|
||||
# report by construction — without one there are no approaches and the file would describe
|
||||
# nothing — so a mandate-less run leaves the outbox byte-identical, which two existing
|
||||
# tests pin as an exact listing. The stop reason comes from the in-flight exception rather
|
||||
# than being inferred: a ``BudgetExceeded`` carries ``kind`` as a field precisely so that
|
||||
# "which cap bound" is readable by machine (kø-(y)), and a run that finished says so with
|
||||
# an empty string rather than with a missing key.
|
||||
if outbox_dir is not None and mandate is not None:
|
||||
assert run_id is not None # narrowed by the step-0 guard (no wall-clock default)
|
||||
_write_or_report(
|
||||
lambda: outbox.write_coverage(
|
||||
outbox_dir,
|
||||
run_id,
|
||||
rows=[
|
||||
{
|
||||
"id": row.id,
|
||||
"label": row.label,
|
||||
"status": row.status,
|
||||
"detail": row.detail,
|
||||
"saving_nok": row.saving_nok,
|
||||
}
|
||||
for row in coverage
|
||||
],
|
||||
stop_reason=(
|
||||
budget_stops[0]
|
||||
if budget_stops
|
||||
else (in_flight.kind if isinstance(in_flight, BudgetExceeded) else "")
|
||||
),
|
||||
),
|
||||
what=f"{run_id}-coverage.json",
|
||||
in_flight=in_flight,
|
||||
)
|
||||
# Same ``finally``, different write rule: IFF a reviewer was given, including when the
|
||||
# list is empty (D4). A reviewer-less run must leave the outbox byte-identical, while a
|
||||
# reviewer that was offered and never consulted is a fact the artefact must be able to
|
||||
|
|
|
|||
|
|
@ -125,6 +125,12 @@ class ApproachVerdict:
|
|||
#: wrote one, and RE-DERIVED with the same classifier when it did not, so rounds 1 and 2 -
|
||||
#: written before the field existed - can be re-judged with the same instrument.
|
||||
prose_codes: tuple[str, ...]
|
||||
#: P19 D2 - WHY this row was not evaluated: ``rounds`` / ``tokens`` when a cap cut the run
|
||||
#: short, ``absent`` when the artefact is simply missing and no coverage file says otherwise,
|
||||
#: and ``""`` for a row that WAS evaluated. Before this, "no artefact" could not be told from
|
||||
#: "an approach nobody ordered" -- the silence ``ApproachOutcome`` exists to remove, one layer
|
||||
#: out, and it reached no file until ``{run_id}-coverage.json``.
|
||||
not_evaluated_reason: str
|
||||
ferdig: bool
|
||||
|
||||
|
||||
|
|
@ -153,6 +159,19 @@ class ContextSetVerdict:
|
|||
#: none" is the measurement, and a missing field would be indistinguishable from a judge that
|
||||
#: did not look.
|
||||
requirements_declared: tuple[str, ...]
|
||||
#: P19 DEL C - how the run asked for its listings. ``filter_calls`` is how many calls narrowed
|
||||
#: a level by word, ``paged_calls`` how many asked for a window other than the default. P18 § 1
|
||||
#: could only infer that the window HAD been widened (five of 31 documents read lay outside the
|
||||
#: default) and never with which knob; these two make it readable directly.
|
||||
filter_calls: int
|
||||
paged_calls: int
|
||||
#: P19 D1 - what the run SPENT, read off ``provenance.token_usage``, which has been stamped on
|
||||
#: every proposal artefact since S3.4 and which P18's report wrongly said could not be given.
|
||||
#: ``0`` when no artefact carried one.
|
||||
token_usage: int
|
||||
#: P19 D2 - ``BudgetExceeded.kind`` when a cap cut the run short, ``""`` when nothing did, and
|
||||
#: ``"absent"`` when the run wrote no coverage file at all (every run before today).
|
||||
stop_reason: str
|
||||
tool_calls_seen: int
|
||||
citations_seen: int
|
||||
approach_rows_seen: int
|
||||
|
|
@ -269,10 +288,18 @@ def score_context_set(
|
|||
hallucinated_reads.append(raw)
|
||||
reads_clean = not hallucinated_reads
|
||||
|
||||
# ---- P19 D1/D2: what the run spent, and why it stopped -----------------------------------
|
||||
coverage_path = outbox / f"{run_id}-coverage.json"
|
||||
stop_reason = (
|
||||
str(_read_json(coverage_path).get("stop_reason", "")) if coverage_path.is_file() else ""
|
||||
)
|
||||
coverage_seen = coverage_path.is_file()
|
||||
|
||||
# ---- per approach ------------------------------------------------------------------------
|
||||
rows: list[ApproachVerdict] = []
|
||||
citations_seen = 0
|
||||
rows_seen = 0
|
||||
token_usage = 0
|
||||
validated_codes: set[str] = set()
|
||||
validated_ids: set[str] = set()
|
||||
|
||||
|
|
@ -299,6 +326,7 @@ def score_context_set(
|
|||
requirement_source=_attributable(approach, declared_paths)[1],
|
||||
requirement_hit=bool(set(_attributable(approach, declared_paths)[0]) & wanted),
|
||||
prose_codes=(),
|
||||
not_evaluated_reason=stop_reason or "absent",
|
||||
ferdig=False,
|
||||
)
|
||||
)
|
||||
|
|
@ -306,6 +334,7 @@ def score_context_set(
|
|||
|
||||
rows_seen += 1
|
||||
payload = _read_json(proposal_path)
|
||||
token_usage = max(token_usage, int(payload.get("provenance", {}).get("token_usage", 0)))
|
||||
proposal = payload.get("proposal", {})
|
||||
citations = payload.get("provenance", {}).get("citations", [])
|
||||
citations_seen += len(citations)
|
||||
|
|
@ -375,6 +404,7 @@ def score_context_set(
|
|||
requirement_source=requirement_source,
|
||||
requirement_hit=requirement_hit,
|
||||
prose_codes=prose_codes,
|
||||
not_evaluated_reason="",
|
||||
ferdig=(
|
||||
grounded
|
||||
and (named_in_measure or named_in_snippet)
|
||||
|
|
@ -421,6 +451,12 @@ def score_context_set(
|
|||
must_refuse=tuple(refusals),
|
||||
hallucinated_reads=tuple(hallucinated_reads),
|
||||
requirements_declared=tuple(declared_paths),
|
||||
token_usage=token_usage,
|
||||
stop_reason=stop_reason if coverage_seen else "absent",
|
||||
filter_calls=sum(1 for c in tool_calls if str(c.get("filter", ""))),
|
||||
paged_calls=sum(
|
||||
1 for c in tool_calls if int(c.get("offset", 0) or 0) or int(c.get("limit", 0) or 0)
|
||||
),
|
||||
tool_calls_seen=len(tool_calls),
|
||||
citations_seen=citations_seen,
|
||||
approach_rows_seen=rows_seen,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue