feat(row6): a proposal whose approach declared no requirement is unsupported
Stress round 6 validated three falsification arms, and every validated approach rested only on run-level declarations nobody can attribute to one approach. declare_requirement now takes a required approach_id (a mandate id or own-proposal; an unknown id is refused naming the valid ones), and a ValidatedProposal whose approach has neither a mandate requirement nor a declaration under its own id becomes validator.Unsupported - a Rejection subclass carrying the validator's own ruling, reported as `unsupported` in coverage, the outcome artefact, the settlement and the judge, and never counted or summed. The rule is active whenever the debate held the declaration tool, the micro base included; the road and pre-pass paths are untouched. Declaration quality is not judged, so the rule can be satisfied by declaring any document the run read. The v1 gate's row 6 probes pass; its artefact half reads IKKE MÅLT because stress round 6 predates approach-addressed declarations, and IKKE MÅLT is never green - it fails the exit code. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
9847e014e7
commit
938a1ca30e
23 changed files with 718 additions and 115 deletions
|
|
@ -42,6 +42,8 @@ _AI_LINE_MIN = 30
|
|||
GREEN = "GRØNN"
|
||||
RED = "RØD"
|
||||
DIAGNOSIS = "DIAGNOSE"
|
||||
#: A row whose evidence could not be read. Never green: on a failing row it fails the exit code.
|
||||
NOT_MEASURED = "IKKE MÅLT"
|
||||
|
||||
ROUNDS_CONTRACT = """\
|
||||
Rundekatalogen (--rounds-dir) har fast form:
|
||||
|
|
@ -498,6 +500,8 @@ class StressMeasure:
|
|||
commissioned: int = 0
|
||||
where: str = ""
|
||||
missing: str = ""
|
||||
#: Declarations with no ``approach_id`` — written before the rule; the row cannot be measured.
|
||||
unaddressed: int = 0
|
||||
undeclared_ids: tuple[str, ...] = field(default=())
|
||||
|
||||
|
||||
|
|
@ -538,6 +542,7 @@ def measure_stress(
|
|||
approaches = [a for v in verdicts for a in v.approaches]
|
||||
validated = [a for a in approaches if a.status == "validated"]
|
||||
undeclared = [a for a in validated if a.requirement_source != "approach"]
|
||||
unaddressed = sum(v.unaddressed_declarations for v in verdicts)
|
||||
commissioned = sum(
|
||||
len(load_mandate(repo_root / c / "mandate.json").approaches) for c in contexts
|
||||
)
|
||||
|
|
@ -549,6 +554,7 @@ def measure_stress(
|
|||
rows=len(approaches),
|
||||
commissioned=commissioned,
|
||||
where=str(stress_root),
|
||||
unaddressed=unaddressed,
|
||||
undeclared_ids=tuple(a.approach_id for a in undeclared),
|
||||
)
|
||||
|
||||
|
|
@ -556,29 +562,54 @@ def measure_stress(
|
|||
def score_undeclared(
|
||||
probes: Sequence[str], outcomes: Mapping[str, str], m: StressMeasure, label: str
|
||||
) -> Row:
|
||||
"""GREEN only when every probe passes AND the artefacts were measured with k = 0. Evidence that
|
||||
could not be read is IKKE MÅLT — never green, and it fails the exit code like red does."""
|
||||
failing = [
|
||||
f"{n.split('::')[-1]}={outcomes.get(n, 'missing')}"
|
||||
for n in probes
|
||||
if outcomes.get(n) != "passed"
|
||||
]
|
||||
if not probes:
|
||||
failing.append("ingen probe registrert")
|
||||
title = "6 validert UTEN erklært krav (tilnærmingens egen)"
|
||||
exceptions = [f"probe {x}" for x in failing]
|
||||
probe_state = "prober røde" if failing else "prober grønne"
|
||||
k: int | None = None
|
||||
n: int | None = None
|
||||
diagnostics: tuple[str, ...] = ()
|
||||
if m.missing:
|
||||
reason = f"{label}: ikke målt, artefakter mangler ({m.missing})"
|
||||
k: int | None = None
|
||||
n: int | None = None
|
||||
reason = f"{probe_state}; {label}: ikke målt, artefakter mangler ({m.missing})"
|
||||
elif m.unaddressed:
|
||||
reason = (
|
||||
f"{probe_state}; {label}: ikke målt: artefaktene er eldre enn regelen "
|
||||
f"(approach_id mangler på {m.unaddressed} erklæring(er))"
|
||||
)
|
||||
diagnostics = (
|
||||
f"før regelen: {m.undeclared} av {m.validated} validerte uten tilnærmingens egen "
|
||||
"erklæring — regelen ville gjort dem unsupported, men modellen fikk aldri spørsmålet",
|
||||
)
|
||||
else:
|
||||
k, n = m.undeclared, m.validated
|
||||
reason = (
|
||||
f"{label} ({m.where}): {k} av {n} validerte uten erklæring fra tilnærmingen; "
|
||||
f"{m.undeclared_anywhere} uten noen erklæring i kjøringen"
|
||||
f"{probe_state}; {label} ({m.where}): {k} av {n} validerte uten erklæring fra "
|
||||
f"tilnærmingen; {m.undeclared_anywhere} uten noen erklæring i kjøringen"
|
||||
)
|
||||
exceptions += [f"validert uten erklæring: {a}" for a in m.undeclared_ids]
|
||||
red = bool(failing) or bool(k)
|
||||
if not probes:
|
||||
red, exceptions = True, [*exceptions, "ingen probe registrert"]
|
||||
if failing or k:
|
||||
status = RED
|
||||
elif k is None:
|
||||
status = NOT_MEASURED
|
||||
else:
|
||||
status = GREEN
|
||||
return Row(
|
||||
"undeclared", title, k, n, RED if red else GREEN, reason, exceptions=tuple(exceptions)
|
||||
"undeclared",
|
||||
title,
|
||||
k,
|
||||
n,
|
||||
status,
|
||||
reason,
|
||||
exceptions=tuple(exceptions),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -596,7 +627,7 @@ def score_named(m: StressMeasure, label: str) -> Row:
|
|||
title,
|
||||
None,
|
||||
None,
|
||||
DIAGNOSIS,
|
||||
NOT_MEASURED,
|
||||
f"{label}: ikke målt, artefakter mangler ({m.missing})",
|
||||
failing=False,
|
||||
diagnostics=(NAMED_WARNING,),
|
||||
|
|
|
|||
|
|
@ -215,7 +215,8 @@ _INSTRUCTIONS: Final = {
|
|||
"BINDS it: pass read_dir a 'filter' word taken from the approach's own label — "
|
||||
"filter='rundkjoring' finds the level's requirements about roundabouts, and one of them is "
|
||||
"the 'Krav 4.1.2-1' you are looking for — read it with read_file, then call "
|
||||
"declare_requirement with the base id, that path and the requirement's own number. The "
|
||||
"declare_requirement with the base id, that path, the requirement's own number and the "
|
||||
"short label of the direction as approach_id. The "
|
||||
"reply gives back the document's own title and number: if they are not about your measure, "
|
||||
"you declared the wrong requirement and should filter again. A direction with no "
|
||||
"requirement behind it is a guess. "
|
||||
|
|
@ -371,6 +372,21 @@ class DeclaredRequirement:
|
|||
bundle_id: str
|
||||
path: str
|
||||
ref: str
|
||||
#: WHICH approach the declaration is for (row 6). A requirement bound at run level cannot be
|
||||
#: attributed to one approach — the judge labelled such a declaration ``run`` — so the rule that
|
||||
#: a validated proposal must rest on its own approach's declaration needs the address on the
|
||||
#: record itself. In a commissioned run it is one of the mandate's ids or ``own-proposal``; the
|
||||
#: exploration, which mints its directions after declaring, records the label verbatim.
|
||||
approach_id: str
|
||||
|
||||
|
||||
class UnknownApproach(ValueError):
|
||||
"""A declaration named an approach this run was not commissioned with (row 6).
|
||||
|
||||
Returned as a refused TURN, never raised out of the run (the ``RequirementNotRead`` rule): the
|
||||
refusal NAMES the valid ids, and naming them is the correction — a declaration filed under an
|
||||
id no approach carries would be recorded against nothing and could never satisfy the rule.
|
||||
"""
|
||||
|
||||
|
||||
class RequirementNotRead(ValueError):
|
||||
|
|
@ -1068,6 +1084,7 @@ def _index_excerpt(body: str) -> tuple[str, bool]:
|
|||
_RETURNABLE_REFUSALS: Final = (
|
||||
ExplorationError,
|
||||
RequirementNotRead,
|
||||
UnknownApproach,
|
||||
okf.BundleIdMismatch,
|
||||
okf.BundlePathNotFound,
|
||||
okf.DocumentPathRefused,
|
||||
|
|
@ -1186,6 +1203,7 @@ def navigator_tools(
|
|||
opened: list[ToolCall] | None = None,
|
||||
requirements: list[DeclaredRequirement] | None = None,
|
||||
labels: Sequence[str] = (),
|
||||
approach_ids: Sequence[str] | None = None,
|
||||
) -> list[FunctionTool]:
|
||||
"""The navigator's tools: survey the catalogue, open one base, read one document — and, when
|
||||
the caller offers the two sinks, DECLARE the requirement that binds a direction.
|
||||
|
|
@ -1241,6 +1259,11 @@ def navigator_tools(
|
|||
the run's own read trace and a second record of it would be free to disagree with the first.
|
||||
Passing one without the other is refused at construction: a log that cannot see what was opened
|
||||
would accept every declaration, which is the vacuous-gate class.
|
||||
|
||||
**Every declaration names the approach it is for** (row 6). ``approach_ids`` is the set a
|
||||
commissioned run can file under — the mandate's ids plus ``own-proposal`` — and an id outside it
|
||||
is refused with the valid ones named. ``None`` (the exploration, which has no ids until it mints
|
||||
them) records any non-empty id verbatim.
|
||||
"""
|
||||
if (opened is None) != (requirements is None):
|
||||
raise ExplorationError(
|
||||
|
|
@ -1462,20 +1485,36 @@ def navigator_tools(
|
|||
"'Krav 4.1.2-1'), read_file to read it, then declare it. The reply gives back the "
|
||||
"document's own "
|
||||
"title and number, so you can see whether you declared the requirement you meant: a "
|
||||
"declaration of a requirement that is not about the measure is worth nothing."
|
||||
"declaration of a requirement that is not about the measure is worth nothing. "
|
||||
"approach_id names the approach the requirement binds: declare once for EACH approach "
|
||||
"you propose for (the run's own proposal is 'own-proposal'). A proposal whose approach "
|
||||
"declared no requirement is not validated, however good its numbers are."
|
||||
),
|
||||
)
|
||||
def declare_requirement(bundle_id: str, path: str, ref: str) -> dict[str, Any]:
|
||||
def declare_requirement(
|
||||
bundle_id: str, path: str, ref: str, approach_id: str
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return _declare_requirement(bundle_id, path, ref)
|
||||
return _declare_requirement(bundle_id, path, ref, approach_id)
|
||||
except _RETURNABLE_REFUSALS as exc:
|
||||
return _refused_mapping(exc)
|
||||
|
||||
def _declare_requirement(bundle_id: str, path: str, ref: str) -> dict[str, Any]:
|
||||
def _declare_requirement(
|
||||
bundle_id: str, path: str, ref: str, approach_id: str
|
||||
) -> dict[str, Any]:
|
||||
assert opened is not None and requirements is not None # the constructor guard above
|
||||
# The base is resolved by the SAME index every read rung uses, so an unknown base is
|
||||
# refused here exactly as it is there rather than being accepted into the record.
|
||||
resolved_dir = _resolve_bundle(index, bundle_id)
|
||||
# Row 6: the address is checked before anything is read, so a declaration filed under no
|
||||
# approach is refused whatever else is right about it.
|
||||
if approach_ids is not None and approach_id not in approach_ids:
|
||||
raise UnknownApproach(
|
||||
f"{approach_id!r}; this run's approaches are {', '.join(map(repr, approach_ids))}. "
|
||||
"Declare the requirement under the id of the approach it binds"
|
||||
)
|
||||
if not approach_id.strip():
|
||||
raise UnknownApproach("an empty approach_id; name the approach this requirement binds")
|
||||
read_paths = [call.path for call in opened if call.name == "read_file" and call.path]
|
||||
if path not in read_paths:
|
||||
raise RequirementNotRead(
|
||||
|
|
@ -1518,7 +1557,9 @@ def navigator_tools(
|
|||
"Use read_dir with a 'filter' word from the approach's own label to find the "
|
||||
"candidates, then read_file the ones that could bind it"
|
||||
)
|
||||
requirements.append(DeclaredRequirement(bundle_id=bundle_id, path=path, ref=ref))
|
||||
requirements.append(
|
||||
DeclaredRequirement(bundle_id=bundle_id, path=path, ref=ref, approach_id=approach_id)
|
||||
)
|
||||
# P20/A1: give back the DOCUMENT's own title and number, read off the base rather than
|
||||
# echoed from the arguments. MEASURED (P19 round 3, P17b): 13 declarations over 5 runs and
|
||||
# NOT ONE named a fasit concept — the tool answered ``{"declared": true, ...}`` to every
|
||||
|
|
@ -1532,6 +1573,7 @@ def navigator_tools(
|
|||
"bundle_id": bundle_id,
|
||||
"path": path,
|
||||
"ref": ref,
|
||||
"approach_id": approach_id,
|
||||
"title": declared[0],
|
||||
"req_number": declared[1],
|
||||
"binds": (
|
||||
|
|
@ -1577,7 +1619,10 @@ def requirement_payload(declared: Sequence[DeclaredRequirement]) -> list[dict[st
|
|||
copies of "what a declaration looks like" would drift into two answers about one run, which is
|
||||
the kø-(p) defect landing in exactly the files an operator reads after a paid run.
|
||||
"""
|
||||
return [{"bundle_id": d.bundle_id, "path": d.path, "ref": d.ref} for d in declared]
|
||||
return [
|
||||
{"bundle_id": d.bundle_id, "path": d.path, "ref": d.ref, "approach_id": d.approach_id}
|
||||
for d in declared
|
||||
]
|
||||
|
||||
|
||||
def _refused(
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ def _budget_payload(exc: BudgetExceeded) -> dict[str, Any]:
|
|||
"""The exhausted-budget body: the ledger's own triple, plus the human line for the log.
|
||||
|
||||
The ``budget_exhausted`` key's PRESENCE is the discriminator — it is not folded into
|
||||
``outcome_type`` (whose values, ``validated``/``rejected``, mean "the run concluded and
|
||||
``outcome_type`` (whose values, ``validated``/``rejected``/``unsupported``, mean "the run concluded and
|
||||
here is the verdict") for the same reason ``BudgetStop`` was given its own field instead of
|
||||
widening ``stop_reason``. Nor could it be: ``outcome_payload`` is the ONE copy of that fork
|
||||
and takes a ``ValidatedProposal | Rejection``, neither of which an exhausted run has."""
|
||||
|
|
|
|||
|
|
@ -334,7 +334,9 @@ class ApproachOutcome:
|
|||
|
||||
id: str
|
||||
label: str
|
||||
status: Literal["validated", "rejected", "not_evaluated"]
|
||||
#: ``unsupported`` (row 6): the numbers held but the approach declared no binding requirement,
|
||||
#: so it is neither a success nor a numeric rejection. Never counted as validated.
|
||||
status: Literal["validated", "rejected", "unsupported", "not_evaluated"]
|
||||
detail: str = ""
|
||||
saving_nok: float | None = None
|
||||
|
||||
|
|
@ -471,6 +473,8 @@ def settle(
|
|||
lines.append(f" {row.id:<20} VALIDATED {amount:>14} {row.label}")
|
||||
elif row.status == "rejected":
|
||||
lines.append(f" {row.id:<20} REJECTED {row.detail}")
|
||||
elif row.status == "unsupported":
|
||||
lines.append(f" {row.id:<20} UNSUPPORTED {row.detail}")
|
||||
else:
|
||||
lines.append(f" {row.id:<20} NOT EVALUATED {row.detail}")
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from collections.abc import Mapping, Sequence
|
|||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from portfolio_optimiser.validator import Rejection, ValidatedProposal
|
||||
from portfolio_optimiser.validator import Rejection, Unsupported, ValidatedProposal
|
||||
|
||||
if TYPE_CHECKING: # provenance imports agent_framework — keep it out of the runtime import graph
|
||||
from portfolio_optimiser.provenance import ProvenanceStamp
|
||||
|
|
@ -113,7 +113,21 @@ def outcome_payload(
|
|||
validated/rejected branching, shared by ``write_outbox`` and the hosted invocations
|
||||
response (``hosting._response_payload``). Two copies of the branch would drift, and a
|
||||
drifted copy would let the HTTP surface describe an outcome the outbox never wrote —
|
||||
the ``to_ore`` single-source rule (kø-(p)) applied to a payload shape."""
|
||||
the ``to_ore`` single-source rule (kø-(p)) applied to a payload shape.
|
||||
|
||||
``unsupported`` (row 6) is checked FIRST because it subclasses ``Rejection``: it carries the
|
||||
validator's percentiles (the numbers held) AND the reason, so a reader sees both halves."""
|
||||
if isinstance(outcome, Unsupported):
|
||||
return {
|
||||
"outcome_type": "unsupported",
|
||||
"reason": outcome.reason,
|
||||
"p10": outcome.validated.p10,
|
||||
"p50": outcome.validated.p50,
|
||||
"p90": outcome.validated.p90,
|
||||
"nominal_feasible": outcome.validated.nominal_feasible,
|
||||
"checker_verdict": checker_verdict,
|
||||
"verdict_id": verdict_id,
|
||||
}
|
||||
if isinstance(outcome, ValidatedProposal):
|
||||
return {
|
||||
"outcome_type": "validated",
|
||||
|
|
|
|||
|
|
@ -123,8 +123,10 @@ from portfolio_optimiser.provenance import ProvenanceStamp
|
|||
from portfolio_optimiser.reference_domain import Project, load_reference_projects
|
||||
from portfolio_optimiser.tracing import TracingConfigError, configure_tracing, tracing_notice
|
||||
from portfolio_optimiser.validator import (
|
||||
UNSUPPORTED_REASON,
|
||||
Grounding,
|
||||
Rejection,
|
||||
Unsupported,
|
||||
ValidatedProposal,
|
||||
baseline_from_project,
|
||||
classify_codes,
|
||||
|
|
@ -410,7 +412,10 @@ def _coverage_row(
|
|||
) -> ApproachOutcome:
|
||||
"""One coverage row from one evaluated approach. A rejection carries the validator's reason
|
||||
verbatim — a bare status would tell the expert their approach failed without telling them why,
|
||||
which is the part they can actually act on."""
|
||||
which is the part they can actually act on. An ``Unsupported`` outcome (row 6) is checked
|
||||
first: it subclasses ``Rejection``, and its own status is the whole point of the class."""
|
||||
if isinstance(outcome, Unsupported):
|
||||
return ApproachOutcome(id=row_id, label=label, status="unsupported", detail=outcome.reason)
|
||||
if isinstance(outcome, ValidatedProposal):
|
||||
return ApproachOutcome(
|
||||
id=row_id,
|
||||
|
|
@ -490,6 +495,15 @@ def evaluate_mandate_candidates(
|
|||
return tuple(rows)
|
||||
|
||||
|
||||
def _declarable_ids(mandate: Mandate | None) -> list[str]:
|
||||
"""The approach ids a declaration may be filed under (row 6): the mandate's own, plus the run's
|
||||
own proposal whenever the run makes one — always without a mandate."""
|
||||
ids = [a.id for a in mandate.approaches] if mandate is not None else []
|
||||
if mandate is None or mandate.allow_own_proposals:
|
||||
ids.append(OWN_PROPOSAL_ID)
|
||||
return ids
|
||||
|
||||
|
||||
def _select_outcome(
|
||||
produced: list[tuple[int, ValidatedProposal | Rejection]],
|
||||
) -> ValidatedProposal | Rejection:
|
||||
|
|
@ -511,6 +525,7 @@ async def _evaluate_mandate(
|
|||
mandate: Mandate,
|
||||
evaluate: Callable[[Approach | None], Awaitable[ValidatedProposal | Rejection]],
|
||||
budget_stops: list[str] | None = None,
|
||||
declared: Sequence[DeclaredRequirement] | None = None,
|
||||
) -> tuple[
|
||||
ValidatedProposal | Rejection,
|
||||
tuple[ApproachOutcome, ...],
|
||||
|
|
@ -532,6 +547,14 @@ async def _evaluate_mandate(
|
|||
become ``not_evaluated`` rows. But if the very first approach exhausts the budget there is
|
||||
nothing honest to return, so ``BudgetExceeded`` propagates exactly as it did before — a run
|
||||
that produced nothing must still fail loudly rather than hand back an empty report.
|
||||
|
||||
**Row 6: a validated outcome must rest on ITS approach's own declaration.** When ``declared`` is
|
||||
given (the rule is active — see ``run_project``), a ``ValidatedProposal`` whose approach carries
|
||||
no ``requirement`` of its own and has no declaration filed under its id becomes
|
||||
``Unsupported``: the numbers held, the ground was never named. A declaration made for another
|
||||
approach does not stand in, and nothing about the declaration's QUALITY is judged — a
|
||||
requirement the model declared is accepted whatever it says (P22 § 4: an overlap gate would
|
||||
refuse legitimate proposals). ``declared`` is read at evaluation time, after the debate.
|
||||
"""
|
||||
plan: list[tuple[str, str, Approach | None]] = [(a.id, a.label, a) for a in mandate.approaches]
|
||||
if mandate.allow_own_proposals:
|
||||
|
|
@ -563,6 +586,15 @@ async def _evaluate_mandate(
|
|||
for rid, lbl, _ in plan[index:]
|
||||
)
|
||||
break
|
||||
if (
|
||||
declared is not None
|
||||
and isinstance(outcome, ValidatedProposal)
|
||||
and not (approach is not None and approach.requirement is not None)
|
||||
and row_id not in {d.approach_id for d in declared}
|
||||
):
|
||||
outcome = Unsupported(
|
||||
proposal=outcome.proposal, reason=UNSUPPORTED_REASON, validated=outcome
|
||||
)
|
||||
produced.append((index, outcome))
|
||||
evaluated.append((row_id, outcome))
|
||||
rows.append(_coverage_row(row_id, label, outcome))
|
||||
|
|
@ -622,8 +654,10 @@ def _bundle_pointer(bundle: okf.Bundle, bundle_id: str, *, dimension: str | None
|
|||
"requirement number or path contains that word, and reports 'total_matches'.\n"
|
||||
"Before you settle on a measure, name the ONE requirement of this base that BINDS it: "
|
||||
"find it with a filter, read it with read_file, then call "
|
||||
f"declare_requirement({bundle_id!r}, path, ref) with the requirement's own number. A "
|
||||
"declaration naming a document this run never opened is refused; reading it is the fix."
|
||||
f"declare_requirement({bundle_id!r}, path, ref, approach_id) with the requirement's own "
|
||||
"number and the id of the approach it binds (the run's own proposal is 'own-proposal'). "
|
||||
"Declare once per approach: a proposal whose approach declared nothing is not validated. "
|
||||
"A declaration naming a document this run never opened is refused; reading it is the fix."
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1184,6 +1218,12 @@ async def run_project(
|
|||
#: documents above, so the gate's vocabulary and the gate's text describe one reading of one
|
||||
#: base. Empty on the road path, which is what keeps the rule unable to fire there.
|
||||
bundle_references: tuple[str, ...] = ()
|
||||
#: Row 6: whether the declaration rung was offered to the debate. The rule that a validated
|
||||
#: proposal needs its approach's own declaration is active exactly when it was: a run that
|
||||
#: could declare and did not is the silence the rule exists for. It is NOT relaxed for a base
|
||||
#: without requirement numbers — any document the run read can be declared, and a base that
|
||||
#: holds nothing worth declaring is a finding about the base, not a reason to validate.
|
||||
requirement_rung = False
|
||||
# S2c: a CALLER-OWNED sink for what the debate opens (the ``parse_failures``/``ExplorationTrace``
|
||||
# shape). A returned value would be lost on exactly the run that most needs the evidence — a
|
||||
# budget stop mid-debate raises out of ``debate.run`` and constructs no ``RunResult`` at all.
|
||||
|
|
@ -1300,8 +1340,11 @@ async def run_project(
|
|||
# names every direction the run carries rather than picking one it cannot
|
||||
# attribute. Without a mandate this is empty and the reply is unchanged.
|
||||
labels=[a.label for a in mandate.approaches] if mandate else (),
|
||||
# Row 6: the ids a declaration may be filed under.
|
||||
approach_ids=_declarable_ids(mandate),
|
||||
)
|
||||
)
|
||||
requirement_rung = True
|
||||
# What the navigation could NOT reach, taken from the run's ONE walk. The road path below
|
||||
# navigates no bundle at all, so its empty tuple is literally true rather than a stand-in.
|
||||
skipped_links: tuple[okf.SkippedLink, ...] = bundle.skipped
|
||||
|
|
@ -1588,7 +1631,10 @@ async def run_project(
|
|||
validator_outcome = await _evaluate(None)
|
||||
else:
|
||||
validator_outcome, coverage, evaluated = await _evaluate_mandate(
|
||||
mandate, _evaluate, budget_stops
|
||||
mandate,
|
||||
_evaluate,
|
||||
budget_stops,
|
||||
declared=debate_requirements if requirement_rung else None,
|
||||
)
|
||||
except BaseException as stop:
|
||||
# Recorded and re-raised UNTOUCHED. This arm decides nothing about the exception itself —
|
||||
|
|
@ -1679,7 +1725,9 @@ async def run_project(
|
|||
model=model,
|
||||
role="proposer",
|
||||
validator_decision=(
|
||||
"validated" if isinstance(validator_outcome, ValidatedProposal) else "rejected"
|
||||
"validated"
|
||||
if isinstance(validator_outcome, (ValidatedProposal, Unsupported))
|
||||
else "rejected"
|
||||
),
|
||||
token_usage=meter.tokens,
|
||||
# Whether stage 0 of the deterministic gate had a baseline to reconcile against. Read off
|
||||
|
|
@ -1799,7 +1847,7 @@ async def run_project(
|
|||
update={
|
||||
"validator_decision": (
|
||||
"validated"
|
||||
if isinstance(approach_outcome, ValidatedProposal)
|
||||
if isinstance(approach_outcome, (ValidatedProposal, Unsupported))
|
||||
else "rejected"
|
||||
),
|
||||
# P20: ``code_forms`` must follow ITS OWN approach too, for
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ import argparse
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -98,7 +98,7 @@ class ApproachVerdict:
|
|||
|
||||
approach_id: str
|
||||
label: str
|
||||
status: str # "validated" | "rejected" | "not_evaluated"
|
||||
status: str # "validated" | "rejected" | "unsupported" | "not_evaluated"
|
||||
#: (a) - grounded by an OPENED path, or by a citation under a NARROWED list. See the module
|
||||
#: docstring: a whole-base citation list is stamped before any model call and grounds nothing.
|
||||
grounded: bool
|
||||
|
|
@ -197,6 +197,10 @@ class ContextSetVerdict:
|
|||
approach_rows_seen: int
|
||||
concepts_in_base: int
|
||||
ferdig: bool
|
||||
#: Row 6 - declarations with no ``approach_id``: written before declarations carried one. A run
|
||||
#: with any cannot be measured against the rule that a validated proposal needs its own
|
||||
#: approach's declaration, and the v1 gate says so instead of counting.
|
||||
unaddressed_declarations: int = 0
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
"""Byte-stable plain data: the ONE rendering, shared by the CLI's stdout and its file."""
|
||||
|
|
@ -265,21 +269,29 @@ def _inside(base: Path, raw: str) -> Path | None:
|
|||
return resolved if resolved == root or root in resolved.parents else None
|
||||
|
||||
|
||||
def _attributable(approach: Any, declared: Sequence[str]) -> tuple[tuple[str, ...], str]:
|
||||
def _attributable(
|
||||
approach: Any, declared: Sequence[Mapping[str, Any]]
|
||||
) -> tuple[tuple[str, ...], str]:
|
||||
"""Which declared requirement paths this approach may be judged on, and where they came from.
|
||||
|
||||
The approach's OWN requirement wins when it has one: the mandate carries it per approach, so
|
||||
it is unambiguous by construction. Otherwise the RUN's declarations are attributable - the
|
||||
debate declares once for the whole run, so the row says ``run`` rather than pretending the
|
||||
declaration was made about it. ``absent`` is the third value and is not the same as "declared
|
||||
nothing that matched": a run that declared nothing is a different finding from one that
|
||||
declared the wrong document."""
|
||||
it is unambiguous by construction. Next, a declaration filed under THIS approach's id (row 6)
|
||||
is the approach's own and says ``approach`` too. A declaration with no ``approach_id`` at all
|
||||
was written before declarations carried one; it can only be attributed to the whole run, and
|
||||
the row says ``run`` rather than pretending it was made about this approach. A declaration
|
||||
filed under ANOTHER approach's id is not this one's, so a run whose declarations all name other
|
||||
approaches reads ``absent`` here — the same value as a run that declared nothing.
|
||||
"""
|
||||
own = getattr(approach, "requirement", None)
|
||||
path = "" if own is None else str(getattr(own, "path", "") or "")
|
||||
if path:
|
||||
return (path,), "approach"
|
||||
if declared:
|
||||
return tuple(declared), "run"
|
||||
addressed = tuple(str(r["path"]) for r in declared if r.get("approach_id") == approach.id)
|
||||
if addressed:
|
||||
return addressed, "approach"
|
||||
legacy = tuple(str(r["path"]) for r in declared if "approach_id" not in r)
|
||||
if legacy:
|
||||
return legacy, "run"
|
||||
return (), "absent"
|
||||
|
||||
|
||||
|
|
@ -359,14 +371,13 @@ def score_context_set(
|
|||
# what every stress round has been) writes no ``{run_id}-exploration.json`` at all - the
|
||||
# hypothesiser never runs - so the debate artefact is the only one that can carry them there.
|
||||
# Both are read, because an ``--explore`` run carries them in the other.
|
||||
declared_paths: list[str] = []
|
||||
declared_records: list[dict[str, Any]] = []
|
||||
for artefact in (debate, outbox / f"{run_id}-exploration.json"):
|
||||
if artefact.is_file():
|
||||
declared_paths += [
|
||||
str(r.get("path", ""))
|
||||
for r in _read_json(artefact).get("requirements", [])
|
||||
if r.get("path")
|
||||
declared_records += [
|
||||
r for r in _read_json(artefact).get("requirements", []) if r.get("path")
|
||||
]
|
||||
declared_paths = [str(r["path"]) for r in declared_records]
|
||||
hallucinated_reads: list[str] = []
|
||||
for call in tool_calls:
|
||||
raw = str(call.get("path", ""))
|
||||
|
|
@ -417,9 +428,11 @@ def score_context_set(
|
|||
named_in_measure=False,
|
||||
named_in_snippet=False,
|
||||
hallucinations=(),
|
||||
requirement_declared=_attributable(approach, declared_paths)[0],
|
||||
requirement_source=_attributable(approach, declared_paths)[1],
|
||||
requirement_hit=bool(set(_attributable(approach, declared_paths)[0]) & wanted),
|
||||
requirement_declared=_attributable(approach, declared_records)[0],
|
||||
requirement_source=_attributable(approach, declared_records)[1],
|
||||
requirement_hit=bool(
|
||||
set(_attributable(approach, declared_records)[0]) & wanted
|
||||
),
|
||||
prose_codes=(),
|
||||
priced=False,
|
||||
not_evaluated_reason=stop_reason or "absent",
|
||||
|
|
@ -472,7 +485,7 @@ def score_context_set(
|
|||
forms = payload.get("provenance", {}).get("code_forms") or classify_codes(codes)
|
||||
prose_codes = tuple(sorted(c for c in codes if forms.get(c) == "prose"))
|
||||
|
||||
attributable, requirement_source = _attributable(approach, declared_paths)
|
||||
attributable, requirement_source = _attributable(approach, declared_records)
|
||||
requirement_hit = bool(set(attributable) & wanted)
|
||||
|
||||
halluc = [f"citation:{f}" for f in sorted(cited_files - concept_names)]
|
||||
|
|
@ -563,6 +576,7 @@ def score_context_set(
|
|||
must_refuse=tuple(refusals),
|
||||
hallucinated_reads=tuple(hallucinated_reads),
|
||||
requirements_declared=tuple(declared_paths),
|
||||
unaddressed_declarations=sum(1 for r in declared_records if "approach_id" not in r),
|
||||
token_usage=token_usage,
|
||||
stop_reason=stop_reason if coverage_seen else "absent",
|
||||
anchored=anchored,
|
||||
|
|
|
|||
|
|
@ -102,6 +102,30 @@ class Rejection:
|
|||
reason: str
|
||||
|
||||
|
||||
#: The ONE sentence an unsupported outcome carries. ``rejection_stage`` keys on it, so the judge can
|
||||
#: tell this falsifier from the numeric ones without a second copy of the wording.
|
||||
UNSUPPORTED_REASON: Final = (
|
||||
"no declared requirement for this approach: the numbers held, but no requirement of the "
|
||||
"knowledge base was declared as binding it"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Unsupported(Rejection):
|
||||
"""A proposal whose NUMBERS held but whose approach declared no binding requirement (row 6).
|
||||
|
||||
Neither ``validated`` (nothing in the knowledge base was said to support the direction) nor an
|
||||
ordinary rejection (every numeric stage passed). It subclasses ``Rejection`` on purpose: every
|
||||
consumer that asks "is this validated?" with ``isinstance(..., ValidatedProposal)`` answers no
|
||||
without being touched, so it is never counted, summed or carried as a success. The consumers
|
||||
that NAME the status — coverage, the outcome artefact, the settlement, the judge — check for
|
||||
this class first. ``validated`` keeps the validator's own ruling, which is what
|
||||
``provenance.validator_decision`` mirrors: the validator said yes, and the record says so.
|
||||
"""
|
||||
|
||||
validated: ValidatedProposal
|
||||
|
||||
|
||||
def _solve_max_feasible(items: list[AffectedItem], fraction: float) -> float:
|
||||
"""Real CBC solve: maximize total saving subject to a per-item upper bound and a
|
||||
global fraction cap. Raises ``CbcUnavailable`` if CBC is genuinely missing."""
|
||||
|
|
@ -728,6 +752,7 @@ _REJECTION_STAGES: Final = (
|
|||
("stage4-p90", ("exceeds P90 feasible",)),
|
||||
("stage4b-nominal", ("exceeds the nominal feasible",)),
|
||||
("stage5-method-cap", ("method cap",)),
|
||||
("unsupported", ("no declared requirement for this approach",)),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue