feat(p19): a direction must NAME the requirement that binds it, and have READ it
Two paid rounds scored 0 of 26 fasit concepts opened -- the same number twice.
P18 closed the navigation side (a listing is a window, an invented path is
refused by name) and it did not move, which makes it a ROLE question: nothing
in the loop ever asked the model to say what requirement binds the direction it
committed to, so opening one was never on the critical path to an answer.
A PREMISE OF THE ORDER WAS FELLED BEFORE ANYTHING WAS BUILT ON IT. A1 places
the demand in _INSTRUCTIONS[HYPOTHESISER_ROLE] alone. Measured: the stress
command sends --mandate and NOT --explore, the two are refused together by
name, and none of the nine round-1/2 outboxes holds a {run_id}-exploration.json
-- the hypothesiser never runs in a stress round, so A3 would have been
unreachable in exactly the paid runs this order commissions.
A2's own sentence resolves it: the refusal goes to the model "som en tur den
kan rette (samme mekanisme som quick_validate's nekt), ikke som en raise" --
and quick_validate IS a tool. declare_requirement therefore lives in
navigator_tools, held by BOTH roles that navigate (the exploration, and since
S2c the debate). It EXISTS only when the caller offers both sinks, which keeps
every pre-P19 call site byte-identical; one sink without the other is refused
at construction. 'opened' is the SAME list ExplorationToolRecorder fills, so
the refusal reads the run's own read trace.
The marked hypothesis carries 'requirement' as a REQUIRED key: omitted is a
hard error, explicit null is legal and needs 'why_none', a half-named one is
refused. A minted approach carries it; a seed never acquires one. The proposer
prompt names it only when the field exists, and the judge counts a hit against
THIS approach's fasit concepts, never against the base.
Load-bearing measured (12 arms), four mutations all red against the whole
suite, green control 1711/5 and demo-transcript.stdout byte-unchanged.
A-iii's predicted signature was FALSIFIED: the golden stays green because the
demo runs without a mandate, so _build_messages' approach branch is never
taken there. A-iv was GREEN first -- the repo's vacuous-gate class, 24th time:
the arm drove _attributable while the hit is computed at the call site.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
e7ba367f9d
commit
c84e8bf6f1
22 changed files with 903 additions and 35 deletions
|
|
@ -69,6 +69,7 @@ import argparse
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -110,6 +111,14 @@ class ApproachVerdict:
|
|||
named_in_snippet: bool
|
||||
#: (c) - attributable hallucinations, ``citation:<file>`` / ``code:<code>``.
|
||||
hallucinations: tuple[str, ...]
|
||||
#: P19 A4 - the binding requirement this row can be attributed, and whether it is one of the
|
||||
#: fasit's own concepts for this approach. ``requirement_source`` says WHICH of the two places
|
||||
#: it came from, because they are different claims: ``approach`` is per-approach by
|
||||
#: construction (the mandate carries it), while ``run`` is the DEBATE's declaration, which is
|
||||
#: written once per run and therefore cannot be attributed to one approach on its own.
|
||||
requirement_declared: tuple[str, ...]
|
||||
requirement_source: str # "approach" | "run" | "absent"
|
||||
requirement_hit: bool
|
||||
ferdig: bool
|
||||
|
||||
|
||||
|
|
@ -133,6 +142,11 @@ class ContextSetVerdict:
|
|||
must_refuse: tuple[RefusalVerdict, ...]
|
||||
#: Run-level: read paths the base does not carry (guessed by the navigator).
|
||||
hallucinated_reads: tuple[str, ...]
|
||||
#: P19 A4: every requirement the RUN declared as binding, in declaration order, with the
|
||||
#: denominator every other field here carries. Reported even when empty - "the run declared
|
||||
#: none" is the measurement, and a missing field would be indistinguishable from a judge that
|
||||
#: did not look.
|
||||
requirements_declared: tuple[str, ...]
|
||||
tool_calls_seen: int
|
||||
citations_seen: int
|
||||
approach_rows_seen: int
|
||||
|
|
@ -171,6 +185,24 @@ def _inside(base: Path, raw: str) -> Path | None:
|
|||
return resolved if resolved == root or root in resolved.parents else None
|
||||
|
||||
|
||||
def _attributable(approach: Any, declared: Sequence[str]) -> tuple[tuple[str, ...], str]:
|
||||
"""Which declared requirement paths this approach may be judged on, and where they came from.
|
||||
|
||||
The approach's OWN requirement wins when it has one: the mandate carries it per approach, so
|
||||
it is unambiguous by construction. Otherwise the RUN's declarations are attributable - the
|
||||
debate declares once for the whole run, so the row says ``run`` rather than pretending the
|
||||
declaration was made about it. ``absent`` is the third value and is not the same as "declared
|
||||
nothing that matched": a run that declared nothing is a different finding from one that
|
||||
declared the wrong document."""
|
||||
own = getattr(approach, "requirement", None)
|
||||
path = "" if own is None else str(getattr(own, "path", "") or "")
|
||||
if path:
|
||||
return (path,), "approach"
|
||||
if declared:
|
||||
return tuple(declared), "run"
|
||||
return (), "absent"
|
||||
|
||||
|
||||
def score_context_set(
|
||||
context_dir: str | Path,
|
||||
outbox_dir: str | Path,
|
||||
|
|
@ -206,6 +238,18 @@ def score_context_set(
|
|||
)
|
||||
opened_paths = {c.get("path", "") for c in tool_calls if c.get("name") == "read_file"}
|
||||
opened_paths.discard("")
|
||||
# P19 A4: WHERE the declarations live was measured, not assumed. A ``--mandate`` run (which is
|
||||
# what every stress round has been) writes no ``{run_id}-exploration.json`` at all - the
|
||||
# hypothesiser never runs - so the debate artefact is the only one that can carry them there.
|
||||
# Both are read, because an ``--explore`` run carries them in the other.
|
||||
declared_paths: list[str] = []
|
||||
for artefact in (debate, outbox / f"{run_id}-exploration.json"):
|
||||
if artefact.is_file():
|
||||
declared_paths += [
|
||||
str(r.get("path", ""))
|
||||
for r in _read_json(artefact).get("requirements", [])
|
||||
if r.get("path")
|
||||
]
|
||||
hallucinated_reads: list[str] = []
|
||||
for call in tool_calls:
|
||||
raw = str(call.get("path", ""))
|
||||
|
|
@ -245,6 +289,9 @@ def score_context_set(
|
|||
named_in_measure=False,
|
||||
named_in_snippet=False,
|
||||
hallucinations=(),
|
||||
requirement_declared=_attributable(approach, declared_paths)[0],
|
||||
requirement_source=_attributable(approach, declared_paths)[1],
|
||||
requirement_hit=bool(set(_attributable(approach, declared_paths)[0]) & wanted),
|
||||
ferdig=False,
|
||||
)
|
||||
)
|
||||
|
|
@ -288,6 +335,9 @@ def score_context_set(
|
|||
# snippets and gave this row ``named`` without the model having said anything.
|
||||
named_in_snippet = scope == "narrowed" and any(m in snippets for m in marks)
|
||||
|
||||
attributable, requirement_source = _attributable(approach, declared_paths)
|
||||
requirement_hit = bool(set(attributable) & wanted)
|
||||
|
||||
halluc = [f"citation:{f}" for f in sorted(cited_files - concept_names)]
|
||||
allowed = set(approach.affected_codes) | baseline_codes
|
||||
halluc += [
|
||||
|
|
@ -310,6 +360,9 @@ def score_context_set(
|
|||
named_in_measure=named_in_measure,
|
||||
named_in_snippet=named_in_snippet,
|
||||
hallucinations=tuple(halluc),
|
||||
requirement_declared=attributable,
|
||||
requirement_source=requirement_source,
|
||||
requirement_hit=requirement_hit,
|
||||
ferdig=(
|
||||
grounded
|
||||
and (named_in_measure or named_in_snippet)
|
||||
|
|
@ -355,6 +408,7 @@ def score_context_set(
|
|||
approaches=tuple(rows),
|
||||
must_refuse=tuple(refusals),
|
||||
hallucinated_reads=tuple(hallucinated_reads),
|
||||
requirements_declared=tuple(declared_paths),
|
||||
tool_calls_seen=len(tool_calls),
|
||||
citations_seen=citations_seen,
|
||||
approach_rows_seen=rows_seen,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue