test(v1-gate): harden the gate against a handwritten green
An independent review made rows 1, 2 and 4 green from a handwritten directory in a minute, and 10 of 20 mutants survived the gate's tests. Rounds now need a new point and their own ids, a timezone-aware given_at in order, and a report the feedback was given on; every outcome must name a run whose own coverage confirms (a)-(d), the feedback must fall between the two runs, and a NOK change under 1 % is noise. Row 4 counts content lines kept unchanged and in order, shows the expert's additions, and calls a byte-identical copy untouched unless round 3 acknowledges it. Row 6 counts the runs' own proposals. Types 3 and 7 are proven through the real flags with the action in the result (still 3 of 8). The contract numbers and the evidence register are pinned to their source. Every run prints that rows 1-2 cannot prove who wrote the feedback. A rounds directory inside the repo that git would commit, and a missing stress or bundle root, are usage errors. The review's 20 mutants, re-run: 20 of 20 killed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
938a1ca30e
commit
9825b2677c
5 changed files with 979 additions and 213 deletions
|
|
@ -15,7 +15,10 @@
|
|||
},
|
||||
"3": {
|
||||
"label": "vinklinger - nye vinklinger",
|
||||
"evidence": ["tests/test_mandate_cli.py::test_run_settles_against_the_mandate_afterwards"]
|
||||
"evidence": [
|
||||
"tests/test_v1_probes.py::test_type_3_a_commissioned_angle_is_evaluated_through_the_cli",
|
||||
"tests/test_v1_probes.py::test_type_3_a_new_angle_changes_the_outcome"
|
||||
]
|
||||
},
|
||||
"4": {
|
||||
"label": "lette paa krav",
|
||||
|
|
@ -32,7 +35,7 @@
|
|||
"7": {
|
||||
"label": "MCP - verktoey i debatten",
|
||||
"evidence": [
|
||||
"tests/test_mcp_run_loadbearing.py::test_configured_server_becomes_a_tool_the_agents_have",
|
||||
"tests/test_v1_probes.py::test_type_7_the_mcp_flag_puts_a_service_the_run_calls_into_the_result",
|
||||
"tests/test_b4_mcp_call_trace_loadbearing.py::test_a_called_mcp_tool_is_recorded_in_provenance"
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -25,12 +25,16 @@ import subprocess
|
|||
import sys
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
import difflib
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from portfolio_optimiser.validator import rejection_stage
|
||||
|
||||
_DATA = Path(__file__).with_name("v1_gate.json")
|
||||
_PACKAGE_SRC = Path(__file__).resolve().parents[1]
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
|
@ -38,6 +42,10 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
|
|||
DEFAULT_ROUNDS_DIR = "v1-rounds"
|
||||
#: A line of an AI-authored document shorter than this is too generic to identify its origin.
|
||||
_AI_LINE_MIN = 30
|
||||
#: Row 2 (d): a change in validated NOK below this share of the earlier figure is noise.
|
||||
_NOK_NOISE = 0.01
|
||||
#: Printed on every run: the one thing rows 1-2 cannot prove.
|
||||
ATTESTATION = "rad 1–2 beviser ikke at en fagperson skrev feedbacken; det bekrefter operatøren"
|
||||
|
||||
GREEN = "GRØNN"
|
||||
RED = "RØD"
|
||||
|
|
@ -46,30 +54,36 @@ DIAGNOSIS = "DIAGNOSE"
|
|||
NOT_MEASURED = "IKKE MÅLT"
|
||||
|
||||
ROUNDS_CONTRACT = """\
|
||||
Rundekatalogen (--rounds-dir) har fast form:
|
||||
Rundekatalogen (--rounds-dir) har fast form. Runde n = tilbakemelding på rapport n-1, så kjøring n:
|
||||
|
||||
<rounds-dir>/0/outcome.json grunnkjøringen (runde 0) som runde 1 måles mot
|
||||
<rounds-dir>/<n>/report.md rapporten slik fagpersonen fikk den (n = 1, 2, 3)
|
||||
<rounds-dir>/<n>/feedback.json fagpersonens tilbakemelding
|
||||
<rounds-dir>/<n>/outcome.json utfallet rapporten er bygget fra
|
||||
<rounds-dir>/0/report.md rapporten fra grunnkjøringen (runde 0)
|
||||
<rounds-dir>/0/outcome.json grunnkjøringen som runde 1 måles mot
|
||||
<rounds-dir>/<n>/feedback.json fagpersonens tilbakemelding på rapport n-1 (n = 1, 2, 3)
|
||||
<rounds-dir>/<n>/outcome.json kjøring n, gjort ETTER den tilbakemeldingen
|
||||
<rounds-dir>/<n>/report.md rapporten bygget fra kjøring n
|
||||
<rounds-dir>/3/report.kept.md runde 3-rapporten slik fagpersonen BEHOLDT den
|
||||
|
||||
feedback.json:
|
||||
{"author": "<fagpersonen>",
|
||||
{"author": "<fagpersonen>", "given_at": "<ISO-8601 med tidssone>",
|
||||
"report_unchanged": true (valgfri, kun runde 3: kvitterer for en urørt rapport),
|
||||
"items": [{"id": "<unik id>", "type": <1-8>, "text": "<tilbakemeldingen>"}]}
|
||||
|
||||
outcome.json:
|
||||
{"run_id": "<kjøringen>",
|
||||
outcome.json (hver rad sjekkes mot kjøringens egen <outbox>/<run_id>-coverage.json):
|
||||
{"run_id": "<kjøringen>", "outbox": "<utboksen, relativ til denne fila eller absolutt>",
|
||||
"approaches": [{"id": "<tilnærming>", "validated": true|false,
|
||||
"stage": "<avvisningsstadium, tom når validert>",
|
||||
"stage": "<avvisningsstadium som validator.rejection_stage gir, tom når validert>",
|
||||
"validated_nok": <tall eller null>,
|
||||
"feedback_ids": ["<id-er fra feedback.json som forklarer raden>"]}],
|
||||
"removed": [{"id": "<tilnærming fjernet siden forrige runde>", "feedback_ids": [...]}]}
|
||||
|
||||
En runde har målbar endring når den skiller seg fra forrige på minst én av (a) settet av
|
||||
tilnærmings-id-er, (b) hvilke som er validert, (c) avvisningsstadium, (d) validert NOK — OG minst
|
||||
én endret rad bærer en feedback-id gitt i DENNE runden. Tekst tatt fra et AI-forfattet dokument
|
||||
(docs/ekspert-svar.md) teller aldri som fagperson-tilbakemelding.
|
||||
En runde har målbar endring når kjøringen skiller seg fra forrige på minst én av (a) settet av
|
||||
tilnærmings-id-er, (b) hvilke som er validert, (c) avvisningsstadium, (d) validert NOK (endring
|
||||
under 1 % er støy) — OG minst én endret rad bærer en feedback-id gitt i DENNE runden, gitt mellom
|
||||
de to kjøringene. Hver runde må ha minst ett nytt punkt og egne id-er. Tekst tatt fra et
|
||||
AI-forfattet dokument (docs/ekspert-svar.md) teller aldri. Rad 1-2 beviser FORM, ikke forfatterskap.
|
||||
Rad 4 teller innholdslinjer (ikke blanke, skillelinjer eller tabellrammer) som står uendret og i
|
||||
samme rekkefølge; fagpersonens tillegg vises som eget tall. --rounds-dir inne i repoet må være
|
||||
gitignored.
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -102,13 +116,18 @@ def load_config(path: Path = _DATA) -> dict[str, Any]:
|
|||
|
||||
|
||||
def _norm(text: str) -> str:
|
||||
return " ".join(text.split())
|
||||
"""Whitespace collapsed and case folded — the AI guard's one normalisation."""
|
||||
return " ".join(text.split()).casefold()
|
||||
|
||||
|
||||
def ai_authored_lines(repo_root: Path, docs: Sequence[str]) -> tuple[str, str] | None:
|
||||
"""The normalised full text and the joined long lines of every AI-authored document, or
|
||||
``None`` when one of them cannot be read — the guard then cannot run, and a round it cannot
|
||||
check is never counted."""
|
||||
check is never counted.
|
||||
|
||||
A KNOWN-TEXT filter, not an authorship detector: it refuses text lifted from the listed
|
||||
documents (case and whitespace ignored), and nothing else. Authorship itself is not verifiable
|
||||
here, and the gate's output says so on every run (``ATTESTATION``)."""
|
||||
texts: list[str] = []
|
||||
for rel in docs:
|
||||
path = repo_root / rel
|
||||
|
|
@ -131,37 +150,109 @@ def _is_ai_text(text: str, ai: tuple[str, str]) -> bool:
|
|||
return any(line in item for line in lines.splitlines() if line)
|
||||
|
||||
|
||||
def read_feedback(round_dir: Path, ai: tuple[str, str] | None) -> tuple[set[str], str]:
|
||||
"""The ids of a round's feedback items, and ``""`` — or an empty set and the reason."""
|
||||
def _parse_time(value: Any) -> datetime | None:
|
||||
try:
|
||||
stamp = datetime.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
return stamp if stamp.tzinfo is not None else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Feedback:
|
||||
ids: frozenset[str]
|
||||
texts: frozenset[str]
|
||||
given_at: datetime
|
||||
report_unchanged: bool
|
||||
|
||||
|
||||
def _read_feedback_file(round_dir: Path, ai: tuple[str, str] | None) -> tuple[Feedback | None, str]:
|
||||
"""One round's feedback file, checked on its own; ``(None, why)`` when it does not hold."""
|
||||
name = f"runde {round_dir.name}"
|
||||
path = round_dir / "feedback.json"
|
||||
if not path.is_file():
|
||||
others = sorted(p.name for p in round_dir.glob("feedback.*")) if round_dir.is_dir() else []
|
||||
extra = f" (fant {', '.join(others)}; kontrakten er feedback.json)" if others else ""
|
||||
return set(), f"runde {round_dir.name}: feedback.json mangler{extra}"
|
||||
return None, f"{name}: feedback.json mangler{extra}"
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
author = str(data["author"]).strip()
|
||||
items = list(data["items"])
|
||||
given_raw = data["given_at"]
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
return set(), f"runde {round_dir.name}: feedback.json uleselig ({exc!r})"
|
||||
return None, f"{name}: feedback.json uleselig ({exc!r})"
|
||||
if not author:
|
||||
return set(), f"runde {round_dir.name}: feedback.json navngir ingen fagperson"
|
||||
return None, f"{name}: feedback.json navngir ingen fagperson"
|
||||
given_at = _parse_time(given_raw)
|
||||
if given_at is None:
|
||||
return None, f"{name}: given_at er ikke et ISO-tidsstempel med tidssone"
|
||||
if ai is None:
|
||||
return set(), f"runde {round_dir.name}: AI-vakten kunne ikke lese sine kilder"
|
||||
return None, f"{name}: AI-vakten kunne ikke lese sine kilder"
|
||||
ids: set[str] = set()
|
||||
texts: set[str] = set()
|
||||
for item in items:
|
||||
try:
|
||||
item_id, item_type, text = str(item["id"]), int(item["type"]), str(item["text"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return set(), f"runde {round_dir.name}: et feedback-punkt mangler id/type/text"
|
||||
return None, f"{name}: et feedback-punkt mangler id/type/text"
|
||||
if not item_id or not text.strip() or not 1 <= item_type <= 8:
|
||||
return set(), f"runde {round_dir.name}: punkt {item_id!r} er tomt eller har ukjent type"
|
||||
return None, f"{name}: punkt {item_id!r} er tomt eller har ukjent type"
|
||||
if item_id in ids:
|
||||
return None, f"{name}: punkt-id {item_id!r} er brukt to ganger"
|
||||
if _is_ai_text(text, ai):
|
||||
return set(), f"runde {round_dir.name}: punkt {item_id!r} er AI-forfattet tekst"
|
||||
return None, f"{name}: punkt {item_id!r} er AI-forfattet tekst"
|
||||
ids.add(item_id)
|
||||
texts.add(_norm(text))
|
||||
if not ids:
|
||||
return set(), f"runde {round_dir.name}: feedback.json har ingen punkter"
|
||||
return ids, ""
|
||||
return None, f"{name}: feedback.json har ingen punkter"
|
||||
unchanged = data.get("report_unchanged") is True
|
||||
return Feedback(frozenset(ids), frozenset(texts), given_at, unchanged), ""
|
||||
|
||||
|
||||
def read_rounds(
|
||||
rounds_dir: Path, required: int, ai: tuple[str, str] | None
|
||||
) -> dict[int, tuple[Feedback | None, str]]:
|
||||
"""Every round's feedback, with the cross-round rules applied in round order: a round must
|
||||
bring at least one point no earlier round gave, may not reuse an earlier round's ids (tracing is
|
||||
per round), must come after the previous round's feedback, and must have been given on a
|
||||
report (``<n-1>/report.md``)."""
|
||||
result: dict[int, tuple[Feedback | None, str]] = {}
|
||||
seen_ids: set[str] = set()
|
||||
seen_texts: set[str] = set()
|
||||
last: datetime | None = None
|
||||
for n in range(1, required + 1):
|
||||
feedback, why = _read_feedback_file(rounds_dir / str(n), ai)
|
||||
if feedback is not None:
|
||||
if not (rounds_dir / str(n - 1) / "report.md").is_file():
|
||||
feedback, why = (
|
||||
None,
|
||||
f"runde {n}: gitt på en rapport som mangler ({n - 1}/report.md)",
|
||||
)
|
||||
elif feedback.ids & seen_ids:
|
||||
reused = ", ".join(sorted(feedback.ids & seen_ids))
|
||||
feedback, why = (
|
||||
None,
|
||||
f"runde {n}: id-er fra en tidligere runde gjenbrukt ({reused})",
|
||||
)
|
||||
elif feedback.texts <= seen_texts:
|
||||
feedback, why = (
|
||||
None,
|
||||
f"runde {n}: ingen punkt som ikke alt er gitt i en tidligere runde",
|
||||
)
|
||||
elif last is not None and feedback.given_at <= last:
|
||||
feedback, why = None, f"runde {n}: given_at er ikke etter forrige rundes"
|
||||
if feedback is not None:
|
||||
seen_ids |= feedback.ids
|
||||
seen_texts |= feedback.texts
|
||||
last = feedback.given_at
|
||||
result[n] = (feedback, why)
|
||||
return result
|
||||
|
||||
|
||||
def read_feedback(round_dir: Path, ai: tuple[str, str] | None) -> tuple[set[str], str]:
|
||||
"""The ids of ONE round's feedback items checked on its own, and ``""`` — or why not."""
|
||||
feedback, why = _read_feedback_file(round_dir, ai)
|
||||
return (set(feedback.ids), "") if feedback is not None else (set(), why)
|
||||
|
||||
|
||||
def score_rounds(rounds_dir: Path, required: int, ai: tuple[str, str] | None) -> Row:
|
||||
|
|
@ -170,14 +261,17 @@ def score_rounds(rounds_dir: Path, required: int, ai: tuple[str, str] | None) ->
|
|||
if not rounds_dir.is_dir():
|
||||
exceptions.append(f"{rounds_dir} finnes ikke")
|
||||
else:
|
||||
for n in range(1, required + 1):
|
||||
ids, why = read_feedback(rounds_dir / str(n), ai)
|
||||
if ids:
|
||||
for _, (feedback, why) in sorted(read_rounds(rounds_dir, required, ai).items()):
|
||||
if feedback is not None:
|
||||
k += 1
|
||||
else:
|
||||
exceptions.append(why)
|
||||
status = GREEN if k == required else RED
|
||||
reason = "alle runder har fagperson-tilbakemelding" if k == required else exceptions[0]
|
||||
reason = (
|
||||
"form verifisert i alle runder (forfatterskap: se attestering)"
|
||||
if k == required
|
||||
else exceptions[0]
|
||||
)
|
||||
return Row(
|
||||
"rounds",
|
||||
"1 runder med ekte fagperson",
|
||||
|
|
@ -189,42 +283,102 @@ def score_rounds(rounds_dir: Path, required: int, ai: tuple[str, str] | None) ->
|
|||
)
|
||||
|
||||
|
||||
def _read_outcome(path: Path) -> tuple[dict[str, dict[str, Any]], dict[str, set[str]], str, str]:
|
||||
"""Rows by approach id, removed ids with their feedback ids, the run id, and ``""`` or why."""
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
rows: dict[str, dict[str, Any]]
|
||||
removed: dict[str, set[str]]
|
||||
run_id: str
|
||||
ran_at: datetime
|
||||
|
||||
|
||||
def _stage_of(status: str, detail: str) -> str:
|
||||
if status == "validated":
|
||||
return ""
|
||||
if status == "not_evaluated":
|
||||
return "not_evaluated"
|
||||
return rejection_stage(detail)
|
||||
|
||||
|
||||
def read_outcome(path: Path) -> tuple[Outcome | None, str]:
|
||||
"""A round's outcome file, VERIFIED against the run it names: ``outbox`` must hold
|
||||
``<run_id>-coverage.json``, and every row's (a)-(d) must equal that run's own coverage. A
|
||||
handwritten outcome with no run behind it is refused; the run's time is the coverage file's."""
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
rows = {str(a["id"]): a for a in data["approaches"]}
|
||||
removed = {
|
||||
str(r["id"]): set(map(str, r.get("feedback_ids", ()))) for r in data.get("removed", ())
|
||||
}
|
||||
return rows, removed, str(data.get("run_id", "")), ""
|
||||
run_id = str(data["run_id"]).strip()
|
||||
outbox = Path(str(data["outbox"])).expanduser()
|
||||
except FileNotFoundError:
|
||||
return {}, {}, "", f"{path} mangler"
|
||||
return None, f"{path} mangler"
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
return {}, {}, "", f"{path} uleselig ({exc!r})"
|
||||
return None, f"{path} uleselig ({exc!r})"
|
||||
if not run_id:
|
||||
return None, f"{path}: run_id er tom"
|
||||
if not outbox.is_absolute():
|
||||
outbox = path.parent / outbox
|
||||
coverage_path = outbox / f"{run_id}-coverage.json"
|
||||
if not coverage_path.is_file():
|
||||
return None, f"{path}: kjøringen {run_id!r} finnes ikke ({coverage_path} mangler)"
|
||||
try:
|
||||
coverage = json.loads(coverage_path.read_text(encoding="utf-8"))["rows"]
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
return None, f"{coverage_path} uleselig ({exc!r})"
|
||||
if not coverage:
|
||||
return None, f"{path}: kjøringen {run_id!r} evaluerte ingen tilnærming"
|
||||
truth = {
|
||||
str(r["id"]): (
|
||||
r["status"] == "validated",
|
||||
_stage_of(str(r["status"]), str(r.get("detail", ""))),
|
||||
r.get("saving_nok") if r["status"] == "validated" else None,
|
||||
)
|
||||
for r in coverage
|
||||
}
|
||||
claimed = {aid: _row_key(row) for aid, row in rows.items()}
|
||||
if claimed != truth:
|
||||
return None, f"{path}: (a)-(d) stemmer ikke med kjøringens egen coverage ({run_id})"
|
||||
ran_at = datetime.fromtimestamp(coverage_path.stat().st_mtime, tz=timezone.utc)
|
||||
return Outcome(rows, removed, run_id, ran_at), ""
|
||||
|
||||
|
||||
def _row_key(row: Mapping[str, Any]) -> tuple[bool, str, Any]:
|
||||
return bool(row.get("validated")), str(row.get("stage") or ""), row.get("validated_nok")
|
||||
nok = row.get("validated_nok")
|
||||
return (
|
||||
bool(row.get("validated")),
|
||||
str(row.get("stage") or ""),
|
||||
None if nok is None else float(nok),
|
||||
)
|
||||
|
||||
|
||||
def round_changed(before: Path, after: Path, feedback_ids: set[str]) -> tuple[bool, str]:
|
||||
"""Whether round ``after`` changed measurably against ``before`` AND the change is traced to
|
||||
feedback given in this round. The second half is what keeps model noise out."""
|
||||
prev, _, _, why = _read_outcome(before)
|
||||
if why:
|
||||
return False, why
|
||||
cur, removed, _, why = _read_outcome(after)
|
||||
if why:
|
||||
return False, why
|
||||
def _nok_changed(before: Any, after: Any) -> bool:
|
||||
"""(d) with a noise floor. ``validated_nok`` is the model's own claim, so two runs on the same
|
||||
input can differ by rounding; a change smaller than 1 % of the earlier figure (and never less
|
||||
than 1 NOK) is not something an expert's feedback asked for, and it does not count."""
|
||||
if (before is None) != (after is None):
|
||||
return True
|
||||
if before is None or after is None:
|
||||
return False
|
||||
return abs(float(after) - float(before)) >= max(1.0, _NOK_NOISE * abs(float(before)))
|
||||
|
||||
|
||||
def _changed(before: Mapping[str, Any], after: Mapping[str, Any]) -> bool:
|
||||
b, a = _row_key(before), _row_key(after)
|
||||
return b[:2] != a[:2] or _nok_changed(b[2], a[2])
|
||||
|
||||
|
||||
def outcomes_changed(prev: Outcome, cur: Outcome, feedback_ids: set[str]) -> tuple[bool, str]:
|
||||
"""Whether ``cur`` changed measurably against ``prev`` AND the change is traced to feedback
|
||||
given before ``cur`` ran. The second half is what keeps model noise out."""
|
||||
changed: dict[str, set[str]] = {}
|
||||
for aid, row in cur.items():
|
||||
if aid not in prev or _row_key(prev[aid]) != _row_key(row):
|
||||
for aid, row in cur.rows.items():
|
||||
if aid not in prev.rows or _changed(prev.rows[aid], row):
|
||||
changed[aid] = set(map(str, row.get("feedback_ids", ())))
|
||||
for aid in prev.keys() - cur.keys():
|
||||
changed[aid] = removed.get(aid, set())
|
||||
for aid in prev.rows.keys() - cur.rows.keys():
|
||||
changed[aid] = cur.removed.get(aid, set())
|
||||
if not changed:
|
||||
return False, "ingen endring i (a)-(d)"
|
||||
return False, "ingen endring i (a)-(d) over støygrensen"
|
||||
traced = sorted(aid for aid, ids in changed.items() if ids & feedback_ids)
|
||||
if not traced:
|
||||
return False, f"{len(changed)} rad(er) endret, ingen sporet til rundens feedback-id-er"
|
||||
|
|
@ -234,17 +388,27 @@ def round_changed(before: Path, after: Path, feedback_ids: set[str]) -> tuple[bo
|
|||
def score_changes(rounds_dir: Path, required: int, ai: tuple[str, str] | None) -> Row:
|
||||
exceptions: list[str] = []
|
||||
k = 0
|
||||
_, _, base_run, base_why = _read_outcome(rounds_dir / "0" / "outcome.json")
|
||||
base = f"runde 0 = {rounds_dir / '0' / 'outcome.json'}"
|
||||
base += f" ({base_run})" if base_run else (f" — {base_why}" if base_why else "")
|
||||
base_path = rounds_dir / "0" / "outcome.json"
|
||||
base_outcome, base_why = read_outcome(base_path)
|
||||
base = f"runde 0 = {base_path}"
|
||||
base += f" (kjøring {base_outcome.run_id})" if base_outcome else f" — {base_why}"
|
||||
feedback_by_round = read_rounds(rounds_dir, required, ai) if rounds_dir.is_dir() else {}
|
||||
for n in range(1, required + 1):
|
||||
ids, why = read_feedback(rounds_dir / str(n), ai)
|
||||
if not ids:
|
||||
feedback, why = feedback_by_round.get(n, (None, f"runde {n}: {rounds_dir} finnes ikke"))
|
||||
if feedback is None:
|
||||
exceptions.append(why)
|
||||
continue
|
||||
ok, detail = round_changed(
|
||||
rounds_dir / str(n - 1) / "outcome.json", rounds_dir / str(n) / "outcome.json", ids
|
||||
)
|
||||
prev, why_prev = read_outcome(rounds_dir / str(n - 1) / "outcome.json")
|
||||
cur, why_cur = read_outcome(rounds_dir / str(n) / "outcome.json")
|
||||
if prev is None or cur is None:
|
||||
exceptions.append(f"runde {n}: {why_prev or why_cur}")
|
||||
continue
|
||||
if not prev.ran_at <= feedback.given_at <= cur.ran_at:
|
||||
exceptions.append(
|
||||
f"runde {n}: feedbacken er ikke gitt mellom kjøring {prev.run_id} og {cur.run_id}"
|
||||
)
|
||||
continue
|
||||
ok, detail = outcomes_changed(prev, cur, set(feedback.ids))
|
||||
if ok:
|
||||
k += 1
|
||||
else:
|
||||
|
|
@ -354,36 +518,74 @@ def score_types(types: Mapping[str, Any], outcomes: Mapping[str, str]) -> Row:
|
|||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def kept_ratio(report: Path, kept: Path) -> tuple[int, int, str]:
|
||||
#: A line that carries no reading: horizontal rules, table rules, and non-breaking-space fillers.
|
||||
_MARKUP_ONLY = re.compile(
|
||||
r"^\s*(?:(?:[-*_=]\s*){3,}|\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?| )\s*$"
|
||||
)
|
||||
|
||||
|
||||
def content_lines(text: str) -> list[str]:
|
||||
"""The lines a reader keeps or rewrites: trailing whitespace stripped (an editor's doing, never
|
||||
the expert's), blank lines and markup-only lines dropped, everything else — headings included —
|
||||
kept in order. Nothing else is normalised."""
|
||||
lines = [line.rstrip() for line in text.splitlines()]
|
||||
return [x for x in lines if re.search(r"\w", x) and not _MARKUP_ONLY.match(x)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Kept:
|
||||
kept: int
|
||||
total: int
|
||||
added: int
|
||||
untouched: bool
|
||||
why: str = ""
|
||||
|
||||
|
||||
def kept_ratio(report: Path, kept: Path) -> Kept:
|
||||
"""How much of the report the expert kept: content lines of ``report`` that survive in
|
||||
``kept`` IN ORDER (each line matched at most once — a multiset, and a reshuffle is a change),
|
||||
plus the expert's additions as their own number. A byte-identical copy is flagged as untouched:
|
||||
nobody can tell it from a report nobody read."""
|
||||
if not report.is_file() or not kept.is_file():
|
||||
return 0, 0, "ingen rapport"
|
||||
lines = [x for x in report.read_text(encoding="utf-8").splitlines() if x.strip()]
|
||||
if not lines:
|
||||
return 0, 0, "tom rapport"
|
||||
pool = Counter(x for x in kept.read_text(encoding="utf-8").splitlines() if x.strip())
|
||||
same = 0
|
||||
for line in lines:
|
||||
if pool[line] > 0:
|
||||
pool[line] -= 1
|
||||
same += 1
|
||||
return same, len(lines), ""
|
||||
return Kept(0, 0, 0, False, "ingen rapport")
|
||||
raw_report = report.read_bytes()
|
||||
raw_kept = kept.read_bytes()
|
||||
before = content_lines(raw_report.decode("utf-8"))
|
||||
after = content_lines(raw_kept.decode("utf-8"))
|
||||
if not before:
|
||||
return Kept(0, 0, 0, False, "tom rapport")
|
||||
matcher = difflib.SequenceMatcher(None, before, after, autojunk=False)
|
||||
same = sum(block.size for block in matcher.get_matching_blocks())
|
||||
return Kept(same, len(before), len(after) - same, raw_report == raw_kept)
|
||||
|
||||
|
||||
def score_kept(rounds_dir: Path, threshold: float) -> Row:
|
||||
same, total, why = kept_ratio(
|
||||
rounds_dir / "3" / "report.md", rounds_dir / "3" / "report.kept.md"
|
||||
)
|
||||
title = f"4 runde 3-rapport beholdt (≥ {threshold:.0%} linjer)"
|
||||
if why:
|
||||
return Row("kept", title, None, total or None, RED, why)
|
||||
ok = same >= threshold * total
|
||||
def score_kept(rounds_dir: Path, threshold: float, ai: tuple[str, str] | None = None) -> Row:
|
||||
result = kept_ratio(rounds_dir / "3" / "report.md", rounds_dir / "3" / "report.kept.md")
|
||||
title = f"4 runde 3-rapport beholdt (≥ {threshold:.0%} innholdslinjer)"
|
||||
if result.why:
|
||||
return Row("kept", title, None, result.total or None, RED, result.why)
|
||||
added = f"; {result.added} linje(r) lagt til av fagpersonen"
|
||||
if result.untouched:
|
||||
feedback, _ = _read_feedback_file(rounds_dir / "3", ai)
|
||||
if feedback is None or not feedback.report_unchanged:
|
||||
return Row(
|
||||
"kept",
|
||||
title,
|
||||
None,
|
||||
result.total,
|
||||
RED,
|
||||
"ikke rørt: report.kept.md er byte-identisk med report.md; kvitter med "
|
||||
'"report_unchanged": true i 3/feedback.json',
|
||||
)
|
||||
ok = result.kept >= threshold * result.total
|
||||
return Row(
|
||||
"kept",
|
||||
title,
|
||||
same,
|
||||
total,
|
||||
result.kept,
|
||||
result.total,
|
||||
GREEN if ok else RED,
|
||||
f"{same / total:.1%} av ikke-tomme linjer uendret",
|
||||
f"{result.kept / result.total:.1%} av innholdslinjene uendret og i rekkefølge{added}",
|
||||
diagnostics=(f"lagt til: {result.added}",),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -400,8 +602,31 @@ def _imports(tree: ast.AST, construct: str, package: str) -> set[str]:
|
|||
return names
|
||||
|
||||
|
||||
def _annotations(node: ast.AST) -> set[int]:
|
||||
"""ids of every node inside a type annotation — a name used only as a type is not a use."""
|
||||
found: set[int] = set()
|
||||
for x in ast.walk(node):
|
||||
parts: list[ast.AST | None] = []
|
||||
if isinstance(x, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
parts.append(x.returns)
|
||||
elif isinstance(x, ast.arg):
|
||||
parts.append(x.annotation)
|
||||
elif isinstance(x, ast.AnnAssign):
|
||||
parts.append(x.annotation)
|
||||
for part in parts:
|
||||
if part is not None:
|
||||
found |= {id(y) for y in ast.walk(part)}
|
||||
return found
|
||||
|
||||
|
||||
def _referenced(nodes: Iterable[ast.AST], names: set[str]) -> bool:
|
||||
return any(isinstance(x, ast.Name) and x.id in names for node in nodes for x in ast.walk(node))
|
||||
for node in nodes:
|
||||
typed = _annotations(node)
|
||||
if any(
|
||||
isinstance(x, ast.Name) and x.id in names and id(x) not in typed for x in ast.walk(node)
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def maf_presence(point: Mapping[str, Any], src: Path) -> tuple[bool, bool]:
|
||||
|
|
@ -502,9 +727,38 @@ class StressMeasure:
|
|||
missing: str = ""
|
||||
#: Declarations with no ``approach_id`` — written before the rule; the row cannot be measured.
|
||||
unaddressed: int = 0
|
||||
#: Of ``validated``, how many were the runs' own proposals (M-3).
|
||||
own_validated: int = 0
|
||||
undeclared_ids: tuple[str, ...] = field(default=())
|
||||
|
||||
|
||||
def _own_proposals(
|
||||
evidence: Mapping[str, Any], stress_root: Path
|
||||
) -> tuple[int, int, tuple[str, ...]]:
|
||||
"""(validated own proposals, of those without an ``own-proposal`` declaration, their labels)."""
|
||||
validated = undeclared = 0
|
||||
ids: list[str] = []
|
||||
for run_spec in evidence["runs"]:
|
||||
outbox = stress_root / run_spec["outbox"]
|
||||
run_id = run_spec["run_id"]
|
||||
outcome = outbox / f"{run_id}-own-proposal-outcome.json"
|
||||
if not outcome.is_file():
|
||||
continue
|
||||
if json.loads(outcome.read_text(encoding="utf-8")).get("outcome_type") != "validated":
|
||||
continue
|
||||
validated += 1
|
||||
debate = outbox / f"{run_id}-debate.json"
|
||||
records = (
|
||||
json.loads(debate.read_text(encoding="utf-8")).get("requirements", [])
|
||||
if debate.is_file()
|
||||
else []
|
||||
)
|
||||
if not any(r.get("approach_id") == "own-proposal" for r in records):
|
||||
undeclared += 1
|
||||
ids.append(f"own-proposal ({run_id})")
|
||||
return validated, undeclared, tuple(ids)
|
||||
|
||||
|
||||
def measure_stress(
|
||||
evidence: Mapping[str, Any], repo_root: Path, stress_root: Path, bundle_root: Path
|
||||
) -> StressMeasure:
|
||||
|
|
@ -542,20 +796,24 @@ def measure_stress(
|
|||
approaches = [a for v in verdicts for a in v.approaches]
|
||||
validated = [a for a in approaches if a.status == "validated"]
|
||||
undeclared = [a for a in validated if a.requirement_source != "approach"]
|
||||
# M-3: the run's OWN proposal is gated by the same rule, but the judge scores only the
|
||||
# commissioned approaches (the fasit has rows for nothing else). Read straight off each outbox.
|
||||
own_validated, own_undeclared, own_ids = _own_proposals(evidence, stress_root)
|
||||
unaddressed = sum(v.unaddressed_declarations for v in verdicts)
|
||||
commissioned = sum(
|
||||
len(load_mandate(repo_root / c / "mandate.json").approaches) for c in contexts
|
||||
)
|
||||
return StressMeasure(
|
||||
validated=len(validated),
|
||||
undeclared=len(undeclared),
|
||||
validated=len(validated) + own_validated,
|
||||
undeclared=len(undeclared) + own_undeclared,
|
||||
undeclared_anywhere=sum(1 for a in validated if a.requirement_source == "absent"),
|
||||
named=sum(1 for a in approaches if a.named),
|
||||
rows=len(approaches),
|
||||
commissioned=commissioned,
|
||||
where=str(stress_root),
|
||||
unaddressed=unaddressed,
|
||||
undeclared_ids=tuple(a.approach_id for a in undeclared),
|
||||
undeclared_ids=tuple(a.approach_id for a in undeclared) + own_ids,
|
||||
own_validated=own_validated,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -585,14 +843,16 @@ def score_undeclared(
|
|||
f"(approach_id mangler på {m.unaddressed} erklæring(er))"
|
||||
)
|
||||
diagnostics = (
|
||||
f"før regelen: {m.undeclared} av {m.validated} validerte uten tilnærmingens egen "
|
||||
"erklæring — regelen ville gjort dem unsupported, men modellen fikk aldri spørsmålet",
|
||||
f"før regelen: {m.undeclared} av {m.validated} validerte (hvorav {m.own_validated} "
|
||||
"egne forslag) uten tilnærmingens egen erklæring — regelen ville gjort dem "
|
||||
"unsupported, men modellen fikk aldri spørsmålet",
|
||||
)
|
||||
else:
|
||||
k, n = m.undeclared, m.validated
|
||||
reason = (
|
||||
f"{probe_state}; {label} ({m.where}): {k} av {n} validerte uten erklæring fra "
|
||||
f"tilnærmingen; {m.undeclared_anywhere} uten noen erklæring i kjøringen"
|
||||
f"tilnærmingen (hvorav {m.own_validated} egne forslag i nevneren); "
|
||||
f"{m.undeclared_anywhere} uten noen erklæring i kjøringen"
|
||||
)
|
||||
exceptions += [f"validert uten erklæring: {a}" for a in m.undeclared_ids]
|
||||
if failing or k:
|
||||
|
|
@ -682,7 +942,7 @@ def evaluate(
|
|||
score_rounds(rounds_dir, required, ai),
|
||||
score_changes(rounds_dir, required, ai),
|
||||
score_types(types, outcomes),
|
||||
score_kept(rounds_dir, float(config["keep_threshold"])),
|
||||
score_kept(rounds_dir, float(config["keep_threshold"]), ai),
|
||||
score_maf(config["maf_points"], green_types(types, outcomes), src),
|
||||
score_undeclared(probes, outcomes, stress_measure, evidence["label"]),
|
||||
score_named(stress_measure, evidence["label"]),
|
||||
|
|
@ -697,6 +957,8 @@ def render(rows: Sequence[Row]) -> str:
|
|||
out = ["rad | k av N | status | grunn"]
|
||||
out += [r.line() for r in rows]
|
||||
out.append("")
|
||||
out.append(f"Attestering: {ATTESTATION}.")
|
||||
out.append("")
|
||||
out.append("Unntak fra 100 %:")
|
||||
for r in rows:
|
||||
for x in r.exceptions:
|
||||
|
|
@ -706,6 +968,20 @@ def render(rows: Sequence[Row]) -> str:
|
|||
return "\n".join(out)
|
||||
|
||||
|
||||
def _safe_rounds_dir(path: Path) -> bool:
|
||||
"""Outside the repository, or inside it and ignored by git."""
|
||||
resolved = path.resolve()
|
||||
try:
|
||||
resolved.relative_to(_REPO_ROOT)
|
||||
except ValueError:
|
||||
return True
|
||||
probe = resolved / "1" / "feedback.json"
|
||||
proc = subprocess.run(
|
||||
["git", "check-ignore", "-q", str(probe)], cwd=_REPO_ROOT, capture_output=True
|
||||
)
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="python -m portfolio_optimiser.evals.v1_gate",
|
||||
|
|
@ -730,6 +1006,14 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
|
||||
if args.rounds_dir is not None and not Path(args.rounds_dir).is_dir():
|
||||
parser.error(f"--rounds-dir {args.rounds_dir} finnes ikke")
|
||||
if args.rounds_dir is not None and not _safe_rounds_dir(Path(args.rounds_dir)):
|
||||
parser.error(
|
||||
f"--rounds-dir {args.rounds_dir} ligger i repoet uten å være gitignored — "
|
||||
"fagpersonens tilbakemelding kunne da bli committet til den offentlige remoten"
|
||||
)
|
||||
for flag, value in (("--stress-root", args.stress_root), ("--bundle-root", args.bundle_root)):
|
||||
if value is not None and not Path(value).expanduser().is_dir():
|
||||
parser.error(f"{flag} {value} finnes ikke")
|
||||
rounds_dir = Path(args.rounds_dir) if args.rounds_dir else _REPO_ROOT / DEFAULT_ROUNDS_DIR
|
||||
rows = evaluate(
|
||||
rounds_dir=rounds_dir,
|
||||
|
|
@ -741,7 +1025,9 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
{"exit": code, "rows": [asdict(r) for r in rows]}, ensure_ascii=False, indent=2
|
||||
{"exit": code, "attestation": ATTESTATION, "rows": [asdict(r) for r in rows]},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue