Kjell Tore Guttormsen
0ed305e979
Row 1: 12 of 16 register entries answer through an existing probe class of the reused edition watch (positive control 4 of 4); unicode and rfc-3986 need a host on its list, the commons repository is private, and the Dual LLM post has no edition. Row 2: the seven documented gaps are frozen in gaps-frozen.tsv and held by tests/test_gap_register.py. docs/gap-sweep.py measures the cheapest rule per gap on the three false-positive corpora and the pinned OKF corpus before anything is built: gaps 4, 1 and 6 are closable without new alarms; 2, 3, 5 and 7 get measured reasons.
59 lines
2.3 KiB
Python
59 lines
2.3 KiB
Python
"""The seven documented gaps are a frozen denominator: row 2 of the 2026-11 done-criterion.
|
|
|
|
Row 2 (`docs/PLAN.md` § V.2) counts, for each documented gap, whether it is closed
|
|
or stands with a measured reason in `docs/LIMITATIONS.md`. The denominator is the
|
|
seven frozen in session 1 (2026-11-01) as `docs/threat-2026/gaps-frozen.tsv`,
|
|
whose sha256 is pinned here and in the plan.
|
|
|
|
This file fails when the frozen list is edited, when a gap disappears from the
|
|
coverage matrix without being recorded as closed below, or when a new gap case
|
|
appears without being frozen. Closing a gap is therefore a deliberate edit of
|
|
``CLOSED`` in the same commit that moves the matrix case from gap to caught.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
from llm_ingestion_guard.coverage import CORE_CASES
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
FROZEN = ROOT / "docs" / "threat-2026" / "gaps-frozen.tsv"
|
|
FROZEN_SHA256 = "b1a6d5be5fbfc81263e55135abf11bc6fe349eac82db0d1e84f9e44949b9d22a"
|
|
|
|
# tag -> the finding label that now catches it. Empty until a gap is closed.
|
|
CLOSED: dict[str, str] = {}
|
|
|
|
|
|
def _rows() -> list[dict[str, str]]:
|
|
with FROZEN.open(encoding="utf-8", newline="") as fh:
|
|
return list(csv.DictReader(fh, delimiter="\t"))
|
|
|
|
|
|
def test_frozen_list_is_unchanged():
|
|
assert hashlib.sha256(FROZEN.read_bytes()).hexdigest() == FROZEN_SHA256
|
|
|
|
|
|
def test_denominator_is_seven():
|
|
rows = _rows()
|
|
assert [r["n"] for r in rows] == [str(i) for i in range(1, 8)]
|
|
assert len({r["tag"] for r in rows}) == 7
|
|
|
|
|
|
def test_every_frozen_gap_is_open_in_the_matrix_or_recorded_closed():
|
|
gaps = {c.expect for c in CORE_CASES if c.status == "gap"}
|
|
caught = {c.expect for c in CORE_CASES if c.status == "caught"}
|
|
for row in _rows():
|
|
tag = row["tag"]
|
|
if tag in CLOSED:
|
|
assert tag not in gaps, f"{tag} recorded closed but still a gap case"
|
|
assert CLOSED[tag] in caught, f"{tag} closed by {CLOSED[tag]}, which no caught case asserts"
|
|
else:
|
|
assert tag in gaps, f"{tag} left the matrix without being recorded in CLOSED"
|
|
|
|
|
|
def test_no_gap_case_outside_the_frozen_list():
|
|
frozen = {r["tag"] for r in _rows()}
|
|
extra = {c.expect for c in CORE_CASES if c.status == "gap"} - frozen
|
|
assert not extra, f"gap cases not in the frozen denominator: {sorted(extra)}"
|