"""False-positive sweep — run *benign* document populations through the persist gate and count the ones it does not wave through. WHAT THE NUMBER MEANS, EXACTLY. The unit is a **document**, not a URL, and the metric is `screen_output(doc, PRESET_USER_UPLOAD).disposition is not WARN`. Every word of that is load-bearing: - **Document, not URL.** `docs/LIMITATIONS.md` also carries URL-level field measurements (16 of 16, 28 of 28, 149 of 1694). Those are a different unit over partly-overlapping corpora. A document rate is NOT comparable to them and must never be combined with them, or quoted as an update to them. - **The upload preset, not the trusted one.** `PRESET_TRUSTED_SOURCE` will hand you a beautiful near-zero and mean nothing: every non-CRITICAL finding WARNs under trust, which is exactly the structural blindness that let the 0.3.0 active-content regression ship through a green suite (see the docstring on `tests/test_corpus.py::test_false_positive_is_not_blocked_on_the_upload_gate`). The trusted door is printed as a footnote, never as the headline. - **`screen_output`, not `_scan_input`.** The output gate is where `active_content` lives; the input path never reaches it. - **not WARN**, not "has findings". A finding is not a false positive — the library reports and the pipeline decides (BRIEF design principle 4). WARN means *persisted, with a note*, which is the benign outcome. GROUND TRUTH for "benign" is **provenance, not inspection**: nobody hand-read these documents. Each population is benign by where it came from — vendor- published documentation, this machine's own generated notes, first-party authored reference material. That is the only ground truth available at this scale, and it is a real caveat, not a formality: an injected document sitting in a harvested corpus would be scored as a false positive here. POPULATIONS ARE NEVER SUMMED. Every population has its own denominator and its own provenance; a pooled rate would be arithmetic over incommensurable things and would inherit the `2400 != 2401` defect one level up. This script refuses to print a total. USAGE — corpus roots are arguments, never hardcoded; the corpora live in private consumer repos and their paths must not reach a public mirror: python docs/fp-sweep.py LABEL=/path/to/corpus [LABEL=/path ...] [--ext=.md,.txt] [--include=/subtree/] Each LABEL should name the population's *class* (`vendor-harvest`, `generated-notes`, `reference-corpus`), not the repo it came from. """ from __future__ import annotations import sys from collections import Counter from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) from llm_ingestion_guard import ( # noqa: E402 DEFAULT_ACTION_MAP, PRESET_TRUSTED_SOURCE, PRESET_USER_UPLOAD, Disposition, Risk, Source, __version__, scan_output, screen_output, severity_rank, ) from llm_ingestion_guard.calibration import RISK_RANK # noqa: E402 BENIGN = Disposition.WARN """The benign outcome: persisted, with a note. Anything else costs a human.""" def check_metric_is_a_risk_statement() -> None: """Fail loudly if "not WARN" has stopped meaning "assessed ELEVATED or worse". The published number is a count of non-WARN documents, but what it *claims* is a statement about assessed risk. The two are the same statement only while the default action map sends exactly `NONE` and `LOW` to WARN. Re-map that and the published number silently changes meaning with no test failing — the method trap this script exists to stay out of. Mirrored in the suite by `tests/test_corpus.py::test_the_published_fp_metric_is_a_risk_statement`. """ elevated = RISK_RANK[Risk.ELEVATED.value] for risk in Risk: benign = DEFAULT_ACTION_MAP[risk] is BENIGN below = RISK_RANK[risk.value] < elevated if benign != below: raise SystemExit( f"metric invalid: {risk.value} maps to " f"{DEFAULT_ACTION_MAP[risk].value}; 'not WARN' no longer means " "'assessed ELEVATED or worse' and the published rate would be " "a different claim than the doc makes" ) def documents(root: Path, exts: tuple[str, ...], include: str = "") -> list[Path]: """Every non-hidden file under ``root`` with a wanted extension. ``include`` is a substring the *relative* path must contain, so a population can be scoped to a subtree (`--include=/references/`) without pretending a differently-scoped count is the same population. Two scopings of one tree are two counts, and the difference between them is exactly the kind of thing `docs/LIMITATIONS.md` has had to correct in public before. """ files = [] for p in sorted(root.rglob("*")): if not p.is_file() or p.suffix not in exts: continue rel = p.relative_to(root) if any(part.startswith(".") for part in rel.parts): continue if include and include not in f"/{rel}": continue files.append(p) return files def measure(label: str, root: Path, exts: tuple[str, ...], include: str = "") -> dict: paths = documents(root, exts, include) dispositions: Counter[str] = Counter() assessments: Counter[str] = Counter() trusted_dispositions: Counter[str] = Counter() labels: Counter[str] = Counter() drivers: Counter[str] = Counter() offenders: list[tuple[str, str, tuple[str, ...]]] = [] empty = 0 for path in paths: text = path.read_text(encoding="utf-8", errors="replace") if not text.strip(): empty += 1 continue result = screen_output(text, PRESET_USER_UPLOAD) dispositions[result.disposition.value] += 1 assessments[result.assessment.value] += 1 trusted_dispositions[ screen_output(text, PRESET_TRUSTED_SOURCE).disposition.value ] += 1 if result.disposition is not BENIGN: findings = scan_output(text, source=Source.OUTPUT).findings found = tuple(sorted({f.label for f in findings})) labels.update(found) # What actually moved this document: the labels at its *worst* # severity. A histogram of every label present would credit the # over-block to whatever else happened to be in the document, which # is how a residual gets blamed on the lexicon. worst = max((severity_rank(f.severity) for f in findings), default=-1) drivers[" + ".join(sorted({ f.label for f in findings if severity_rank(f.severity) == worst })) or "(no findings)"] += 1 offenders.append((path.name, result.disposition.value, found)) n = sum(dispositions.values()) return { "label": label, "n": n, "empty": empty, "dispositions": dispositions, "assessments": assessments, "trusted": trusted_dispositions, "labels": labels, "drivers": drivers, "offenders": offenders, "non_warn": n - dispositions[BENIGN.value], } def report(m: dict) -> None: n, non_warn = m["n"], m["non_warn"] rate = f"{non_warn / n:.1%}" if n else "n/a" print(f"\n## {m['label']} — {non_warn} of {n} documents disposed non-WARN ({rate})") if m["empty"]: print(f" ({m['empty']} empty file(s) skipped — no document, no verdict)") print(" upload gate :", dict(m["dispositions"])) print(" assessment :", dict(m["assessments"])) print(" trusted gate :", dict(m["trusted"]), " <- footnote only, structurally blind") if m["drivers"]: print(" what MOVED them (labels at each document's worst severity):") for label, count in m["drivers"].most_common(): print(f" {count:5d} {label}") if m["labels"]: print(" what was merely present on them:") for label, count in m["labels"].most_common(): print(f" {count:5d} {label}") for name, disposition, found in m["offenders"][:10]: print(f" - {name}: {disposition} via {', '.join(found) or '(no labels)'}") if len(m["offenders"]) > 10: print(f" ... and {len(m['offenders']) - 10} more") def main() -> None: argv = sys.argv[1:] exts = (".md", ".txt") include = "" rest = [] for arg in argv: if arg.startswith("--ext="): exts = tuple(e if e.startswith(".") else f".{e}" for e in arg.split("=", 1)[1].split(",")) elif arg.startswith("--include="): include = arg.split("=", 1)[1] else: rest.append(arg) if not rest: print(__doc__) raise SystemExit(2) check_metric_is_a_risk_statement() print(f"llm-ingestion-guard {__version__} — persist gate, PRESET_USER_UPLOAD") print("metric: disposition is not WARN (== assessed ELEVATED or worse)") print(f"extensions: {', '.join(exts)}") measurements = [] for spec in rest: if "=" not in spec: raise SystemExit(f"expected LABEL=PATH, got {spec!r}") label, _, path = spec.partition("=") root = Path(path).expanduser() if not root.is_dir(): raise SystemExit(f"{label}: {root} is not a directory") measurements.append(measure(label, root, exts, include)) for m in measurements: report(m) print("\n---") print("Populations are reported separately by construction. They have " "different\nprovenance and different denominators; a pooled rate would " "be arithmetic over\nincommensurable things. This script prints no total.") if __name__ == "__main__": main()