test(m1): seed anonymised fixture corpus with hygiene scan

tests/test_fixture_hygiene.py

@@ -0,0 +1,189 @@
+"""The fixture corpus is anonymised, and this test is what makes that a fact
+rather than an intention (plan Step 7).
+
+This repository has a PUBLIC remote. A fixture carrying the operator's real
+name, home address, employer or a real job-board URL would be published the
+moment it is pushed, and no later cleanup un-publishes it. So the corpus is
+scanned on every run, before it grows past the point where anyone reads it
+whole.
+
+The scan reports its denominator, and that is the point rather than a
+courtesy. "Found nothing" is a measurement, and a measurement without a
+denominator is indistinguishable from a scan that ran over an empty
+directory, matched nothing because its patterns were wrong, or never ran at
+all. So every run says how many files and how many bytes it looked at, and
+`test_the_scan_can_actually_find` seeds a canary into a temporary corpus and
+requires the scan to catch it -- a scanner that cannot find is not evidence of
+absence.
+
+Two deliberate deviations from the plan text, both stated rather than
+smuggled:
+
+1. The operator's home address is not hard-coded here. Writing a real home
+   address into a deny list on a public remote would publish the very thing
+   the list exists to keep out. Private tokens go in the environment variable
+   `JOBBSOK_HYGIENE_DENY_EXTRA` (newline-separated) instead, and the scan
+   picks them up if it is set. The name and employer ARE listed literally,
+   because both already appear in every commit's author metadata on the same
+   public remote -- listing them costs nothing that is not already public and
+   buys a check that actually runs.
+2. The provenance-header rule covers the directories this step seeds.
+   `cowork-probe/` is exempt by name: it is the Step 1 probe rig, machine
+   config with no person in it, and it predates this rule.
+
+Style note: this file follows tests/test_helpers_selfcheck.py.
+"""
+
+import os
+import re
+import warnings
+
+FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures")
+
+#: Directories seeded by this step, and the ones the provenance rule governs.
+CORPUS_DIRS = ("profiles", "listings")
+
+#: Exempt from the provenance rule, with the reason in the module docstring.
+PROVENANCE_EXEMPT = ("cowork-probe",)
+
+#: Literal tokens that must never appear. Real employer names and real job
+#: boards; the person tokens are already public in the commit metadata.
+DENY_TOKENS = (
+    "Kjell Tore",
+    "Guttormsen",
+    "Statens vegvesen",
+    "vegvesen",
+    "finn.no",
+    "arbeidsplassen.nav.no",
+    "jobbnorge.no",
+    "linkedin.com",
+    "indeed.com",
+)
+
+#: Extra deny tokens supplied at run time, one per line. This is where a home
+#: address belongs -- checked, never committed.
+DENY_EXTRA_ENV = "JOBBSOK_HYGIENE_DENY_EXTRA"
+
+#: Every host and mail domain in the corpus must sit under the reserved
+#: `.example` TLD (RFC 2606), so nothing in here can resolve to a real place.
+SAFE_TLD = ".example"
+
+EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})")
+URL_HOST_RE = re.compile(r"https?://([A-Za-z0-9.:-]+)")
+
+PROVENANCE_MARKER = "Proveniens:"
+
+
+class KorpusSkannet(UserWarning):
+    """Not a defect: the denominator the hygiene scan actually looked at.
+
+    Emitted as a warning because pytest shows warnings even under ``-q``,
+    while a passing test's stdout is captured and never reaches the operator.
+    """
+
+
+def deny_tokens():
+    tokens = list(DENY_TOKENS)
+    extra = os.environ.get(DENY_EXTRA_ENV, "")
+    tokens.extend(line.strip() for line in extra.splitlines() if line.strip())
+    return tokens
+
+
+def corpus_files(root):
+    found = []
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames.sort()
+        for name in sorted(filenames):
+            found.append(os.path.join(dirpath, name))
+    return found
+
+
+def scan(root):
+    """Scan every file under ``root`` and report findings AND the denominator."""
+    tokens = deny_tokens()
+    findings = []
+    files = corpus_files(root)
+    total_bytes = 0
+
+    for path in files:
+        with open(path, "r", encoding="utf-8") as handle:
+            text = handle.read()
+        total_bytes += len(text.encode("utf-8"))
+        relative = os.path.relpath(path, root)
+
+        lowered = text.lower()
+        for token in tokens:
+            if token.lower() in lowered:
+                findings.append((relative, "deny-token %r" % token))
+        for domain in EMAIL_RE.findall(text):
+            if not domain.lower().endswith(SAFE_TLD):
+                findings.append((relative, "mail domain %r is not %s" % (domain, SAFE_TLD)))
+        for host in URL_HOST_RE.findall(text):
+            if not host.lower().split(":")[0].endswith(SAFE_TLD):
+                findings.append((relative, "url host %r is not %s" % (host, SAFE_TLD)))
+
+    return {"files": len(files), "bytes": total_bytes, "findings": findings}
+
+
+def report(result, root):
+    message = "hygiene scan: %d files, %d bytes scanned under %s; %d finding(s)" % (
+        result["files"],
+        result["bytes"],
+        os.path.basename(root.rstrip(os.sep)),
+        len(result["findings"]),
+    )
+    print(message)
+    warnings.warn(message, KorpusSkannet)
+    return message
+
+
+def test_every_corpus_fixture_carries_a_provenance_header():
+    checked = 0
+    for name in CORPUS_DIRS:
+        directory = os.path.join(FIXTURES, name)
+        assert os.path.isdir(directory), "corpus directory %r is missing" % name
+        files = corpus_files(directory)
+        assert files, "corpus directory %r is empty" % name
+        for path in files:
+            with open(path, "r", encoding="utf-8") as handle:
+                text = handle.read()
+            assert PROVENANCE_MARKER in text, "%s has no %r header" % (path, PROVENANCE_MARKER)
+            checked += 1
+    # Eight profiles and eight listings, per the plan. A rule that checked
+    # zero files would pass just as quietly.
+    assert checked == 16, "expected 16 corpus fixtures, provenance-checked %d" % checked
+
+
+def test_no_deny_listed_token_appears_anywhere_in_the_corpus():
+    result = scan(FIXTURES)
+    assert result["findings"] == [], "hygiene findings: %r" % (result["findings"],)
+
+
+def test_the_scan_states_the_denominator_it_looked_at():
+    result = scan(FIXTURES)
+    message = report(result, FIXTURES)
+    assert result["files"] >= 16
+    assert result["bytes"] > 0
+    assert str(result["files"]) in message and "bytes scanned" in message
+
+
+def test_the_scan_can_actually_find(tmp_path):
+    # A canary corpus, deliberately dirty. It lives in tmp_path and never in
+    # tests/fixtures, because a canary committed to a public remote is the
+    # leak it is supposed to warn about.
+    (tmp_path / "cv.md").write_text(
+        "Proveniens: kanarifugl.\nTidligere hos Statens vegvesen.\n", encoding="utf-8"
+    )
+    (tmp_path / "kontakt.md").write_text(
+        "Proveniens: kanarifugl.\nSkriv til rekruttering@fjordtek.no\n", encoding="utf-8"
+    )
+    (tmp_path / "lenke.md").write_text(
+        "Proveniens: kanarifugl.\nSe https://www.finn.no/job/123\n", encoding="utf-8"
+    )
+
+    result = scan(str(tmp_path))
+    reasons = " ".join(reason for _path, reason in result["findings"])
+    assert result["files"] == 3
+    assert "vegvesen" in reasons
+    assert "fjordtek.no" in reasons
+    assert "finn.no" in reasons